17801 contents -r should return only highest-ranked match by default
authorShawn Walker <shawn.walker@oracle.com>
Fri, 09 Sep 2011 09:48:43 -0700
changeset 2547 032eaff05522
parent 2546 a9487a95b5cc
child 2548 5f4b99b9ae56
17801 contents -r should return only highest-ranked match by default 17804 info -r should return only highest-ranked match by default
doc/client_api_versions.txt
src/client.py
src/modules/catalog.py
src/modules/client/api.py
src/modules/gui/misc_non_gui.py
src/modules/lint/engine.py
src/pkgdep.py
src/sysrepo.py
src/tests/cli/t_pkg_contents.py
src/tests/cli/t_pkg_info.py
src/tests/pkg5unittest.py
src/util/distro-import/importer.py
--- a/doc/client_api_versions.txt	Thu Sep 08 17:50:27 2011 -0700
+++ b/doc/client_api_versions.txt	Fri Sep 09 09:48:43 2011 -0700
@@ -1,3 +1,12 @@
+Version 69:
+Compatible with clients using versions 66-68.
+
+    pkg.client.api.ImageInterface has changed as follows:
+        * get_pkg_list() and info() now have an optional boolean
+          parameter named 'ranked' that controls whether only
+          packages for the highest-ranked matching publisher should
+          be returned. See 'pydoc pkg.client.api' for details.
+
 Version 68:
 Compatible with clients using versions 66-67.
 
--- a/src/client.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/client.py	Fri Sep 09 09:48:43 2011 -0700
@@ -87,7 +87,7 @@
         import sys
         sys.exit(1)
 
-CLIENT_API_VERSION = 68
+CLIENT_API_VERSION = 69
 PKG_CLIENT_NAME = "pkg"
 
 JUST_UNKNOWN = 0
@@ -3113,7 +3113,7 @@
 
         try:
                 ret = api_inst.info(pargs, info_local, info_needed,
-                    repos=origins)
+                    ranked=info_remote, repos=origins)
         except api_errors.ImageFormatUpdateNeeded, e:
                 format_update_error(e)
                 return EXIT_OOPS
@@ -3618,8 +3618,8 @@
         notfound = EmptyI
         try:
                 res = api_inst.get_pkg_list(pkg_list, patterns=pargs,
-                    raise_unmatched=True, return_fmris=True, variants=True,
-                    repos=origins)
+                    raise_unmatched=True, ranked=remote, return_fmris=True,
+                    variants=True, repos=origins)
                 manifests = []
 
                 for pfmri, summ, cats, states, pattrs in res:
--- a/src/modules/catalog.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/modules/catalog.py	Fri Sep 09 09:48:43 2011 -0700
@@ -703,7 +703,8 @@
                 (pub, stem) as it iterates over the contents of the CatalogPart.
 
                 'pubs' is an optional list of publisher prefixes to restrict
-                the results to.
+                the results to.  If specified, publishers will be sorted in
+                the order given.
 
                 Results are always returned sorted by stem and then by
                 publisher."""
@@ -718,7 +719,19 @@
                         for stem in self.__data[pub]
                 ]
 
-                for entry in sorted(pkg_list):
+                pub_sort = None
+                if pubs:
+                        pos = dict((p, i) for (i, p) in enumerate(pubs))
+                        def pos_sort(a, b):
+                                astem, apub = a.split("!", 1)
+                                bstem, bpub = b.split("!", 1)
+                                res = cmp(astem, bstem)
+                                if res != 0:
+                                        return res
+                                return cmp(pos[apub], pos[bpub])
+                        pub_sort = pos_sort
+
+                for entry in sorted(pkg_list, cmp=pub_sort):
                         stem, pub = entry.split("!", 1)
                         yield pub, stem
 
--- a/src/modules/client/api.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/modules/client/api.py	Fri Sep 09 09:48:43 2011 -0700
@@ -77,8 +77,8 @@
 from pkg.client.pkgdefs import *
 from pkg.smf import NonzeroExitException
 
-CURRENT_API_VERSION = 68
-COMPATIBLE_API_VERSIONS = frozenset([66, 67, CURRENT_API_VERSION])
+CURRENT_API_VERSION = 69
+COMPATIBLE_API_VERSIONS = frozenset([66, 67, 68, CURRENT_API_VERSION])
 CURRENT_P5I_VERSION = 1
 
 # Image type constants.
@@ -2926,7 +2926,7 @@
         @_LockedGenerator()
         def get_pkg_list(self, pkg_list, cats=None, collect_attrs=False,
             patterns=misc.EmptyI, pubs=misc.EmptyI, raise_unmatched=False,
-            repos=None, return_fmris=False, variants=False):
+            ranked=False, repos=None, return_fmris=False, variants=False):
                 """A generator function that produces tuples of the form:
 
                     (
@@ -2991,6 +2991,11 @@
                 (after applying all other filtering and returning all results)
                 didn't match any packages.
 
+                'ranked' is an optional boolean value that indicates whether
+                only the matching package versions from the highest-ranked
+                publisher should be returned.  This option is ignored for
+                patterns that explicitly specify the publisher to match.
+
                 'repos' is a list of URI strings or RepositoryURI objects that
                 represent the locations of package repositories to list packages
                 for.
@@ -3008,12 +3013,12 @@
 
                 return self.__get_pkg_list(pkg_list, cats=cats,
                     collect_attrs=collect_attrs, patterns=patterns, pubs=pubs,
-                    raise_unmatched=raise_unmatched, repos=repos,
+                    raise_unmatched=raise_unmatched, ranked=ranked, repos=repos,
                     return_fmris=return_fmris, variants=variants)
 
         def __get_pkg_list(self, pkg_list, cats=None, collect_attrs=False,
             inst_cat=None, known_cat=None, patterns=misc.EmptyI,
-            pubs=misc.EmptyI, raise_unmatched=False, repos=None,
+            pubs=misc.EmptyI, raise_unmatched=False, ranked=False, repos=None,
             return_fmris=False, variants=False):
                 """This is the implementation of get_pkg_list.  The other
                 function is a wrapper that uses locking.  The separation was
@@ -3205,6 +3210,30 @@
                 # to be filtered.)
                 use_last = newest and not pat_versioned and variants
 
+                if ranked:
+                        # If caller requested results to be ranked by publisher,
+                        # then the list of publishers to return must be passed
+                        # to entry_actions() in rank order.
+                        pub_ranks = self._img.get_publisher_ranks()
+                        if not pubs:
+                                # It's important that the list of possible
+                                # publishers is gleaned from the catalog
+                                # directly and not image configuration so
+                                # that temporary sources (archives, etc.)
+                                # work as expected.
+                                pubs = pkg_cat.publishers()
+                        for p in pubs:
+                                pub_ranks.setdefault(p, (99, (p, False, False)))
+
+                        def pub_order(a, b):
+                                res = cmp(pub_ranks[a], pub_ranks[b])
+                                if res != 0:
+                                        return res
+                                return cmp(a, b)
+
+                        pubs = sorted(pubs, cmp=pub_order)
+
+                ranked_stems = {}
                 for t, entry, actions in pkg_cat.entry_actions(cat_info,
                     cb=filter_cb, excludes=excludes, last=use_last,
                     ordered=True, pubs=pubs):
@@ -3219,6 +3248,15 @@
                                 # any additional entries need to be marked for
                                 # omission before continuing.
                                 omit_package = True
+                        elif ranked and not patterns and \
+                            ranked_stems.get(stem, pub) != pub:
+                                # A different version from a higher-ranked
+                                # publisher has been returned already, so skip
+                                # this one.  This can only be done safely at
+                                # this point if no patterns have been specified,
+                                # since publisher-specific patterns override
+                                # ranking behaviour.
+                                omit_package = True
                         else:
                                 nlist[pkg_stem] += 1
 
@@ -3236,6 +3274,16 @@
                                                 if omit_package is None:
                                                         omit_package = True
                                                 continue
+                                        elif ranked and not pat_pub and \
+                                            ranked_stems.get(stem, pub) != pub:
+                                                # A different version from a
+                                                # higher-ranked publisher has
+                                                # been returned already, so skip
+                                                # this one since no publisher
+                                                # was specified for the pattern.
+                                                if omit_package is None:
+                                                        omit_package = True
+                                                continue
 
                                         if matcher == self.MATCH_EXACT:
                                                 if pat_stem != stem:
@@ -3463,6 +3511,11 @@
                                 # applied are the patterns that the package
                                 # matched considered "matching".
                                 matched_pats.update(pkg_matching_pats)
+                        if ranked:
+                                # Only after all other filtering has been
+                                # applied is the stem considered to have been
+                                # a "ranked" match.
+                                ranked_stems.setdefault(stem, pub)
 
                         if return_fmris:
                                 pfmri = fmri.PkgFmri("%s@%s" % (stem, ver),
@@ -3480,7 +3533,8 @@
                                 raise apx.InventoryException(notfound=notfound)
 
         @_LockedCancelable()
-        def info(self, fmri_strings, local, info_needed, repos=None):
+        def info(self, fmri_strings, local, info_needed, ranked=False,
+            repos=None):
                 """Gathers information about fmris.  fmri_strings is a list
                 of fmri_names for which information is desired.  local
                 determines whether to retrieve the information locally
@@ -3489,6 +3543,11 @@
                 definition.  The values are lists of PackageInfo objects or
                 strings.
 
+                'ranked' is an optional boolean value that indicates whether
+                only the matching package versions from the highest-ranked
+                publisher should be returned.  This option is ignored for
+                patterns that explicitly specify the publisher to match.
+
                 'repos' is a list of URI strings or RepositoryURI objects that
                 represent the locations of packages to return information for.
                 """
@@ -3556,7 +3615,7 @@
                             ilist, collect_attrs=collect_attrs,
                             inst_cat=inst_cat, known_cat=known_cat,
                             patterns=fmri_strings, raise_unmatched=True,
-                            return_fmris=True, variants=True):
+                            ranked=ranked, return_fmris=True, variants=True):
                                 release = build_release = branch = \
                                     packaging_date = None
 
--- a/src/modules/gui/misc_non_gui.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/modules/gui/misc_non_gui.py	Fri Sep 09 09:48:43 2011 -0700
@@ -41,7 +41,7 @@
 
 # The current version of the Client API the PM, UM and
 # WebInstall GUIs have been tested against and are known to work with.
-CLIENT_API_VERSION = 68
+CLIENT_API_VERSION = 69
 LOG_DIR = "/var/tmp"
 LOG_ERROR_EXT = "_error.log"
 LOG_INFO_EXT = "_info.log"
--- a/src/modules/lint/engine.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/modules/lint/engine.py	Fri Sep 09 09:48:43 2011 -0700
@@ -39,7 +39,7 @@
 import sys
 
 PKG_CLIENT_NAME = "pkglint"
-CLIENT_API_VERSION = 68
+CLIENT_API_VERSION = 69
 pkg.client.global_settings.client_name = PKG_CLIENT_NAME
 
 class LintEngineException(Exception):
--- a/src/pkgdep.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/pkgdep.py	Fri Sep 09 09:48:43 2011 -0700
@@ -42,7 +42,7 @@
 import pkg.publish.dependencies as dependencies
 from pkg.misc import msg, emsg, PipeError
 
-CLIENT_API_VERSION = 68
+CLIENT_API_VERSION = 69
 PKG_CLIENT_NAME = "pkgdepend"
 
 DEFAULT_SUFFIX = ".res"
--- a/src/sysrepo.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/sysrepo.py	Fri Sep 09 09:48:43 2011 -0700
@@ -53,7 +53,7 @@
 orig_cwd = None
 
 PKG_CLIENT_NAME = "pkg.sysrepo"
-CLIENT_API_VERSION = 68
+CLIENT_API_VERSION = 69
 pkg.client.global_settings.client_name = PKG_CLIENT_NAME
 
 # exit codes
--- a/src/tests/cli/t_pkg_contents.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/tests/cli/t_pkg_contents.py	Fri Sep 09 09:48:43 2011 -0700
@@ -214,6 +214,66 @@
 
                 self.assertEqualDiff(expected_res, self.output)
 
+        def test_ranked(self):
+                """Verify that pkg contents -r returns expected results
+                when multiple publishers provide the same package based
+                on publisher search order."""
+
+                # Create an isolated repository for this test
+                repodir = os.path.join(self.test_root, "test-ranked")
+                self.create_repo(repodir)
+                self.pkgrepo("add-publisher -s %s test" % repodir)
+                self.pkgsend_bulk(repodir, self.bronze10)
+
+                self.pkgrepo("add-publisher -s %s test2" % repodir)
+                self.pkgrepo("set -s %s publisher/prefix=test2" % repodir)
+                self.pkgsend_bulk(repodir, self.bronze10)
+
+                self.pkgrepo("add-publisher -s %s test3" % repodir)
+                self.pkgrepo("set -s %s publisher/prefix=test3" % repodir)
+                self.pkgsend_bulk(repodir, self.bronze10)
+
+                # Create a test image.
+                self.image_create()
+                self.pkg("set-publisher -p %s" % repodir)
+
+                # Test should be higher ranked than test2 since the default
+                # for auto-configuration is to use lexical order when
+                # multiple publishers are found.  As such, info -r should
+                # return results for 'test' by default.
+                self.pkg("contents -H -r -t set -o pkg.fmri bronze")
+                self.assert_(self.output.startswith("pkg://test/bronze"))
+                self.assert_("pkg://test2/bronze" not in self.output)
+                self.assert_("pkg://test3/bronze" not in self.output)
+
+                # Verify that if the publisher is specified, that is preferred
+                # over rank.
+                self.pkg("contents -H -r -t set -o pkg.fmri //test2/bronze")
+                self.assert_("pkg://test/bronze" not in self.output)
+                self.assert_(self.output.startswith("pkg://test2/bronze"))
+                self.assert_("pkg://test3/bronze" not in self.output)
+
+                # Verify that if stem is specified with and without publisher,
+                # both matches are listed if the higher-ranked publisher differs
+                # from the publisher specified.
+                self.pkg("contents -H -r -t set -o pkg.fmri //test/bronze "
+                    "bronze")
+                self.assert_(self.output.startswith("pkg://test/bronze"))
+                self.assert_("pkg://test2/bronze" not in self.output)
+                self.assert_("pkg://test3/bronze" not in self.output)
+
+                self.pkg("contents -H -r -t set -o pkg.fmri //test2/bronze "
+                    "bronze")
+                self.assert_(self.output.startswith("pkg://test/bronze"))
+                self.assert_("pkg://test2/bronze" in self.output)
+                self.assert_("pkg://test3/bronze" not in self.output)
+
+                self.pkg("contents -H -r -t set -o pkg.fmri //test3/bronze "
+                    "//test2/bronze bronze")
+                self.assert_(self.output.startswith("pkg://test/bronze"))
+                self.assert_("pkg://test2/bronze" in self.output)
+                self.assert_("pkg://test3/bronze" in self.output)
+
 
 if __name__ == "__main__":
         unittest.main()
--- a/src/tests/cli/t_pkg_info.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/tests/cli/t_pkg_info.py	Fri Sep 09 09:48:43 2011 -0700
@@ -38,7 +38,7 @@
         persistent_setup = True
 
         bronze10 = """
-            open [email protected],5.11-0
+            open [email protected],5.11-0:20110908T004546Z
             add dir mode=0755 owner=root group=bin path=/usr
             add dir mode=0755 owner=root group=bin path=/usr/bin
             add file tmp/sh mode=0555 owner=root group=bin path=/usr/bin/sh
@@ -63,7 +63,7 @@
         """
 
         human = """
-            open [email protected],5.11-0
+            open [email protected],5.11-0:20110908T004546Z
             add set name=pkg.human-version value=0.9.8r
             close
         """
@@ -270,6 +270,157 @@
                 self.image_create(self.rurl)
                 self.pkg("info -r human | grep 'Version: 0.9.8.18 (0.9.8r)'")
 
+        def test_ranked(self):
+                """Verify that pkg info -r returns expected results when
+                multiple publishers provide the same package based on
+                publisher search order."""
+
+                # Create an isolated repository for this test
+                repodir = os.path.join(self.test_root, "test-ranked")
+                self.create_repo(repodir)
+                self.pkgrepo("add-publisher -s %s test" % repodir)
+                self.pkgsend_bulk(repodir, (self.bronze10, self.human))
+
+                self.pkgrepo("add-publisher -s %s test2" % repodir)
+                self.pkgrepo("set -s %s publisher/prefix=test2" % repodir)
+                self.pkgsend_bulk(repodir, self.bronze10)
+
+                self.pkgrepo("add-publisher -s %s test3" % repodir)
+                self.pkgrepo("set -s %s publisher/prefix=test3" % repodir)
+                self.pkgsend_bulk(repodir, self.bronze10)
+
+                # Create a test image.
+                self.image_create()
+                self.pkg("set-publisher -p %s" % repodir)
+
+                # Test should be higher ranked than test2 since the default
+                # for auto-configuration is to use lexical order when
+                # multiple publishers are found.  As such, info -r should
+                # return results for 'test' by default.
+                self.pkg("info -r bronze human")
+                expected = """\
+ Name: bronze
+ Summary: 
+ State: Not installed
+ Publisher: test
+ Version: 1.0
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 54.00 B
+ FMRI: pkg://test/[email protected],5.11-0:20110908T004546Z
+
+ Name: human
+ Summary: 
+ State: Not installed
+ Publisher: test
+ Version: 0.9.8.18 (0.9.8r)
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 0.00 B
+ FMRI: pkg://test/[email protected],5.11-0:20110908T004546Z
+"""
+                self.assertEqualDiff(expected, self.reduceSpaces(self.output))
+
+                # Verify that if the publisher is specified, that is preferred
+                # over rank.
+                self.pkg("info -r //test2/bronze")
+                expected = """\
+ Name: bronze
+ Summary: 
+ State: Not installed
+ Publisher: test2
+ Version: 1.0
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 54.00 B
+ FMRI: pkg://test2/[email protected],5.11-0:20110908T004546Z
+"""
+                self.assertEqualDiff(expected, self.reduceSpaces(self.output))
+
+                # Verify that if stem is specified with and without publisher,
+                # both matches are listed if the higher-ranked publisher differs
+                # from the publisher specified.
+                self.pkg("info -r //test/bronze bronze")
+                expected = """\
+ Name: bronze
+ Summary: 
+ State: Not installed
+ Publisher: test
+ Version: 1.0
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 54.00 B
+ FMRI: pkg://test/[email protected],5.11-0:20110908T004546Z
+"""
+                self.assertEqualDiff(expected, self.reduceSpaces(self.output))
+
+                self.pkg("info -r //test2/bronze bronze")
+                expected = """\
+ Name: bronze
+ Summary: 
+ State: Not installed
+ Publisher: test
+ Version: 1.0
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 54.00 B
+ FMRI: pkg://test/[email protected],5.11-0:20110908T004546Z
+
+ Name: bronze
+ Summary: 
+ State: Not installed
+ Publisher: test2
+ Version: 1.0
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 54.00 B
+ FMRI: pkg://test2/[email protected],5.11-0:20110908T004546Z
+"""
+                self.assertEqualDiff(expected, self.reduceSpaces(self.output))
+
+                self.pkg("info -r //test3/bronze //test2/bronze bronze")
+                expected = """\
+ Name: bronze
+ Summary: 
+ State: Not installed
+ Publisher: test
+ Version: 1.0
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 54.00 B
+ FMRI: pkg://test/[email protected],5.11-0:20110908T004546Z
+
+ Name: bronze
+ Summary: 
+ State: Not installed
+ Publisher: test2
+ Version: 1.0
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 54.00 B
+ FMRI: pkg://test2/[email protected],5.11-0:20110908T004546Z
+
+ Name: bronze
+ Summary: 
+ State: Not installed
+ Publisher: test3
+ Version: 1.0
+ Build Release: 5.11
+ Branch: 0
+Packaging Date: Thu Sep 08 00:45:46 2011
+ Size: 54.00 B
+ FMRI: pkg://test3/[email protected],5.11-0:20110908T004546Z
+"""
+                self.assertEqualDiff(expected, self.reduceSpaces(self.output))
+
         def test_renamed_packages(self):
                 """Verify that info returns the expected output for renamed
                 packages."""
--- a/src/tests/pkg5unittest.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/tests/pkg5unittest.py	Fri Sep 09 09:48:43 2011 -0700
@@ -126,7 +126,7 @@
 
 # Version test suite is known to work with.
 PKG_CLIENT_NAME = "pkg"
-CLIENT_API_VERSION = 68
+CLIENT_API_VERSION = 69
 
 ELIDABLE_ERRORS = [ TestSkippedException, depotcontroller.DepotStateException ]
 
--- a/src/util/distro-import/importer.py	Thu Sep 08 17:50:27 2011 -0700
+++ b/src/util/distro-import/importer.py	Fri Sep 09 09:48:43 2011 -0700
@@ -56,7 +56,7 @@
 from pkg.misc import emsg
 from pkg.portable import PD_LOCAL_PATH, PD_PROTO_DIR, PD_PROTO_DIR_LIST
 
-CLIENT_API_VERSION = 68
+CLIENT_API_VERSION = 69
 PKG_CLIENT_NAME = "importer.py"
 pkg.client.global_settings.client_name = PKG_CLIENT_NAME