18105 api should support multiple repositories (origins) with different package data
18107 pkg search returns no results and fails if a publisher with no origins is present
--- a/src/modules/catalog.py Thu May 12 16:55:35 2011 -0700
+++ b/src/modules/catalog.py Thu May 12 19:11:16 2011 -0700
@@ -3387,27 +3387,44 @@
return self._attrs.updates
- def update_entry(self, pfmri, metadata):
+ def update_entry(self, metadata, pfmri=None, pub=None, stem=None,
+ ver=None):
"""Updates the metadata stored in a package's BASE catalog
- record for the specified FMRI. Cannot be used when read_only
+ record for the specified package. Cannot be used when read_only
or log_updates is enabled; should never be used with a Catalog
intended for incremental update usage.
+ 'metadata' must be a dict of additional metadata to store with
+ the package's BASE record.
+
'pfmri' is the FMRI of the package to update the entry for.
- 'metadata' must be a dict of additional metadata to store with
- the package's BASE record."""
-
+ 'pub' is the publisher of the package.
+
+ 'stem' is the stem of the package.
+
+ 'ver' is the version string of the package.
+
+ 'pfmri' or 'pub', 'stem', and 'ver' must be provided.
+ """
+
+ assert pfmri or (pub and stem and ver)
assert not self.log_updates and not self.read_only
base = self.get_part(self.__BASE_PART, must_exist=True)
if base is None:
+ if not pfmri:
+ pfmri = fmri.PkgFmri("%s@%s" % (stem, ver),
+ publisher=pub)
raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
# get_entry returns the actual catalog entry, so updating it
# simply requires reassignment.
- entry = base.get_entry(pfmri)
+ entry = base.get_entry(pfmri=pfmri, pub=pub, stem=stem, ver=ver)
if entry is None:
+ if not pfmri:
+ pfmri = fmri.PkgFmri("%s@%s" % (stem, ver),
+ publisher=pub)
raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
if metadata is None:
if "metadata" in entry:
--- a/src/modules/client/api.py Thu May 12 16:55:35 2011 -0700
+++ b/src/modules/client/api.py Thu May 12 19:11:16 2011 -0700
@@ -3518,28 +3518,39 @@
incorp_info, inst_stems = self.get_incorp_info()
- for pub in servers:
+ slist = []
+ for entry in servers:
descriptive_name = None
-
- if self.__canceling:
- raise apx.CanceledException()
-
- if isinstance(pub, dict):
- origin = pub["origin"]
+ if isinstance(entry, dict):
+ origin = entry["origin"]
try:
pub = self._img.get_publisher(
origin=origin)
except apx.UnknownPublisher:
pub = publisher.RepositoryURI(origin)
descriptive_name = origin
-
- if not descriptive_name:
- descriptive_name = pub.prefix
+ slist.append((pub, None, descriptive_name))
+ continue
+
+ # Must be a publisher object.
+ osets = entry.get_origin_sets()
+ if not osets:
+ unsupported.append((entry.prefix,
+ apx.NoPublisherRepositories(
+ entry.prefix)))
+ continue
+ for repo in osets:
+ slist.append((entry, repo, entry.prefix))
+
+ for pub, alt_repo, descriptive_name in slist:
+ if self.__canceling:
+ raise apx.CanceledException()
try:
res = self._img.transport.do_search(pub,
query_str_and_args_lst,
- ccancel=self.__check_cancel)
+ ccancel=self.__check_cancel,
+ alt_repo=alt_repo)
except apx.CanceledException:
raise
except apx.NegativeSearchResult:
--- a/src/modules/client/image.py Thu May 12 16:55:35 2011 -0700
+++ b/src/modules/client/image.py Thu May 12 19:11:16 2011 -0700
@@ -2336,7 +2336,7 @@
mdata["states"] = list(states)
# Now record the package state.
- kcat.update_entry(pfmri, metadata=mdata)
+ kcat.update_entry(mdata, pfmri=pfmri)
# If the package is being marked as installed,
# then it shouldn't already exist in the
@@ -2575,6 +2575,44 @@
return True
return False
+ def get_pkg_repo(self, pfmri):
+ """Returns the repository object containing the origins that
+ should be used to retrieve the specified package or None if
+ it can be retrieved from all sources or is not a known package.
+ """
+
+ assert pfmri.publisher
+ cat = self.get_catalog(self.IMG_CATALOG_KNOWN)
+ entry = cat.get_entry(pfmri)
+ if entry is None:
+ # Package not known.
+ return
+
+ try:
+ slist = entry["metadata"]["sources"]
+ except KeyError:
+ # Can be retrieved from any source.
+ return
+ else:
+ if not slist:
+ # Can be retrieved from any source.
+ return
+
+ pub = self.get_publisher(prefix=pfmri.publisher)
+ repo = copy.copy(pub.repository)
+ norigins = [
+ o for o in repo.origins
+ if o.uri in slist
+ ]
+
+ if not norigins:
+ # Known sources don't match configured; return so that
+ # caller can fallback to default behaviour.
+ return
+
+ repo.origins = norigins
+ return repo
+
def get_pkg_state(self, pfmri):
"""Returns the list of states a package is in for this image."""
@@ -2776,11 +2814,13 @@
# Only the base catalog part stores package
# state information and/or other metadata.
- mdata = entry["metadata"] = {}
- states = [self.PKG_STATE_KNOWN]
+ mdata = entry.setdefault("metadata", {})
+ states = mdata.setdefault("states", [])
+ states.append(self.PKG_STATE_KNOWN)
+
if cat_ver == 0:
states.append(self.PKG_STATE_V0)
- else:
+ elif self.PKG_STATE_V0 not in states:
# Assume V1 catalog source.
states.append(self.PKG_STATE_V1)
--- a/src/modules/client/publisher.py Thu May 12 16:55:35 2011 -0700
+++ b/src/modules/client/publisher.py Thu May 12 19:11:16 2011 -0700
@@ -35,6 +35,7 @@
#
import calendar
+import collections
import copy
import cStringIO
import datetime as dt
@@ -802,6 +803,7 @@
__client_uuid = None
__disabled = False
__meta_root = None
+ __origin_root = None
__prefix = None
__repository = None
__sticky = True
@@ -1049,6 +1051,8 @@
if self.__catalog:
self.__catalog.meta_root = self.catalog_root
if self.__meta_root:
+ self.__origin_root = os.path.join(self.__meta_root,
+ "origins")
self.cert_root = os.path.join(self.__meta_root, "certs")
self.__subj_root = os.path.join(self.cert_root,
"subject_hashes")
@@ -1077,12 +1081,12 @@
def __str__(self):
return self.prefix
- def __validate_metadata(self):
+ def __validate_metadata(self, croot, repo):
"""Private helper function to check the publisher's metadata
for configuration or other issues and log appropriate warnings
or errors. Currently only checks catalog metadata."""
- c = self.catalog
+ c = pkg.catalog.Catalog(meta_root=croot, read_only=True)
if not c.exists:
# Nothing to validate.
return
@@ -1096,10 +1100,10 @@
# XXX For now, perform this check using the catalog data.
# In the future, it should be done using the output of the
# publisher/0 operation.
- pubs = self.catalog.publishers()
+ pubs = c.publishers()
if self.prefix not in pubs:
- origins = self.repository.origins
+ origins = repo.origins
origin = origins[0]
logger.error(_("""
Unable to retrieve package data for publisher '%(prefix)s' from one
@@ -1210,7 +1214,8 @@
# Otherwise, raise the exception.
raise
# Optional roots not needed for all operations.
- for path in (self.cert_root, self.__subj_root, self.__crl_root):
+ for path in (self.cert_root, self.__origin_root,
+ self.__subj_root, self.__crl_root):
try:
os.makedirs(path)
except EnvironmentError, e:
@@ -1221,6 +1226,43 @@
# Otherwise, raise the exception.
raise
+ def get_origin_sets(self):
+ """Returns a list of Repository objects representing the unique
+ groups of origins available. Each group is based on the origins
+ that share identical package catalog data."""
+
+ if not self.repository or not self.repository.origins:
+ # Guard against failure for publishers with no
+ # transport information.
+ return []
+
+ if not self.meta_root or not os.path.exists(self.__origin_root):
+ # No way to identify unique sets.
+ return [self.repository]
+
+ # Index origins by tuple of (catalog creation, catalog modified)
+ osets = collections.defaultdict(list)
+
+ for origin, opath in self.__gen_origin_paths():
+ cat = pkg.catalog.Catalog(meta_root=opath,
+ read_only=True)
+ if not cat.exists:
+ key = None
+ else:
+ key = (str(cat.created), str(cat.last_modified))
+ osets[key].append(origin)
+
+ # Now return a list of Repository objects (copies of the
+ # currently selected one) assigning each set of origins.
+ # Sort by index to ensure consistent ordering.
+ rval = []
+ for k in sorted(osets):
+ nrepo = copy.copy(self.repository)
+ nrepo.origins = osets[k]
+ rval.append(nrepo)
+
+ return rval
+
def has_configuration(self):
"""Returns whether this publisher has any configuration which
should prevent its removal."""
@@ -1228,7 +1270,7 @@
return bool(self.__repository.origins or
self.__repository.mirrors or self.__sig_policy or
self.approved_ca_certs or self.revoked_ca_certs)
-
+
@property
def needs_refresh(self):
"""A boolean value indicating whether the publisher's
@@ -1263,7 +1305,195 @@
return True
return False
- def __convert_v0_catalog(self, v0_cat):
+ def __get_origin_path(self, origin):
+ if not os.path.exists(self.__origin_root):
+ return
+ # A digest of the URI string is used here to attempt to avoid
+ # path length problems.
+ return os.path.join(self.__origin_root,
+ hashlib.sha1(origin.uri).hexdigest())
+
+ def __gen_origin_paths(self):
+ if not os.path.exists(self.__origin_root):
+ return
+ for origin in self.repository.origins:
+ yield origin, self.__get_origin_path(origin)
+
+ def __rebuild_catalog(self):
+ """Private helper function that builds publisher catalog based
+ on catalog from each origin."""
+
+ # First, remove catalogs for any origins that no longer exist.
+ ohashes = [
+ hashlib.sha1(o.uri).hexdigest()
+ for o in self.repository.origins
+ ]
+
+ for entry in os.listdir(self.__origin_root):
+ opath = os.path.join(self.__origin_root, entry)
+ try:
+ if entry in ohashes:
+ continue
+ except Exception:
+ # Discard anything that isn't an origin.
+ pass
+
+ # Not an origin or origin no longer exists; either way,
+ # it shouldn't exist here.
+ try:
+ if os.path.isdir(opath):
+ shutil.rmtree(opath)
+ else:
+ portable.remove(opath)
+ except EnvironmentError, e:
+ raise api_errors._convert_error(e)
+
+ # Discard existing catalog.
+ self.catalog.destroy()
+ self.__catalog = None
+
+ # Ensure all old catalog files are removed.
+ for entry in os.listdir(self.catalog_root):
+ if entry == "attrs" or entry == "catalog" or \
+ entry.startswith("catalog."):
+ try:
+ portable.remove(os.path.join(
+ self.catalog_root, entry))
+ except EnvironmentError, e:
+ raise apx._convert_error(e)
+
+ # If there's only one origin, then just symlink its catalog
+ # files into place.
+ opaths = [entry for entry in self.__gen_origin_paths()]
+ if len(opaths) == 1:
+ opath = opaths[0][1]
+ for fname in os.listdir(opath):
+ if fname.startswith("catalog."):
+ src = os.path.join(opath, fname)
+ dest = os.path.join(self.catalog_root,
+ fname)
+ os.symlink(misc.relpath(src,
+ self.catalog_root), dest)
+ return
+
+ # If there's more than one origin, then create a new catalog
+ # based on a composite of the catalogs for all origins.
+ ncat = pkg.catalog.Catalog(batch_mode=True,
+ meta_root=self.catalog_root, sign=False)
+
+ # Mark all operations as occurring at this time.
+ op_time = dt.datetime.utcnow()
+
+ # Copied from pkg.client.image.Image to avoid circular
+ # dependency.
+ PKG_STATE_V0 = 6
+
+ for origin, opath in opaths:
+ src_cat = pkg.catalog.Catalog(meta_root=opath,
+ read_only=True)
+ for name in src_cat.parts:
+ spart = src_cat.get_part(name, must_exist=True)
+ if spart is None:
+ # Client hasn't retrieved this part.
+ continue
+
+ npart = ncat.get_part(name)
+ base = name.startswith("catalog.base.")
+
+ # Avoid accessor overhead since these will be
+ # used for every entry.
+ cat_ver = src_cat.version
+
+ for t, sentry in spart.tuple_entries(
+ pubs=[self.prefix]):
+ pub, stem, ver = t
+
+ entry = dict(sentry.iteritems())
+ try:
+ npart.add(metadata=entry,
+ op_time=op_time, pub=pub,
+ stem=stem, ver=ver)
+ except api_errors.DuplicateCatalogEntry:
+ if not base:
+ # Don't care.
+ continue
+
+ # Destination entry is in
+ # catalog already.
+ entry = npart.get_entry(
+ pub=pub, stem=stem, ver=ver)
+
+ src_sigs = set(
+ s
+ for s in sentry
+ if s.startswith("signature-")
+ )
+ dest_sigs = set(
+ s
+ for s in entry
+ if s.startswith("signature-")
+ )
+
+ if src_sigs != dest_sigs:
+ # Ignore any packages
+ # that are different
+ # from the first
+ # encountered for this
+ # package version.
+ # The client expects
+ # these to always be
+ # the same. This seems
+ # saner than failing.
+ continue
+ else:
+ if not base:
+ # Nothing to do.
+ continue
+
+ # Destination entry is one just
+ # added.
+ entry["metadata"] = {
+ "sources": [],
+ "states": [],
+ }
+
+ entry["metadata"]["sources"].append(
+ origin.uri)
+
+ states = entry["metadata"]["states"]
+ if src_cat.version == 0:
+ states.append(PKG_STATE_V0)
+
+ # Now go back and trim each entry to minimize footprint. This
+ # ensures each package entry only has state and source info
+ # recorded when needed.
+ for t, entry in ncat.tuple_entries():
+ pub, stem, ver = t
+ mdata = entry["metadata"]
+ if len(mdata["sources"]) == len(opaths):
+ # Package is available from all origins, so
+ # there's no need to require which ones
+ # have it.
+ del mdata["sources"]
+
+ if len(mdata["states"]) < len(opaths):
+ # At least one source is not V0, so the lazy-
+ # load fallback for the package metadata isn't
+ # needed.
+ del mdata["states"]
+ elif len(mdata["states"]) > 1:
+ # Ensure only one instance of state value.
+ mdata["states"] = [PKG_STATE_V0]
+ if not mdata:
+ mdata = None
+ ncat.update_entry(mdata, pub=pub, stem=stem, ver=ver)
+
+ # Finally, write out publisher catalog.
+ ncat.batch_mode = False
+ ncat.finalize()
+ ncat.save()
+
+ def __convert_v0_catalog(self, v0_cat, v1_root):
"""Transforms the contents of the provided version 0 Catalog
into a version 1 Catalog, replacing the current Catalog."""
@@ -1272,11 +1502,10 @@
# last_modified can be none if the catalog is empty.
v0_lm = pkg.catalog.ts_to_datetime(v0_lm)
- v1_cat = self.catalog
-
# There's no point in signing this catalog since it's simply
# a transformation of a v0 catalog.
- v1_cat.sign = False
+ v1_cat = pkg.catalog.Catalog(batch_mode=True,
+ meta_root=v1_root, sign=False)
# A check for a previous non-zero package count is made to
# determine whether the last_modified date alone can be
@@ -1288,8 +1517,8 @@
except (TypeError, ValueError):
n0_pkgs = 0
- if n0_pkgs != v1_cat.package_version_count:
- if v0_lm == self.catalog.last_modified:
+ if v1_cat.exists and n0_pkgs != v1_cat.package_version_count:
+ if v0_lm == v1_cat.last_modified:
# Already converted.
return
# Simply rebuild the entire v1 catalog every time, this
@@ -1297,10 +1526,10 @@
# deficiencies in the v0 implementation.
v1_cat.destroy()
self.__catalog = None
- v1_cat = self.catalog
+ v1_cat = pkg.catalog.Catalog(meta_root=v1_root,
+ sign=False)
# Now populate the v1 Catalog with the v0 Catalog's data.
- v1_cat.batch_mode = True
for f in v0_cat.fmris():
v1_cat.add_package(f)
@@ -1320,20 +1549,23 @@
v1_cat.batch_mode = False
v1_cat.finalize()
v1_cat.save()
- self.__catalog = v1_cat
-
- def __refresh_v0(self, full_refresh, immediate):
+
+ def __refresh_v0(self, croot, full_refresh, immediate, repo):
"""The method to refresh the publisher's metadata against
a catalog/0 source. If the more recent catalog/1 version
- isn't supported, this routine gets invoked as a fallback."""
+ isn't supported, this routine gets invoked as a fallback.
+ Returns a tuple of (changed, refreshed) where 'changed'
+ indicates whether new catalog data was found and 'refreshed'
+ indicates that catalog data was actually retrieved to determine
+ if there were any updates."""
if full_refresh:
immediate = True
# Catalog needs v0 -> v1 transformation if repository only
# offers v0 catalog.
- v0_cat = old_catalog.ServerCatalog(self.catalog_root,
- read_only=True, publisher=self.prefix)
+ v0_cat = old_catalog.ServerCatalog(croot, read_only=True,
+ publisher=self.prefix)
new_cat = True
v0_lm = None
@@ -1341,7 +1573,7 @@
repo = self.repository
if full_refresh or v0_cat.origin() not in repo.origins:
try:
- v0_cat.destroy(root=self.catalog_root)
+ v0_cat.destroy(root=croot)
except EnvironmentError, e:
if e.errno == errno.EACCES:
raise api_errors.PermissionsException(
@@ -1357,18 +1589,19 @@
if not immediate and not self.needs_refresh:
# No refresh needed.
- return False
+ return False, False
import pkg.updatelog as old_ulog
try:
# Note that this currently retrieves a v0 catalog that
# has to be converted to v1 format.
- self.transport.get_catalog(self, v0_lm)
+ self.transport.get_catalog(self, v0_lm, path=croot,
+ alt_repo=repo)
except old_ulog.UpdateLogException:
# If an incremental update fails, attempt a full
# catalog retrieval instead.
try:
- v0_cat.destroy(root=self.catalog_root)
+ v0_cat.destroy(root=croot)
except EnvironmentError, e:
if e.errno == errno.EACCES:
raise api_errors.PermissionsException(
@@ -1377,25 +1610,28 @@
raise api_errors.ReadOnlyFileSystemException(
e.filename)
raise
- self.transport.get_catalog(self)
-
- v0_cat = pkg.server.catalog.ServerCatalog(
- self.catalog_root, read_only=True,
+ self.transport.get_catalog(self, path=croot,
+ alt_repo=repo)
+
+ v0_cat = pkg.server.catalog.ServerCatalog(croot, read_only=True,
publisher=self.prefix)
- self.__convert_v0_catalog(v0_cat)
- self.last_refreshed = dt.datetime.utcnow()
-
+ self.__convert_v0_catalog(v0_cat, croot)
if new_cat or v0_lm != v0_cat.last_modified():
# If the catalog was rebuilt, or the timestamp of the
# catalog changed, then an update has occurred.
- return True
- return False
-
- def __refresh_v1(self, tempdir, full_refresh, immediate, mismatched):
+ return True, True
+ return False, True
+
+ def __refresh_v1(self, croot, tempdir, full_refresh, immediate,
+ mismatched, repo):
"""The method to refresh the publisher's metadata against
a catalog/1 source. If the more recent catalog/1 version
- isn't supported, __refresh_v0 is invoked as a fallback."""
+ isn't supported, __refresh_v0 is invoked as a fallback.
+ Returns a tuple of (changed, refreshed) where 'changed'
+ indicates whether new catalog data was found and 'refreshed'
+ indicates that catalog data was actually retrieved to determine
+ if there were any updates."""
# If full_refresh is True, then redownload should be True to
# ensure a non-cached version of the catalog is retrieved.
@@ -1406,35 +1642,35 @@
redownload = full_refresh
revalidate = not redownload and mismatched
+ v1_cat = pkg.catalog.Catalog(meta_root=croot)
try:
self.transport.get_catalog1(self, ["catalog.attrs"],
path=tempdir, redownload=redownload,
- revalidate=revalidate)
+ revalidate=revalidate, alt_repo=repo)
except api_errors.UnsupportedRepositoryOperation:
# No v1 catalogs available.
- if self.catalog.exists:
+ if v1_cat.exists:
# Ensure v1 -> v0 transition works right.
- self.catalog.destroy()
+ v1_cat.destroy()
self.__catalog = None
- return self.__refresh_v0(full_refresh, immediate)
+ return self.__refresh_v0(croot, full_refresh, immediate,
+ repo)
# If a v0 catalog is present, remove it before proceeding to
# ensure transitions between catalog versions work correctly.
- v0_cat = old_catalog.ServerCatalog(self.catalog_root,
- read_only=True, publisher=self.prefix)
+ v0_cat = old_catalog.ServerCatalog(croot, read_only=True,
+ publisher=self.prefix)
if v0_cat.exists:
- v0_cat.destroy(root=self.catalog_root)
+ v0_cat.destroy(root=croot)
# If above succeeded, we now have a catalog.attrs file. Parse
# this to determine what other constituent parts need to be
# downloaded.
flist = []
- if not full_refresh and self.catalog.exists:
- flist = self.catalog.get_updates_needed(tempdir)
+ if not full_refresh and v1_cat.exists:
+ flist = v1_cat.get_updates_needed(tempdir)
if flist == None:
- # Catalog has not changed.
- self.last_refreshed = dt.datetime.utcnow()
- return False
+ return False, True
else:
attrs = pkg.catalog.CatalogAttrs(meta_root=tempdir)
for name in attrs.parts:
@@ -1450,31 +1686,34 @@
try:
self.transport.get_catalog1(self, flist,
path=tempdir, redownload=redownload,
- revalidate=revalidate)
+ revalidate=revalidate, alt_repo=repo)
except api_errors.UnsupportedRepositoryOperation:
# Couldn't find a v1 catalog after getting one
# before. This would be a bizzare error, but we
# can try for a v0 catalog anyway.
- return self.__refresh_v0(full_refresh,
- immediate)
+ return self.__refresh_v0(croot, full_refresh,
+ immediate, repo)
+
+ # Clear __catalog, so we'll read in the new catalog.
+ self.__catalog = None
+ v1_cat = pkg.catalog.Catalog(meta_root=croot)
# At this point the client should have a set of the constituent
# pieces that are necessary to construct a catalog. If a
# catalog already exists, call apply_updates. Otherwise,
# move the files to the appropriate location.
validate = False
- if not full_refresh and self.catalog.exists:
- self.catalog.apply_updates(tempdir)
+ if not full_refresh and v1_cat.exists:
+ v1_cat.apply_updates(tempdir)
else:
- if self.catalog.exists:
+ if v1_cat.exists:
# This is a full refresh. Destroy
# the existing catalog.
- self.catalog.destroy()
- self.__catalog = None
+ v1_cat.destroy()
for fn in os.listdir(tempdir):
srcpath = os.path.join(tempdir, fn)
- dstpath = os.path.join(self.catalog_root, fn)
+ dstpath = os.path.join(croot, fn)
pkg.portable.rename(srcpath, dstpath)
# Apply_updates validates the newly constructed catalog.
@@ -1482,15 +1721,10 @@
# have the new catalog validated.
validate = True
- # Update refresh time.
- self.last_refreshed = dt.datetime.utcnow()
-
- # Clear __catalog, so we'll read in the new catalog.
- self.__catalog = None
-
if validate:
try:
- self.catalog.validate()
+ v1_cat = pkg.catalog.Catalog(meta_root=croot)
+ v1_cat.validate()
except api_errors.BadCatalogSignatures:
# If signature validation fails here, that means
# that the attributes and individual parts were
@@ -1499,39 +1733,27 @@
# be the result of a broken source providing
# an attributes file that is much older or newer
# than the catalog parts being provided.
- self.catalog.destroy()
- self.__catalog = None
+ v1_cat.destroy()
raise api_errors.MismatchedCatalog(self.prefix)
- return True
-
- def __refresh(self, full_refresh, immediate, mismatched=False):
- """The method to handle the overall refresh process. It
- determines if a refresh is actually needed, and then calls
- the first version-specific refresh method in the chain."""
-
- assert self.catalog_root
- assert self.transport
-
- if full_refresh:
- immediate = True
-
- # Ensure consistent directory structure.
- self.create_meta_root()
-
- # Check if we already have a v1 catalog on disk.
- if not full_refresh and self.catalog.exists:
- # If catalog is on disk, check if refresh is necessary.
- if not immediate and not self.needs_refresh:
- # No refresh needed.
- return False
-
- if not self.repository.origins:
- # Nothing to do.
- return False
+ return True, True
+
+ def __refresh_origin(self, croot, full_refresh, immediate, mismatched,
+ origin):
+ """Private helper method used to refresh catalog data for each
+ origin. Returns a tuple of (changed, refreshed) where 'changed'
+ indicates whether new catalog data was found and 'refreshed'
+ indicates that catalog data was actually retrieved to determine
+ if there were any updates."""
+
+ # Create a copy of the current repository object that only
+ # contains the origin specified.
+ repo = copy.copy(self.repository)
+ repo.origins = [origin]
# Create temporary directory for assembly of catalog pieces.
try:
- tempdir = tempfile.mkdtemp(dir=self.catalog_root)
+ misc.makedirs(croot)
+ tempdir = tempfile.mkdtemp(dir=croot)
except EnvironmentError, e:
if e.errno == errno.EACCES:
raise api_errors.PermissionsException(
@@ -1544,17 +1766,68 @@
# Ensure that the temporary directory gets removed regardless
# of success or failure.
try:
- rval = self.__refresh_v1(tempdir, full_refresh,
- immediate, mismatched)
+ rval = self.__refresh_v1(croot, tempdir,
+ full_refresh, immediate, mismatched, repo)
# Perform publisher metadata sanity checks.
- self.__validate_metadata()
+ self.__validate_metadata(croot, repo)
return rval
finally:
# Cleanup tempdir.
shutil.rmtree(tempdir, True)
+ def __refresh(self, full_refresh, immediate, mismatched=False):
+ """The method to handle the overall refresh process. It
+ determines if a refresh is actually needed, and then calls
+ the first version-specific refresh method in the chain."""
+
+ assert self.transport
+
+ if full_refresh:
+ immediate = True
+
+ for origin, opath in self.__gen_origin_paths():
+ misc.makedirs(opath)
+ cat = pkg.catalog.Catalog(meta_root=opath,
+ read_only=True)
+ if not cat.exists:
+ # If a catalog hasn't been retrieved for
+ # any of the origins, then a refresh is
+ # needed now.
+ immediate = True
+ break
+
+ # Ensure consistent directory structure.
+ self.create_meta_root()
+
+ # Check if we already have a v1 catalog on disk.
+ if not full_refresh and self.catalog.exists:
+ # If catalog is on disk, check if refresh is necessary.
+ if not immediate and not self.needs_refresh:
+ # No refresh needed.
+ return False
+
+ any_changed = False
+ any_refreshed = False
+ for origin, opath in self.__gen_origin_paths():
+ changed, refreshed = self.__refresh_origin(opath,
+ full_refresh, immediate, mismatched, origin)
+ if changed:
+ any_changed = True
+ if refreshed:
+ any_refreshed = True
+
+ if any_refreshed:
+ # Update refresh time.
+ self.last_refreshed = dt.datetime.utcnow()
+
+ # Finally, build a new catalog for this publisher based on a
+ # composite of the catalogs from all origins.
+ self.__rebuild_catalog()
+
+ return any_changed
+
def refresh(self, full_refresh=False, immediate=False):
"""Refreshes the publisher's metadata, returning a boolean
value indicating whether any updates to the publisher's
--- a/src/modules/client/transport/transport.py Thu May 12 16:55:35 2011 -0700
+++ b/src/modules/client/transport/transport.py Thu May 12 19:11:16 2011 -0700
@@ -196,6 +196,25 @@
"""
raise NotImplementedError
+ def get_pkg_alt_repo(self, pfmri):
+ """Returns the repository object containing the origins that
+ should be used to retrieve the specified package or None.
+
+ 'pfmri' is the FMRI object for the package."""
+
+ if not self.pkg_pub_map:
+ return
+
+ # Package data should be retrieved from an alternative location.
+ pfx, stem, ver = pfmri.tuple()
+ sver = str(ver)
+ pmap = self.pkg_pub_map
+ try:
+ return pmap[pfx][stem][sver].repository
+ except KeyError:
+ # No alternate known for source.
+ return
+
def get_publisher(self, publisher_name):
raise NotImplementedError
@@ -296,6 +315,17 @@
return self.__img.get_manifest_path(pfmri)
+ def get_pkg_alt_repo(self, pfmri):
+ """Returns the repository object containing the origins that
+ should be used to retrieve the specified package or None.
+
+ 'pfmri' is the FMRI object for the package."""
+
+ alt_repo = TransportCfg.get_pkg_alt_repo(self, pfmri)
+ if not alt_repo:
+ alt_repo = self.__img.get_pkg_repo(pfmri)
+ return alt_repo
+
def get_property(self, property_name):
if not self.__img.cfg:
raise KeyError
@@ -606,7 +636,8 @@
return self.__cadir
@LockedTransport()
- def get_catalog(self, pub, ts=None, ccancel=None):
+ def get_catalog(self, pub, ts=None, ccancel=None, path=None,
+ alt_repo=None):
"""Get the catalog for the specified publisher. If
ts is defined, request only changes newer than timestamp
ts."""
@@ -614,7 +645,11 @@
failures = tx.TransportFailures()
retry_count = global_settings.PKG_CLIENT_MAX_TIMEOUT
header = self.__build_header(uuid=self.__get_uuid(pub))
- croot = pub.catalog_root
+ download_dir = self.cfg.incoming_root
+ if path:
+ croot = path
+ else:
+ croot = pub.catalog_root
# Call setup if the transport isn't configured or was shutdown.
if not self.__engine:
@@ -624,7 +659,8 @@
# prior to this operation.
self._captive_portal_test(ccancel=ccancel)
- for d in self.__gen_repo(pub, retry_count, origin_only=True):
+ for d in self.__gen_repo(pub, retry_count, origin_only=True,
+ alt_repo=alt_repo):
repostats = self.stats[d.get_url()]
@@ -995,9 +1031,8 @@
for o in tpub.repository.origins:
if not alt_repo.has_origin(o):
alt_repo.add_origin(o)
- elif self.cfg.pkg_pub_map:
- alt_repo = self.__get_alt_repo(fmri,
- self.cfg.pkg_pub_map)
+ elif fmri:
+ alt_repo = self.cfg.get_pkg_alt_repo(fmri)
for d, v in self.__gen_repo(pub, retry_count, operation="file",
versions=[0, 1], alt_repo=alt_repo):
@@ -1109,9 +1144,8 @@
header = self.__build_header(intent=intent,
uuid=self.__get_uuid(pub))
- pmap = self.cfg.pkg_pub_map
- if not alt_repo and pmap:
- alt_repo = self.__get_alt_repo(fmri, pmap)
+ if not alt_repo:
+ alt_repo = self.cfg.get_pkg_alt_repo(fmri)
for d in self.__gen_repo(pub, retry_count, origin_only=True,
alt_repo=alt_repo):
@@ -1170,9 +1204,8 @@
# the directories.
self._makedirs(download_dir)
- pmap = self.cfg.pkg_pub_map
- if not alt_repo and pmap:
- alt_repo = self.__get_alt_repo(fmri, pmap)
+ if not alt_repo:
+ alt_repo = self.cfg.get_pkg_alt_repo(fmri)
for d in self.__gen_repo(pub, retry_count, origin_only=True,
alt_repo=alt_repo):
@@ -1221,15 +1254,6 @@
raise failures
- def __get_alt_repo(self, pfmri, pmap):
- # Package data should be retrieved from an
- # alternate location.
- pfx, stem, ver = pfmri.tuple()
- sver = str(ver)
- if pfx in pmap and stem in pmap[pfx] and \
- sver in pmap[pfx][stem]:
- return pmap[pfx][stem][sver].repository
-
@LockedTransport()
def prefetch_manifests(self, fetchlist, excludes=misc.EmptyI,
progtrack=None, ccancel=None, alt_repo=None):
@@ -1285,13 +1309,10 @@
# this routine must process.
mx_pub = {}
- pmap = None
- if not alt_repo:
- pmap = self.cfg.pkg_pub_map
-
+ get_alt = not alt_repo
for fmri, intent in fetchlist:
- if pmap:
- alt_repo = self.__get_alt_repo(fmri, pmap)
+ if get_alt:
+ alt_repo = self.cfg.get_pkg_alt_repo(fmri)
# Multi transfer object must be created for each unique
# publisher or repository.
@@ -1314,10 +1335,6 @@
# fmri. Value contains (header, fmri) tuple.
mx_pub[eid].add_hash(fmri, (header, fmri))
- # Must reset every cycle if pmap is set.
- if pmap:
- alt_repo = None
-
for mxfr in mx_pub.values():
namelist = [k for k in mxfr]
while namelist:
@@ -2167,9 +2184,8 @@
if not self.__engine:
self.__setup()
- pmap = self.cfg.pkg_pub_map
- if not alt_repo and pmap:
- alt_repo = self.__get_alt_repo(fmri, pmap)
+ if not alt_repo:
+ alt_repo = self.cfg.get_pkg_alt_repo(fmri)
try:
pub = self.cfg.get_publisher(fmri.publisher)
--- a/src/pull.py Thu May 12 16:55:35 2011 -0700
+++ b/src/pull.py Thu May 12 19:11:16 2011 -0700
@@ -269,14 +269,13 @@
sendb = 0
sendcb = 0
- for atype in ("file", "license", "signature"):
- for a in mfst.gen_actions_by_type(atype):
- if a.needsdata(None, None):
- multi.add_action(a)
- getb += get_pkg_otw_size(a)
- getf += 1
- sendb += int(a.attrs.get("pkg.size", 0))
- sendcb += int(a.attrs.get("pkg.csize", 0))
+ for a in mfst.gen_actions():
+ if a.has_payload:
+ multi.add_action(a)
+ getb += get_pkg_otw_size(a)
+ getf += 1
+ sendb += int(a.attrs.get("pkg.size", 0))
+ sendcb += int(a.attrs.get("pkg.csize", 0))
return getb, getf, sendb, sendcb
def prune(fmri_list, all_versions, all_timestamps):
--- a/src/tests/api/t_catalog.py Thu May 12 16:55:35 2011 -0700
+++ b/src/tests/api/t_catalog.py Thu May 12 19:11:16 2011 -0700
@@ -21,7 +21,7 @@
# CDDL HEADER END
#
-# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved.
import testutils
if __name__ == "__main__":
@@ -972,7 +972,7 @@
# Update logging has to be disabled for this to work.
cat.log_updates = False
- cat.update_entry(p2_fmri, { "foo": True })
+ cat.update_entry({ "foo": True }, pfmri=p2_fmri)
entry = cat.get_entry(p2_fmri)
self.assertEqual(entry["metadata"], { "foo": True })
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/tests/cli/t_pkg_composite.py Thu May 12 19:11:16 2011 -0700
@@ -0,0 +1,545 @@
+#!/usr/bin/python
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+# Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+
+import testutils
+if __name__ == "__main__":
+ testutils.setup_environment("../../../proto")
+import pkg5unittest
+
+import os
+import pkg.fmri as fmri
+import pkg.portable as portable
+import pkg.misc as misc
+import pkg.p5p
+import shutil
+import stat
+import tempfile
+import unittest
+
+
+class TestPkgCompositePublishers(pkg5unittest.ManyDepotTestCase):
+
+ # Don't discard repository or setUp() every test.
+ persistent_setup = True
+ # Tests in this suite use the read only data directory.
+ need_ro_data = True
+
+ foo_pkg = """
+ open pkg://test/[email protected]
+ add set name=pkg.summary value="Example package foo."
+ add dir mode=0755 owner=root group=bin path=lib
+ add dir mode=0755 owner=root group=bin path=usr
+ add dir mode=0755 owner=root group=bin path=usr/bin
+ add dir mode=0755 owner=root group=bin path=usr/local
+ add dir mode=0755 owner=root group=bin path=usr/local/bin
+ add dir mode=0755 owner=root group=bin path=usr/share
+ add dir mode=0755 owner=root group=bin path=usr/share/doc
+ add dir mode=0755 owner=root group=bin path=usr/share/doc/foo
+ add dir mode=0755 owner=root group=bin path=usr/share/man
+ add dir mode=0755 owner=root group=bin path=usr/share/man/man1
+ add file tmp/foo mode=0755 owner=root group=bin path=usr/bin/foo
+ add file tmp/libfoo.so.1 mode=0755 owner=root group=bin path=lib/libfoo.so.1 variant.debug.foo=false
+ add file tmp/libfoo_debug.so.1 mode=0755 owner=root group=bin path=lib/libfoo.so.1 variant.debug.foo=true
+ add file tmp/foo.1 mode=0444 owner=root group=bin path=usr/share/man/man1/foo.1 facet.doc.man=true
+ add file tmp/README mode=0444 owner=root group=bin path=/usr/share/doc/foo/README
+ add link path=usr/local/bin/soft-foo target=usr/bin/foo
+ add hardlink path=usr/local/bin/hard-foo target=/usr/bin/foo
+ close """
+
+ incorp_pkg = """
+ open pkg://test/[email protected]
+ add set name=pkg.summary value="Incorporation"
+ add depend type=incorporate [email protected],5.11-0.1
+ close
+ open pkg://test/[email protected]
+ add set name=pkg.summary value="Incorporation"
+ add depend type=incorporate [email protected],5.11-0.2
+ close """
+
+ signed_pkg = """
+ open pkg://test/[email protected]
+ add depend type=require [email protected]
+ add dir mode=0755 owner=root group=bin path=usr/bin
+ add file tmp/quux mode=0755 owner=root group=bin path=usr/bin/quark
+ add set name=authorized.species value=bobcat
+ close """
+
+ quux_pkg = """
+ open pkg://test2/[email protected],5.11-0.1
+ add set name=pkg.summary value="Example package quux."
+ add depend type=require fmri=pkg:/incorp
+ close
+ open pkg://test2/[email protected],5.11-0.2
+ add set name=pkg.summary value="Example package quux."
+ add depend type=require fmri=pkg:/incorp
+ add dir mode=0755 owner=root group=bin path=usr
+ add dir mode=0755 owner=root group=bin path=usr/bin
+ add file tmp/quux mode=0755 owner=root group=bin path=usr/bin/quux
+ close """
+
+ misc_files = ["tmp/foo", "tmp/libfoo.so.1", "tmp/libfoo_debug.so.1",
+ "tmp/foo.1", "tmp/README", "tmp/LICENSE", "tmp/quux"]
+
+ def __seed_ta_dir(self, certs, dest_dir=None):
+ if isinstance(certs, basestring):
+ certs = [certs]
+ if not dest_dir:
+ dest_dir = self.ta_dir
+ self.assert_(dest_dir)
+ self.assert_(self.raw_trust_anchor_dir)
+ for c in certs:
+ name = "%s_cert.pem" % c
+ portable.copyfile(
+ os.path.join(self.raw_trust_anchor_dir, name),
+ os.path.join(dest_dir, name))
+
+ def image_create(self, *args, **kwargs):
+ pkg5unittest.ManyDepotTestCase.image_create(self,
+ *args, **kwargs)
+ self.ta_dir = os.path.join(self.img_path(), "etc/certs/CA")
+ os.makedirs(self.ta_dir)
+
+ def __publish_packages(self, rurl):
+ """Private helper function to publish packages needed for
+ testing.
+ """
+
+ pkgs = "".join([self.foo_pkg, self.incorp_pkg, self.signed_pkg,
+ self.quux_pkg])
+
+ # Publish packages needed for tests.
+ plist = self.pkgsend_bulk(rurl, pkgs)
+
+ # Sign the 'signed' package.
+ r = self.get_repo(self.dcs[1].get_repodir())
+ sign_args = "-k %(key)s -c %(cert)s -i %(i1)s -i %(i2)s " \
+ "-i %(i3)s -i %(i4)s -i %(i5)s -i %(i6)s %(pkg)s" % {
+ "key": os.path.join(self.keys_dir, "cs1_ch5_ta1_key.pem"),
+ "cert": os.path.join(self.cs_dir, "cs1_ch5_ta1_cert.pem"),
+ "i1": os.path.join(self.chain_certs_dir,
+ "ch1_ta1_cert.pem"),
+ "i2": os.path.join(self.chain_certs_dir,
+ "ch2_ta1_cert.pem"),
+ "i3": os.path.join(self.chain_certs_dir,
+ "ch3_ta1_cert.pem"),
+ "i4": os.path.join(self.chain_certs_dir,
+ "ch4_ta1_cert.pem"),
+ "i5": os.path.join(self.chain_certs_dir,
+ "ch5_ta1_cert.pem"),
+ "i6": os.path.join(self.chain_certs_dir,
+ "ch1_ta3_cert.pem"),
+ "pkg": plist[3]
+ }
+ self.pkgsign(rurl, sign_args)
+
+ # This is just a test assertion to verify that the
+ # package was signed as expected.
+ self.image_create(rurl, prefix=None)
+ self.__seed_ta_dir("ta1")
+ self.pkg("set-property signature-policy verify")
+ self.pkg("install signed")
+ self.image_destroy()
+
+ return [
+ fmri.PkgFmri(sfmri)
+ for sfmri in plist
+ ]
+
+ def __archive_packages(self, arc_name, repo, plist):
+ """Private helper function to archive packages needed for
+ testing.
+ """
+
+ arc_path = os.path.join(self.test_root, arc_name)
+ assert not os.path.exists(arc_path)
+
+ arc = pkg.p5p.Archive(arc_path, mode="w")
+ for pfmri in plist:
+ arc.add_repo_package(pfmri, repo)
+ arc.close()
+
+ return arc_path
+
+ def setUp(self):
+ pkg5unittest.ManyDepotTestCase.setUp(self, ["test", "test",
+ "test", "empty"])
+ self.make_misc_files(self.misc_files)
+
+ # First repository will contain all packages.
+ self.all_rurl = self.dcs[1].get_repo_url()
+
+ # Second repository will contain only foo.
+ self.foo_rurl = self.dcs[2].get_repo_url()
+
+ # Third repository will contain only signed.
+ self.signed_rurl = self.dcs[3].get_repo_url()
+
+ # Fourth will be empty.
+ self.empty_rurl = self.dcs[4].get_repo_url()
+ self.pkgrepo("refresh -s %s" % self.empty_rurl)
+
+ # Setup base test paths.
+ self.path_to_certs = os.path.join(self.ro_data_root,
+ "signing_certs", "produced")
+ self.keys_dir = os.path.join(self.path_to_certs, "keys")
+ self.cs_dir = os.path.join(self.path_to_certs,
+ "code_signing_certs")
+ self.chain_certs_dir = os.path.join(self.path_to_certs,
+ "chain_certs")
+ self.pub_cas_dir = os.path.join(self.path_to_certs,
+ "publisher_cas")
+ self.inter_certs_dir = os.path.join(self.path_to_certs,
+ "inter_certs")
+ self.raw_trust_anchor_dir = os.path.join(self.path_to_certs,
+ "trust_anchors")
+ self.crl_dir = os.path.join(self.path_to_certs, "crl")
+
+ # Publish packages.
+ plist = self.__publish_packages(self.all_rurl)
+ self.pkgrepo("refresh -s %s" % self.all_rurl)
+
+ # Copy foo to second repository and build index.
+ self.pkgrecv(self.all_rurl, "-d %s foo" % self.foo_rurl)
+ self.pkgrepo("refresh -s %s" % self.foo_rurl)
+
+ # Copy incorp and quux to third repository and build index.
+ self.pkgrecv(self.all_rurl, "-d %s signed" % self.signed_rurl)
+ self.pkgrepo("refresh -s %s" % self.signed_rurl)
+
+ # Now create a package archive containing all packages, and
+ # then one for each.
+ repo = self.dcs[1].get_repo()
+ self.all_arc = self.__archive_packages("all_pkgs.p5p", repo,
+ plist)
+
+ for alist in ([plist[0]], [plist[1], plist[2]], [plist[3]],
+ [plist[4], plist[5]]):
+ arc_path = self.__archive_packages(
+ "%s.p5p" % alist[0].pkg_name, repo, alist)
+ setattr(self, "%s_arc" % alist[0].pkg_name, arc_path)
+
+ self.ta_dir = None
+
+ # Store FMRIs for later use.
+ self.foo10 = plist[0]
+ self.incorp10 = plist[1]
+ self.incorp20 = plist[2]
+ self.signed10 = plist[3]
+ self.quux01 = plist[4]
+ self.quux10 = plist[5]
+
+ def test_00_list(self):
+ """Verify that the list operation works as expected when
+ compositing publishers.
+ """
+
+ # Create an image and verify no packages are known.
+ self.image_create(self.empty_rurl, prefix=None)
+ self.pkg("list -a", exit=1)
+
+ # Verify list output for multiple, disparate sources using
+ # different combinations of archives and repositories.
+ self.pkg("set-publisher -g %s -g %s test" % (self.signed_arc,
+ self.foo_rurl))
+ self.pkg("list -afH ")
+ expected = \
+ ("foo (test) 1.0 ---\n"
+ "signed (test) 1.0 ---\n")
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ self.pkg("set-publisher -G %s -g %s test" % (self.foo_rurl,
+ self.foo_arc))
+ self.pkg("set-publisher -g %s test" % self.incorp_arc)
+ self.pkg("set-publisher -g %s test2" % self.quux_arc)
+ self.pkg("list -afH")
+ expected = \
+ ("foo (test) 1.0 ---\n"
+ "incorp (test) 2.0 ---\n"
+ "incorp (test) 1.0 ---\n"
+ "quux (test2) 1.0-0.2 ---\n"
+ "quux (test2) 0.1-0.1 ---\n"
+ "signed (test) 1.0 ---\n")
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ self.pkg("set-publisher -G %s -g %s test" % (self.foo_arc,
+ self.foo_rurl))
+ self.pkg("list -afH")
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ self.pkg("set-publisher -G \* -g %s test" % self.all_arc)
+ self.pkg("set-publisher -G %s -g %s -g %s test2" % (
+ self.quux_arc, self.all_arc, self.all_rurl))
+ self.pkg("list -afH -g %s -g %s" % (self.all_arc,
+ self.all_rurl))
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ # Verify packages can be installed from disparate sources and
+ # show in default list output.
+ self.pkg("install [email protected] quux signed")
+ self.pkg("list -H")
+ expected = \
+ ("foo (test) 1.0 i--\n"
+ "incorp (test) 1.0 i--\n"
+ "quux (test2) 0.1-0.1 i--\n"
+ "signed (test) 1.0 i--\n")
+
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ def test_01_info(self):
+ """Verify that the info operation works as expected when
+ compositing publishers.
+ """
+
+ # Create an image and verify no packages are known.
+ self.image_create(self.empty_rurl, prefix=None)
+ self.pkg("list -a", exit=1)
+
+ # Verify info result for multiple disparate sources using
+ # different combinations of archives and repositories.
+ self.pkg("set-publisher -g %s -g %s test" % (self.signed_arc,
+ self.foo_rurl))
+ self.pkg("info -r [email protected] [email protected]")
+
+ self.pkg("set-publisher -G %s -g %s -g %s test" %
+ (self.foo_rurl, self.foo_arc, self.incorp_arc))
+ self.pkg("set-publisher -g %s test2" % self.quux_arc)
+ self.pkg("info -r [email protected] [email protected] [email protected] [email protected]")
+
+ self.pkg("set-publisher -G %s -g %s test" % (self.foo_arc,
+ self.foo_rurl))
+ self.pkg("info -g %s -g %s -g %s -g %s [email protected] [email protected] "
+ "[email protected] [email protected]" % (
+ self.signed_arc, self.incorp_arc, self.quux_arc,
+ self.foo_rurl))
+
+ self.pkg("set-publisher -G \* -g %s -g %s test" %
+ (self.all_arc, self.all_rurl))
+ self.pkg("set-publisher -G \* -g %s -g %s test2" %
+ (self.all_arc, self.all_rurl))
+ self.pkg("info -r [email protected] [email protected] [email protected] [email protected]")
+
+ # Verify package installed from archive shows in default info
+ # output.
+ self.pkg("install [email protected]")
+ self.pkg("info")
+ expected = """\
+ Name: foo
+ Summary: Example package foo.
+ State: Installed
+ Publisher: test
+ Version: 1.0
+ Build Release: 5.11
+ Branch: None
+Packaging Date: %(pkg_date)s
+ Size: 41.00 B
+ FMRI: %(pkg_fmri)s
+""" % { "pkg_date": self.foo10.version.get_timestamp().strftime("%c"),
+ "pkg_fmri": self.foo10 }
+ self.assertEqualDiff(expected, self.output)
+
+ def test_02_contents(self):
+ """Verify that the contents operation works as expected when
+ compositing publishers.
+ """
+
+ # Create an image and verify no packages are known.
+ self.image_create(self.empty_rurl, prefix=None)
+ self.pkg("list -a", exit=1)
+
+ # Verify contents result for multiple disparate sources using
+ # different combinations of archives and repositories.
+ self.pkg("set-publisher -g %s -g %s test" % (self.signed_arc,
+ self.foo_rurl))
+ self.pkg("contents -r [email protected] [email protected]")
+
+ self.pkg("set-publisher -G %s -g %s -g %s test" %
+ (self.foo_rurl, self.foo_arc, self.incorp_arc))
+ self.pkg("set-publisher -g %s test2" % self.quux_arc)
+ self.pkg("contents -r [email protected] [email protected] [email protected] [email protected]")
+
+ self.pkg("set-publisher -G %s -g %s test" % (self.foo_arc,
+ self.foo_rurl))
+ self.pkg("contents -r [email protected] [email protected] [email protected] [email protected]")
+
+ self.pkg("set-publisher -G \* -g %s -g %s test" %
+ (self.all_arc, self.all_rurl))
+ self.pkg("set-publisher -G \* -g %s -g %s test2" %
+ (self.all_arc, self.all_rurl))
+ self.pkg("contents -r [email protected] [email protected] [email protected] [email protected]")
+
+ # Verify package installed from archive can be used with
+ # contents.
+ self.pkg("install [email protected]")
+ self.pkg("contents foo")
+
+ def test_03_install_update(self):
+ """Verify that install and update work as expected when
+ compositing publishers.
+ """
+
+ #
+ # Create an image and verify no packages are known.
+ #
+ self.image_create(self.empty_rurl, prefix=None)
+ self.pkg("list -a", exit=1)
+
+ # Verify that packages with dependencies can be installed when
+ # using multiple, disparate sources.
+ self.pkg("set-publisher -g %s -g %s test" % (self.foo_arc,
+ self.signed_arc))
+ self.pkg("install signed")
+ self.pkg("list foo signed")
+ self.pkg("uninstall \*")
+
+ # Verify publisher can be removed.
+ self.pkg("unset-publisher test")
+
+ #
+ # Create an image using the signed archive.
+ #
+ self.image_create(misc.parse_uri(self.signed_arc), prefix=None)
+ self.__seed_ta_dir("ta1")
+
+ # Verify that signed package can be installed and the archive
+ # configured for the publisher allows dependencies to be
+ # satisfied.
+ self.pkg("set-publisher -g %s test" % self.foo_arc)
+ self.pkg("set-property signature-policy verify")
+ self.pkg("publisher test")
+ self.pkg("install signed")
+ self.pkg("list foo signed")
+
+ # Verify that removing all packages and the signed archive as
+ # a source leaves only foo known.
+ self.pkg("uninstall \*")
+ self.pkg("set-publisher -G %s test" % self.signed_arc)
+ self.pkg("list -aH")
+ expected = \
+ ("foo 1.0 ---\n"
+ "signed 1.0 ---\n")
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ #
+ # Create an image and verify no packages are known.
+ #
+ self.image_create(self.empty_rurl, prefix=None)
+ self.pkg("list -a", exit=1)
+
+ # Install an older version of a known package.
+ self.pkg("set-publisher -g %s test" % self.all_arc)
+ self.pkg("set-publisher -g %s test2" % self.all_arc)
+ self.pkg("install [email protected]")
+ self.pkg("list [email protected] [email protected]")
+
+ # Verify that packages can be updated when using multiple,
+ # disparate sources (that have some overlap).
+ self.pkg("set-publisher -g %s test" % self.incorp_arc)
+ self.pkg("update")
+ self.pkg("list [email protected] [email protected]")
+
+ #
+ # Create an image using the signed archive.
+ #
+ self.image_create(misc.parse_uri(self.signed_arc), prefix=None)
+ self.__seed_ta_dir("ta1")
+
+ # Add the incorp archive as a source.
+ self.pkg("set-publisher -g %s test" % self.incorp_arc)
+
+ # Now verify that temporary package sources can be used during
+ # package operations when multiple, disparate sources are
+ # already configured for the same publisher.
+ self.pkg("install -g %s incorp signed" % self.foo_rurl)
+ self.pkg("list incorp foo signed")
+
+ def test_04_search(self):
+ """Verify that search works as expected when compositing
+ publishers.
+ """
+
+ #
+ # Create an image and verify no packages are known.
+ #
+ self.image_create(self.empty_rurl, prefix=None)
+ self.pkg("list -a", exit=1)
+
+ # Add multiple, different sources.
+ self.pkg("set-publisher -g %s -g %s test" % (self.foo_rurl,
+ self.signed_rurl))
+
+ # Verify a remote search that should only match one of the
+ # sources works as expected.
+ self.pkg("search -Hpr -o pkg.shortfmri /usr/bin/foo")
+ expected = "pkg:/[email protected]\n"
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ # Verify a remote search for multiple terms that should match
+ # each source works as expected.
+ self.pkg("search -Hpr -o pkg.shortfmri /usr/bin/foo OR "
+ "/usr/bin/quark")
+ expected = \
+ ("pkg:/[email protected]\n"
+ "pkg:/[email protected]\n")
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ # Add a source that partially overlaps with the existing ones
+ # (provides some of the same packages) and verify that some
+ # of the results are duplicated (since search across sources
+ # is a simple aggregation of all sources).
+ self.pkg("set-publisher -g %s test" % self.all_rurl)
+ self.pkg("search -Hpr -o pkg.shortfmri /usr/bin/foo OR "
+ "/usr/bin/quark OR Incorporation")
+ expected = \
+ ("pkg:/[email protected]\n"
+ "pkg:/[email protected]\n"
+ "pkg:/[email protected]\n"
+ "pkg:/[email protected]\n"
+ "pkg:/[email protected]\n"
+ "pkg:/[email protected]\n")
+ output = self.reduceSpaces(self.output)
+ self.assertEqualDiff(expected, output)
+
+ # Add a publisher with no origins and verify output still
+ # matches expected (although it will currently exit 3).
+ self.pkg("set-publisher no-origins")
+ self.pkg("search -Hpr -o pkg.shortfmri /usr/bin/foo OR "
+ "/usr/bin/quark OR Incorporation", exit=3)
+ output = self.reduceSpaces(self.output)
+
+ # Elide error output from client to verify that search
+ # results were returned despite error.
+ output = output[:output.find("pkg: ")] + "\n"
+ self.assertEqualDiff(expected, output)
+
+
+if __name__ == "__main__":
+ unittest.main()
--- a/src/tests/cli/t_pkg_publisher.py Thu May 12 16:55:35 2011 -0700
+++ b/src/tests/cli/t_pkg_publisher.py Thu May 12 19:11:16 2011 -0700
@@ -451,32 +451,38 @@
durl4 = self.dcs[4].get_depot_url()
durl5 = self.dcs[5].get_depot_url()
- # Test single add.
- self.pkg("set-publisher %s http://%s1 test1" % (add_opt,
- self.bogus_url))
- self.pkg("set-publisher %s http://%s2 test1" % (add_opt,
- self.bogus_url))
- self.pkg("set-publisher %s http://%s5" % (add_opt,
+ # Test single add; --no-refresh must be used here since the URI
+ # being added is for a non-existent repository.
+ self.pkg("set-publisher --no-refresh %s http://%s1 test1" %
+ (add_opt, self.bogus_url))
+ self.pkg("set-publisher --no-refresh %s http://%s2 test1" %
+ (add_opt, self.bogus_url))
+ self.pkg("set-publisher --no-refresh %s http://%s5" % (add_opt,
self.bogus_url), exit=2)
self.pkg("set-publisher %s test1" % add_opt, exit=2)
- self.pkg("set-publisher %s http://%s1 test1" % (add_opt,
- self.bogus_url), exit=1)
+ self.pkg("set-publisher --no-refresh %s http://%s1 test1" %
+ (add_opt, self.bogus_url), exit=1)
self.pkg("set-publisher %s http://%s5 test11" % (add_opt,
self.bogus_url), exit=1)
if etype == "origin":
- self.pkg("set-publisher %s %s7 test1" % (add_opt,
- self.bogus_url), exit=1)
+ self.pkg("set-publisher %s %s7 test1" %
+ (add_opt, self.bogus_url), exit=1)
# Test single remove.
- self.pkg("set-publisher %s http://%s1 test1" % (remove_opt,
- self.bogus_url))
- self.pkg("set-publisher %s http://%s2 test1" % (remove_opt,
- self.bogus_url))
+ self.pkg("set-publisher --no-refresh %s http://%s1 test1" %
+ (remove_opt, self.bogus_url))
+ self.pkg("set-publisher --no-refresh %s http://%s2 test1" %
+ (remove_opt, self.bogus_url))
+ # URIs to remove not specified using options, so they are seen
+ # as publisher names -- only one publisher name may be
+ # specified at a time.
self.pkg("set-publisher %s test11 http://%s2 http://%s4" % (
remove_opt, self.bogus_url, self.bogus_url), exit=2)
self.pkg("set-publisher %s http://%s5" % (remove_opt,
self.bogus_url), exit=2)
+ # publisher name specified to remove as URI.
self.pkg("set-publisher %s test1" % remove_opt, exit=2)
+ # URI already removed or never existed.
self.pkg("set-publisher %s http://%s5 test11" % (remove_opt,
self.bogus_url), exit=1)
self.pkg("set-publisher %s http://%s6 test1" % (remove_opt,
--- a/src/tests/cli/t_pkg_sysrepo.py Thu May 12 16:55:35 2011 -0700
+++ b/src/tests/cli/t_pkg_sysrepo.py Thu May 12 19:11:16 2011 -0700
@@ -550,10 +550,10 @@
# go through the user configured origin.
self.sc.conf = self.apache_confs["none"]
- # Check that the catalog can be refreshed and that the
- # communcation with the repository works.
+ # Check that the catalog can't be refreshed and that the
+ # communcation with the repository fails.
self.pkg("contents -rm example_pkg")
- self.pkg("refresh --full")
+ self.pkg("refresh --full", exit=1)
# Check that removing the system configured origin fails.
self.pkg("set-publisher -G %s test1" % self.durl1, exit=1)