17908 pkgmerge should have a blend mode
18051 pkgmerge may attempt to get files from wrong repo
18060 pkgmerge doesn't use latest fmri of packages being merged like it thinks it does
--- a/src/man/pkgmerge.1.txt Tue Mar 29 15:08:42 2011 -0700
+++ b/src/man/pkgmerge.1.txt Thu Mar 31 16:42:11 2011 -0700
@@ -10,12 +10,23 @@
DESCRIPTION
pkgmerge is a package publication tool for creating multi-variant
- packages. It does this by merging packages with identical names and
- versions (excluding timestamp), tagging actions that are unique
- in the versions being merged with the specified variant name and value
- for the given source, and then publishing the new packages to the
- target repository. Only the newest version of every package from each
- source is used.
+ packages. It does this by merging packages with identical names
+ and versions (excluding timestamp), tagging actions that are
+ unique in the versions being merged with the specified variant
+ name and value for the given source, and then publishing the new
+ packages to the target repository. Only the newest version of
+ every package from each source is used.
+
+ If an action has the attribute "pkg.merge.blend" set to the name
+ of the variant being merged, that action is copied to the other
+ manifests prior to merging so that the action will appear without
+ any added variant tags in the final output. Note that the
+ attribute "pkg.merge.blend" itself will be removed from any
+ actions in the the output manifest. This attribute may be
+ repeated with different values for multi-pass merges.
+
+ Non-identical actions that deliver to the same path in an input
+ manifest will result in pkgmerge exiting with an error.
OPTIONS
The following options are supported:
@@ -128,6 +139,40 @@
file d285ada5f3cae14ea00e97a8d99bd3e357caadc0 mode=0555 owner=root group=bin path=usr/bin/foo variant.arch=i386 variant.debug=true
dir group=sys mode=0755 owner=root path=usr
+ Example 4: Merge packages for two architectures that don't collide together using
+ the pkg.merge.blend attribute.
+
+ $ pkgmerge -s arch=sparc,http://src1.example.com \
+ -s arch=i386,http://src2.example.com \
+ -d /path/to/target/repository
+
+ Sample package from source 1 (sparc):
+ set name=pkg.fmri value=pkg://example.com/[email protected],5.11-0.200:20381001T121410Z
+ file 1d5eac1aab628317f9c088d21e4afda9c754bb76 mode=0555 owner=root \
+ group=bin path=usr/bin/sparc/foo pkg.merge.blend=arch
+ file d285ada5f3cae14ea00e97a8d99bd3e357caadc0 mode=0555 owner=root \
+ group=bin path=usr/bin/foo
+ dir group=sys mode=0755 owner=root path=usr
+
+ Sample package from source 2 (i386):
+ set name=pkg.fmri value=pkg://example.com/[email protected],5.11-0.200:20381001T163427Z
+ file a285ada5f3cae14ea00e97a8d99bd3e357cb0dca mode=0555 owner=root \
+ group=bin path=usr/bin/i386/foo pkg.merge.blend=arch
+ file d285ada5f3cae14ea00e97a8d99bd3e357caadc0 mode=0555 owner=root \
+ group=bin path=usr/bin/foo
+ dir group=sys mode=0755 owner=root path=usr
+
+ Merged package:
+ set name=pkg.fmri value=pkg://example.com/[email protected],5.11-0.200:20381001T163427Z
+ set name=variant.arch value=sparc value=i386
+ file d285ada5f3cae14ea00e97a8d99bd3e357caadc0 mode=0555 owner=root \
+ group=bin path=usr/bin/foo
+ file a285ada5f3cae14ea00e97a8d99bd3e357cb0dca mode=0555 owner=root \
+ group=bin path=usr/bin/i386/foo
+ file 1d5eac1aab628317f9c088d21e4afda9c754bb76 mode=0555 owner=root \
+ group=bin path=usr/bin/sparc/foo
+ dir group=sys mode=0755 owner=root path=usr
+
EXIT STATUS
The following exit values are returned:
--- a/src/modules/manifest.py Tue Mar 29 15:08:42 2011 -0700
+++ b/src/modules/manifest.py Thu Mar 31 16:42:11 2011 -0700
@@ -182,6 +182,7 @@
# Must specify at least one manifest.
assert compare_m
+ dups = []
# construct list of dictionaries of actions in each
# manifest, indexed by unique key and variant combination
@@ -205,13 +206,15 @@
# id for the action as its identifier.
key = (id(a),)
- # Only use the first action found for each
- # unique combination; this does mean that
- # duplicate actions will be silently discarded
- # for each manifest.
- m_dict.setdefault((a.name, key), a)
+ # catch duplicate actions here...
+ if m_dict.setdefault((a.name, key), a) != a:
+ dups.append((m_dict[(a.name, key)], a))
+
m_dicts.append(m_dict)
+ if dups:
+ raise ManifestError(duplicates=dups)
+
# construct list of key sets in each dict
m_sets = [
set(m.keys())
@@ -357,7 +360,7 @@
'excludes' is an optional list of variants to exclude from the
manifest.
-
+
'pathname' is an optional filename containing the location of
the manifest content.
@@ -608,7 +611,7 @@
if e.errno == errno.EACCES:
raise apx.PermissionsException(e.filename)
if e.errno == errno.EROFS:
- raise apx.ReadOnlyFileSystemException(
+ raise apx.ReadOnlyFileSystemException(
e.filename)
raise
@@ -1080,3 +1083,18 @@
return []
NullFactoredManifest = EmptyFactoredManifest()
+
+class ManifestError(Exception):
+ """Simple Exception class to handle manifest specific errors"""
+
+ def __init__(self, duplicates=EmptyI):
+ self.__duplicates=duplicates
+
+ def __str__(self):
+ ret = []
+ for d in self.__duplicates:
+ ret.append("%s\n%s\n\n" % d)
+
+ return "\n".join(ret)
+
+
--- a/src/tests/cli/t_pkgmerge.py Tue Mar 29 15:08:42 2011 -0700
+++ b/src/tests/cli/t_pkgmerge.py Thu Mar 31 16:42:11 2011 -0700
@@ -42,30 +42,33 @@
import unittest
import zlib
+import sys
class TestUtilMerge(pkg5unittest.ManyDepotTestCase):
persistent_setup = True
scheme10 = """
open pkg:/[email protected],5.11-0
- close
+ add file tmp/sparc-only mode=0444 owner=root group=bin path=/etc/tree
+ close
"""
tree10 = """
open [email protected],5.11-0
- close
+ add file tmp/sparc-only mode=0444 owner=root group=bin path=/etc/tree
+ close
"""
amber10 = """
open [email protected],5.11-0
add depend fmri=pkg:/[email protected] type=require
- close
+ close
"""
amber20 = """
open [email protected],5.11-0
add depend fmri=pkg:/[email protected] type=require
- close
+ close
"""
bronze10 = """
@@ -96,7 +99,7 @@
add license tmp/copyright3 license=copyright
add file tmp/bronzeA2 mode=0444 owner=root group=bin path=/A1/B2/C3/D4/E5/F6/bronzeA2
add depend fmri=pkg:/[email protected] type=require
- close
+ close
"""
bronze20b = """
@@ -113,7 +116,7 @@
add file tmp/bronzeA2 mode=0444 owner=root group=bin path=/A1/B2/C3/D4/E5/F6/bronzeA2
add depend fmri=pkg:/[email protected] type=require
add depend fmri=pkg:/[email protected] type=require
- close
+ close
"""
bronze20c = """
@@ -132,15 +135,70 @@
add depend fmri=pkg:/[email protected] type=require
add depend fmri=pkg:/[email protected] type=require
add depend fmri=pkg:/[email protected] type=require
- close
+ close
+ """
+
+ silverA = """
+ open [email protected],5.11-0
+ add file tmp/bronze1 mode=0444 owner=root group=bin path=/etc/bronze1
+ add file tmp/sh mode=0444 owner=root group=bin path=/etc/tree pkg.merge.blend=arch
+ close
+ """
+ silverB = """
+ open [email protected],5.11-0
+ add file tmp/bronze1 mode=0555 owner=root group=bin path=/etc/bronze1
+ close
+ """
+
+ multiA = """
+ open [email protected],5.11-0
+ add file tmp/sparc1 mode=0444 owner=root group=bin path=/etc/debug-notes pkg.merge.blend=arch
+ add file tmp/sparc2 mode=0444 owner=root group=bin path=/etc/sparc/debug-notes
+ add file tmp/sparc3 mode=0444 owner=root group=bin path=/etc/binary
+ close
+ """
+
+ multiB = """
+ open [email protected],5.11-0
+ add file tmp/sparc4 mode=0444 owner=root group=bin path=/etc/everywhere-notes pkg.merge.blend=arch pkg.merge.blend=debug
+ add file tmp/sparc4 mode=0444 owner=root group=bin path=/etc/binary
+ close
+ """
+
+ multiC = """
+ open [email protected],5.11-0
+ add file tmp/i3862 mode=0444 owner=root group=bin path=/etc/binary
+ close
+ """
+
+ multiD = """
+ open [email protected],5.11-0
+ add file tmp/i3861 mode=0444 owner=root group=bin path=/etc/nondebug-notes pkg.merge.blend=variant.arch
+ add file tmp/i3863 mode=0444 owner=root group=bin path=/etc/binary
+ close
+ """
+
+ tinA = """
+ open [email protected],5.11-0
+ add file tmp/bronze1 mode=0444 owner=root group=bin path=/etc/bronze1
+ add file tmp/sh mode=0444 owner=root group=bin path=/etc/tree pkg.merge.blend=arch
+ close
+ """
+ tinB = """
+ open [email protected],5.11-0
+ add file tmp/bronze1 mode=0555 owner=root group=bin path=/etc/bronze1
+ add file tmp/scheme mode=0444 owner=root group=bin path=/etc/tree pkg.merge.blend=arch
+ close
"""
misc_files = [ "tmp/bronzeA1", "tmp/bronzeA2", "tmp/bronze1",
"tmp/bronze2", "tmp/copyright2", "tmp/copyright3", "tmp/libc.so.1",
- "tmp/sh", "tmp/scheme"]
+ "tmp/sh", "tmp/scheme", "tmp/sparc-only", "tmp/sparc1", "tmp/sparc2",
+ "tmp/sparc3", "tmp/sparc4", "tmp/i3861", "tmp/i3862", "tmp/i3863"]
def setUp(self):
pkg5unittest.ManyDepotTestCase.setUp(self, ["os.org", "os.org",
+ "os.org", "os.org", "os.org", "os.org", "os.org", "os.org",
"os.org", "os.org", "os.org", "os.org", "os.org"])
self.make_misc_files(self.misc_files)
@@ -153,7 +211,14 @@
# Empty repository.
self.rurl7 = self.dcs[7].get_repo_url()
-
+
+ # a bunch for testing blending
+ self.rurl8 = self.dcs[8].get_repo_url()
+ self.rurl9 = self.dcs[9].get_repo_url()
+ self.rurl10 = self.dcs[10].get_repo_url()
+ self.rurl11 = self.dcs[11].get_repo_url()
+ self.rurl12 = self.dcs[12].get_repo_url()
+ self.rurl13 = self.dcs[13].get_repo_url()
# Publish a set of packages to one repository.
self.published = self.pkgsend_bulk(self.rurl1, (self.amber10,
self.amber20, self.bronze10, self.bronze20, self.tree10,
@@ -201,6 +266,21 @@
self.amber10, self.amber20, self.bronze10, self.bronze20c,
self.tree10))
+ self.published_blend = self.pkgsend_bulk(self.rurl8, (self.silverA,
+ self.tinA))
+ time.sleep(1)
+ self.published_blend += self.pkgsend_bulk(self.rurl9, (self.silverB,
+ self.tinB))
+
+ time.sleep(1)
+ self.published_blend += self.pkgsend_bulk(self.rurl10, (self.multiA,))
+ time.sleep(1)
+ self.published_blend += self.pkgsend_bulk(self.rurl11, (self.multiB,))
+ time.sleep(1)
+ self.published_blend += self.pkgsend_bulk(self.rurl12, (self.multiC,))
+ time.sleep(1)
+ self.published_blend += self.pkgsend_bulk(self.rurl13, (self.multiD,))
+
def test_0_options(self):
"""Verify that pkgmerge gracefully fails when given bad option
values."""
@@ -293,6 +373,7 @@
self.published[9], # pkg://os.org/[email protected]
]
actual = [str(f) for f in sorted(cat.fmris())]
+
self.assertEqualDiff(expected, actual)
# Verify that each package was merged correctly.
@@ -399,8 +480,8 @@
# Merge the packages.
self.pkgmerge(" ".join([
+ "-s arch=i386,%s" % self.rurl2,
"-s arch=sparc,%s" % self.rurl1,
- "-s arch=i386,%s" % self.rurl2,
"-d %s" % repodir
]))
@@ -419,6 +500,7 @@
expected = [str(f) for f in nlist]
actual = [str(f) for f in sorted(cat.fmris())]
+
self.assertEqualDiff(expected, actual)
# Verify that each package was merged correctly.
@@ -426,7 +508,7 @@
"amber": """\
depend fmri=pkg:/[email protected] type=require
set name=pkg.fmri value=%s
-set name=variant.arch value=sparc value=i386\
+set name=variant.arch value=i386 value=sparc\
""" % self.published[7], # pkg://os.org/[email protected]
"bronze": """\
depend fmri=pkg:/[email protected] type=require
@@ -443,13 +525,15 @@
license 995ad376b9c7ae79d67e673504fc4199fbfb32eb chash=9374d402ed3034a553119e179d0ae00386bb5206 license=copyright pkg.csize=34 pkg.size=14
link path=usr/bin/jsh target=./sh
set name=pkg.fmri value=%s
-set name=variant.arch value=sparc value=i386\
+set name=variant.arch value=i386 value=sparc\
""" % self.published[9], # pkg://os.org/[email protected]
"scheme": """\
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14
set name=pkg.fmri value=%s
set name=variant.arch value=sparc\
""" % self.published[5], # pkg://os.org/[email protected]
"tree": """\
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14
set name=pkg.fmri value=%s
set name=variant.arch value=sparc\
""" % self.published[4], # pkg://os.org/[email protected]
@@ -524,10 +608,12 @@
set name=variant.arch value=sparc value=i386 value=arm\
""" % self.published[13], # pkg://os.org/[email protected]
"scheme": """\
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14
set name=pkg.fmri value=%s
set name=variant.arch value=sparc\
""" % self.published[5], # pkg://os.org/[email protected]
"tree": """\
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14
set name=pkg.fmri value=%s
set name=variant.arch value=sparc value=arm\
""" % self.published[14], # pkg://os.org/[email protected]
@@ -678,11 +764,15 @@
set name=variant.debug value=false value=true\
""" % self.published_debug[13], # pkg://os.org/[email protected]
"scheme": """\
+file 3a06aa547ffe0186a2b9db55b8853874a048fb47 chash=ab50364de4ce8f847d765d402d80e37431e1f0aa group=bin mode=0444 owner=root path=etc/tree pkg.csize=40 pkg.size=20 variant.debug=true
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14 variant.debug=false
set name=pkg.fmri value=%s
set name=variant.arch value=sparc
set name=variant.debug value=false value=true\
""" % self.published_debug[5], # pkg://os.org/[email protected]
"tree": """\
+file 3a06aa547ffe0186a2b9db55b8853874a048fb47 chash=ab50364de4ce8f847d765d402d80e37431e1f0aa group=bin mode=0444 owner=root path=etc/tree pkg.csize=40 pkg.size=20 variant.debug=true
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14 variant.debug=false
set name=pkg.fmri value=%s
set name=variant.arch value=sparc value=arm
set name=variant.debug value=false value=true\
@@ -796,11 +886,16 @@
set name=variant.debug value=false variant.arch=i386\
""" % self.published_debug[3], # pkg://os.org/[email protected]
"scheme": """\
+file 3a06aa547ffe0186a2b9db55b8853874a048fb47 chash=ab50364de4ce8f847d765d402d80e37431e1f0aa group=bin mode=0444 owner=root path=etc/tree pkg.csize=40 pkg.size=20 variant.debug=true
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14 variant.debug=false
set name=pkg.fmri value=%s
set name=variant.arch value=sparc
set name=variant.debug value=false value=true\
""" % self.published_debug[5], # pkg://os.org/[email protected]
"tree": """\
+file 3a06aa547ffe0186a2b9db55b8853874a048fb47 chash=ab50364de4ce8f847d765d402d80e37431e1f0aa group=bin mode=0444 owner=root path=etc/tree pkg.csize=40 pkg.size=20 variant.arch=sparc variant.debug=true
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14 variant.arch=arm
+file 3b7cee8797632f83a11b66d028016946b4fa47fa chash=00621927edeb8e5b96ef63a93b4c5d125f2a3298 group=bin mode=0444 owner=root path=etc/tree pkg.csize=34 pkg.size=14 variant.arch=sparc variant.debug=false
set name=pkg.fmri value=%s
set name=variant.arch value=arm value=sparc
set name=variant.debug value=false value=true variant.arch=sparc
@@ -818,6 +913,87 @@
# Cleanup.
shutil.rmtree(repodir)
+ def test_4_blend(self):
+ """Make sure simple blending works"""
+
+ # Create the target repository.
+ repodir = os.path.join(self.test_root, "4merge_repo")
+ self.create_repo(repodir)
+
+ # Merge the silver packages.
+ self.pkgmerge(" ".join([
+ "-s arch=sparc,%s" % self.rurl8,
+ "-s arch=i386,%s" % self.rurl9,
+ "-d %s silver" % repodir
+ ]))
+
+ # get target repo
+ repo = self.get_repo(repodir)
+ cat = repo.get_catalog(pub="os.org")
+ expected = """\
+file 1abe1a7084720f501912eceb1312ddd799fb2a34 chash=ea7230676e13986491d7405c5a9298e074930575 group=bin mode=0444 owner=root path=etc/bronze1 pkg.csize=37 pkg.size=17 variant.arch=sparc
+file 1abe1a7084720f501912eceb1312ddd799fb2a34 chash=ea7230676e13986491d7405c5a9298e074930575 group=bin mode=0555 owner=root path=etc/bronze1 pkg.csize=37 pkg.size=17 variant.arch=i386
+file 34f88965d55d3a730fa7683bc0f370fc6e42bf95 chash=66eebb69ee0299dcb495162336db81a3188de037 group=bin mode=0444 owner=root path=etc/tree pkg.csize=32 pkg.size=12
+set name=pkg.fmri value=%s
+set name=variant.arch value=sparc value=i386\
+""" % self.published_blend[2]
+
+ for f in cat.fmris():
+ with open(repo.manifest(f), "rb") as m:
+ actual = "".join(sorted(l for l in m)).strip()
+ self.assertEqualDiff(expected, actual)
+ shutil.rmtree(repodir)
+
+ def test_5_blend(self):
+ """test duplicate action detection during blending"""
+ repodir = os.path.join(self.test_root, "5merge_repo")
+ self.create_repo(repodir)
+
+ # Merge the tin packages - whoops
+ self.pkgmerge(" ".join([
+ "-s arch=sparc,%s" % self.rurl8,
+ "-s arch=i386,%s" % self.rurl9,
+ "-d %s tin" % repodir
+ ]), exit=1)
+ shutil.rmtree(repodir)
+
+ def test_6_blend(self):
+ """check complex blending"""
+ repodir = os.path.join(self.test_root, "6merge_repo")
+ self.create_repo(repodir)
+
+ # merge the multi packages
+ self.pkgmerge(" ".join([
+ "-s arch=sparc,debug=true,%s" % self.dcs[10].get_repodir(),
+ "-s arch=i386,debug=true,%s" % self.dcs[12].get_repodir(),
+ "-s arch=sparc,debug=false,%s" % self.dcs[11].get_repodir(),
+ "-s arch=i386,debug=false,%s" % self.dcs[13].get_repodir(),
+ "-d %s" % repodir]))
+
+ actual = self.get_manifest(repodir)
+ expected = """\
+file 24bb3b46361cf7d180d0227beea4f75a872b6ff4 chash=b91fb7bdd4d35779bbd70c6b0367198e48290373 group=bin mode=0444 owner=root path=etc/nondebug-notes pkg.csize=35 pkg.size=15 variant.debug=false
+file 3dfdd5c4f64e2005e7913ba8444c8ee6fa70f238 chash=05beb59e279eb2c9146c6547f1c4b94536f4b2b9 group=bin mode=0444 owner=root path=etc/binary pkg.csize=35 pkg.size=15 variant.arch=i386 variant.debug=false
+file 4c12fa38950b7a5580c2715725f0ea980354b407 chash=61801db07f048941675ab3951cace0899b571430 group=bin mode=0444 owner=root path=etc/binary pkg.csize=35 pkg.size=15 variant.arch=i386 variant.debug=true
+file 6b7161cb29262ea4924a8874818da189bb70da09 chash=77e271370cec04931346c969a85d6af37c1ea83f group=bin mode=0444 owner=root path=etc/binary pkg.csize=36 pkg.size=16 variant.arch=sparc variant.debug=false
+file 6b7161cb29262ea4924a8874818da189bb70da09 chash=77e271370cec04931346c969a85d6af37c1ea83f group=bin mode=0444 owner=root path=etc/everywhere-notes pkg.csize=36 pkg.size=16
+file 9e837a70edd530a88c88f8a58b8a5bf2a8f3943c chash=d0323533586e1153bd1701254f45d2eb2c7eb0c4 group=bin mode=0444 owner=root path=etc/debug-notes pkg.csize=36 pkg.size=16 variant.debug=true
+file a10f11b8559a723bea9ee0cf5980811a9d51afbb chash=9fb8079898da8a2a9faad65c8df4c4a42095f25a group=bin mode=0444 owner=root path=etc/sparc/debug-notes pkg.csize=36 pkg.size=16 variant.arch=sparc variant.debug=true
+file aab699c6424ed1fc258b6b39eb113e624a9ee368 chash=43c3b9a83a112727264390002c3db3fcebec2e76 group=bin mode=0444 owner=root path=etc/binary pkg.csize=36 pkg.size=16 variant.arch=sparc variant.debug=true
+set name=pkg.fmri value=%s
+set name=variant.arch value=sparc value=i386
+set name=variant.debug value=true value=false\
+""" % self.published_blend[-1]
+ self.assertEqualDiff(expected, actual)
+ shutil.rmtree(repodir)
+
+ def get_manifest(self, repodir):
+ repo = self.get_repo(repodir)
+ cat = repo.get_catalog(pub="os.org")
+ for f in cat.fmris():
+ with open(repo.manifest(f), "rb") as m:
+ actual = "".join(sorted(l for l in m)).strip()
+ return actual
if __name__ == "__main__":
unittest.main()
--- a/src/util/publish/pkgmerge.py Tue Mar 29 15:08:42 2011 -0700
+++ b/src/util/publish/pkgmerge.py Thu Mar 31 16:42:11 2011 -0700
@@ -465,6 +465,9 @@
# Merge each variant one at a time.
merged = {}
+ # where to find files...
+ hash_source = {}
+
for i, variant in enumerate(variants):
# Build the unique list of remaining variant combinations to
# use for merging this variant.
@@ -509,6 +512,7 @@
slist = []
flist = []
vlist = []
+ sindex = []
new_fmri = None
for j, src in enumerate(source_list):
if combo:
@@ -530,15 +534,16 @@
# merged with another package.
pfmri = fmri_list[j]
if not pfmri or \
- merged.get(pfmri, None) == null_manifest:
+ merged.get(id(pfmri), None) == null_manifest:
continue
# The newest FMRI in the set of manifests being
# merged will be used as the new FMRI of the
# merged package.
- if new_fmri is None or pfmri.version > new_fmri:
+ if new_fmri is None or pfmri.version > new_fmri.version:
new_fmri = pfmri
+ sindex.append(j)
slist.append(src)
flist.append(pfmri)
vlist.append(variant_list[j][variant])
@@ -549,32 +554,36 @@
# Build the list of manifests to be merged.
mlist = []
- for s, f in zip(slist, flist):
- if f in merged:
+ for j, s, f in zip(sindex, slist, flist):
+ if id(f) in merged:
# Manifest already merged before, use
# the merged version.
- m = merged[f]
+ m = merged[id(f)]
else:
# Manifest not yet merged, retrieve
- # from source.
+ # from source; record those w/ payloads
+ # so we know from where to get them..
m = get_manifest(s, f)
+ for a in m.gen_actions():
+ if a.has_payload:
+ hash_source.setdefault(a.hash, j)
mlist.append(m)
m = __merge_fmris(new_fmri, mlist, flist, vlist,
variant)
for f in flist:
- if f == new_fmri:
+ if id(f) == id(new_fmri):
# This FMRI was used for the merged
# manifest; any future merges should
# use the merged manifest for this
# FMRI.
- merged[f] = m
+ merged[id(f)] = m
else:
# This package has been merged with
# another so shouldn't be retrieved
# or merged again.
- merged[f] = null_manifest
+ merged[id(f)] = null_manifest
# Merge process should have resulted in a single non-null manifest.
m = [v for v in merged.values() if v != null_manifest]
@@ -583,53 +592,14 @@
# Finally, build a list of actions to retrieve based on position in
# source_list.
- already_seen = set()
- def repeated(a, d):
- if a in d:
- return True
- already_seen.add(a)
- return False
retrievals = [list() for i in source_list]
+
for a in m.gen_actions():
- if not a.has_payload or repeated(a.hash, already_seen):
- continue
-
- avars, afacets = a.get_varcet_keys()
-
- # Only evaluate retrievals based on the variants declared for
- # each source; the action may have been tagged with variants
- # from a previous merge.
- avars = [v for v in avars if v in variants]
-
- if not avars:
- # Action can be retrieved from any source; so use the
- # first one.
- retrievals[0].append(a)
- continue
-
- found = False
- for i, s in enumerate(source_list):
- for vname in avars:
- if not vname in variants:
- continue
- if not vname in variant_list[i]:
- found = False
- break
- if not variant_list[i][vname] == a.attrs[vname]:
- found = False
- break
- found = True
-
- if found:
- # Retrieve action from first source that matches
- # all variants the action is tagged with.
- retrievals[i].append(a)
- break
-
- # Should have found a source to retrieve the action from.
- assert found
-
+ if a.has_payload:
+ source = hash_source.pop(a.hash, None)
+ if source is not None:
+ retrievals[source].append(a)
return m, retrievals
@@ -639,6 +609,10 @@
# Remove variant tags, package variant metadata, and signatures
# from manifests since we're reassigning. This allows merging
# pre-tagged, already merged pkgs, or signed packages.
+
+ blended_actions = []
+ blend_names = set([variant, variant[8:]])
+
for j, m in enumerate(manifest_list):
deleted_count = 0
vval = variant_list[j]
@@ -672,13 +646,27 @@
"var_value": vval })
del m.actions[i - deleted_count]
deleted_count += 1
+ # checking if we're supposed to blend this action
+ # for this variant. Handle prepended "variant.".
+ if blend_names & set(a.attrlist("pkg.merge.blend")):
+ blended_actions.append((j, a))
+
+ # add blended actions to other manifests
+ for j, m in enumerate(manifest_list):
+ for k, a in blended_actions:
+ if k != j:
+ m.actions.append(a)
# Like the unix utility comm, except that this function
# takes an arbitrary number of manifests and compares them,
# returning a tuple consisting of each manifest's actions
# that are not the same for all manifests, followed by a
# list of actions that are the same in each manifest.
- action_lists = list(manifest.Manifest.comm(manifest_list))
+ try:
+ action_lists = list(manifest.Manifest.comm(manifest_list))
+ except manifest.ManifestError, e:
+ error("Duplicate action(s) in package \"%s\": \n%s" %
+ (new_fmri.pkg_name, e))
# Declare new package FMRI.
action_lists[-1].insert(0,
@@ -687,7 +675,15 @@
for a_list, v in zip(action_lists[:-1], variant_list):
for a in a_list:
a.attrs[variant] = v
-
+ # discard any blend tags for this variant from common list
+ for a in action_lists[-1]:
+ blend_attrs = set(a.attrlist("pkg.merge.blend"))
+ match = blend_names & blend_attrs
+ for m in list(match):
+ if len(blend_attrs) == 1:
+ del a.attrs["pkg.merge.blend"]
+ else:
+ a.attrlist("pkg.merge.blend").remove(m)
# combine actions into single list
allactions = reduce(lambda a, b: a + b, action_lists)
@@ -887,7 +883,8 @@
developers know about this problem by including the information above (and
this message) when filing a bug at:
-%(bug_uri)s""") % { "version": pkg.VERSION, "bug_uri": misc.BUG_URI_CLI })
+%(bug_uri)s""") % { "version": pkg.VERSION, "bug_uri": misc.BUG_URI_CLI },
+ exitcode=None)
__ret = 99
finally:
cleanup()