[OE-core] [PATCH 1/2] package_manager: Filter to only rpms we depend upon
Denys Dmytriyenko
denis at denix.org
Tue Feb 27 21:36:38 UTC 2018
Is it specific and/or limited to RPMs? How about other package managers?
On Tue, Feb 27, 2018 at 07:56:44PM +0000, Richard Purdie wrote:
> Currently do_rootfs gets to see all rpms in the deploy directory. This filters
> that view to only rpms which the image recipe has actual depends upon which
> potentially removes some sources of confusion in the image construction.
>
> This makes builds more reproducibile and also fixes contamination issues
> where dnf picks up packages it shouldn't be able to 'see'.
>
> [YOCTO #12039]
>
> Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
> ---
> meta/lib/oe/package_manager.py | 114 ++++++++++++++++++++++++++++++++-
> meta/lib/oeqa/utils/package_manager.py | 3 +-
> 2 files changed, 113 insertions(+), 4 deletions(-)
>
> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
> index f7e0134..f59eaf7 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -454,6 +454,114 @@ class PackageManager(object, metaclass=ABCMeta):
> return res
> return _append(uris, base_paths)
>
> +def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependencies):
> + """
> + Go through our do_package_write_X dependencies and hardlink the packages we depend
> + upon into the repo directory. This prevents us seeing other packages that may
> + have been built that we don't depend upon and also packages for architectures we don't
> + support.
> + """
> + import errno
> +
> + taskdepdata = d.getVar("BB_TASKDEPDATA", False)
> + mytaskname = d.getVar("BB_RUNTASK")
> + pn = d.getVar("PN")
> + seendirs = set()
> + multilibs = {}
> +
> + rpm_subrepo_dir = oe.path.join(rpm_repo_dir, "rpm")
> +
> + bb.utils.remove(rpm_subrepo_dir, recurse=True)
> + bb.utils.mkdirhier(rpm_subrepo_dir)
> +
> + # Detect bitbake -b usage
> + nodeps = d.getVar("BB_LIMITEDDEPS") or False
> + if nodeps or not filterbydependencies:
> + oe.path.symlink(deploydir, rpm_subrepo_dir, True)
> + return
> +
> + start = None
> + for dep in taskdepdata:
> + data = taskdepdata[dep]
> + if data[1] == mytaskname and data[0] == pn:
> + start = dep
> + break
> + if start is None:
> + bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
> + rpmdeps = set()
> + start = [start]
> + seen = set(start)
> + # Support direct dependencies (do_rootfs -> rpms)
> + # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> rpms)
> + while start:
> + next = []
> + for dep2 in start:
> + for dep in taskdepdata[dep2][3]:
> + if taskdepdata[dep][0] != pn:
> + if "do_" + taskname in dep:
> + rpmdeps.add(dep)
> + elif dep not in seen:
> + next.append(dep)
> + seen.add(dep)
> + start = next
> +
> + for dep in rpmdeps:
> + c = taskdepdata[dep][0]
> +
> + d2 = d
> + variant = ''
> + if taskdepdata[dep][2].startswith("virtual:multilib"):
> + variant = taskdepdata[dep][2].split(":")[2]
> + if variant not in multilibs:
> + multilibs[variant] = oe.utils.get_multilib_datastore(variant, d)
> + d2 = multilibs[variant]
> +
> + if c.endswith("-native"):
> + pkgarchs = ["${BUILD_ARCH}"]
> + elif c.startswith("nativesdk-"):
> + pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"]
> + elif "-cross-canadian" in c:
> + pkgarchs = ["${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}"]
> + elif "-cross-" in c:
> + pkgarchs = ["${BUILD_ARCH}_${TARGET_ARCH}"]
> + elif "-crosssdk" in c:
> + pkgarchs = ["${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"]
> + else:
> + pkgarchs = ['${MACHINE_ARCH}']
> + pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split()))
> + pkgarchs.append('allarch')
> + pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}')
> +
> + for pkgarch in pkgarchs:
> + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, c, taskname))
> + if os.path.exists(manifest):
> + break
> + if not os.path.exists(manifest):
> + bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant))
> + continue
> + with open(manifest, "r") as f:
> + for l in f:
> + l = l.strip()
> + dest = l.replace(deploydir, "")
> + dest = rpm_subrepo_dir + dest
> + if l.endswith("/"):
> + if dest not in seendirs:
> + bb.utils.mkdirhier(dest)
> + seendirs.add(dest)
> + continue
> + # Try to hardlink the file, copy if that fails
> + destdir = os.path.dirname(dest)
> + if destdir not in seendirs:
> + bb.utils.mkdirhier(destdir)
> + seendirs.add(destdir)
> + try:
> + os.link(l, dest)
> + except OSError as err:
> + if err.errno == errno.EXDEV:
> + bb.utils.copyfile(l, dest)
> + else:
> + raise
> +
> class RpmPM(PackageManager):
> def __init__(self,
> d,
> @@ -462,7 +570,8 @@ class RpmPM(PackageManager):
> task_name='target',
> arch_var=None,
> os_var=None,
> - rpm_repo_workdir="oe-rootfs-repo"):
> + rpm_repo_workdir="oe-rootfs-repo",
> + filterbydependencies=True):
> super(RpmPM, self).__init__(d)
> self.target_rootfs = target_rootfs
> self.target_vendor = target_vendor
> @@ -477,8 +586,7 @@ class RpmPM(PackageManager):
> self.primary_arch = self.d.getVar('MACHINE_ARCH')
>
> self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), rpm_repo_workdir)
> - bb.utils.mkdirhier(self.rpm_repo_dir)
> - oe.path.symlink(self.d.getVar('DEPLOY_DIR_RPM'), oe.path.join(self.rpm_repo_dir, "rpm"), True)
> + create_packages_dir(self.d, self.rpm_repo_dir, d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies)
>
> self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name)
> if not os.path.exists(self.d.expand('${T}/saved_packaging_data')):
> diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py
> index 724afb2..afd5b8e 100644
> --- a/meta/lib/oeqa/utils/package_manager.py
> +++ b/meta/lib/oeqa/utils/package_manager.py
> @@ -14,7 +14,8 @@ def get_package_manager(d, root_path):
> if pkg_class == "rpm":
> pm = RpmPM(d,
> root_path,
> - d.getVar('TARGET_VENDOR'))
> + d.getVar('TARGET_VENDOR'),
> + filterbydependencies=False)
> pm.create_configs()
>
> elif pkg_class == "ipk":
> --
> 2.7.4
>
> --
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core at lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
More information about the Openembedded-core
mailing list