[OE-core] [PATCH] package.bbclass: Optimise the per file rpm handling
Joshua Lock
josh at linux.intel.com
Fri Feb 10 02:23:22 UTC 2012
On 09/02/12 16:10, Richard Purdie wrote:
> Currently a process was being forked off for each individual file
> this class wanted to inspect with rpmdeps. This converts it to use
> rpmdeps-oecore which allows batch processing of these dependencies.
>
> For do_package for perl, this reduced the time by about 1 minute (33%).
Awesome! Sounds like this addresses #1718
http://bugzilla.pokylinux.org/show_bug.cgi?id=1718
>
> Signed-off-by: Richard Purdie<richard.purdie at linuxfoundation.org>
A minor nit below, otherwise:
Acked-by: Joshua Lock <josh at linux.intel.com>
> ---
> diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
> index 348c13c..dfabcf8 100644
> --- a/meta/classes/package.bbclass
> +++ b/meta/classes/package.bbclass
> @@ -1125,7 +1125,7 @@ if [ x"$D" = "x" ]; then
> fi
> }
>
> -RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt"
> +RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps-oecore --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt"
>
> # Collect perfile run-time dependency metadata
> # Output:
> @@ -1136,7 +1136,7 @@ RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NA
> # FILERDEPENDS_filepath_pkg - per file dep
>
> python package_do_filedeps() {
> - import os, re
> + import re, subprocess
Was this accidentally left in? I don't see any calls to subprocess?
>
> pkgdest = d.getVar('PKGDEST', True)
> packages = d.getVar('PACKAGES', True)
> @@ -1145,39 +1145,49 @@ python package_do_filedeps() {
> r = re.compile(r'[<>=]+ +[^ ]*')
>
> # Quick routine to process the results of the rpmdeps call...
> - def process_deps(pipe, pkg, f, provides_files, requires_files):
> - provides = []
> - requires = []
> - file = f.replace(pkgdest + "/" + pkg, "")
> - file = file.replace("@", "@at@")
> - file = file.replace(" ", "@space@")
> - file = file.replace("\t", "@tab@")
> - file = file.replace("[", "@openbrace@")
> - file = file.replace("]", "@closebrace@")
> - file = file.replace("_", "@underscore@")
> + def process_deps(pipe, pkg, provides_files, requires_files):
> + provides = {}
> + requires = {}
>
> for line in pipe:
> + f = line.split(" ", 1)[0].strip()
> + line = line.split(" ", 1)[1].strip()
> +
> if line.startswith("Requires:"):
> i = requires
> elif line.startswith("Provides:"):
> i = provides
> else:
> continue
> +
> + file = f.replace(pkgdest + "/" + pkg, "")
> + file = file.replace("@", "@at@")
> + file = file.replace(" ", "@space@")
> + file = file.replace("\t", "@tab@")
> + file = file.replace("[", "@openbrace@")
> + file = file.replace("]", "@closebrace@")
> + file = file.replace("_", "@underscore@")
> value = line.split(":", 1)[1].strip()
> value = r.sub(r'(\g<0>)', value)
> +
> if value.startswith("rpmlib("):
> continue
> - i.append(value)
> + if file not in i:
> + i[file] = []
> + i[file].append(value)
>
> - if len(provides)> 0:
> + for file in provides:
> provides_files.append(file)
> key = "FILERPROVIDES_" + file + "_" + pkg
> - d.setVar(key, " ".join(provides))
> + d.setVar(key, " ".join(provides[file]))
>
> - if len(requires)> 0:
> + for file in requires:
> requires_files.append(file)
> key = "FILERDEPENDS_" + file + "_" + pkg
> - d.setVar(key, " ".join(requires))
> + d.setVar(key, " ".join(requires[file]))
> +
> + def chunks(files, n):
> + return [files[i:i+n] for i in range(0, len(files), n)]
>
> # Determine dependencies
> for pkg in packages.split():
> @@ -1186,13 +1196,15 @@ python package_do_filedeps() {
>
> provides_files = []
> requires_files = []
> + rpfiles = []
> for root, dirs, files in os.walk(pkgdest + "/" + pkg):
> for file in files:
> - f = os.path.join(root, file)
> + rpfiles.append(os.path.join(root, file))
>
> - dep_pipe = os.popen(rpmdeps + " --provides --requires -v " + f)
> + for files in chunks(rpfiles, 100):
> + dep_pipe = os.popen(rpmdeps + " " + " ".join(files))
>
> - process_deps(dep_pipe, pkg, f, provides_files, requires_files)
> + process_deps(dep_pipe, pkg, provides_files, requires_files)
>
> d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files))
> d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files))
> diff --git a/meta/recipes-core/busybox/busybox.inc b/meta/recipes-core/busybox/busybox.inc
> index f35779d..5fb436e 100644
> --- a/meta/recipes-core/busybox/busybox.inc
> +++ b/meta/recipes-core/busybox/busybox.inc
> @@ -216,9 +216,9 @@ python package_do_filedeps_append () {
> # Load/backup original set
> filerprovides = d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or ""
>
> - dep_pipe = os.popen('sed -e "s,^,Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox_links))
> + dep_pipe = os.popen('sed -e "s,^,%s/%s%s Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox, pkgdest, pkg, f_busybox_links))
>
> - process_deps(dep_pipe, pkg, "%s/%s%s" % (pkgdest, pkg, f_busybox), provides_files, requires_files)
> + process_deps(dep_pipe, pkg, provides_files, requires_files)
>
> # Add the new set
> filerprovides += d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or ""
>
>
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core at lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core
--
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology Centre
More information about the Openembedded-core
mailing list