[oe-commits] [openembedded-core] 29/34: package: Call file to determine elf status in parallel
git at git.openembedded.org
git at git.openembedded.org
Tue Jul 24 10:54:21 UTC 2018
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master
in repository openembedded-core.
commit bbe0d3e26484f3f347262d40a8a9d415ce21fb43
Author: Richard Purdie <richard.purdie at linuxfoundation.org>
AuthorDate: Fri Jul 20 09:36:06 2018 +0000
package: Call file to determine elf status in parallel
This allows the calls to is_elf (which calls file) to happen in parallel
allowing a speedup of do_package and do_populate_sysroot for native
recipes.
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
meta/classes/package.bbclass | 85 ++++++++++++++++++++++++++------------------
meta/classes/staging.bbclass | 4 +--
meta/lib/oe/package.py | 18 ++++++----
3 files changed, 64 insertions(+), 43 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index f121acc..81cb0c0 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -949,6 +949,8 @@ python split_and_strip_files () {
skipfiles = (d.getVar("INHIBIT_PACKAGE_STRIP_FILES") or "").split()
if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \
d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
+ checkelf = {}
+ checkelflinks = {}
for root, dirs, files in cpath.walk(dvar):
for f in files:
file = os.path.join(root, f)
@@ -982,44 +984,57 @@ python split_and_strip_files () {
# Check its an executable
if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in f or ".node" in f)):
- # If it's a symlink, and points to an ELF file, we capture the readlink target
+
if cpath.islink(file):
- target = os.readlink(file)
- if oe.package.is_elf(ltarget):
- #bb.note("Sym: %s (%d)" % (ltarget, oe.package.is_elf(ltarget)))
- symlinks[file] = target
+ checkelflinks[file] = ltarget
continue
+ # Use a reference of device ID and inode number to identify files
+ file_reference = "%d_%d" % (s.st_dev, s.st_ino)
+ checkelf[file] = (file, file_reference)
+
+ results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelflinks.values(), d)
+ results_map = {}
+ for (ltarget, elf_file) in results:
+ results_map[ltarget] = elf_file
+ for file in checkelflinks:
+ ltarget = checkelflinks[file]
+ # If it's a symlink, and points to an ELF file, we capture the readlink target
+ if results_map[ltarget]:
+ target = os.readlink(file)
+ #bb.note("Sym: %s (%d)" % (ltarget, results_map[ltarget]))
+ symlinks[file] = target
+
+ results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelf.keys(), d)
+ for (file, elf_file) in results:
+ # It's a file (or hardlink), not a link
+ # ...but is it ELF, and is it already stripped?
+ if elf_file & 1:
+ if elf_file & 2:
+ if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split():
+ bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn))
+ else:
+ msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)
+ package_qa_handle_error("already-stripped", msg, d)
+ continue
- # It's a file (or hardlink), not a link
- # ...but is it ELF, and is it already stripped?
- elf_file = oe.package.is_elf(file)
- if elf_file & 1:
- if elf_file & 2:
- if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split():
- bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn))
- else:
- msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)
- package_qa_handle_error("already-stripped", msg, d)
- continue
-
- # At this point we have an unstripped elf file. We need to:
- # a) Make sure any file we strip is not hardlinked to anything else outside this tree
- # b) Only strip any hardlinked file once (no races)
- # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks
-
- # Use a reference of device ID and inode number to identify files
- file_reference = "%d_%d" % (s.st_dev, s.st_ino)
- if file_reference in inodes:
- os.unlink(file)
- os.link(inodes[file_reference][0], file)
- inodes[file_reference].append(file)
- else:
- inodes[file_reference] = [file]
- # break hardlink
- bb.utils.copyfile(file, file)
- elffiles[file] = elf_file
- # Modified the file so clear the cache
- cpath.updatecache(file)
+ # At this point we have an unstripped elf file. We need to:
+ # a) Make sure any file we strip is not hardlinked to anything else outside this tree
+ # b) Only strip any hardlinked file once (no races)
+ # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks
+
+ # Use a reference of device ID and inode number to identify files
+ file_reference = checkelf[file]
+ if file_reference in inodes:
+ os.unlink(file)
+ os.link(inodes[file_reference][0], file)
+ inodes[file_reference].append(file)
+ else:
+ inodes[file_reference] = [file]
+ # break hardlink
+ bb.utils.copyfile(file, file)
+ elffiles[file] = elf_file
+ # Modified the file so clear the cache
+ cpath.updatecache(file)
#
# First lets process debug splitting
diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index 939042e..41df883 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -70,7 +70,7 @@ sysroot_stage_all() {
python sysroot_strip () {
inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP')
if inhibit_sysroot and oe.types.boolean(inhibit_sysroot):
- return 0
+ return
dstdir = d.getVar('SYSROOT_DESTDIR')
pn = d.getVar('PN')
@@ -79,7 +79,7 @@ python sysroot_strip () {
qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split()
strip_cmd = d.getVar("STRIP")
- oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir,
+ oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d,
qa_already_stripped=qa_already_stripped)
}
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index a435d66..4255143 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -74,7 +74,7 @@ def is_elf(path):
if "relocatable" in result:
if path.endswith(".ko") and path.find("/lib/modules/") != -1 and is_kernel_module(path):
exec_type |= 16
- return exec_type
+ return (path, exec_type)
def is_static_lib(path):
if path.endswith('.a') and not os.path.islink(path):
@@ -86,7 +86,7 @@ def is_static_lib(path):
return start == magic
return False
-def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=False):
+def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripped=False):
"""
Strip executable code (like executables, shared libraries) _in_place_
- Based on sysroot_strip in staging.bbclass
@@ -107,6 +107,8 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
#
# First lets figure out all of the files we may have to process
#
+ checkelf = []
+ inodecache = {}
for root, dirs, files in os.walk(dstdir):
for f in files:
file = os.path.join(root, f)
@@ -132,7 +134,11 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
# It's a file (or hardlink), not a link
# ...but is it ELF, and is it already stripped?
- elf_file = is_elf(file)
+ checkelf.append(file)
+ inodecache[file] = s.st_ino
+ results = oe.utils.multiprocess_launch(is_elf, checkelf, d)
+ for (file, elf_file) in results:
+ #elf_file = is_elf(file)
if elf_file & 1:
if elf_file & 2:
if qa_already_stripped:
@@ -141,12 +147,12 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn))
continue
- if s.st_ino in inodes:
+ if inodecache[file] in inodes:
os.unlink(file)
- os.link(inodes[s.st_ino], file)
+ os.link(inodes[inodecache[file]], file)
else:
# break hardlinks so that we do not strip the original.
- inodes[s.st_ino] = file
+ inodes[inodecache[file]] = file
bb.utils.copyfile(file, file)
elffiles[file] = elf_file
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list