[OE-core] [PATCH v2 2/6] base.bbclass: initial support for binary reproducibility
Martin Jansa
martin.jansa at gmail.com
Wed Jun 14 20:30:55 UTC 2017
For some recipes (llvm, chromium, chromium-wayland) I've noticed this
function to fail with:
ERROR: llvm3.3-3.3-r0 do_unpack: Error executing a python function in
exec_python_func() autogenerated:
The stack trace of python calls that resulted in this exception/failure was:
File: 'exec_python_func() autogenerated', lineno: 2, function: <module>
0001:
*** 0002:base_do_unpack(d)
0003:
File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass',
lineno: 215, function: base_do_unpack
0211:
0212: try:
0213: fetcher = bb.fetch2.Fetch(src_uri, d)
0214: fetcher.unpack(d.getVar('WORKDIR'))
*** 0215: create_src_date_epoch_stamp(d)
0216: except bb.fetch2.BBFetchException as e:
0217: bb.fatal(str(e))
0218:}
0219:
File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass',
lineno: 34, function: create_src_date_epoch_stamp
0030: exclude = set(["temp", "licenses", "patches",
"recipe-sysroot-native", "recipe-sysroot" ])
0031: for root, dirs, files in os.walk(path, topdown=True):
0032: dirs[:] = [d for d in dirs if d not in exclude]
0033: if root.endswith('/git'):
*** 0034: src_date_epoch = get_git_src_date_epoch(d, root)
0035: break
0036:
0037: for fname in files:
0038: filename = os.path.join(root, fname)
File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass',
lineno: 17, function: get_git_src_date_epoch
0013:def get_git_src_date_epoch(d, path):
0014: import subprocess
0015: saved_cwd = os.getcwd()
0016: os.chdir(path)
*** 0017: src_date_epoch =
int(subprocess.check_output(['git','log','-1','--pretty=%ct']))
0018: os.chdir(saved_cwd)
0019: return src_date_epoch
0020:
0021:def create_src_date_epoch_stamp(d):
File: '/usr/lib/python3.5/subprocess.py', lineno: 626, function: check_output
0622: # empty string. That is maintained here for
backwards compatibility.
0623: kwargs['input'] = '' if
kwargs.get('universal_newlines', False) else b''
0624:
0625: return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
*** 0626: **kwargs).stdout
0627:
0628:
0629:class CompletedProcess(object):
0630: """A process that has finished running.
File: '/usr/lib/python3.5/subprocess.py', lineno: 708, function: run
0704: raise
0705: retcode = process.poll()
0706: if check and retcode:
0707: raise CalledProcessError(retcode, process.args,
*** 0708: output=stdout, stderr=stderr)
0709: return CompletedProcess(process.args, retcode, stdout, stderr)
0710:
0711:
0712:def list2cmdline(seq):
Exception: subprocess.CalledProcessError: Command '['git', 'log',
'-1', '--pretty=%ct']' returned non-zero exit status 128
ERROR: llvm3.3-3.3-r0 do_unpack: Function failed: base_do_unpack
ERROR: Logfile of failure stored in:
/home/jenkins/oe/world/shr-core/tmp-glibc/work/i586-oe-linux/llvm3.3/3.3-r0/temp/log.do_unpack.25005
NOTE: recipe llvm3.3-3.3-r0: task do_unpack: Failed
ERROR: Task (/home/jenkins/oe/world/shr-core/meta-openembedded/meta-oe/recipes-core/llvm/llvm3.3_3.3.bb:do_unpack)
failed with exit code '1'
Maybe I don't have the latest version of this patch (I'm not using
your poky-contrib branch yet), but it should fail with nicer message
when git log fails for whatever reason.
On Mon, May 1, 2017 at 10:59 PM, Juro Bystricky <juro.bystricky at intel.com>
wrote:
> Conditionally set some environment variables in order to achieve
> improved binary reproducibility. Providing BUILD_REPRODUCIBLE_BINARIES is
> set to "1", we set the following environment variables:
>
> export PYTHONHASHSEED=0
> export PERL_HASH_SEED=0
> export TZ="UTC"
>
> We also export and set SOURCE_DATE_EPOCH. The value for this variable
> is obtained after source code for a recipe has been unpacked, but before
> it is
> patched. If the code comes from a GIT repo, we get the timestamp from the
> top
> commit. (This usually corresponds to the mktime of "changelog".)
> Otherwise we go through all files and get the timestamp from the youngest
> one. We create a timestamp for each recipe. The timestamp is stored in the
> file
> 'src_date_epoch.txt'. Later on, each task reads this file and sets
> SOURCE_DATE_EPOCH
> based on the value found in the file.
>
> [YOCTO#11178]
> [YOCTO#11179]
>
> Signed-off-by: Juro Bystricky <juro.bystricky at intel.com>
> ---
> meta/classes/base.bbclass | 82 ++++++++++++++++++++++++++++++
> +++++++++++++++++
> 1 file changed, 82 insertions(+)
>
> diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
> index e29821f..f2b2d97 100644
> --- a/meta/classes/base.bbclass
> +++ b/meta/classes/base.bbclass
> @@ -10,6 +10,52 @@ inherit utility-tasks
> inherit metadata_scm
> inherit logging
>
> +def get_git_src_date_epoch(d, path):
> + import subprocess
> + saved_cwd = os.getcwd()
> + os.chdir(path)
> + src_date_epoch = int(subprocess.check_output(['
> git','log','-1','--pretty=%ct']))
> + os.chdir(saved_cwd)
> + return src_date_epoch
> +
> +def create_src_date_epoch_stamp(d):
> + if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
> + path = d.getVar('S')
> + src_date_epoch = 0
> + filename_dbg = None
> +
> + if path.endswith('/git'):
> + src_date_epoch = get_git_src_date_epoch(d, path)
> + else:
> + exclude = set(["temp", "licenses", "patches",
> "recipe-sysroot-native", "recipe-sysroot" ])
> + for root, dirs, files in os.walk(path, topdown=True):
> + dirs[:] = [d for d in dirs if d not in exclude]
> + if root.endswith('/git'):
> + src_date_epoch = get_git_src_date_epoch(d, root)
> + break
> +
> + for fname in files:
> + filename = os.path.join(root, fname)
> + try:
> + mtime = int(os.path.getmtime(filename))
> + except:
> + mtime = 0
> + if mtime > src_date_epoch:
> + src_date_epoch = mtime
> + filename_dbg = filename
> +
> + # Most likely an empty folder
> + if src_date_epoch == 0:
> + bb.warn("Unable to determine src_date_epoch! path:%s" % path)
> +
> + f = open(os.path.join(path,'src_date_epoch.txt'), 'w')
> + f.write(str(src_date_epoch))
> + f.close()
> +
> + if filename_dbg != None:
> + bb.debug(1," src_date_epoch %d derived from: %s" %
> (src_date_epoch, filename_dbg))
> +
> +
> OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package
> oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath oe.license"
> OE_IMPORTS[type] = "list"
>
> @@ -173,6 +219,7 @@ python base_do_unpack() {
> try:
> fetcher = bb.fetch2.Fetch(src_uri, d)
> fetcher.unpack(d.getVar('WORKDIR'))
> + create_src_date_epoch_stamp(d)
> except bb.fetch2.BBFetchException as e:
> bb.fatal(str(e))
> }
> @@ -383,9 +430,43 @@ def set_packagetriplet(d):
>
> settriplet(d, "PKGMLTRIPLETS", archs, tos, tvs)
>
> +
> +export PYTHONHASHSEED
> +export PERL_HASH_SEED
> +export SOURCE_DATE_EPOCH
> +
> +BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH PYTHONHASHSEED PERL_HASH_SEED
> "
> +
> python () {
> import string, re
>
> + # Create reproducible_environment
> +
> + if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
> + import subprocess
> + d.setVar('PYTHONHASHSEED', '0')
> + d.setVar('PERL_HASH_SEED', '0')
> + d.setVar('TZ', 'UTC')
> +
> + path = d.getVar('S')
> + epochfile = os.path.join(path,'src_date_epoch.txt')
> + if os.path.isfile(epochfile):
> + f = open(epochfile, 'r')
> + src_date_epoch = f.read()
> + f.close()
> + bb.debug(1, "src_date_epoch stamp found ---> stamp %s" %
> src_date_epoch)
> + d.setVar('SOURCE_DATE_EPOCH', src_date_epoch)
> + else:
> + bb.debug(1, "src_date_epoch stamp not found.")
> + d.setVar('SOURCE_DATE_EPOCH', '0')
> + else:
> + if 'PYTHONHASHSEED' in os.environ:
> + del os.environ['PYTHONHASHSEED']
> + if 'PERL_HASH_SEED' in os.environ:
> + del os.environ['PERL_HASH_SEED']
> + if 'SOURCE_DATE_EPOCH' in os.environ:
> + del os.environ['SOURCE_DATE_EPOCH']
> +
> # Handle PACKAGECONFIG
> #
> # These take the form:
> @@ -678,6 +759,7 @@ python () {
> bb.warn("Recipe %s is marked as only being architecture
> specific but seems to have machine specific packages?! The recipe may as
> well mark itself as machine specific directly." % d.getVar("PN"))
> }
>
> +
> addtask cleansstate after do_clean
> python do_cleansstate() {
> sstate_clean_cachefiles(d)
> --
> 2.7.4
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openembedded.org/pipermail/openembedded-core/attachments/20170614/44000679/attachment-0002.html>
More information about the Openembedded-core
mailing list