<html dir="ltr">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<style type="text/css" id="owaParaStyle">P {margin-top:0;margin-bottom:0;}</style>
</head>
<body fpstyle="1" ocsi="0">
<div style="direction: ltr;font-family: Tahoma;color: #000000;font-size: 10pt;">Thanks, it seems there is not-too-great error checking implemented. There is an assumption if there is
<br>
a git folder there is also a git repo (and "git" binary in PATH). Seems the assumption is wrong.<br>
I'll add some error checking/improved error message. The problem with git repos is they modify the
<br>
file mod timestamps upon checkout, so it is not possible to find the one that is truly the youngest.<br>
So the code tries to get the timestamp of the top commit instead.<br>
<br>
<div style="font-family: Times New Roman; color: #000000; font-size: 16px">
<hr tabindex="-1">
<div id="divRpF897323" style="direction: ltr;"><font size="2" color="#000000" face="Tahoma"><b>From:</b> Martin Jansa [martin.jansa@gmail.com]<br>
<b>Sent:</b> Wednesday, June 14, 2017 1:30 PM<br>
<b>To:</b> Bystricky, Juro<br>
<b>Cc:</b> Patches and discussions about the oe-core layer; Richard Purdie; Joshua G Lock; Burton, Ross; Khem Raj; Juro Bystricky<br>
<b>Subject:</b> Re: [PATCH v2 2/6] base.bbclass: initial support for binary reproducibility<br>
</font><br>
</div>
<div></div>
<div>
<div dir="ltr">For some recipes (llvm, chromium, chromium-wayland) I've noticed this function to fail with:
<div><br>
</div>
<div>
<pre style="white-space:pre-wrap; word-wrap:break-word; margin-top:0px; margin-bottom:0px; color:rgb(51,51,51); font-size:14px">ERROR: llvm3.3-3.3-r0 do_unpack: Error executing a python function in exec_python_func() autogenerated:
The stack trace of python calls that resulted in this exception/failure was:
File: 'exec_python_func() autogenerated', lineno: 2, function: <module>
0001:
*** 0002:base_do_unpack(d)
0003:
File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass', lineno: 215, function: base_do_unpack
0211:
0212: try:
0213: fetcher = bb.fetch2.Fetch(src_uri, d)
0214: fetcher.unpack(d.getVar('WORKDIR'))
*** 0215: create_src_date_epoch_stamp(d)
0216: except bb.fetch2.BBFetchException as e:
0217: bb.fatal(str(e))
0218:}
0219:
File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass', lineno: 34, function: create_src_date_epoch_stamp
0030: exclude = set(["temp", "licenses", "patches", "recipe-sysroot-native", "recipe-sysroot" ])
0031: for root, dirs, files in os.walk(path, topdown=True):
0032: dirs[:] = [d for d in dirs if d not in exclude]
0033: if root.endswith('/git'):
*** 0034: src_date_epoch = get_git_src_date_epoch(d, root)
0035: break
0036:
0037: for fname in files:
0038: filename = os.path.join(root, fname)
File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass', lineno: 17, function: get_git_src_date_epoch
0013:def get_git_src_date_epoch(d, path):
0014: import subprocess
0015: saved_cwd = os.getcwd()
0016: os.chdir(path)
*** 0017: src_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct']))
0018: os.chdir(saved_cwd)
0019: return src_date_epoch
0020:
0021:def create_src_date_epoch_stamp(d):
File: '/usr/lib/python3.5/subprocess.py', lineno: 626, function: check_output
0622: # empty string. That is maintained here for backwards compatibility.
0623: kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
0624:
0625: return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
*** 0626: **kwargs).stdout
0627:
0628:
0629:class CompletedProcess(object):
0630: """A process that has finished running.
File: '/usr/lib/python3.5/subprocess.py', lineno: 708, function: run
0704: raise
0705: retcode = process.poll()
0706: if check and retcode:
0707: raise CalledProcessError(retcode, process.args,
*** 0708: output=stdout, stderr=stderr)
0709: return CompletedProcess(process.args, retcode, stdout, stderr)
0710:
0711:
0712:def list2cmdline(seq):
Exception: subprocess.CalledProcessError: Command '['git', 'log', '-1', '--pretty=%ct']' returned non-zero exit status 128
ERROR: llvm3.3-3.3-r0 do_unpack: Function failed: base_do_unpack
ERROR: Logfile of failure stored in: /home/jenkins/oe/world/shr-core/tmp-glibc/work/i586-oe-linux/llvm3.3/3.3-r0/temp/log.do_unpack.25005
NOTE: recipe llvm3.3-3.3-r0: task do_unpack: Failed
ERROR: Task (/home/jenkins/oe/world/shr-core/meta-openembedded/meta-oe/recipes-core/llvm/llvm3.3_3.3.bb:do_unpack) failed with exit code '1'</pre>
<pre style="white-space:pre-wrap; word-wrap:break-word; margin-top:0px; margin-bottom:0px; color:rgb(51,51,51); font-size:14px"><br></pre>
<pre style="white-space:pre-wrap; word-wrap:break-word; margin-top:0px; margin-bottom:0px; color:rgb(51,51,51); font-size:14px">Maybe I don't have the latest version of this patch (I'm not using your poky-contrib branch yet), but it should fail with nicer message when git log fails for whatever reason.</pre>
</div>
</div>
<div class="gmail_extra"><br>
<div class="gmail_quote">On Mon, May 1, 2017 at 10:59 PM, Juro Bystricky <span dir="ltr">
<<a href="mailto:juro.bystricky@intel.com" target="_blank">juro.bystricky@intel.com</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="margin:0 0 0 .8ex; border-left:1px #ccc solid; padding-left:1ex">
Conditionally set some environment variables in order to achieve<br>
improved binary reproducibility. Providing BUILD_REPRODUCIBLE_BINARIES is<br>
set to "1", we set the following environment variables:<br>
<br>
export PYTHONHASHSEED=0<br>
export PERL_HASH_SEED=0<br>
export TZ="UTC"<br>
<br>
We also export and set SOURCE_DATE_EPOCH. The value for this variable<br>
is obtained after source code for a recipe has been unpacked, but before it is<br>
patched. If the code comes from a GIT repo, we get the timestamp from the top<br>
commit. (This usually corresponds to the mktime of "changelog".)<br>
Otherwise we go through all files and get the timestamp from the youngest<br>
one. We create a timestamp for each recipe. The timestamp is stored in the file<br>
'src_date_epoch.txt'. Later on, each task reads this file and sets SOURCE_DATE_EPOCH<br>
based on the value found in the file.<br>
<br>
[YOCTO#11178]<br>
[YOCTO#11179]<br>
<br>
Signed-off-by: Juro Bystricky <<a href="mailto:juro.bystricky@intel.com" target="_blank">juro.bystricky@intel.com</a>><br>
---<br>
meta/classes/base.bbclass | 82 ++++++++++++++++++++++++++++++<wbr>+++++++++++++++++<br>
1 file changed, 82 insertions(+)<br>
<br>
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass<br>
index e29821f..f2b2d97 100644<br>
--- a/meta/classes/base.bbclass<br>
+++ b/meta/classes/base.bbclass<br>
@@ -10,6 +10,52 @@ inherit utility-tasks<br>
inherit metadata_scm<br>
inherit logging<br>
<br>
+def get_git_src_date_epoch(d, path):<br>
+ import subprocess<br>
+ saved_cwd = os.getcwd()<br>
+ os.chdir(path)<br>
+ src_date_epoch = int(subprocess.check_output(['<wbr>git','log','-1','--pretty=%ct'<wbr>]))<br>
+ os.chdir(saved_cwd)<br>
+ return src_date_epoch<br>
+<br>
+def create_src_date_epoch_stamp(d)<wbr>:<br>
+ if d.getVar('BUILD_REPRODUCIBLE_<wbr>BINARIES') == '1':<br>
+ path = d.getVar('S')<br>
+ src_date_epoch = 0<br>
+ filename_dbg = None<br>
+<br>
+ if path.endswith('/git'):<br>
+ src_date_epoch = get_git_src_date_epoch(d, path)<br>
+ else:<br>
+ exclude = set(["temp", "licenses", "patches", "recipe-sysroot-native", "recipe-sysroot" ])<br>
+ for root, dirs, files in os.walk(path, topdown=True):<br>
+ dirs[:] = [d for d in dirs if d not in exclude]<br>
+ if root.endswith('/git'):<br>
+ src_date_epoch = get_git_src_date_epoch(d, root)<br>
+ break<br>
+<br>
+ for fname in files:<br>
+ filename = os.path.join(root, fname)<br>
+ try:<br>
+ mtime = int(os.path.getmtime(filename)<wbr>)<br>
+ except:<br>
+ mtime = 0<br>
+ if mtime > src_date_epoch:<br>
+ src_date_epoch = mtime<br>
+ filename_dbg = filename<br>
+<br>
+ # Most likely an empty folder<br>
+ if src_date_epoch == 0:<br>
+ bb.warn("Unable to determine src_date_epoch! path:%s" % path)<br>
+<br>
+ f = open(os.path.join(path,'src_<wbr>date_epoch.txt'), 'w')<br>
+ f.write(str(src_date_epoch))<br>
+ f.close()<br>
+<br>
+ if filename_dbg != None:<br>
+ bb.debug(1," src_date_epoch %d derived from: %s" % (src_date_epoch, filename_dbg))<br>
+<br>
+<br>
OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath oe.license"<br>
OE_IMPORTS[type] = "list"<br>
<br>
@@ -173,6 +219,7 @@ python base_do_unpack() {<br>
try:<br>
fetcher = bb.fetch2.Fetch(src_uri, d)<br>
fetcher.unpack(d.getVar('<wbr>WORKDIR'))<br>
+ create_src_date_epoch_stamp(d)<br>
except bb.fetch2.BBFetchException as e:<br>
bb.fatal(str(e))<br>
}<br>
@@ -383,9 +430,43 @@ def set_packagetriplet(d):<br>
<br>
settriplet(d, "PKGMLTRIPLETS", archs, tos, tvs)<br>
<br>
+<br>
+export PYTHONHASHSEED<br>
+export PERL_HASH_SEED<br>
+export SOURCE_DATE_EPOCH<br>
+<br>
+BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH PYTHONHASHSEED PERL_HASH_SEED "<br>
+<br>
python () {<br>
import string, re<br>
<br>
+ # Create reproducible_environment<br>
+<br>
+ if d.getVar('BUILD_REPRODUCIBLE_<wbr>BINARIES') == '1':<br>
+ import subprocess<br>
+ d.setVar('PYTHONHASHSEED', '0')<br>
+ d.setVar('PERL_HASH_SEED', '0')<br>
+ d.setVar('TZ', 'UTC')<br>
+<br>
+ path = d.getVar('S')<br>
+ epochfile = os.path.join(path,'src_date_<wbr>epoch.txt')<br>
+ if os.path.isfile(epochfile):<br>
+ f = open(epochfile, 'r')<br>
+ src_date_epoch = f.read()<br>
+ f.close()<br>
+ bb.debug(1, "src_date_epoch stamp found ---> stamp %s" % src_date_epoch)<br>
+ d.setVar('SOURCE_DATE_EPOCH', src_date_epoch)<br>
+ else:<br>
+ bb.debug(1, "src_date_epoch stamp not found.")<br>
+ d.setVar('SOURCE_DATE_EPOCH', '0')<br>
+ else:<br>
+ if 'PYTHONHASHSEED' in os.environ:<br>
+ del os.environ['PYTHONHASHSEED']<br>
+ if 'PERL_HASH_SEED' in os.environ:<br>
+ del os.environ['PERL_HASH_SEED']<br>
+ if 'SOURCE_DATE_EPOCH' in os.environ:<br>
+ del os.environ['SOURCE_DATE_EPOCH'<wbr>]<br>
+<br>
# Handle PACKAGECONFIG<br>
#<br>
# These take the form:<br>
@@ -678,6 +759,7 @@ python () {<br>
bb.warn("Recipe %s is marked as only being architecture specific but seems to have machine specific packages?! The recipe may as well mark itself as machine specific directly." % d.getVar("PN"))<br>
}<br>
<br>
+<br>
addtask cleansstate after do_clean<br>
python do_cleansstate() {<br>
sstate_clean_cachefiles(d)<br>
<span class="HOEnZb"><font color="#888888">--<br>
2.7.4<br>
<br>
</font></span></blockquote>
</div>
<br>
</div>
</div>
</div>
</div>
</body>
</html>