<div dir="ltr">Would it make sense to create <span style="font-size:12.8px">src_date_</span><wbr style="font-size:12.8px"><span class="gmail-il" style="font-size:12.8px;background-color:rgb(255,255,255)">epoch</span><span style="font-size:12.8px">.txt outside ${S} directory?<br></span><div><span style="font-size:12.8px"><br></span></div><div><span style="font-size:12.8px">In some recipes (which weren't picky what they install from ${S} to ${D}) I've noticed that </span><span style="font-size:12.8px">src_date_</span><wbr style="font-size:12.8px"><span class="gmail-il" style="background-color:rgb(255,255,255);font-size:12.8px">epoch</span><span style="font-size:12.8px">.txt is now included in their package (I've noticed, because 2 different packages were trying to install this file on the same path in rootfs).</span></div><div><span style="font-size:12.8px"><br></span></div><div><span style="font-size:12.8px">Can we use temp or ${WORKDIR} for this file? </span><span style="font-size:12.8px">${S} might be also be completely archived with archiver.bbclass, not sure if you want src_date_epoch.txt part of that or not.</span></div><div><span style="font-size:12.8px"><br></span></div><div><span style="font-size:12.8px">Regards,</span></div></div><div class="gmail_extra"><br><div class="gmail_quote">On Wed, Aug 9, 2017 at 7:48 PM, Juro Bystricky <span dir="ltr"><<a href="mailto:juro.bystricky@intel.com" target="_blank">juro.bystricky@intel.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Conditionally set some environment variables in order to achieve<br>
improved binary reproducibility. Providing BUILD_REPRODUCIBLE_BINARIES is<br>
set to "1", we set the following environment variables:<br>
<br>
export PYTHONHASHSEED=0<br>
export PERL_HASH_SEED=0<br>
export TZ="UTC"<br>
<br>
Additionally, we export and set SOURCE_DATE_EPOCH. This is the most crucial step to<br>
achieve binary reproducibility. The value for this variable (timestamp) is<br>
obtained after source code for a recipe has been unpacked, but before it is patched.<br>
If the code sources come from a GIT repo, we get the timestamp from the top<br>
commit. (GIT repo does not preserve file mktime timestamps). It is not safe to assume<br>
folders named "git" contain git repositories, so we check for presence of .git folder in ${S}.<br>
Otherwise, if GIT repo is not present, we get mtime from known files suche as NEWS,<br>
ChangeLog, etc. If this fails, we go through all files and get the timestamp<br>
from the youngest one. We create an individual timestamp for each recipe.<br>
The timestamp is stored in the file 'src_date_epoch.txt'. Later on, each task<br>
reads this file and sets SOURCE_DATE_EPOCH based on the value found in the file.<br>
<br>
The file src_date_epoch.txt file is re-used if found. This can be the file<br>
we previously created ourselves, or it can be a file provided by a user via<br>
a recipe.<br>
<br>
[YOCTO#11178]<br>
[YOCTO#11179]<br>
<br>
Signed-off-by: Juro Bystricky <<a href="mailto:juro.bystricky@intel.com">juro.bystricky@intel.com</a>><br>
---<br>
meta/classes/base.bbclass | 4 ++<br>
meta/classes/reproducible_<wbr>build.bbclass | 108 ++++++++++++++++++++++++++++++<wbr>++<br>
2 files changed, 112 insertions(+)<br>
create mode 100644 meta/classes/reproducible_<wbr>build.bbclass<br>
<br>
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass<br>
index 8c86977..bf79eb9 100644<br>
--- a/meta/classes/base.bbclass<br>
+++ b/meta/classes/base.bbclass<br>
@@ -9,6 +9,7 @@ inherit utils<br>
inherit utility-tasks<br>
inherit metadata_scm<br>
inherit logging<br>
+inherit reproducible_build<br>
<br>
OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath oe.license"<br>
OE_IMPORTS[type] = "list"<br>
@@ -166,6 +167,7 @@ python base_do_unpack() {<br>
try:<br>
fetcher = bb.fetch2.Fetch(src_uri, d)<br>
fetcher.unpack(d.getVar('<wbr>WORKDIR'))<br>
+ create_src_date_epoch_stamp(d)<br>
except bb.fetch2.BBFetchException as e:<br>
bb.fatal(str(e))<br>
}<br>
@@ -386,6 +388,7 @@ def set_packagetriplet(d):<br>
<br>
settriplet(d, "PKGMLTRIPLETS", archs, tos, tvs)<br>
<br>
+<br>
python () {<br>
import string, re<br>
<br>
@@ -685,6 +688,7 @@ python () {<br>
bb.warn("Recipe %s is marked as only being architecture specific but seems to have machine specific packages?! The recipe may as well mark itself as machine specific directly." % d.getVar("PN"))<br>
}<br>
<br>
+<br>
addtask cleansstate after do_clean<br>
python do_cleansstate() {<br>
sstate_clean_cachefiles(d)<br>
diff --git a/meta/classes/reproducible_<wbr>build.bbclass b/meta/classes/reproducible_<wbr>build.bbclass<br>
new file mode 100644<br>
index 0000000..af8db95<br>
--- /dev/null<br>
+++ b/meta/classes/reproducible_<wbr>build.bbclass<br>
@@ -0,0 +1,108 @@<br>
+<br>
+BUILD_REPRODUCIBLE_BINARIES ??= "0"<br>
+BUILD_REPRODUCIBLE_BINARIES[<wbr>export] = "1"<br>
+<br>
+# Unix timestamp<br>
+REPRODUCIBLE_TIMESTAMP_ROOTFS ??= ""<br>
+<br>
+def get_src_date_epoch_quick(d, path):<br>
+ import subprocess<br>
+ src_date_epoch = 0<br>
+ saved_cwd = os.getcwd()<br>
+ os.chdir(path)<br>
+ if os.path.isdir(".git"):<br>
+ try:<br>
+ src_date_epoch = int(subprocess.check_output(['<wbr>git','log','-1','--pretty=%ct'<wbr>]))<br>
+ except subprocess.CalledProcessError as grepexc:<br>
+ bb.warn("Not a git repository in .git folder? error:%d" % (grepexc.returncode))<br>
+ else:<br>
+ known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"])<br>
+<br>
+ for file in known_files:<br>
+ if os.path.isfile(file):<br>
+ mtime = int(os.path.getmtime(file))<br>
+<br>
+ # There may be more than one "known_file" present.<br>
+ # If so, use the youngest one<br>
+ if mtime > src_date_epoch:<br>
+ src_date_epoch = mtime<br>
+<br>
+ os.chdir(saved_cwd)<br>
+ return src_date_epoch<br>
+<br>
+<br>
+def create_src_date_epoch_stamp(d)<wbr>:<br>
+ if d.getVar('BUILD_REPRODUCIBLE_<wbr>BINARIES') == '1':<br>
+ path = d.getVar('S')<br>
+<br>
+ epochfile = os.path.join(path,'src_date_<wbr>epoch.txt')<br>
+ if os.path.isfile(epochfile):<br>
+ bb.debug(1, " path: %s reusing src_date_epoch.txt" % epochfile)<br>
+ return<br>
+<br>
+ filename_dbg = None<br>
+ src_date_epoch = get_src_date_epoch_quick(d, path)<br>
+<br>
+ if src_date_epoch == 0:<br>
+ exclude = set(["temp", "licenses", "patches", "recipe-sysroot-native", "recipe-sysroot", "pseudo"])<br>
+ for root, dirs, files in os.walk(path, topdown=True):<br>
+ files = [f for f in files if not f[0] == '.']<br>
+ dirs[:] = [d for d in dirs if d not in exclude]<br>
+<br>
+ for fname in files:<br>
+ filename = os.path.join(root, fname)<br>
+ try:<br>
+ mtime = int(os.path.getmtime(filename)<wbr>)<br>
+ except:<br>
+ mtime = 0<br>
+ if mtime > src_date_epoch:<br>
+ src_date_epoch = mtime<br>
+ filename_dbg = filename<br>
+<br>
+ # Most likely an empty folder<br>
+ if src_date_epoch == 0:<br>
+ bb.warn("Unable to determine src_date_epoch! path:%s" % path)<br>
+<br>
+ f = open(os.path.join(path,'src_<wbr>date_epoch.txt'), 'w')<br>
+ f.write(str(src_date_epoch))<br>
+ f.close()<br>
+<br>
+ if filename_dbg != None:<br>
+ bb.debug(1," src_date_epoch %d derived from: %s" % (src_date_epoch, filename_dbg))<br>
+ #bb.warn(" src_date_epoch %d derived from: %s" % (src_date_epoch, filename_dbg))<br>
+<br>
+export PYTHONHASHSEED<br>
+export PERL_HASH_SEED<br>
+export SOURCE_DATE_EPOCH<br>
+<br>
+BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH PYTHONHASHSEED PERL_HASH_SEED "<br>
+<br>
+python () {<br>
+ import string, re<br>
+<br>
+ # Create reproducible_environment<br>
+<br>
+ if d.getVar('BUILD_REPRODUCIBLE_<wbr>BINARIES') == '1':<br>
+ import subprocess<br>
+ d.setVar('PYTHONHASHSEED', '0')<br>
+ d.setVar('PERL_HASH_SEED', '0')<br>
+ d.setVar('TZ', 'UTC')<br>
+ path = d.getVar('S')<br>
+ epochfile = os.path.join(path,'src_date_<wbr>epoch.txt')<br>
+ if os.path.isfile(epochfile):<br>
+ f = open(epochfile, 'r')<br>
+ src_date_epoch = f.read()<br>
+ f.close()<br>
+ bb.debug(1, "src_date_epoch stamp found ---> stamp %s" % src_date_epoch)<br>
+ d.setVar('SOURCE_DATE_EPOCH', src_date_epoch)<br>
+ else:<br>
+ bb.debug(1, "src_date_epoch stamp not found.")<br>
+ d.setVar('SOURCE_DATE_EPOCH', '0')<br>
+ else:<br>
+ if 'PYTHONHASHSEED' in os.environ:<br>
+ del os.environ['PYTHONHASHSEED']<br>
+ if 'PERL_HASH_SEED' in os.environ:<br>
+ del os.environ['PERL_HASH_SEED']<br>
+ if 'SOURCE_DATE_EPOCH' in os.environ:<br>
+ del os.environ['SOURCE_DATE_EPOCH'<wbr>]<br>
+}<br>
<span class="HOEnZb"><font color="#888888">--<br>
2.7.4<br>
<br>
</font></span></blockquote></div><br></div>