[oe-commits] [bitbake] 01/01: EXPERIMENT: hashserv: Allow equiv mappings
git at git.openembedded.org
git at git.openembedded.org
Thu Nov 28 21:04:54 UTC 2019
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
commit 23abf4d63da579eace87ec0c360beb497d340a79
Author: Richard Purdie <richard.purdie at linuxfoundation.org>
AuthorDate: Mon Nov 25 17:21:13 2019 +0000
EXPERIMENT: hashserv: Allow equiv mappings
Experiment with allowing equivalent hashes to be mapped to each other.
Adds the server extension and updates the code to call into it when
the scenario is detected.
(gdb-cross dependency of meta-extsdk-toolchain which can be x86 or arm
yet the downstream hash needs to match)
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/runqueue.py | 9 +++++++-
lib/bb/siggen.py | 18 +++++++++++++++
lib/hashserv/client.py | 8 +++++++
lib/hashserv/server.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 95 insertions(+), 1 deletion(-)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 485d934..e53c8b3 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1718,6 +1718,7 @@ class RunQueueExecute:
self.sq_deferred = {}
self.stampcache = {}
+ self.rehashes = {}
self.holdoff_tasks = set()
self.holdoff_need_update = True
@@ -2286,6 +2287,11 @@ class RunQueueExecute:
self.rqdata.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
origuni = self.rqdata.runtaskentries[tid].unihash
self.rqdata.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
+ self.rehashes[tid] = origuni
+ #if orighash == self.rqdata.runtaskentries[tid].hash and origuni == self.rqdata.runtaskentries[tid].unihash:
+ #logger.info("Task hash didn't change for: %s" % (tid))
+ # continue
+ #else:
logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, self.rqdata.runtaskentries[tid].hash, origuni, self.rqdata.runtaskentries[tid].unihash))
next |= self.rqdata.runtaskentries[tid].revdeps
changed.add(tid)
@@ -2314,7 +2320,8 @@ class RunQueueExecute:
if tid in self.scenequeue_covered or tid in self.sq_live:
# Already ran this setscene task or it running
- # Potentially risky, should we report this hash as a match?
+ # Potentially risky, should we report this hash as a match
+ bb.parse.siggen.report_unihash_equiv(tid, self.rehashes[tid], self.rqdata.runtaskentries[tid].unihash, self.rqdata.dataCaches)
logger.info("Already covered setscene for %s so ignoring rehash" % (tid))
self.pending_migrations.remove(tid)
continue
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index e19812b..3d0be35 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -525,6 +525,24 @@ class SignatureGeneratorUniHashMixIn(object):
except OSError:
pass
+ def report_unihash_equiv(self, tid, unihash, newunihash, datacaches):
+ try:
+ extra_data = {}
+ data = self.client().report_unihash_equiv(newunihash, self.method, unihash, extra_data)
+ bb.warn('Reported task %s as unihash %s to %s (%s)' % (tid, newunihash, self.server, str(data)))
+
+ # FIXME, only rename if there is an upstream hash?
+ (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
+ stamp = bb.build.stampfile(taskname + "_setscene", datacaches[mc], taskfn, noextra=True)
+ oldstamp = stamp.replace(newunihash, unihash)
+ if os.path.exists(oldstamp):
+ bb.note("Renaming %s to %s" % (oldstamp, stamp))
+ os.rename(oldstamp, stamp)
+ else:
+ bb.note("Not renaming %s to %s" % (oldstamp, stamp))
+
+ except hashserv.client.HashConnectionError as e:
+ bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
#
# Dummy class used for bitbake-selftest
diff --git a/lib/hashserv/client.py b/lib/hashserv/client.py
index f659566..ae0cce9 100644
--- a/lib/hashserv/client.py
+++ b/lib/hashserv/client.py
@@ -148,6 +148,14 @@ class Client(object):
m['unihash'] = unihash
return self.send_message({'report': m})
+ def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
+ self._set_mode(self.MODE_NORMAL)
+ m = extra.copy()
+ m['taskhash'] = taskhash
+ m['method'] = method
+ m['unihash'] = unihash
+ return self.send_message({'report-equiv': m})
+
def get_stats(self):
self._set_mode(self.MODE_NORMAL)
return self.send_message({'get-stats': None})
diff --git a/lib/hashserv/server.py b/lib/hashserv/server.py
index 0aff776..0a7d932 100644
--- a/lib/hashserv/server.py
+++ b/lib/hashserv/server.py
@@ -143,6 +143,7 @@ class ServerClient(object):
handlers = {
'get': self.handle_get,
'report': self.handle_report,
+ 'report-equiv': self.handle_equivreport,
'get-stream': self.handle_get_stream,
'get-stats': self.handle_get_stats,
'reset-stats': self.handle_reset_stats,
@@ -303,6 +304,66 @@ class ServerClient(object):
self.write_message(d)
+ async def handle_equivreport(self, data):
+ with closing(self.db.cursor()) as cursor:
+ cursor.execute('''
+ -- Find the task entry for the matching taskhash
+ SELECT unihash, outhash FROM tasks_v2 WHERE method=:method AND taskhash=:unihash
+
+ -- Only return one row
+ LIMIT 1
+ ''', {k: data[k] for k in ('method', 'unihash')})
+
+ row = cursor.fetchone()
+
+ if row is None:
+ self.write_message(None)
+ return
+
+ insert_data = {
+ 'method': data['method'],
+ 'outhash': row['outhash'],
+ 'taskhash': data['taskhash'],
+ 'unihash': row['unihash'],
+ 'created': datetime.now()
+ }
+
+
+ cursor.execute('''
+ -- Find the task entry for the matching taskhash
+ SELECT outhash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash AND outhash=:outhash
+
+ -- Only return one row
+ LIMIT 1
+ ''', {k: insert_data[k] for k in ('method', 'taskhash', 'outhash')})
+
+ row2 = cursor.fetchone()
+
+ if row2 is None:
+ for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
+ if k in data:
+ insert_data[k] = data[k]
+
+ cursor.execute('''INSERT INTO tasks_v2 (%s) VALUES (%s)''' % (
+ ', '.join(sorted(insert_data.keys())),
+ ', '.join(':' + k for k in sorted(insert_data.keys()))),
+ insert_data)
+
+ self.db.commit()
+
+ logger.info('Adding taskhash equivaence for %s with unihash %s',
+ data['taskhash'], row['unihash'])
+
+ d = {
+ 'taskhash': data['taskhash'],
+ 'method': data['method'],
+ 'unihash': row['unihash'],
+ 'outhash': row['outhash'],
+ }
+
+ self.write_message(d)
+
+
async def handle_get_stats(self, request):
d = {
'requests': self.request_stats.todict(),
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list