[oe-commits] [bitbake] 01/02: bitbake: Fix up equivalent hash reporting
git at git.openembedded.org
git at git.openembedded.org
Tue Dec 3 18:37:14 UTC 2019
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
commit c942fd95171a5f87e5c4b43e48d9a8e83a7e7831
Author: Joshua Watt <JPEWhacker at gmail.com>
AuthorDate: Tue Dec 3 10:46:13 2019 -0600
bitbake: Fix up equivalent hash reporting
Equivalent hashes should be reported to the server as a taskhash that
needs to map to an specific unihash. Update the API to reflect this, and
also do some other cleanup like renaming variables to make more sense
and improving logging.
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/runqueue.py | 17 ++++++++-------
lib/bb/siggen.py | 20 ++++++++++-------
lib/hashserv/server.py | 59 +++++++++++++++-----------------------------------
3 files changed, 38 insertions(+), 58 deletions(-)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 2813462..f505488 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -2289,12 +2289,13 @@ class RunQueueExecute:
newuni = bb.parse.siggen.get_unihash(tid)
# FIXME, need to check it can come from sstate at all for determinism?
remapped = False
- if tid in self.scenequeue_covered or tid in self.sq_live:
- # Already ran this setscene task or it running
- remapped = bb.parse.siggen.report_unihash_equiv(tid, origuni, newuni, self.rqdata.dataCaches)
- logger.info("Already covered setscene for %s so ignoring rehash" % (tid))
+ if (tid in self.scenequeue_covered or tid in self.sq_live) and newuni != origuni:
+ # Already ran this setscene task or it running. Report the new taskhash
+ remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
+ logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
+
if not remapped:
- logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, self.rqdata.runtaskentries[tid].hash, origuni, self.rqdata.runtaskentries[tid].unihash))
+ logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
self.rqdata.runtaskentries[tid].hash = newhash
self.rqdata.runtaskentries[tid].unihash = newuni
self.rehashes[tid] = origuni
@@ -2331,9 +2332,9 @@ class RunQueueExecute:
if tid in self.scenequeue_covered or tid in self.sq_live:
# Already ran this setscene task or it running
- # Potentially risky, should we report this hash as a match
- bb.parse.siggen.report_unihash_equiv(tid, self.rehashes[tid], self.rqdata.runtaskentries[tid].unihash, self.rqdata.dataCaches)
- logger.info("Already covered setscene for %s so ignoring rehash" % (tid))
+ # "Force" the new taskhash to be equivalent to the old unihash
+ bb.parse.siggen.report_unihash_equiv(tid, self.rqdata.runtaskentries[tid].hash, self.rehashes[tid], self.rqdata.runtaskentries[tid].unihash, self.rqdata.dataCaches)
+ logger.info("Already covered setscene for %s so ignoring rehash (pending migration)" % (tid))
self.pending_migrations.remove(tid)
continue
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index bc91203..bc8bdee 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -525,29 +525,33 @@ class SignatureGeneratorUniHashMixIn(object):
except OSError:
pass
- def report_unihash_equiv(self, tid, oldunihash, newunihash, datacaches):
+ def report_unihash_equiv(self, tid, taskhash, wanted_unihash, current_unihash, datacaches):
try:
extra_data = {}
- data = self.client().report_unihash_equiv(newunihash, self.method, oldunihash, extra_data)
- bb.note('Reported task %s as unihash %s to %s (%s)' % (tid, newunihash, self.server, str(data)))
+ data = self.client().report_unihash_equiv(taskhash, self.method, wanted_unihash, extra_data)
+ bb.note('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
if data is None:
bb.warn("Server unable to handle unihash report")
- return
+ return False
finalunihash = data['unihash']
- if newunihash != finalunihash:
- bb.note('Task %s unihash changed %s -> %s (%s)' % (tid, newunihash, finalunihash, oldunihash))
+ if finalunihash == current_unihash:
+ bb.note('Task %s unihash %s unchanged by server' % (tid, finalunihash))
+ elif finalunihash == wanted_unihash:
+ bb.note('Task %s unihash changed %s -> %s as wanted' % (tid, current_unihash, finalunihash))
self.set_unihash(tid, finalunihash)
return True
else:
- bb.note('Task %s unihash %s unchanged by server' % (tid, finalunihash))
- return False
+ # TODO: What to do here?
+ bb.note('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
except hashserv.client.HashConnectionError as e:
bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
+ return False
+
#
# Dummy class used for bitbake-selftest
#
diff --git a/lib/hashserv/server.py b/lib/hashserv/server.py
index 0a7d932..cc7e482 100644
--- a/lib/hashserv/server.py
+++ b/lib/hashserv/server.py
@@ -306,60 +306,35 @@ class ServerClient(object):
async def handle_equivreport(self, data):
with closing(self.db.cursor()) as cursor:
- cursor.execute('''
- -- Find the task entry for the matching taskhash
- SELECT unihash, outhash FROM tasks_v2 WHERE method=:method AND taskhash=:unihash
-
- -- Only return one row
- LIMIT 1
- ''', {k: data[k] for k in ('method', 'unihash')})
-
- row = cursor.fetchone()
-
- if row is None:
- self.write_message(None)
- return
-
insert_data = {
'method': data['method'],
- 'outhash': row['outhash'],
+ 'outhash': "",
'taskhash': data['taskhash'],
- 'unihash': row['unihash'],
+ 'unihash': data['unihash'],
'created': datetime.now()
}
+ for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
+ if k in data:
+ insert_data[k] = data[k]
- cursor.execute('''
- -- Find the task entry for the matching taskhash
- SELECT outhash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash AND outhash=:outhash
+ cursor.execute('''INSERT OR IGNORE INTO tasks_v2 (%s) VALUES (%s)''' % (
+ ', '.join(sorted(insert_data.keys())),
+ ', '.join(':' + k for k in sorted(insert_data.keys()))),
+ insert_data)
- -- Only return one row
- LIMIT 1
- ''', {k: insert_data[k] for k in ('method', 'taskhash', 'outhash')})
+ self.db.commit()
- row2 = cursor.fetchone()
+ # Fetch the unihash that will be reported for the taskhash. If the
+ # unihash matches, it means this row was inserted (or the mapping
+ # was already valid)
+ row = self.query_equivalent(data['method'], data['taskhash'])
- if row2 is None:
- for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
- if k in data:
- insert_data[k] = data[k]
-
- cursor.execute('''INSERT INTO tasks_v2 (%s) VALUES (%s)''' % (
- ', '.join(sorted(insert_data.keys())),
- ', '.join(':' + k for k in sorted(insert_data.keys()))),
- insert_data)
-
- self.db.commit()
-
- logger.info('Adding taskhash equivaence for %s with unihash %s',
+ if row['unihash'] == data['unihash']:
+ logger.info('Adding taskhash equivalence for %s with unihash %s',
data['taskhash'], row['unihash'])
- d = {
- 'taskhash': data['taskhash'],
- 'method': data['method'],
- 'unihash': row['unihash'],
- 'outhash': row['outhash'],
- }
+ d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
self.write_message(d)
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list