[oe-commits] [bitbake] 03/04: runqueue: Rework process_possible_migrations() to improve performance
git at git.openembedded.org
git at git.openembedded.org
Fri Dec 13 16:14:16 UTC 2019
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
commit 5e208a91f104dcd3b357ed0be3974ffabeece6e3
Author: Richard Purdie <richard.purdie at linuxfoundation.org>
AuthorDate: Fri Dec 13 16:06:47 2019 +0000
runqueue: Rework process_possible_migrations() to improve performance
The looping over multiple changed hashes causes many calls to get_taskhash
and get_unihash which are potentially slow and then overwritten.
Instead, batch up all the tasks which have changed unihashes and then
do one big loop over the changed tasks rather than each in turn.
This makes worlds of difference to the performance graphs and should speed
up build where many tasks are being rehashed.
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/runqueue.py | 99 +++++++++++++++++++++++++++++-------------------------
1 file changed, 53 insertions(+), 46 deletions(-)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index cb499a1..bbd5853 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -2248,6 +2248,7 @@ class RunQueueExecute:
def process_possible_migrations(self):
changed = set()
+ toprocess = set()
for tid, unihash in self.updated_taskhash_queue.copy():
if tid in self.runq_running and tid not in self.runq_complete:
continue
@@ -2258,53 +2259,59 @@ class RunQueueExecute:
logger.info("Task %s unihash changed to %s" % (tid, unihash))
self.rqdata.runtaskentries[tid].unihash = unihash
bb.parse.siggen.set_unihash(tid, unihash)
+ toprocess.add(tid)
+
+ # Work out all tasks which depend upon these
+ total = set()
+ for p in toprocess:
+ next = set(self.rqdata.runtaskentries[p].revdeps)
+ while next:
+ current = next.copy()
+ total = total | next
+ next = set()
+ for ntid in current:
+ next |= self.rqdata.runtaskentries[ntid].revdeps
+ next.difference_update(total)
- # Work out all tasks which depend on this one
- total = set()
- next = set(self.rqdata.runtaskentries[tid].revdeps)
- while next:
- current = next.copy()
- total = total |next
- next = set()
- for ntid in current:
- next |= self.rqdata.runtaskentries[ntid].revdeps
- next.difference_update(total)
-
- # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
- done = set()
- next = set(self.rqdata.runtaskentries[tid].revdeps)
- while next:
- current = next.copy()
- next = set()
- for tid in current:
- if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
- continue
- procdep = []
- for dep in self.rqdata.runtaskentries[tid].depends:
- procdep.append(dep)
- orighash = self.rqdata.runtaskentries[tid].hash
- newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
- origuni = self.rqdata.runtaskentries[tid].unihash
- newuni = bb.parse.siggen.get_unihash(tid)
- # FIXME, need to check it can come from sstate at all for determinism?
- remapped = False
- if newuni == origuni:
- # Nothing to do, we match, skip code below
- remapped = True
- elif tid in self.scenequeue_covered or tid in self.sq_live:
- # Already ran this setscene task or it running. Report the new taskhash
- remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
- logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
-
- if not remapped:
- logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
- self.rqdata.runtaskentries[tid].hash = newhash
- self.rqdata.runtaskentries[tid].unihash = newuni
- changed.add(tid)
-
- next |= self.rqdata.runtaskentries[tid].revdeps
- total.remove(tid)
- next.intersection_update(total)
+ # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
+ done = set()
+ next = set()
+ for p in total:
+ if not self.rqdata.runtaskentries[p].depends.isdisjoint(total):
+ next.add(p)
+
+ while next:
+ current = next.copy()
+ next = set()
+ for tid in current:
+ if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
+ continue
+ procdep = []
+ for dep in self.rqdata.runtaskentries[tid].depends:
+ procdep.append(dep)
+ orighash = self.rqdata.runtaskentries[tid].hash
+ newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
+ origuni = self.rqdata.runtaskentries[tid].unihash
+ newuni = bb.parse.siggen.get_unihash(tid)
+ # FIXME, need to check it can come from sstate at all for determinism?
+ remapped = False
+ if newuni == origuni:
+ # Nothing to do, we match, skip code below
+ remapped = True
+ elif tid in self.scenequeue_covered or tid in self.sq_live:
+ # Already ran this setscene task or it running. Report the new taskhash
+ remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
+ logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
+
+ if not remapped:
+ logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
+ self.rqdata.runtaskentries[tid].hash = newhash
+ self.rqdata.runtaskentries[tid].unihash = newuni
+ changed.add(tid)
+
+ next |= self.rqdata.runtaskentries[tid].revdeps
+ total.remove(tid)
+ next.intersection_update(total)
if changed:
for mc in self.rq.worker:
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list