[oe-commits] [bitbake] 01/03: runqueue: Rework process_possible_migrations() to improve performance
git at git.openembedded.org
git at git.openembedded.org
Sat Dec 14 13:01:49 UTC 2019
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
commit c9c68d898985cf0bec6fc95f54c151cc50255cac
Author: Richard Purdie <richard.purdie at linuxfoundation.org>
AuthorDate: Fri Dec 13 16:06:47 2019 +0000
runqueue: Rework process_possible_migrations() to improve performance
The looping over multiple changed hashes causes many calls to get_taskhash
and get_unihash which are potentially slow and then overwritten.
Instead, batch up all the tasks which have changed unihashes and then
do one big loop over the changed tasks rather than each in turn.
This makes worlds of difference to the performance graphs and should speed
up build where many tasks are being rehashed.
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/runqueue.py | 103 +++++++++++++++++++++++++++++------------------------
1 file changed, 56 insertions(+), 47 deletions(-)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index cb499a1..a45b27c 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -2248,6 +2248,7 @@ class RunQueueExecute:
def process_possible_migrations(self):
changed = set()
+ toprocess = set()
for tid, unihash in self.updated_taskhash_queue.copy():
if tid in self.runq_running and tid not in self.runq_complete:
continue
@@ -2258,53 +2259,61 @@ class RunQueueExecute:
logger.info("Task %s unihash changed to %s" % (tid, unihash))
self.rqdata.runtaskentries[tid].unihash = unihash
bb.parse.siggen.set_unihash(tid, unihash)
-
- # Work out all tasks which depend on this one
- total = set()
- next = set(self.rqdata.runtaskentries[tid].revdeps)
- while next:
- current = next.copy()
- total = total |next
- next = set()
- for ntid in current:
- next |= self.rqdata.runtaskentries[ntid].revdeps
- next.difference_update(total)
-
- # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
- done = set()
- next = set(self.rqdata.runtaskentries[tid].revdeps)
- while next:
- current = next.copy()
- next = set()
- for tid in current:
- if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
- continue
- procdep = []
- for dep in self.rqdata.runtaskentries[tid].depends:
- procdep.append(dep)
- orighash = self.rqdata.runtaskentries[tid].hash
- newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
- origuni = self.rqdata.runtaskentries[tid].unihash
- newuni = bb.parse.siggen.get_unihash(tid)
- # FIXME, need to check it can come from sstate at all for determinism?
- remapped = False
- if newuni == origuni:
- # Nothing to do, we match, skip code below
- remapped = True
- elif tid in self.scenequeue_covered or tid in self.sq_live:
- # Already ran this setscene task or it running. Report the new taskhash
- remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
- logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
-
- if not remapped:
- logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
- self.rqdata.runtaskentries[tid].hash = newhash
- self.rqdata.runtaskentries[tid].unihash = newuni
- changed.add(tid)
-
- next |= self.rqdata.runtaskentries[tid].revdeps
- total.remove(tid)
- next.intersection_update(total)
+ toprocess.add(tid)
+
+ # Work out all tasks which depend upon these
+ total = set()
+ for p in toprocess:
+ next = set(self.rqdata.runtaskentries[p].revdeps)
+ while next:
+ current = next.copy()
+ total = total | next
+ next = set()
+ for ntid in current:
+ next |= self.rqdata.runtaskentries[ntid].revdeps
+ next.difference_update(total)
+
+ # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
+ next = set()
+ for p in total:
+ if len(self.rqdata.runtaskentries[p].depends) == 0:
+ next.add(p)
+ elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
+ next.add(p)
+
+ # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
+ while next:
+ current = next.copy()
+ next = set()
+ for tid in current:
+ if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
+ continue
+ procdep = []
+ for dep in self.rqdata.runtaskentries[tid].depends:
+ procdep.append(dep)
+ orighash = self.rqdata.runtaskentries[tid].hash
+ newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
+ origuni = self.rqdata.runtaskentries[tid].unihash
+ newuni = bb.parse.siggen.get_unihash(tid)
+ # FIXME, need to check it can come from sstate at all for determinism?
+ remapped = False
+ if newuni == origuni:
+ # Nothing to do, we match, skip code below
+ remapped = True
+ elif tid in self.scenequeue_covered or tid in self.sq_live:
+ # Already ran this setscene task or it running. Report the new taskhash
+ remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
+ logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
+
+ if not remapped:
+ #logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
+ self.rqdata.runtaskentries[tid].hash = newhash
+ self.rqdata.runtaskentries[tid].unihash = newuni
+ changed.add(tid)
+
+ next |= self.rqdata.runtaskentries[tid].revdeps
+ total.remove(tid)
+ next.intersection_update(total)
if changed:
for mc in self.rq.worker:
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list