[oe-commits] [bitbake] 01/02: runqueue: Optimise multiconfig with overlapping setscene
git at git.openembedded.org
git at git.openembedded.org
Tue Jul 16 12:54:29 UTC 2019
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master
in repository bitbake.
commit a75c5fd6d4ec56836de0be2fe679c81297a080ad
Author: Richard Purdie <richard.purdie at linuxfoundation.org>
AuthorDate: Thu Jul 11 17:05:19 2019 +0100
runqueue: Optimise multiconfig with overlapping setscene
Currently if a multiconfig build contains different configurations which
have overlapping sstate artefacts, it will build them multiple times.
This is clearly suboptimal and not what users want/expect.
This adds code to detect this and stall all but one of the setscne tasks
so that once its built, it can be found by the other tasks.
We take care to iterate the multiconfigs in order so try and avoid
dependency loops. We also match on PN+taskname+taskhash since this is
what we know sstate in OE-Core would use. There are some tasks even within
a multiconfig which match hashes (mostly do_populate_lic tasks) but those
have a much higher chance of circular dependency so aren't work attempting
to optimise.
If a deadlock does occur the build will be slower but there is code to
unbreak such a deadlock so it hopefully doens't break anything.
Comments are injected into the test tasks so they have different task
hashes and a new test for this optimisation is added.
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/runqueue.py | 60 +++++++++++++++++++---
lib/bb/tests/runqueue-tests/classes/base.bbclass | 19 ++++++-
lib/bb/tests/runqueue-tests/conf/bitbake.conf | 9 +++-
.../tests/runqueue-tests/conf/multiconfig/mc1.conf | 1 +
.../tests/runqueue-tests/conf/multiconfig/mc2.conf | 1 +
lib/bb/tests/runqueue.py | 19 +++++++
6 files changed, 98 insertions(+), 11 deletions(-)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 00c7107..fa84832 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -68,6 +68,14 @@ def build_tid(mc, fn, taskname):
return "mc:" + mc + ":" + fn + ":" + taskname
return fn + ":" + taskname
+# Index used to pair up potentially matching multiconfig tasks
+# We match on PN, taskname and hash being equal
+def pending_hash_index(tid, rqdata):
+ (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+ pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
+ h = rqdata.runtaskentries[tid].hash
+ return pn + ":" + "taskname" + h
+
class RunQueueStats:
"""
Holds statistics on the tasks handled by the associated runQueue
@@ -1717,6 +1725,7 @@ class RunQueueExecute:
self.build_stamps = {}
self.build_stamps2 = []
self.failed_tids = []
+ self.sq_deferred = {}
self.stampcache = {}
@@ -1921,17 +1930,32 @@ class RunQueueExecute:
# Find the next setscene to run
for nexttask in self.rqdata.runq_setscene_tids:
if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
- if nexttask in self.sqdata.unskippable:
- logger.debug(2, "Setscene task %s is unskippable" % nexttask)
if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
if nexttask not in self.rqdata.target_tids:
logger.debug(2, "Skipping setscene for task %s" % nexttask)
self.sq_task_skip(nexttask)
self.scenequeue_notneeded.add(nexttask)
+ if nexttask in self.sq_deferred:
+ del self.sq_deferred[nexttask]
+ return True
+ if nexttask in self.sq_deferred:
+ if self.sq_deferred[nexttask] not in self.runq_complete:
+ continue
+ logger.debug(1, "Task %s no longer deferred" % nexttask)
+ del self.sq_deferred[nexttask]
+ valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, None, False)
+ if not valid:
+ logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask)
+ self.sq_task_failoutright(nexttask)
return True
+ else:
+ self.sqdata.outrightfail.remove(nexttask)
if nexttask in self.sqdata.outrightfail:
+ logger.debug(2, 'No package found, so skipping setscene task %s', nexttask)
self.sq_task_failoutright(nexttask)
return True
+ if nexttask in self.sqdata.unskippable:
+ logger.debug(2, "Setscene task %s is unskippable" % nexttask)
task = nexttask
break
if task is not None:
@@ -1982,7 +2006,7 @@ class RunQueueExecute:
if self.can_start_task():
return True
- if not self.sq_live and not self.sqdone:
+ if not self.sq_live and not self.sqdone and not self.sq_deferred:
logger.info("Setscene tasks completed")
logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
@@ -2083,6 +2107,13 @@ class RunQueueExecute:
self.rq.read_workers()
return self.rq.active_fds()
+ # No more tasks can be run. If we have deferred setscene tasks we should run them.
+ if self.sq_deferred:
+ tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0])
+ logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid)
+ self.sq_task_failoutright(tid)
+ return True
+
if len(self.failed_tids) != 0:
self.rq.state = runQueueFailed
return True
@@ -2347,7 +2378,7 @@ class SQData(object):
# Setscene tasks directly depended upon by the build
self.unskippable = set()
# List of setscene tasks which aren't present
- self.outrightfail = []
+ self.outrightfail = set()
# A list of normal tasks a setscene task covers
self.sq_covered_tasks = {}
@@ -2510,7 +2541,9 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
rqdata.init_progress_reporter.next_stage()
+ multiconfigs = set()
for tid in sqdata.sq_revdeps:
+ multiconfigs.add(mc_from_tid(tid))
if len(sqdata.sq_revdeps[tid]) == 0:
sqrq.sq_buildable.add(tid)
@@ -2552,10 +2585,21 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
for v in valid:
valid_new.append(v)
- for tid in sqdata.sq_revdeps:
- if tid not in valid_new and tid not in noexec:
- logger.debug(2, 'No package found, so skipping setscene task %s', tid)
- sqdata.outrightfail.append(tid)
+ hashes = {}
+ for mc in sorted(multiconfigs):
+ for tid in sqdata.sq_revdeps:
+ if mc_from_tid(tid) != mc:
+ continue
+ if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered:
+ sqdata.outrightfail.add(tid)
+
+ h = pending_hash_index(tid, rqdata)
+ if h not in hashes:
+ hashes[h] = tid
+ else:
+ sqrq.sq_deferred[tid] = hashes[h]
+ bb.warn("Deferring %s after %s" % (tid, hashes[h]))
+
class TaskFailure(Exception):
"""
diff --git a/lib/bb/tests/runqueue-tests/classes/base.bbclass b/lib/bb/tests/runqueue-tests/classes/base.bbclass
index e174c02..cf38d09 100644
--- a/lib/bb/tests/runqueue-tests/classes/base.bbclass
+++ b/lib/bb/tests/runqueue-tests/classes/base.bbclass
@@ -4,7 +4,9 @@ SSTATEVALID ??= ""
def stamptask(d):
import time
- thistask = d.expand("${PN}:${BB_CURRENTTASK}")
+ thistask = d.expand("${PN}:${BB_CURRENTTASK}")
+ if d.getVar("BB_CURRENT_MC") != "default":
+ thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
if thistask in d.getVar("SLOWTASKS").split():
bb.note("Slowing task %s" % thistask)
time.sleep(0.5)
@@ -13,48 +15,63 @@ def stamptask(d):
f.write(thistask + "\n")
python do_fetch() {
+ # fetch
stamptask(d)
}
python do_unpack() {
+ # unpack
stamptask(d)
}
python do_patch() {
+ # patch
stamptask(d)
}
python do_populate_lic() {
+ # populate_lic
stamptask(d)
}
python do_prepare_recipe_sysroot() {
+ # prepare_recipe_sysroot
stamptask(d)
}
python do_configure() {
+ # configure
stamptask(d)
}
python do_compile() {
+ # compile
stamptask(d)
}
python do_install() {
+ # install
stamptask(d)
}
python do_populate_sysroot() {
+ # populate_sysroot
stamptask(d)
}
python do_package() {
+ # package
stamptask(d)
}
python do_package_write_ipk() {
+ # package_write_ipk
stamptask(d)
}
python do_package_write_rpm() {
+ # package_write_rpm
stamptask(d)
}
python do_packagedata() {
+ # packagedata
stamptask(d)
}
python do_package_qa() {
+ # package_qa
stamptask(d)
}
python do_build() {
+ # build
stamptask(d)
}
do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
diff --git a/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index 8c7b754..96ee1cd 100644
--- a/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -6,6 +6,11 @@ PROVIDES = "${PN}"
PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
PF = "${BB_CURRENT_MC}:${PN}"
export PATH
-STAMP = "${TOPDIR}/stamps/${PN}"
-T = "${TOPDIR}/workdir/${PN}/temp"
+TMPDIR ??= "${TOPDIR}"
+STAMP = "${TMPDIR}/stamps/${PN}"
+T = "${TMPDIR}/workdir/${PN}/temp"
BB_NUMBER_THREADS = "4"
+
+BB_HASHBASE_WHITELIST = "BB_CURRENT_MC"
+
+include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
new file mode 100644
index 0000000..ecf23e1
--- /dev/null
+++ b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc1/"
diff --git a/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
new file mode 100644
index 0000000..eef338e
--- /dev/null
+++ b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc2/"
diff --git a/lib/bb/tests/runqueue.py b/lib/bb/tests/runqueue.py
index 4a65b5b..f0cea64 100644
--- a/lib/bb/tests/runqueue.py
+++ b/lib/bb/tests/runqueue.py
@@ -198,3 +198,22 @@ class RunQueueTests(unittest.TestCase):
'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
self.assertEqual(set(tasks), set(expected))
+ def test_multiconfig_setscene_optimise(self):
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BBMULTICONFIG" : "mc1 mc2",
+ "BB_SIGNATURE_HANDLER" : "basic"
+ }
+ cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"]
+ setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+ 'populate_sysroot_setscene', 'package_qa_setscene']
+ sstatevalid = ""
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \
+ ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \
+ ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \
+ ['mc1:b1:build', 'mc2:b1:build']
+ for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
+ expected.remove(x)
+ self.assertEqual(set(tasks), set(expected))
+
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list