[oe-commits] [bitbake] 02/05: runqueue: Optimise multiconfig with overlapping setscene
git at git.openembedded.org
git at git.openembedded.org
Fri Jul 12 13:17:54 UTC 2019
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
commit 2135232fb9c031fb655b5d12e34cedb2d82b909c
Author: Richard Purdie <richard.purdie at linuxfoundation.org>
AuthorDate: Thu Jul 11 17:00:10 2019 +0100
runqueue: Optimise multiconfig with overlapping setscene
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/runqueue.py | 33 +++++++++++++++++++---
lib/bb/tests/runqueue-tests/classes/base.bbclass | 19 ++++++++++++-
lib/bb/tests/runqueue-tests/conf/bitbake.conf | 9 ++++--
.../tests/runqueue-tests/conf/multiconfig/mc1.conf | 1 +
.../tests/runqueue-tests/conf/multiconfig/mc2.conf | 1 +
lib/bb/tests/runqueue.py | 19 +++++++++++++
6 files changed, 75 insertions(+), 7 deletions(-)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 1cc2bf2..3cf4c0b 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1717,6 +1717,8 @@ class RunQueueExecute:
self.build_stamps = {}
self.build_stamps2 = []
self.failed_tids = []
+ self.pending_hashes = {}
+ self.sq_deferred = set()
self.stampcache = {}
@@ -1884,6 +1886,10 @@ class RunQueueExecute:
if alldeps:
self.setbuildable(revdep)
logger.debug(1, "Marking task %s as buildable", revdep)
+ if task in self.rqdata.runq_setscene_tids:
+ h = self.rqdata.runtaskentries[task].hash
+ if h in self.pending_hashes:
+ del self.pending_hashes[h]
def task_complete(self, task):
self.stats.taskCompleted()
@@ -1921,14 +1927,25 @@ class RunQueueExecute:
# Find the next setscene to run
for nexttask in self.rqdata.runq_setscene_tids:
if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
- if nexttask in self.sqdata.unskippable:
- logger.debug(2, "Setscene task %s is unskippable" % nexttask)
if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
if nexttask not in self.rqdata.target_tids:
logger.debug(2, "Skipping setscene for task %s" % nexttask)
self.sq_task_skip(nexttask)
self.scenequeue_notneeded.add(nexttask)
return True
+ if self.rqdata.runtaskentries[nexttask].hash in self.pending_hashes:
+ # This setscene task has a hash matching something we're currently building, so wait
+ continue
+ if nexttask in self.sq_deferred:
+ logger.debug(1, "Task %s no longer deferred" % nexttask)
+ self.sq_deferred.remove(nexttask)
+ valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, None, False)
+ if not valid:
+ logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask)
+ self.sq_task_failoutright(nexttask)
+ return True
+ if nexttask in self.sqdata.unskippable:
+ logger.debug(2, "Setscene task %s is unskippable" % nexttask)
task = nexttask
break
if task is not None:
@@ -1979,7 +1996,7 @@ class RunQueueExecute:
if self.can_start_task():
return True
- if not self.sq_live and not self.sqdone:
+ if not self.sq_live and not self.sqdone and not self.sq_deferred:
logger.info("Setscene tasks completed")
logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
@@ -2248,6 +2265,8 @@ class RunQueueExecute:
bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData)
self.scenequeue_notcovered.add(task)
self.tasks_notcovered.add(task)
+ h = self.rqdata.runtaskentries[task].hash
+ self.pending_hashes[h] = task
self.scenequeue_updatecounters(task, True)
self.sq_check_taskfail(task)
@@ -2258,6 +2277,8 @@ class RunQueueExecute:
self.sq_stats.taskCompleted()
self.scenequeue_notcovered.add(task)
self.tasks_notcovered.add(task)
+ h = self.rqdata.runtaskentries[task].hash
+ self.pending_hashes[h] = task
self.scenequeue_updatecounters(task, True)
def sq_task_skip(self, task):
@@ -2548,7 +2569,11 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
valid_new.append(v)
for tid in sqdata.sq_revdeps:
- if tid not in valid_new and tid not in noexec:
+ if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered:
+ if rqdata.runtaskentries[tid].hash in sqrq.pending_hashes:
+ logger.debug(1, "Deferring %s due to %s" % (tid, sqrq.pending_hashes[rqdata.runtaskentries[tid].hash]))
+ sqrq.sq_deferred.add(tid)
+ continue
logger.debug(2, 'No package found, so skipping setscene task %s', tid)
sqrq.sq_task_failoutright(tid)
diff --git a/lib/bb/tests/runqueue-tests/classes/base.bbclass b/lib/bb/tests/runqueue-tests/classes/base.bbclass
index e81df7a..b966568 100644
--- a/lib/bb/tests/runqueue-tests/classes/base.bbclass
+++ b/lib/bb/tests/runqueue-tests/classes/base.bbclass
@@ -4,7 +4,9 @@ SSTATEVALID ??= ""
def stamptask(d):
import time
- thistask = d.expand("${PN}:${BB_CURRENTTASK}")
+ thistask = d.expand("${PN}:${BB_CURRENTTASK}")
+ if d.getVar("BB_CURRENT_MC") != "default":
+ thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
if thistask in d.getVar("SLOWTASKS").split():
bb.note("Slowing task %s" % thistask)
time.sleep(0.5)
@@ -13,48 +15,63 @@ def stamptask(d):
f.write(thistask + "\n")
python do_fetch() {
+ # fetch
stamptask(d)
}
python do_unpack() {
+ # unpack
stamptask(d)
}
python do_patch() {
+ # patch
stamptask(d)
}
python do_populate_lic() {
+ # populate_lic
stamptask(d)
}
python do_prepare_recipe_sysroot() {
+ # prepare_recipe_sysroot
stamptask(d)
}
python do_configure() {
+ # configure
stamptask(d)
}
python do_compile() {
+ # compile
stamptask(d)
}
python do_install() {
+ # install
stamptask(d)
}
python do_populate_sysroot() {
+ # populate_sysroot
stamptask(d)
}
python do_package() {
+ # package
stamptask(d)
}
python do_package_write_ipk() {
+ # package_write_ipk
stamptask(d)
}
python do_package_write_rpm() {
+ # package_write_rpm
stamptask(d)
}
python do_packagedata() {
+ # packagedata
stamptask(d)
}
python do_package_qa() {
+ # package_qa
stamptask(d)
}
python do_build() {
+ # build
stamptask(d)
}
do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
diff --git a/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index cccd677..d87ded2 100644
--- a/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -5,6 +5,11 @@ BBFILES = "${COREBASE}/recipes/*.bb"
PROVIDES = "${PN}"
PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
export PATH
-STAMP = "${TOPDIR}/stamps/${PN}"
-T = "${TOPDIR}/workdir/${PN}/temp"
+TMPDIR ??= "${TOPDIR}"
+STAMP = "${TMPDIR}/stamps/${PN}"
+T = "${TMPDIR}/workdir/${PN}/temp"
BB_NUMBER_THREADS = "4"
+
+BB_HASHBASE_WHITELIST = "BB_CURRENT_MC"
+
+include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
new file mode 100644
index 0000000..ecf23e1
--- /dev/null
+++ b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc1/"
diff --git a/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
new file mode 100644
index 0000000..eef338e
--- /dev/null
+++ b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc2/"
diff --git a/lib/bb/tests/runqueue.py b/lib/bb/tests/runqueue.py
index b1a23bc..8b29fde 100644
--- a/lib/bb/tests/runqueue.py
+++ b/lib/bb/tests/runqueue.py
@@ -189,3 +189,22 @@ class RunQueueTests(unittest.TestCase):
'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
self.assertEqual(set(tasks), set(expected))
+ def test_multiconfig_setscene_optimise(self):
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BBMULTICONFIG" : "mc1 mc2",
+ "BB_SIGNATURE_HANDLER" : "basic"
+ }
+ cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"]
+ setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+ 'populate_sysroot_setscene', 'package_qa_setscene']
+ sstatevalid = ""
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \
+ ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \
+ ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \
+ ['mc1:b1:build', 'mc2:b1:build']
+ for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
+ expected.remove(x)
+ self.assertEqual(set(tasks), set(expected))
+
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list