[oe-commits] [bitbake] 04/10: runqueue: add ability to enforce that tasks are setscened
git at git.openembedded.org
git at git.openembedded.org
Tue Jul 5 12:40:14 UTC 2016
rpurdie pushed a commit to branch master-next
in repository bitbake.
commit 624722c067a7fdd0c0f5d8be611e1f6666ecc4a2
Author: Paul Eggleton <paul.eggleton at linux.intel.com>
AuthorDate: Thu Jun 23 22:59:11 2016 +1200
runqueue: add ability to enforce that tasks are setscened
Add the ability to enter a mode where only a specified whitelist of
tasks can be executed outright; everything else must be successfully
provided in the form of a setscene task (or covered by a setscene task).
Any setscene failure outside of the whitelist will cause the build to
fail immediately instead of running the real task, and any real tasks
that would execute outside of the whitelist cause an immediate build
failure when it comes to executing the runqueue as well.
The mode is enabled by setting BB_SETSCENE_ENFORCE="1", and the
whitelist is specified through BB_SETSCENE_ENFORCE_WHITELIST, consisting
of pn:taskname pairs. A single % character can be substituted for the pn
value to match any target explicitly specified on the bitbake command
line. Wildcards * and ? can also be used as per standard unix file name
matching for both pn and taskname.
Part of the implementation of [YOCTO #9367].
Signed-off-by: Paul Eggleton <paul.eggleton at linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/runqueue.py | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 80 insertions(+), 1 deletion(-)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index c9b6b84..b62a28a 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -240,6 +240,7 @@ class RunQueueData:
self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
+ self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
self.reset()
@@ -1596,8 +1597,51 @@ class RunQueueExecuteTasks(RunQueueExecute):
Run the tasks in a queue prepared by rqdata.prepare()
"""
- self.rq.read_workers()
+ if self.rqdata.setscenewhitelist:
+ # Check tasks that are going to run against the whitelist
+ def check_norun_task(tid, showerror=False):
+ fn = fn_from_tid(tid)
+ taskname = taskname_from_tid(tid)
+ # Ignore covered tasks
+ if tid in self.rq.scenequeue_covered:
+ return False
+ # Ignore stamped tasks
+ if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache):
+ return False
+ # Ignore noexec tasks
+ taskdep = self.rqdata.dataCache.task_deps[fn]
+ if 'noexec' in taskdep and taskname in taskdep['noexec']:
+ return False
+ pn = self.rqdata.dataCache.pkg_fn[fn]
+ if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+ if showerror:
+ if tid in self.rqdata.runq_setscene_tids:
+ logger.error('Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname))
+ else:
+ logger.error('Task %s.%s attempted to execute unexpectedly' % (pn, taskname))
+ return True
+ return False
+ # Look to see if any tasks that we think shouldn't run are going to
+ unexpected = False
+ for tid in self.rqdata.runtaskentries:
+ if check_norun_task(tid):
+ unexpected = True
+ break
+ if unexpected:
+ # Run through the tasks in the rough order they'd have executed and print errors
+ # (since the order can be useful - usually missing sstate for the last few tasks
+ # is the cause of the problem)
+ task = self.sched.next()
+ while task is not None:
+ check_norun_task(task, showerror=True)
+ self.task_skip(task, 'Setscene enforcement check')
+ task = self.sched.next()
+
+ self.rq.state = runQueueCleanUp
+ return True
+
+ self.rq.read_workers()
if self.stats.total == 0:
# nothing to do
@@ -1940,6 +1984,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.scenequeue_covered.add(task)
self.scenequeue_updatecounters(task)
+ def check_taskfail(self, task):
+ if self.rqdata.setscenewhitelist:
+ realtask = task.split('_setscene')[0]
+ fn = fn_from_tid(realtask)
+ taskname = taskname_from_tid(realtask)
+ pn = self.rqdata.dataCache.pkg_fn[fn]
+ if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+ logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
+ self.rq.state = runQueueCleanUp
+
def task_complete(self, task):
self.stats.taskCompleted()
bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
@@ -1950,6 +2004,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
self.scenequeue_notcovered.add(task)
self.scenequeue_updatecounters(task, True)
+ self.check_taskfail(task)
def task_failoutright(self, task):
self.runq_running.add(task)
@@ -2231,3 +2286,27 @@ class runQueuePipe():
if len(self.queue) > 0:
print("Warning, worker left partial message: %s" % self.queue)
self.input.close()
+
+def get_setscene_enforce_whitelist(d):
+ if d.getVar('BB_SETSCENE_ENFORCE', True) != '1':
+ return None
+ whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST", True) or "").split()
+ outlist = []
+ for item in whitelist[:]:
+ if item.startswith('%:'):
+ for target in sys.argv[1:]:
+ if not target.startswith('-'):
+ outlist.append(target.split(':')[0] + ':' + item.split(':')[1])
+ else:
+ outlist.append(item)
+ return outlist
+
+def check_setscene_enforce_whitelist(pn, taskname, whitelist):
+ import fnmatch
+ if whitelist:
+ item = '%s:%s' % (pn, taskname)
+ for whitelist_item in whitelist:
+ if fnmatch.fnmatch(item, whitelist_item):
+ return True
+ return False
+ return True
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list