[oe-commits] [bitbake] 01/02: bitbake: Add support for multiconfig dependencies
git at git.openembedded.org
git at git.openembedded.org
Tue Jul 31 21:57:35 UTC 2018
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
commit 844d069dacbe80d58c4dcf484df233fe73a3f8d4
Author: Alejandro Enedino Hernandez Samaniego <alejandro.enedino.hernandez-samaniego at xilinx.com>
AuthorDate: Wed Jul 25 09:01:01 2018 -0700
bitbake: Add support for multiconfig dependencies
This patch adds the capability for tasks from different
multiconfigs to depend on one another.
These dependencies can be enabled using the following format:
task[mcdepends] = "multiconfig:FROM-MC:TO-MC:PN:task-to-depend-on"
For the sake of simplicity consider the following example:
Assuming we have set up multiconfig builds, one for qemux86 and one for
qemuarm, named x86 and arm respectively.
Adding the following line to an image recipe (core-image-sato):
do_image[mcdepends] = "multiconfig:x86:arm:core-image-minimal:do_rootfs"
Would state that core-image-sato:do_image from x86 will depend on
core-image-minimal:do_rootfs from arm so it can be executed.
This patch makes modifications to:
- cooker: To glue both multiconfigs in one place and make sure
the dependencies can be provided.
- taskdata: To parse and add a new kind of dependency (mcdepends) to
the taskdata object.
- runqueue: To differentiate tasks from different multiconfigs,
add the specified dependencies to the corresponding tasks, and
create a working runqueue that contains tasks from both multiconfigs.
- siggen: To avoid looking for tasks from different multiconfigs on
objects where they dont belong.
The taskdata objects are still not aware of the concept of multiconfig,
so each object doesnt know which multiconfig its building, hence why
the mcdepends are added to all taskdata objects equally (we really
dont expect many of these), but the actual dependencies are added only
to the required tasks by the runqueue.
Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandr at xilinx.com>
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/build.py | 1 +
lib/bb/cooker.py | 34 +++++++++++++++++++++++++++++++---
lib/bb/runqueue.py | 21 +++++++++++++++++++--
lib/bb/siggen.py | 13 +++++++++++--
lib/bb/taskdata.py | 18 +++++++++++++++++-
5 files changed, 79 insertions(+), 8 deletions(-)
diff --git a/lib/bb/build.py b/lib/bb/build.py
index 4631abd..c79354b 100644
--- a/lib/bb/build.py
+++ b/lib/bb/build.py
@@ -803,6 +803,7 @@ def add_tasks(tasklist, d):
if name in flags:
deptask = d.expand(flags[name])
task_deps[name][task] = deptask
+ getTask('mcdepends')
getTask('depends')
getTask('rdepends')
getTask('deptask')
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 1fda40d..946ba9c 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -609,7 +609,14 @@ class BBCooker:
k2 = k.split(":do_")
k = k2[0]
ktask = k2[1]
- taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
+ if mc:
+ # Provider might be from another mc
+ for mcavailable in self.multiconfigs:
+ # The first element is empty
+ if mcavailable:
+ taskdata[mcavailable].add_provider(localdata[mcavailable], self.recipecaches[mcavailable], k)
+ else:
+ taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
current += 1
if not ktask.startswith("do_"):
ktask = "do_%s" % ktask
@@ -620,6 +627,27 @@ class BBCooker:
runlist.append([mc, k, ktask, fn])
bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
+ mcdeps = taskdata[mc].get_mcdepends()
+ # No need to do check providers if there are no mcdeps or not an mc build
+ if mcdeps and mc:
+ # Make sure we can provide the multiconfig dependency
+ seen = set()
+ new = True
+ while new:
+ new = False
+ for mc in self.multiconfigs:
+ for k in mcdeps:
+ if k in seen:
+ continue
+ l = k.split(':')
+ depmc = l[2]
+ if depmc not in self.multiconfigs:
+ bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
+ else:
+ logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
+ taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
+ seen.add(k)
+ new = True
for mc in self.multiconfigs:
taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
@@ -706,8 +734,8 @@ class BBCooker:
if not dotname in depend_tree["tdepends"]:
depend_tree["tdepends"][dotname] = []
for dep in rq.rqdata.runtaskentries[tid].depends:
- (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
- deppn = self.recipecaches[mc].pkg_fn[deptaskfn]
+ (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
+ deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
if taskfn not in seen_fns:
seen_fns.append(taskfn)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index ba9bebe..a43c998 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -602,6 +602,19 @@ class RunQueueData:
if t in taskData[mc].taskentries:
depends.add(t)
+ def add_mc_dependencies(mc, tid):
+ mcdeps = taskData[mc].get_mcdepends()
+ for dep in mcdeps:
+ mcdependency = dep.split(':')
+ pn = mcdependency[3]
+ frommc = mcdependency[1]
+ mcdep = mcdependency[2]
+ deptask = mcdependency[4]
+ if mc == frommc:
+ fn = taskData[mcdep].build_targets[pn][0]
+ newdep = '%s:%s' % (fn,deptask)
+ taskData[mc].taskentries[tid].tdepends.append(newdep)
+
for mc in taskData:
for tid in taskData[mc].taskentries:
@@ -618,12 +631,16 @@ class RunQueueData:
if fn in taskData[mc].failed_fns:
continue
+ # We add multiconfig dependencies before processing internal task deps (tdepends)
+ if 'mcdepends' in task_deps and taskname in task_deps['mcdepends']:
+ add_mc_dependencies(mc, tid)
+
# Resolve task internal dependencies
#
# e.g. addtask before X after Y
for t in taskData[mc].taskentries[tid].tdepends:
- (_, depfn, deptaskname, _) = split_tid_mcfn(t)
- depends.add(build_tid(mc, depfn, deptaskname))
+ (depmc, depfn, deptaskname, _) = split_tid_mcfn(t)
+ depends.add(build_tid(depmc, depfn, deptaskname))
# Resolve 'deptask' dependencies
#
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index ab228e4..e9bb51d 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -193,15 +193,24 @@ class SignatureGeneratorBasic(SignatureGenerator):
return taint
def get_taskhash(self, fn, task, deps, dataCache):
+
+ mc = ''
+ if fn.startswith('multiconfig:'):
+ mc = fn.split(':')[1]
k = fn + "." + task
+
data = dataCache.basetaskhash[k]
self.basehash[k] = data
self.runtaskdeps[k] = []
self.file_checksum_values[k] = []
recipename = dataCache.pkg_fn[fn]
-
for dep in sorted(deps, key=clean_basepath):
- depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')]
+ pkgname = self.pkgnameextract.search(dep).group('fn')
+ if mc:
+ depmc = pkgname.split(':')[1]
+ if mc != depmc:
+ continue
+ depname = dataCache.pkg_fn[pkgname]
if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
continue
if dep not in self.taskhash:
diff --git a/lib/bb/taskdata.py b/lib/bb/taskdata.py
index 0ea6c0b..94e822c 100644
--- a/lib/bb/taskdata.py
+++ b/lib/bb/taskdata.py
@@ -70,6 +70,8 @@ class TaskData:
self.skiplist = skiplist
+ self.mcdepends = []
+
def add_tasks(self, fn, dataCache):
"""
Add tasks for a given fn to the database
@@ -88,6 +90,13 @@ class TaskData:
self.add_extra_deps(fn, dataCache)
+ def add_mcdepends(task):
+ for dep in task_deps['mcdepends'][task].split():
+ if len(dep.split(':')) != 5:
+ bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'multiconfig:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
+ if dep not in self.mcdepends:
+ self.mcdepends.append(dep)
+
# Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
def handle_deps(task, dep_name, depends, seen):
if dep_name in task_deps and task in task_deps[dep_name]:
@@ -110,16 +119,20 @@ class TaskData:
parentids = []
for dep in task_deps['parents'][task]:
if dep not in task_deps['tasks']:
- bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep))
+ bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
continue
parentid = "%s:%s" % (fn, dep)
parentids.append(parentid)
self.taskentries[tid].tdepends.extend(parentids)
+
# Touch all intertask dependencies
handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
+ if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
+ add_mcdepends(task)
+
# Work out build dependencies
if not fn in self.depids:
dependids = set()
@@ -537,6 +550,9 @@ class TaskData:
provmap[name] = provider[0]
return provmap
+ def get_mcdepends(self):
+ return self.mcdepends
+
def dump_data(self):
"""
Dump some debug information on the internal data structures
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list