[bitbake-devel] [OE-core] [PATCH v5 3/8] bitbake: runqueue: Track task unique hash

[Joshua Watt]

On Sat, Jan 5, 2019 at 1:50 AM Alejandro Hernandez
<alejandro.enedino.hernandez-samaniego at xilinx.com> wrote:
>
> Hey Joshua,
>
> This is breaking multiconfig builds with the following error (trimmed),
> I believe it is not taking into account that the Key could contain
> "mc:..." if it is a multiconfig build.

Hmm, yes that seems likely. I'll take a look, would you mind opening a
bug in Bugzilla and assigning it to me? I'm not very familiar with
multiconfig, so some instructions to help reproduce would be very
helpful.

Does anyone know if mutliconfig is tested on the autobuilders?

>
>
> ERROR: Running idle function
>   File "poky/bitbake/lib/bb/runqueue.py", line 1170, in
> RunQueueData.prepare_task_hash(tid='multiconfig:x86:poky/meta/recipes-support/attr/acl_2.2.52.bb:do_fetch'):
>               self.runtaskentries[tid].hash =
> bb.parse.siggen.get_taskhash(taskfn, taskname, procdep, self.dataCaches[mc])
>      >        self.runtaskentries[tid].unihash =
> bb.parse.siggen.get_unihash(fn + "." + taskname)
>
>    File "poky/bitbake/lib/bb/siggen.py", line 45, in
> SignatureGeneratorOEBasicHash.get_unihash(task='poky/meta/recipes-support/attr/acl_2.2.52.bb.do_fetch'):
>           def get_unihash(self, task):
>      >        return self.taskhash[task]
>
> KeyError: 'poky/meta/recipes-support/attr/acl_2.2.52.bb.do_fetch'
>
>
> Cheers,
>
> Alejandro
>
>
> On 12/18/2018 7:10 PM, Joshua Watt wrote:
> > Requests the task unique hash from siggen and tracks it
> >
> > [YOCTO #13030]
> >
> > Signed-off-by: Joshua Watt <JPEWhacker at gmail.com>
> > ---
> >   bitbake/lib/bb/runqueue.py | 25 +++++++++++++++++--------
> >   1 file changed, 17 insertions(+), 8 deletions(-)
> >
> > diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
> > index f2b95a9829b..27b188256dd 100644
> > --- a/bitbake/lib/bb/runqueue.py
> > +++ b/bitbake/lib/bb/runqueue.py
> > @@ -346,6 +346,7 @@ class RunTaskEntry(object):
> >           self.depends = set()
> >           self.revdeps = set()
> >           self.hash = None
> > +        self.unihash = None
> >           self.task = None
> >           self.weight = 1
> >
> > @@ -385,6 +386,9 @@ class RunQueueData:
> >       def get_task_hash(self, tid):
> >           return self.runtaskentries[tid].hash
> >
> > +    def get_task_unihash(self, tid):
> > +        return self.runtaskentries[tid].unihash
> > +
> >       def get_user_idstring(self, tid, task_name_suffix = ""):
> >           return tid + task_name_suffix
> >
> > @@ -1150,18 +1154,21 @@ class RunQueueData:
> >                   if len(self.runtaskentries[tid].depends - dealtwith) == 0:
> >                       dealtwith.add(tid)
> >                       todeal.remove(tid)
> > -                    procdep = []
> > -                    for dep in self.runtaskentries[tid].depends:
> > -                        procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep))
> > -                    (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
> > -                    self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(taskfn, taskname, procdep, self.dataCaches[mc])
> > -                    task = self.runtaskentries[tid].task
> > +                    self.prepare_task_hash(tid)
> >
> >           bb.parse.siggen.writeout_file_checksum_cache()
> >
> >           #self.dump_data()
> >           return len(self.runtaskentries)
> >
> > +    def prepare_task_hash(self, tid):
> > +        procdep = []
> > +        for dep in self.runtaskentries[tid].depends:
> > +            procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep))
> > +        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
> > +        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(taskfn, taskname, procdep, self.dataCaches[mc])
> > +        self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(fn + "." + taskname)
> > +
> >       def dump_data(self):
> >           """
> >           Dump some debug information on the internal data structures
> > @@ -2081,7 +2088,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
> >                   deps = self.rqdata.runtaskentries[revdep].depends
> >                   provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
> >                   taskhash = self.rqdata.runtaskentries[revdep].hash
> > -                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash]
> > +                unihash = self.rqdata.runtaskentries[revdep].unihash
> > +                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash]
> >                   for revdep2 in deps:
> >                       if revdep2 not in taskdepdata:
> >                           additional.append(revdep2)
> > @@ -2524,7 +2532,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
> >                   deps = getsetscenedeps(revdep)
> >                   provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
> >                   taskhash = self.rqdata.runtaskentries[revdep].hash
> > -                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash]
> > +                unihash = self.rqdata.runtaskentries[revdep].unihash
> > +                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash]
> >                   for revdep2 in deps:
> >                       if revdep2 not in taskdepdata:
> >                           additional.append(revdep2)