[bitbake-devel] [OE-core] [PATCH v5 3/8] bitbake: runqueue: Track task unique hash

[Joshua Watt]

On Mon, 2019-01-07 at 08:16 -0800, akuster808 wrote:
> 
> On 1/5/19 7:09 PM, Joshua Watt wrote:
> > On Sat, Jan 5, 2019 at 1:50 AM Alejandro Hernandez
> > <alejandro.enedino.hernandez-samaniego at xilinx.com> wrote:
> > > Hey Joshua,
> > > 
> > > This is breaking multiconfig builds with the following error
> > > (trimmed),
> > > I believe it is not taking into account that the Key could
> > > contain
> > > "mc:..." if it is a multiconfig build.
> > Hmm, yes that seems likely. I'll take a look, would you mind
> > opening a
> > bug in Bugzilla and assigning it to me? I'm not very familiar with
> > multiconfig, so some instructions to help reproduce would be very
> > helpful.
> > 
> > Does anyone know if mutliconfig is tested on the autobuilders?
> We don't as far as I know.
> 
> Do we need to bug this as this patch in in master.

I don't know what the exact criteria is for when a bug gets created.
Based on my (limited) previous experience, I thought it deserved a bug
because it was on master and we are past 2.7 M1.

Anyway, it's already been created: 
https://bugzilla.yoctoproject.org/show_bug.cgi?id=13124

> 
> - armin
> > > ERROR: Running idle function
> > >   File "poky/bitbake/lib/bb/runqueue.py", line 1170, in
> > > RunQueueData.prepare_task_hash(tid='multiconfig:x86:poky/meta/rec
> > > ipes-support/attr/acl_2.2.52.bb:do_fetch'):
> > >               self.runtaskentries[tid].hash =
> > > bb.parse.siggen.get_taskhash(taskfn, taskname, procdep,
> > > self.dataCaches[mc])
> > >      >        self.runtaskentries[tid].unihash =
> > > bb.parse.siggen.get_unihash(fn + "." + taskname)
> > > 
> > >    File "poky/bitbake/lib/bb/siggen.py", line 45, in
> > > SignatureGeneratorOEBasicHash.get_unihash(task='poky/meta/recipes
> > > -support/attr/acl_2.2.52.bb.do_fetch'):
> > >           def get_unihash(self, task):
> > >      >        return self.taskhash[task]
> > > 
> > > KeyError: 'poky/meta/recipes-support/attr/acl_2.2.52.bb.do_fetch'
> > > 
> > > 
> > > Cheers,
> > > 
> > > Alejandro
> > > 
> > > 
> > > On 12/18/2018 7:10 PM, Joshua Watt wrote:
> > > > Requests the task unique hash from siggen and tracks it
> > > > 
> > > > [YOCTO #13030]
> > > > 
> > > > Signed-off-by: Joshua Watt <JPEWhacker at gmail.com>
> > > > ---
> > > >   bitbake/lib/bb/runqueue.py | 25 +++++++++++++++++--------
> > > >   1 file changed, 17 insertions(+), 8 deletions(-)
> > > > 
> > > > diff --git a/bitbake/lib/bb/runqueue.py
> > > > b/bitbake/lib/bb/runqueue.py
> > > > index f2b95a9829b..27b188256dd 100644
> > > > --- a/bitbake/lib/bb/runqueue.py
> > > > +++ b/bitbake/lib/bb/runqueue.py
> > > > @@ -346,6 +346,7 @@ class RunTaskEntry(object):
> > > >           self.depends = set()
> > > >           self.revdeps = set()
> > > >           self.hash = None
> > > > +        self.unihash = None
> > > >           self.task = None
> > > >           self.weight = 1
> > > > 
> > > > @@ -385,6 +386,9 @@ class RunQueueData:
> > > >       def get_task_hash(self, tid):
> > > >           return self.runtaskentries[tid].hash
> > > > 
> > > > +    def get_task_unihash(self, tid):
> > > > +        return self.runtaskentries[tid].unihash
> > > > +
> > > >       def get_user_idstring(self, tid, task_name_suffix = ""):
> > > >           return tid + task_name_suffix
> > > > 
> > > > @@ -1150,18 +1154,21 @@ class RunQueueData:
> > > >                   if len(self.runtaskentries[tid].depends -
> > > > dealtwith) == 0:
> > > >                       dealtwith.add(tid)
> > > >                       todeal.remove(tid)
> > > > -                    procdep = []
> > > > -                    for dep in
> > > > self.runtaskentries[tid].depends:
> > > > -                        procdep.append(fn_from_tid(dep) + "."
> > > > + taskname_from_tid(dep))
> > > > -                    (mc, fn, taskname, taskfn) =
> > > > split_tid_mcfn(tid)
> > > > -                    self.runtaskentries[tid].hash =
> > > > bb.parse.siggen.get_taskhash(taskfn, taskname, procdep,
> > > > self.dataCaches[mc])
> > > > -                    task = self.runtaskentries[tid].task
> > > > +                    self.prepare_task_hash(tid)
> > > > 
> > > >           bb.parse.siggen.writeout_file_checksum_cache()
> > > > 
> > > >           #self.dump_data()
> > > >           return len(self.runtaskentries)
> > > > 
> > > > +    def prepare_task_hash(self, tid):
> > > > +        procdep = []
> > > > +        for dep in self.runtaskentries[tid].depends:
> > > > +            procdep.append(fn_from_tid(dep) + "." +
> > > > taskname_from_tid(dep))
> > > > +        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
> > > > +        self.runtaskentries[tid].hash =
> > > > bb.parse.siggen.get_taskhash(taskfn, taskname, procdep,
> > > > self.dataCaches[mc])
> > > > +        self.runtaskentries[tid].unihash =
> > > > bb.parse.siggen.get_unihash(fn + "." + taskname)
> > > > +
> > > >       def dump_data(self):
> > > >           """
> > > >           Dump some debug information on the internal data
> > > > structures
> > > > @@ -2081,7 +2088,8 @@ class
> > > > RunQueueExecuteTasks(RunQueueExecute):
> > > >                   deps =
> > > > self.rqdata.runtaskentries[revdep].depends
> > > >                   provides =
> > > > self.rqdata.dataCaches[mc].fn_provides[taskfn]
> > > >                   taskhash =
> > > > self.rqdata.runtaskentries[revdep].hash
> > > > -                taskdepdata[revdep] = [pn, taskname, fn, deps,
> > > > provides, taskhash]
> > > > +                unihash =
> > > > self.rqdata.runtaskentries[revdep].unihash
> > > > +                taskdepdata[revdep] = [pn, taskname, fn, deps,
> > > > provides, taskhash, unihash]
> > > >                   for revdep2 in deps:
> > > >                       if revdep2 not in taskdepdata:
> > > >                           additional.append(revdep2)
> > > > @@ -2524,7 +2532,8 @@ class
> > > > RunQueueExecuteScenequeue(RunQueueExecute):
> > > >                   deps = getsetscenedeps(revdep)
> > > >                   provides =
> > > > self.rqdata.dataCaches[mc].fn_provides[taskfn]
> > > >                   taskhash =
> > > > self.rqdata.runtaskentries[revdep].hash
> > > > -                taskdepdata[revdep] = [pn, taskname, fn, deps,
> > > > provides, taskhash]
> > > > +                unihash =
> > > > self.rqdata.runtaskentries[revdep].unihash
> > > > +                taskdepdata[revdep] = [pn, taskname, fn, deps,
> > > > provides, taskhash, unihash]
> > > >                   for revdep2 in deps:
> > > >                       if revdep2 not in taskdepdata:
> > > >                           additional.append(revdep2)
-- 
Joshua Watt <JPEWhacker at gmail.com>