[bitbake-devel] [PATCH] runqueue: Fix setscene hard dependency problems

[Richard Purdie]

Commit c54e738e2b5dc0d8e6fd8e93b284ed96e7a83051 added in the idea of hard dependencies
such as the case a setscene has a hard dependency on pseudo-native and that
dependency wasn't available from sstate for some reason.

Unfortunately the implementation was a bit too enthusiastic, causing rebuilds
of things when it wasn't necessary. A test case was:

bitbake quilt-native
bitbake quilt-native -c clean
bitbake <some-image>

and then you'd watch quilt-native get rebuilt for no good reason.

The clue to the problem is in the for loop where it never depends on
the item being iterated over.

The fix is to include the exact list of hard dependencies rather than
guessing. With these changes, the use case above works, the one in
the original commit also works.

This patch also adds in or cleans up various pieces of logging to
allow issues like this to be more easily debugged in future.

Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 26a0d86..a83fc6a 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1637,7 +1637,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         sq_revdeps = []
         sq_revdeps_new = []
         sq_revdeps_squash = []
-        self.sq_harddeps = []
+        self.sq_harddeps = {}
 
         # We need to construct a dependency graph for the setscene functions. Intermediate
         # dependencies between the setscene tasks only complicate the code. This code
@@ -1748,15 +1748,19 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                     dep = self.rqdata.taskData.fn_index[depdata]
                     taskid = self.rqdata.get_task_id(self.rqdata.taskData.getfn_id(dep), idependtask.replace("_setscene", ""))
                     if taskid is None:
-                        bb.msg.fatal("RunQueue", "Task %s:%s depends upon non-existent task %s:%s" % (self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realid]], self.rqdata.taskData.tasks_name[realid], dep, idependtask))
+                        bb.msg.fatal("RunQueue", "Task %s_setscene depends upon non-existent task %s:%s" % (self.self.rqdata.get_user_idstring(task), dep, idependtask))
+
+                    if not self.rqdata.runq_setscene.index(taskid) in self.sq_harddeps:
+                        self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)] = set()
+                    self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)].add(self.rqdata.runq_setscene.index(task))
 
-                    self.sq_harddeps.append(self.rqdata.runq_setscene.index(taskid))
                     sq_revdeps_squash[self.rqdata.runq_setscene.index(task)].add(self.rqdata.runq_setscene.index(taskid))
                     # Have to zero this to avoid circular dependencies
                     sq_revdeps_squash[self.rqdata.runq_setscene.index(taskid)] = set()
 
         #for task in xrange(len(sq_revdeps_squash)):
-        #    print "Task %s: %s.%s is %s " % (task, self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[self.rqdata.runq_setscene[task]]], self.rqdata.runq_task[self.rqdata.runq_setscene[task]] + "_setscene", sq_revdeps_squash[task])
+        #    realtask = self.rqdata.runq_setscene[task]
+        #    bb.warn("Task %s: %s_setscene is %s " % (task, self.rqdata.get_user_idstring(realtask) , sq_revdeps_squash[task]))
 
         self.sq_deps = []
         self.sq_revdeps = sq_revdeps_squash
@@ -1830,7 +1834,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
     def scenequeue_updatecounters(self, task, fail = False):
         for dep in self.sq_deps[task]:
-            if fail and task in self.sq_harddeps:
+            if fail and task in self.sq_harddeps and dep in self.sq_harddeps[task]:
+                realtask = self.rqdata.runq_setscene[task]
+                realdep = self.rqdata.runq_setscene[dep]
+                logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (self.rqdata.get_user_idstring(realtask), self.rqdata.get_user_idstring(realdep)))
                 continue
             self.sq_revdeps2[dep].remove(task)
             if len(self.sq_revdeps2[dep]) == 0:
@@ -1943,6 +1950,12 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             self.rq.read_workers()
             return self.rq.active_fds()
 
+        #for task in xrange(self.stats.total):
+        #    if self.runq_running[task] != 1:
+        #        buildable = self.runq_buildable[task]
+        #        revdeps = self.sq_revdeps[task]
+        #        bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task])))
+
         # Convert scenequeue_covered task numbers into full taskgraph ids
         oldcovered = self.scenequeue_covered
         self.rq.scenequeue_covered = set()