[OE-core] [PATCH 5/5] sstatesig: incremental dump lockedsigs

[Hongxu Jia]

While including an existed lockedsigs file to build from
locked sstate-cache, and we want to incremental build the
locked sstate-cache. Config with:
...
SIGGEN_DUMP_LOCKEDSIGS = '1'
SIGGEN_LOCKEDSIGS_CONFIG = "${TOPDIR}/locked-sigs.inc"
require ${SIGGEN_LOCKEDSIGS_CONFIG}
...
So we support to dump lockedsigs file incrementally

Since we improve to handle locking of multiple machines and
add a new SIGGEN_LOCKEDSIGS_TYPES variable which lists the
package architectures to load in. We should add package
architectures as type to self.lockedsigs to reflect that.
Such as:
{recipename:{task:{hash1}}} --> {type:{recipename:{task:{hash1}}}}

Signed-off-by: Hongxu Jia <hongxu.jia at windriver.com>
---
 meta/lib/oe/sstatesig.py | 67 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 45 insertions(+), 22 deletions(-)

diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index c9edd80..8b491a6 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -65,12 +65,15 @@ def sstate_lockedsigs(d):
     sigs = {}
     types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split()
     for t in types:
+        if t not in sigs:
+            sigs[t] = {}
+
         lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split()
         for ls in lockedsigs:
             pn, task, h = ls.split(":", 2)
-            if pn not in sigs:
-                sigs[pn] = {}
-            sigs[pn][task] = h
+            if pn not in sigs[t]:
+                sigs[t][pn] = {}
+            sigs[t][pn][task] = h
     return sigs
 
 class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
@@ -115,10 +118,14 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
         recipename = dataCache.pkg_fn[fn]
         self.lockedpnmap[fn] = recipename
         self.lockedhashfn[fn] = dataCache.hashfn[fn]
-        if recipename in self.lockedsigs:
-            if task in self.lockedsigs[recipename]:
+
+        t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
+        if t not in self.lockedsigs:
+            return h
+        if recipename in self.lockedsigs[t]:
+            if task in self.lockedsigs[t][recipename]:
                 k = fn + "." + task
-                h_locked = self.lockedsigs[recipename][task]
+                h_locked = self.lockedsigs[t][recipename][task]
                 self.lockedhashes[k] = h_locked
                 self.taskhash[k] = h_locked
                 #bb.warn("Using %s %s %s" % (recipename, task, h))
@@ -137,6 +144,19 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
             return
         super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
 
+    def update_lockedsigs(self, type, recipename, task, hash):
+        if type not in self.lockedsigs:
+            self.lockedsigs[type] = {}
+
+        if recipename not in self.lockedsigs[type]:
+            self.lockedsigs[type][recipename] = {}
+
+        if task not in self.lockedsigs[type][recipename]:
+            self.lockedsigs[type][recipename][task] = hash
+        elif hash != self.lockedsigs[type][recipename][task]:
+            bb.warn('Conflict %s %s %s (new %s, old %s)' %
+                    (type, recipename, task, hash, self.lockedsigs[type][recipename][task]))
+
     def dump_lockedsigs(self, where_to_dump=None):
         if not where_to_dump:
             where_to_dump = os.getcwd() + "/locked-sigs.inc"
@@ -146,30 +166,33 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
         for k in self.runtaskdeps:
             fn = k.rsplit(".",1)[0]
             t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
-            if t not in types:
-                types[t] = []
-            types[t].append(k)
+            recipename = self.lockedpnmap[fn]
+            task = k.rsplit(".",1)[1]
+            hash = self.taskhash[k]
+            self.update_lockedsigs(t, recipename, task, hash)
 
         with open(where_to_dump, "w") as f:
-            for t in types:
-                f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
-                types[t].sort()
-                sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
-                for k in sortedk:
-                    fn = k.rsplit(".",1)[0]
-                    task = k.rsplit(".",1)[1]
-                    if k not in self.taskhash:
-                        continue
-                    f.write("    " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n")
+            for type in sorted(self.lockedsigs):
+                f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % type)
+
+                for recipename in sorted(self.lockedsigs[type]):
+                    for task in sorted(self.lockedsigs[type][recipename]):
+                        hash = self.lockedsigs[type][recipename][task]
+                        f.write("    " + recipename + ":" + task + ":" + hash + " \\\n")
                 f.write('    "\n')
-            f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
+
+            f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(self.lockedsigs.keys())))
 
     def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
         checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True)
         for task in range(len(sq_fn)):
             if task not in ret:
-                for pn in self.lockedsigs:
-                    if sq_hash[task] in self.lockedsigs[pn].itervalues():
+                t = sq_hashfn[task].split(" ")[1].split(":")[5]
+                if t not in self.lockedsigs:
+                    continue
+
+                for pn in self.lockedsigs[t]:
+                    if sq_hash[task] in self.lockedsigs[t][pn].itervalues():
                         self.checkmsgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
                                                % (pn, sq_task[task], sq_hash[task]))
 
-- 
1.9.1