[bitbake-devel] FW: [PATCH] Checksums for local files now stored using partial recipe path

[Jate Sujjavanich]

This allows sstate-cache to be shared between builds in different
directories.

Differences in the full path were triggering a false positive when
there were actually no changes.

Signed-off-by: Jate Sujjavanich <jate.sujjavanich at myfuelmaster.com>
---
Additional comments (cross post from oe-core mailing list)
On the same machine, I had two identical poky directories. One, I built first to use as the cache source. I then configured the second one to use a SSTATE_MIRRORS pointing to the first.

The build of the second would just detect changes and force a rebuild. Running the bitbake-diffsigs tool on the do_fetch...sigdata showed something like the following on all of the local files:

Removed("/home/user/poky1/meta/recipes...file1", chksum1)
Added("/home/user/poky2/meta/recipes...file1", chksum1)

Upon applying my patch, the components of core-minimal-image were pulled from the sstate-cache.

I will have to see about the other local file cases that I don't know.
---
 bitbake/lib/bb/fetch2/__init__.py |   14 +++++++++-----
 bitbake/lib/bb/siggen.py          |    3 ++-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index dd1cc93..7ab44d7 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -900,8 +900,7 @@ def get_checksum_file_list(d):

     return " ".join(filelist)

-
-def get_file_checksums(filelist, pn):
+def get_file_checksums(filelist, pn, topdir):
     """Get a list of the checksums for a list of local files

     Returns the checksums for a list of local files, caching the results as
@@ -917,7 +916,12 @@ def get_file_checksums(filelist, pn):
             bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e))
             return None
         return checksum
+
+    (recipe_root, _) = os.path.split(topdir)

+    def remove_recipe_parent(data):
+        return data.replace(recipe_root, '').strip('/')
+
     checksums = []
     for pth in filelist.split():
         checksum = None
@@ -927,7 +931,7 @@ def get_file_checksums(filelist, pn):
             for f in glob.glob(pth):
                 checksum = checksum_file(f)
                 if checksum:
-                    checksums.append((f, checksum))
+                    checksums.append((remove_recipe_parent(f), checksum))
         elif os.path.isdir(pth):
             # Handle directories
             for root, dirs, files in os.walk(pth):
@@ -935,12 +939,12 @@ def get_file_checksums(filelist, pn):
                     fullpth = os.path.join(root, name)
                     checksum = checksum_file(fullpth)
                     if checksum:
-                        checksums.append((fullpth, checksum))
+                        checksums.append((remove_recipe_parent(fullpth), checksum))
         else:
             checksum = checksum_file(pth)

         if checksum:
-            checksums.append((pth, checksum))
+            checksums.append((remove_recipe_parent(pth), checksum))

     checksums.sort(key=operator.itemgetter(1))
     return checksums
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 8861337..c64acfe 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -74,6 +74,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
         self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split())
         self.taskwhitelist = None
+        self.topdir = data.getVar("TOPDIR", True)
         self.init_rundepcheck(data)

     def init_rundepcheck(self, data):
@@ -187,7 +188,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
             self.runtaskdeps[k].append(dep)

         if task in dataCache.file_checksums[fn]:
-            checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
+            checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename, self.topdir)
             for (f,cs) in checksums:
                 self.file_checksum_values[k][f] = cs
                 data = data + cs
--
1.7.0.4