[oe-commits] [openembedded-core] 01/02: sstate: Add extra directory level

git at git.openembedded.org git at git.openembedded.org
Sun Jan 5 14:08:26 UTC 2020


This is an automated email from the git hooks/post-receive script.

rpurdie pushed a commit to branch master-next
in repository openembedded-core.

commit 4afd02834b0b3091da120c4525321501a356c414
Author: Richard Purdie <richard.purdie at linuxfoundation.org>
AuthorDate: Fri Jan 3 18:03:35 2020 +0000

    sstate: Add extra directory level
    
    We're having speed issues on the autobuilder due to the numbers of files in sstate
    directories. We previously split these by the first two characters of the hash.
    This change extends this to split by the next two characters as well.
    
    The hope is this signifiantly speeds up eSDK builds on the autobuilder in partcular
    as the current sstate layout simply isn't scaling there.
    
    gen-lockedsig-cache: Update for split level sstate
    
    Update tetss for new layout
    
    Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
 meta/classes/sstate.bbclass             |  4 ++--
 meta/lib/oeqa/selftest/cases/signing.py |  4 ++--
 meta/lib/oeqa/selftest/cases/sstate.py  |  4 ++--
 scripts/gen-lockedsig-cache             | 13 ++++++++++---
 4 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index c0329cd..7d2cb9e 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -6,7 +6,7 @@ SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
 def generate_sstatefn(spec, hash, d):
     if not hash:
         hash = "INVALID"
-    return hash[:2] + "/" + spec + hash
+    return hash[:2] + "/" + hash[2:4] + "/" + spec + hash
 
 SSTATE_PKGARCH    = "${PACKAGE_ARCH}"
 SSTATE_PKGSPEC    = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
@@ -15,7 +15,7 @@ SSTATE_PKGNAME    = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
 SSTATE_PKG        = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
 SSTATE_EXTRAPATH   = ""
 SSTATE_EXTRAPATHWILDCARD = ""
-SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
+SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}"
 
 # explicitly make PV to depend on evaluated value of PV variable
 PV[vardepvalue] = "${PV}"
diff --git a/meta/lib/oeqa/selftest/cases/signing.py b/meta/lib/oeqa/selftest/cases/signing.py
index b65f3de..9ea3132 100644
--- a/meta/lib/oeqa/selftest/cases/signing.py
+++ b/meta/lib/oeqa/selftest/cases/signing.py
@@ -157,8 +157,8 @@ class Signing(OESelftestTestCase):
             bitbake('-c clean %s' % test_recipe)
             bitbake('-c populate_lic %s' % test_recipe)
 
-            recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz.sig')
-            recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz')
+            recipe_sig = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz.sig')
+            recipe_tgz = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz')
 
             self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.')
             self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.')
diff --git a/meta/lib/oeqa/selftest/cases/sstate.py b/meta/lib/oeqa/selftest/cases/sstate.py
index 410dec6..80ce9e3 100644
--- a/meta/lib/oeqa/selftest/cases/sstate.py
+++ b/meta/lib/oeqa/selftest/cases/sstate.py
@@ -56,11 +56,11 @@ class SStateBase(OESelftestTestCase):
     def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True):
         result = []
         for root, dirs, files in os.walk(self.sstate_path):
-            if distro_specific and re.search("%s/[a-z0-9]{2}$" % self.hostdistro, root):
+            if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root):
                 for f in files:
                     if re.search(filename_regex, f):
                         result.append(f)
-            if distro_nonspecific and re.search("%s/[a-z0-9]{2}$" % self.sstate_path, root):
+            if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root):
                 for f in files:
                     if re.search(filename_regex, f):
                         result.append(f)
diff --git a/scripts/gen-lockedsig-cache b/scripts/gen-lockedsig-cache
index 9bfae9d..6a7d285 100755
--- a/scripts/gen-lockedsig-cache
+++ b/scripts/gen-lockedsig-cache
@@ -78,11 +78,18 @@ files = set()
 sstate_content_cache = {}
 for s in sigs:
     prefix = s[:2]
+    prefix2 = s[2:4]
     if prefix not in sstate_content_cache:
         sstate_content_cache[prefix] = build_sha_cache(prefix)
-
-    for f in sstate_content_cache[prefix][s]:
-        files.add(f)
+    if prefix2 not in sstate_content_cache[prefix]:
+        sstate_content_cache[prefix][prefix2] = build_sha_cache(prefix + "/" + prefix2)
+
+    if s in sstate_content_cache[prefix]:
+        for f in sstate_content_cache[prefix][s]:
+            files.add(f)
+    if s in sstate_content_cache[prefix][prefix2]:
+        for f in sstate_content_cache[prefix][prefix2][s]:
+            files.add(f)
 
 elapsed = time.perf_counter() - start_time
 print("Gathering file list took %.1fs" % elapsed)

-- 
To stop receiving notification emails like this one, please contact
the administrator of this repository.


More information about the Openembedded-commits mailing list