[OE-core] [PATCH 1/4] sstate: Add extra directory level

[akuster808]

On 1/5/20 4:06 PM, Richard Purdie wrote:
> We're having speed issues on the autobuilder due to the numbers of files in sstate
> directories. We previously split these by the first two characters of the hash.
> This change extends this to split by the next two characters as well, creating
> more layers of directories.
>
> This should signifiantly speed up eSDK builds on the autobuilder as the current
> sstate layout simply isn't scaling there but addresses a general complaint.
>
> gen-lockedsig-cache needed to be updated for the new split level sstateI.

I assume zeus worthy?

> Also update tests for new layout.
>
> Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
> ---
>  meta/classes/sstate.bbclass             |  4 ++--
>  meta/lib/oeqa/selftest/cases/signing.py |  4 ++--
>  meta/lib/oeqa/selftest/cases/sstate.py  |  4 ++--
>  scripts/gen-lockedsig-cache             | 13 ++++++++++---
>  4 files changed, 16 insertions(+), 9 deletions(-)
>
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index c0329cd5d18..7d2cb9eb840 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -6,7 +6,7 @@ SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
>  def generate_sstatefn(spec, hash, d):
>      if not hash:
>          hash = "INVALID"
> -    return hash[:2] + "/" + spec + hash
> +    return hash[:2] + "/" + hash[2:4] + "/" + spec + hash
>  
>  SSTATE_PKGARCH    = "${PACKAGE_ARCH}"
>  SSTATE_PKGSPEC    = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
> @@ -15,7 +15,7 @@ SSTATE_PKGNAME    = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
>  SSTATE_PKG        = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
>  SSTATE_EXTRAPATH   = ""
>  SSTATE_EXTRAPATHWILDCARD = ""
> -SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
> +SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}"
>  
>  # explicitly make PV to depend on evaluated value of PV variable
>  PV[vardepvalue] = "${PV}"
> diff --git a/meta/lib/oeqa/selftest/cases/signing.py b/meta/lib/oeqa/selftest/cases/signing.py
> index b65f3de64c4..9ea31328eff 100644
> --- a/meta/lib/oeqa/selftest/cases/signing.py
> +++ b/meta/lib/oeqa/selftest/cases/signing.py
> @@ -157,8 +157,8 @@ class Signing(OESelftestTestCase):
>              bitbake('-c clean %s' % test_recipe)
>              bitbake('-c populate_lic %s' % test_recipe)
>  
> -            recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz.sig')
> -            recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz')
> +            recipe_sig = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz.sig')
> +            recipe_tgz = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz')
>  
>              self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.')
>              self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.')
> diff --git a/meta/lib/oeqa/selftest/cases/sstate.py b/meta/lib/oeqa/selftest/cases/sstate.py
> index 410dec64fcd..80ce9e353c0 100644
> --- a/meta/lib/oeqa/selftest/cases/sstate.py
> +++ b/meta/lib/oeqa/selftest/cases/sstate.py
> @@ -56,11 +56,11 @@ class SStateBase(OESelftestTestCase):
>      def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True):
>          result = []
>          for root, dirs, files in os.walk(self.sstate_path):
> -            if distro_specific and re.search("%s/[a-z0-9]{2}$" % self.hostdistro, root):
> +            if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root):
>                  for f in files:
>                      if re.search(filename_regex, f):
>                          result.append(f)
> -            if distro_nonspecific and re.search("%s/[a-z0-9]{2}$" % self.sstate_path, root):
> +            if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root):
>                  for f in files:
>                      if re.search(filename_regex, f):
>                          result.append(f)
> diff --git a/scripts/gen-lockedsig-cache b/scripts/gen-lockedsig-cache
> index 9bfae9d8323..6a7d2859104 100755
> --- a/scripts/gen-lockedsig-cache
> +++ b/scripts/gen-lockedsig-cache
> @@ -78,11 +78,18 @@ files = set()
>  sstate_content_cache = {}
>  for s in sigs:
>      prefix = s[:2]
> +    prefix2 = s[2:4]
>      if prefix not in sstate_content_cache:
>          sstate_content_cache[prefix] = build_sha_cache(prefix)
> -
> -    for f in sstate_content_cache[prefix][s]:
> -        files.add(f)
> +    if prefix2 not in sstate_content_cache[prefix]:
> +        sstate_content_cache[prefix][prefix2] = build_sha_cache(prefix + "/" + prefix2)
> +
> +    if s in sstate_content_cache[prefix]:
> +        for f in sstate_content_cache[prefix][s]:
> +            files.add(f)
> +    if s in sstate_content_cache[prefix][prefix2]:
> +        for f in sstate_content_cache[prefix][prefix2][s]:
> +            files.add(f)
>  
>  elapsed = time.perf_counter() - start_time
>  print("Gathering file list took %.1fs" % elapsed)