[oe] [PATCH] sstate: Add a two character subdirectory to the sstate directory layout

[McClintock Matthew-B29882]

On Wed, Jul 25, 2012 at 4:09 PM, Richard Purdie
<richard.purdie at linuxfoundation.org> wrote:
> Currently all sstate files are placed into one directory. This does not scale and
> causes a variety of filesystem issues. This patch adds a two character subdirectory
> to the layout (based on the first two characters of the hash) so that files
> can be split into several directories.
>
> This should help performance of sstate in most cases by avoding creating directories with
> huge numbers of files.
>
> The SSTATE_MIRRORS syntax needs updating to account for the extra path element by
> the addition of a PATH item, for example:
>
> SSTATE_MIRRORS = "file://.* file:///some/path/to/sstate-cache/PATH"
> SSTATE_MIRRORS = "file://.* http://192.168.1.23/sstate-cache/PATH"
>
> This change also sets the scene for using things like lsb-release in
> the
>
> Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>

What about backwards compatibility?

-M

> ---
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 570b371..d00779a 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -4,16 +4,21 @@ SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
>  SSTATE_MANFILEBASE = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-"
>  SSTATE_MANFILEPREFIX = "${SSTATE_MANFILEBASE}${PN}"
>
> +def generate_sstatefn(spec, hash, d):
> +    if not hash:
> +        hash = "INVALID"
> +    return hash[:2] + "/" + spec + hash
>
>  SSTATE_PKGARCH    = "${PACKAGE_ARCH}"
>  SSTATE_PKGSPEC    = "sstate-${PN}-${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}-${PV}-${PR}-${SSTATE_PKGARCH}-${SSTATE_VERSION}-"
> -SSTATE_PKGNAME    = "${SSTATE_PKGSPEC}${BB_TASKHASH}"
> +SSTATE_PKGNAME    = "${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC', True), d.getVar('BB_TASKHASH', True), d)}"
>  SSTATE_PKG        = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
> +SSTATE_PATHSPEC   = "${SSTATE_DIR}/*/${SSTATE_PKGSPEC}"
>
>  SSTATE_SCAN_FILES ?= "*.la *-config *_config"
>  SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f'
>
> -BB_HASHFILENAME = "${SSTATE_PKGNAME}"
> +BB_HASHFILENAME = "${SSTATE_PKGSPEC}"
>
>  SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
>
> @@ -158,10 +163,11 @@ def sstate_installpkg(ss, d):
>          oe.path.remove(dir)
>
>      sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['name'])
> +    sstatefetch = d.getVar('SSTATE_PKGNAME', True) + '_' + ss['name'] + ".tgz"
>      sstatepkg = d.getVar('SSTATE_PKG', True) + '_' + ss['name'] + ".tgz"
>
>      if not os.path.exists(sstatepkg):
> -       pstaging_fetch(sstatepkg, d)
> +       pstaging_fetch(sstatefetch, sstatepkg, d)
>
>      if not os.path.isfile(sstatepkg):
>          bb.note("Staging package %s does not exist" % sstatepkg)
> @@ -223,8 +229,7 @@ def sstate_installpkg(ss, d):
>  def sstate_clean_cachefile(ss, d):
>      import oe.path
>
> -    sstatepkgdir = d.getVar('SSTATE_DIR', True)
> -    sstatepkgfile = sstatepkgdir + '/' + d.getVar('SSTATE_PKGSPEC', True) + "*_" + ss['name'] + ".tgz*"
> +    sstatepkgfile = d.getVar('SSTATE_PATHSPEC', True) + "*_" + ss['name'] + ".tgz*"
>      bb.note("Removing %s" % sstatepkgfile)
>      oe.path.remove(sstatepkgfile)
>
> @@ -417,7 +422,7 @@ def sstate_package(ss, d):
>
>      return
>
> -def pstaging_fetch(sstatepkg, d):
> +def pstaging_fetch(sstatefetch, sstatepkg, d):
>      import bb.fetch2
>
>      # Only try and fetch if the user has configured a mirror
> @@ -430,7 +435,7 @@ def pstaging_fetch(sstatepkg, d):
>      bb.data.update_data(localdata)
>
>      dldir = localdata.expand("${SSTATE_DIR}")
> -    srcuri = "file://" + os.path.basename(sstatepkg)
> +    srcuri = "file://" + sstatefetch
>
>      bb.mkdirhier(dldir)
>
> @@ -519,8 +524,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d):
>      }
>
>      for task in range(len(sq_fn)):
> -        sstatefile = d.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + mapping[sq_task[task]] + ".tgz")
> -        sstatefile = sstatefile.replace("${BB_TASKHASH}", sq_hash[task])
> +        sstatefile = d.expand("${SSTATE_DIR}/" + generate_sstatefn(sq_hashfn[task], sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz")
>          if os.path.exists(sstatefile):
>              bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
>              ret.append(task)
> @@ -544,10 +548,9 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d):
>              if task in ret:
>                  continue
>
> -            sstatefile = d.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + mapping[sq_task[task]] + ".tgz")
> -            sstatefile = sstatefile.replace("${BB_TASKHASH}", sq_hash[task])
> +            sstatefile = d.expand(generate_sstatefn(sq_hashfn[task], sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz")
>
> -            srcuri = "file://" + os.path.basename(sstatefile)
> +            srcuri = "file://" + sstatefile
>              localdata.setVar('SRC_URI', srcuri)
>              bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
>
>
>
>
>
> _______________________________________________
> Openembedded-devel mailing list
> Openembedded-devel at lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-devel