[OE-core] [oe] [PATCH] sstate: Add a two character subdirectory to the sstate directory layout
Martin Jansa
martin.jansa at gmail.com
Thu Aug 2 13:53:35 UTC 2012
On Wed, Jul 25, 2012 at 10:09:22PM +0100, Richard Purdie wrote:
> Currently all sstate files are placed into one directory. This does not scale and
> causes a variety of filesystem issues. This patch adds a two character subdirectory
> to the layout (based on the first two characters of the hash) so that files
> can be split into several directories.
>
> This should help performance of sstate in most cases by avoding creating directories with
> huge numbers of files.
>
> The SSTATE_MIRRORS syntax needs updating to account for the extra path element by
> the addition of a PATH item, for example:
>
> SSTATE_MIRRORS = "file://.* file:///some/path/to/sstate-cache/PATH"
> SSTATE_MIRRORS = "file://.* http://192.168.1.23/sstate-cache/PATH"
>
> This change also sets the scene for using things like lsb-release in
> the
Is it possible to create 2nd level cache with this?
I have some server with slow upload but fully populated sstate-cache.
So on server with faster upload which could be used as offical
SSTATE_MIRROR for SHR distro I would like to add
SSTATE_MIRRORS ?= "file://.* http://slow-server/sstate-cache/PATH"
And then sync my sstate-cache directory to public accessible web root (with rsync).
Problem is that now sstate-cache has all files in slightly different
layout then original sstate-cache on slow server. From what I see I guess
it finds URL with correct prefix "sstate-cache/Gentoo-2.1/0d" and downloads it
directly to sstate-cache dir (and adds .done)
OE @ ~/oe-core $ ll sstate-cache/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-*populate-lic*
-rw-r--r-- 1 bitbake bitbake 9257 Jul 30 12:31 sstate-cache/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-0d2ed24b90d50bf83e5fe94536596e50_populate-lic.tgz
-rw-r--r-- 1 bitbake bitbake 0 Aug 2 15:40 sstate-cache/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-0d2ed24b90d50bf83e5fe94536596e50_populate-lic.tgz.done
And then creates symlink in right prefix back to absolute path of sstate-cache/file:
OE @ ~/oe-core $ ll sstate-cache/Gentoo-2.1/0d/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-*populate-lic*
lrwxrwxrwx 1 bitbake bitbake 123 Aug 2 15:40 sstate-cache/Gentoo-2.1/0d/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-0d2ed24b90d50bf83e5fe94536596e50_populate-lic.tgz ->
/OE/oe-core/sstate-cache/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-0d2ed24b90d50bf83e5fe94536596e50_populate-lic.tgz
But after sstate-cache directory is rsynced somewhere else and oe-core/sstate-cache is removed,
all those symlinks point nowhere and public sstate-cache is unusable.
Can we have relative paths used in symlinks or even instruct fetcher to download that
file directly to right prefix?
Cheers,
> Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
> ---
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 570b371..d00779a 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -4,16 +4,21 @@ SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
> SSTATE_MANFILEBASE = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-"
> SSTATE_MANFILEPREFIX = "${SSTATE_MANFILEBASE}${PN}"
>
> +def generate_sstatefn(spec, hash, d):
> + if not hash:
> + hash = "INVALID"
> + return hash[:2] + "/" + spec + hash
>
> SSTATE_PKGARCH = "${PACKAGE_ARCH}"
> SSTATE_PKGSPEC = "sstate-${PN}-${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}-${PV}-${PR}-${SSTATE_PKGARCH}-${SSTATE_VERSION}-"
> -SSTATE_PKGNAME = "${SSTATE_PKGSPEC}${BB_TASKHASH}"
> +SSTATE_PKGNAME = "${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC', True), d.getVar('BB_TASKHASH', True), d)}"
> SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
> +SSTATE_PATHSPEC = "${SSTATE_DIR}/*/${SSTATE_PKGSPEC}"
>
> SSTATE_SCAN_FILES ?= "*.la *-config *_config"
> SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f'
>
> -BB_HASHFILENAME = "${SSTATE_PKGNAME}"
> +BB_HASHFILENAME = "${SSTATE_PKGSPEC}"
>
> SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
>
> @@ -158,10 +163,11 @@ def sstate_installpkg(ss, d):
> oe.path.remove(dir)
>
> sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['name'])
> + sstatefetch = d.getVar('SSTATE_PKGNAME', True) + '_' + ss['name'] + ".tgz"
> sstatepkg = d.getVar('SSTATE_PKG', True) + '_' + ss['name'] + ".tgz"
>
> if not os.path.exists(sstatepkg):
> - pstaging_fetch(sstatepkg, d)
> + pstaging_fetch(sstatefetch, sstatepkg, d)
>
> if not os.path.isfile(sstatepkg):
> bb.note("Staging package %s does not exist" % sstatepkg)
> @@ -223,8 +229,7 @@ def sstate_installpkg(ss, d):
> def sstate_clean_cachefile(ss, d):
> import oe.path
>
> - sstatepkgdir = d.getVar('SSTATE_DIR', True)
> - sstatepkgfile = sstatepkgdir + '/' + d.getVar('SSTATE_PKGSPEC', True) + "*_" + ss['name'] + ".tgz*"
> + sstatepkgfile = d.getVar('SSTATE_PATHSPEC', True) + "*_" + ss['name'] + ".tgz*"
> bb.note("Removing %s" % sstatepkgfile)
> oe.path.remove(sstatepkgfile)
>
> @@ -417,7 +422,7 @@ def sstate_package(ss, d):
>
> return
>
> -def pstaging_fetch(sstatepkg, d):
> +def pstaging_fetch(sstatefetch, sstatepkg, d):
> import bb.fetch2
>
> # Only try and fetch if the user has configured a mirror
> @@ -430,7 +435,7 @@ def pstaging_fetch(sstatepkg, d):
> bb.data.update_data(localdata)
>
> dldir = localdata.expand("${SSTATE_DIR}")
> - srcuri = "file://" + os.path.basename(sstatepkg)
> + srcuri = "file://" + sstatefetch
>
> bb.mkdirhier(dldir)
>
> @@ -519,8 +524,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d):
> }
>
> for task in range(len(sq_fn)):
> - sstatefile = d.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + mapping[sq_task[task]] + ".tgz")
> - sstatefile = sstatefile.replace("${BB_TASKHASH}", sq_hash[task])
> + sstatefile = d.expand("${SSTATE_DIR}/" + generate_sstatefn(sq_hashfn[task], sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz")
> if os.path.exists(sstatefile):
> bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
> ret.append(task)
> @@ -544,10 +548,9 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d):
> if task in ret:
> continue
>
> - sstatefile = d.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + mapping[sq_task[task]] + ".tgz")
> - sstatefile = sstatefile.replace("${BB_TASKHASH}", sq_hash[task])
> + sstatefile = d.expand(generate_sstatefn(sq_hashfn[task], sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz")
>
> - srcuri = "file://" + os.path.basename(sstatefile)
> + srcuri = "file://" + sstatefile
> localdata.setVar('SRC_URI', srcuri)
> bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
>
>
>
>
>
> _______________________________________________
> Openembedded-devel mailing list
> Openembedded-devel at lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-devel
--
Martin 'JaMa' Jansa jabber: Martin.Jansa at gmail.com
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 198 bytes
Desc: Digital signature
URL: <http://lists.openembedded.org/pipermail/openembedded-core/attachments/20120802/4b77af58/attachment-0002.sig>
More information about the Openembedded-core
mailing list