[bitbake-devel] [PATCH] lib/bb: Add BB_SIGNATURE_LOCAL_DIRS_EXCLUDE to speed-up taskhash on directories

[Khem Raj]

On Wed, Nov 13, 2019 at 3:23 PM Aníbal Limón <anibal.limon at linaro.org> wrote:
>
> The new BB_SIGNATURE_LOCAL_DIRS_EXCLUDE allows you to specify a list
> of directories to exclude when making taskhash, our specific case
> is using SRC_URI that points local VCS directory.
>
> In OE-Core this variable could be set by default to:
>
> BB_SIGNATURE_LOCAL_DIRS_EXCLUDE = ".cvs .git .svn"
>

I like the approach.  perhaps a sane default would be nice.

> Signed-off-by: Aníbal Limón <anibal.limon at linaro.org>
> ---
>  lib/bb/checksum.py        | 5 +++--
>  lib/bb/fetch2/__init__.py | 4 ++--
>  lib/bb/siggen.py          | 5 +++--
>  3 files changed, 8 insertions(+), 6 deletions(-)
>
> diff --git a/lib/bb/checksum.py b/lib/bb/checksum.py
> index 5bc8a8fc..677020f4 100644
> --- a/lib/bb/checksum.py
> +++ b/lib/bb/checksum.py
> @@ -74,7 +74,7 @@ class FileChecksumCache(MultiProcessCache):
>              else:
>                  dest[0][h] = source[0][h]
>
> -    def get_checksums(self, filelist, pn):
> +    def get_checksums(self, filelist, pn, localdirsexclude):
>          """Get checksums for a list of files"""
>
>          def checksum_file(f):
> @@ -90,7 +90,8 @@ class FileChecksumCache(MultiProcessCache):
>              if pth == "/":
>                  bb.fatal("Refusing to checksum /")
>              dirchecksums = []
> -            for root, dirs, files in os.walk(pth):
> +            for root, dirs, files in os.walk(pth, topdown=True):
> +                [dirs.remove(d) for d in list(dirs) if d in localdirsexclude]
>                  for name in files:
>                      fullpth = os.path.join(root, name)
>                      checksum = checksum_file(fullpth)
> diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
> index 07de6c26..731c1608 100644
> --- a/lib/bb/fetch2/__init__.py
> +++ b/lib/bb/fetch2/__init__.py
> @@ -1197,14 +1197,14 @@ def get_checksum_file_list(d):
>
>      return " ".join(filelist)
>
> -def get_file_checksums(filelist, pn):
> +def get_file_checksums(filelist, pn, localdirsexclude):
>      """Get a list of the checksums for a list of local files
>
>      Returns the checksums for a list of local files, caching the results as
>      it proceeds
>
>      """
> -    return _checksum_cache.get_checksums(filelist, pn)
> +    return _checksum_cache.get_checksums(filelist, pn, localdirsexclude)
>
>
>  class FetchData(object):
> diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
> index a4bb1ff7..f110fd72 100644
> --- a/lib/bb/siggen.py
> +++ b/lib/bb/siggen.py
> @@ -123,6 +123,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
>
>          self.unihash_cache = bb.cache.SimpleCache("1")
>          self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {})
> +        self.localdirsexclude = (data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE") or "").split()
>
>      def init_rundepcheck(self, data):
>          self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
> @@ -221,9 +222,9 @@ class SignatureGeneratorBasic(SignatureGenerator):
>
>          if task in dataCache.file_checksums[fn]:
>              if self.checksum_cache:
> -                checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename)
> +                checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude)
>              else:
> -                checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
> +                checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude)
>              for (f,cs) in checksums:
>                  self.file_checksum_values[tid].append((f,cs))
>                  if cs:
> --
> 2.24.0
>
> --
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel at lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/bitbake-devel