[bitbake-devel] [PATCH] gitsm: The fetcher did not process some recursive submodules properly.

[Stefan Agner]

On 27.01.2019 17:57, Mark Hatle wrote:
> Move the submodule processing outside of the if statement to avoid any
> optimizations that may happen.
> 
> Update the test cases to include the additional case, and split the other
> test cases into individual tests to make it easier to figure out what
> the failure may be.
> 
> Signed-off-by: Mark Hatle <mark.hatle at windriver.com>

I can confirm that this resolves the issue we have been seeing:

Tested-by: Stefan Agner <stefan.agner at toradex.com>

Thanks Mark for looking into this!

--
Stefan

> ---
>  lib/bb/fetch2/gitsm.py |  4 +++-
>  lib/bb/tests/fetch.py  | 20 ++++++++++++++++++++
>  2 files changed, 23 insertions(+), 1 deletion(-)
> 
> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> index f45546b..b21fed2 100644
> --- a/lib/bb/fetch2/gitsm.py
> +++ b/lib/bb/fetch2/gitsm.py
> @@ -198,6 +198,8 @@ class GitSM(Git):
>  
>          Git.unpack(self, ud, destdir, d)
>  
> -        if not ud.bareclone and self.process_submodules(ud,
> ud.destdir, unpack_submodules, d):
> +        ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
> +
> +        if not ud.bareclone and ret:
>              # Run submodule update, this sets up the directories --
> without touching the config
>              runfetchcmd("%s submodule update --recursive --no-fetch"
> % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
> diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
> index e9ad807..522d202 100644
> --- a/lib/bb/tests/fetch.py
> +++ b/lib/bb/tests/fetch.py
> @@ -912,6 +912,7 @@ class FetcherNetworkTest(FetcherTest):
>          if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
>              self.assertTrue(os.path.exists(os.path.join(repo_path,
> 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule
> missing')
>  
> +    def test_git_submodule_dbus_broker(self):
>          # The following external repositories have show failures in
> fetch and unpack operations
>          # We want to avoid regressions!
>          url =
> "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"
> @@ -928,16 +929,35 @@ class FetcherNetworkTest(FetcherTest):
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule
> config "subprojects/c-sundry"')
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule
> config "subprojects/c-utf8"')
>  
> +    def test_git_submodule_CLI11(self):
>          url =
> "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
>          fetcher = bb.fetch.Fetch([url], self.d)
>          fetcher.download()
>          # Previous cwd has been deleted
>          os.chdir(os.path.dirname(self.unpackdir))
>          fetcher.unpack(self.unpackdir)
> +
> +        repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/extern/googletest/config')), msg='Missing submodule
> config "extern/googletest"')
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/extern/json/config')), msg='Missing submodule config
> "extern/json"')
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/extern/sanitizers/config')), msg='Missing submodule
> config "extern/sanitizers"')
>  
> +    def test_git_submodule_aktualizr(self):
> +        url =
> "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
> +        fetcher = bb.fetch.Fetch([url], self.d)
> +        fetcher.download()
> +        # Previous cwd has been deleted
> +        os.chdir(os.path.dirname(self.unpackdir))
> +        fetcher.unpack(self.unpackdir)
> +
> +        repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule
> config "partial/extern/isotp-c/config"')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')),
> msg='Missing submodule config
> "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of
> submodule isotp-c did not unpack properly")
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule
> config "tests/tuf-test-vectors/config"')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/third_party/googletest/config')), msg='Missing submodule
> config "third_party/googletest/config"')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing
> submodule config "third_party/HdrHistogram_c/config"')
> +
>  class TrustedNetworksTest(FetcherTest):
>      def test_trusted_network(self):
>          # Ensure trusted_network returns False when the host IS in the list.