<div dir="ltr">Hi;<div><br></div><div>I was just wondering what the expected procedure is here among the bitbake devs. Should I be contacting the specific people that have historically been the committers for the particular code affected by the change?</div><div><br></div><div>Thanks,</div><div>Matt</div><br><div class="gmail_quote"><div dir="ltr">On Fri, May 25, 2018 at 8:45 AM Matt Hoosier <<a href="mailto:matt.hoosier@gmail.com">matt.hoosier@gmail.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Although the submodules' histories have been fetched during the<br>
do_fetch() phase, the mechanics used to clone the workdir copy<br>
of the repo haven't been transferring the actual .git/modules<br>
directory from the repo fetched into downloads/ during the<br>
fetch task.<br>
<br>
Fix that, and for good measure also explicitly tell Git to avoid<br>
hitting the network during do_unpack() of the submodules.<br>
<br>
[YOCTO #12739]<br>
<br>
Signed-off-by: Matt Hoosier <<a href="mailto:matt.hoosier@gmail.com" target="_blank">matt.hoosier@gmail.com</a>><br>
---<br>
lib/bb/fetch2/gitsm.py | 84 +++++++++++++++++++++++++++++++++++++++++++++-----<br>
1 file changed, 76 insertions(+), 8 deletions(-)<br>
<br>
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py<br>
index 0aff1008..7ac0dbb1 100644<br>
--- a/lib/bb/fetch2/gitsm.py<br>
+++ b/lib/bb/fetch2/gitsm.py<br>
@@ -98,7 +98,7 @@ class GitSM(Git):<br>
for line in lines:<br>
f.write(line)<br>
<br>
- def update_submodules(self, ud, d):<br>
+ def update_submodules(self, ud, d, allow_network):<br>
# We have to convert bare -> full repo, do the submodule bit, then convert back<br>
tmpclonedir = ud.clonedir + ".tmp"<br>
gitdir = tmpclonedir + os.sep + ".git"<br>
@@ -108,11 +108,47 @@ class GitSM(Git):<br>
runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)<br>
runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)<br>
runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)<br>
- runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=tmpclonedir)<br>
- self._set_relative_paths(tmpclonedir)<br>
- runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)<br>
- os.rename(gitdir, ud.clonedir,)<br>
- bb.utils.remove(tmpclonedir, True)<br>
+<br>
+ try:<br>
+ if allow_network:<br>
+ fetch_flags = ""<br>
+ else:<br>
+ fetch_flags = "--no-fetch"<br>
+<br>
+ # The 'git submodule sync' sandwiched between two successive 'git submodule update' commands is<br>
+ # intentional. See the notes on the similar construction in download() for an explanation.<br>
+ runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags' : fetch_flags}, d, workdir=tmpclonedir)<br>
+ except bb.fetch.FetchError:<br>
+ if allow_network:<br>
+ raise<br>
+ else:<br>
+ # This method was called as a probe to see whether the submodule history<br>
+ # is complete enough to allow the current working copy to have its<br>
+ # modules filled in. It's not, so swallow up the exception and report<br>
+ # the negative result.<br>
+ return False<br>
+ finally:<br>
+ self._set_relative_paths(tmpclonedir)<br>
+ runfetchcmd(ud.basecmd + " submodule deinit -f --all", d, workdir=tmpclonedir)<br>
+ runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)<br>
+ os.rename(gitdir, ud.clonedir,)<br>
+ bb.utils.remove(tmpclonedir, True)<br>
+<br>
+ return True<br>
+<br>
+ def need_update(self, ud, d):<br>
+ main_repo_needs_update = Git.need_update(self, ud, d)<br>
+<br>
+ # First check that the main repository has enough history fetched. If it doesn't, then we don't<br>
+ # even have the .gitmodules and gitlinks for the submodules to attempt asking whether the<br>
+ # submodules' histories are recent enough.<br>
+ if main_repo_needs_update:<br>
+ return True<br>
+<br>
+ # Now check that the submodule histories are new enough. The git-submodule command doesn't have<br>
+ # any clean interface for doing this aside from just attempting the checkout (with network<br>
+ # fetched disabled).<br>
+ return not self.update_submodules(ud, d, allow_network=False)<br>
<br>
def download(self, ud, d):<br>
Git.download(self, ud, d)<br>
@@ -120,7 +156,7 @@ class GitSM(Git):<br>
if not ud.shallow or ud.localpath != ud.fullshallow:<br>
submodules = self.uses_submodules(ud, d, ud.clonedir)<br>
if submodules:<br>
- self.update_submodules(ud, d)<br>
+ self.update_submodules(ud, d, allow_network=True)<br>
<br>
def clone_shallow_local(self, ud, dest, d):<br>
super(GitSM, self).clone_shallow_local(ud, dest, d)<br>
@@ -132,4 +168,36 @@ class GitSM(Git):<br>
<br>
if self.uses_submodules(ud, d, ud.destdir):<br>
runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir)<br>
- runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=ud.destdir)<br>
+<br>
+ # Copy over the submodules' fetched histories too.<br>
+ if ud.bareclone:<br>
+ repo_conf = ud.destdir<br>
+ else:<br>
+ repo_conf = os.path.join(ud.destdir, '.git')<br>
+<br>
+ if os.path.exists(ud.clonedir):<br>
+ # This is not a copy unpacked from a shallow mirror clone. So<br>
+ # the manual intervention to populate the .git/modules done<br>
+ # in clone_shallow_local() won't have been done yet.<br>
+ runfetchcmd("cp -fpPRH %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d)<br>
+ fetch_flags = "--no-fetch"<br>
+ elif os.path.exists(os.path.join(repo_conf, 'modules')):<br>
+ # Unpacked from a shallow mirror clone. Manual population of<br>
+ # .git/modules is already done.<br>
+ fetch_flags = "--no-fetch"<br>
+ else:<br>
+ # This isn't fatal; git-submodule will just fetch it<br>
+ # during do_unpack().<br>
+ fetch_flags = ""<br>
+ bb.error("submodule history not retrieved during do_fetch()")<br>
+<br>
+ # Careful not to hit the network during unpacking; all history should already<br>
+ # be fetched.<br>
+ #<br>
+ # The repeated attempts to do the submodule initialization sandwiched around a sync to<br>
+ # install the correct remote URLs into the submodules' .git/config metadata are deliberate.<br>
+ # Bad remote URLs are leftover in the modules' .git/config files from the unpack of bare<br>
+ # clone tarballs and an initial 'git submodule update' is necessary to prod them back to<br>
+ # enough life so that the 'git submodule sync' realizes the existing module .git/config<br>
+ # files exist to be updated.<br>
+ runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags': fetch_flags}, d, workdir=ud.destdir)<br>
-- <br>
2.13.6<br>
<br>
</blockquote></div></div>