[bitbake-devel] [PATCH 4/4] fetch2/repo: Use multiple jobs to fetch and sync

[Oleksandr Andrushchenko]

From: Oleksandr Andrushchenko <oleksandr_andrushchenko at epam.com>

Google repo supports "--jobs" parameter, so one can
speed up the process of fetching and syncing. According to
https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/ny3nkgE9AE8
it is even more gets to speed if used separately for
network jobs and "local" ones.
Use BB_NUMBER_THREADS to run parallel repo sync.

Signed-off-by: Oleksandr Andrushchenko <oleksandr_andrushchenko at epam.com>
---
 lib/bb/fetch2/repo.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/bb/fetch2/repo.py b/lib/bb/fetch2/repo.py
index 3c28fe04ab1e..08331acd6706 100644
--- a/lib/bb/fetch2/repo.py
+++ b/lib/bb/fetch2/repo.py
@@ -57,6 +57,13 @@ class Repo(FetchMethod):
         ud.codir = os.path.join(repodir, gitsrcname, ud.manifest)
         ud.repodir = os.path.join(ud.codir, "repo")
 
+    def sync(self, ud, d):
+        # repo can separate the "network" sync (the git fetch part, network bound)
+        # from the local sync (the git rebase part, compute & i/o bound).
+        num_jobs = d.getVar("BB_NUMBER_THREADS", True) or "1"
+        runfetchcmd("repo sync -c -n -j%s" % num_jobs, d, workdir=ud.repodir)
+        runfetchcmd("repo sync -c -l -j%s" % num_jobs, d, workdir=ud.repodir)
+
     def download(self, ud, d):
         """Fetch url"""
 
@@ -74,7 +81,7 @@ class Repo(FetchMethod):
         runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=ud.repodir)
 
         bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
-        runfetchcmd("repo sync", d, workdir=ud.repodir)
+        self.sync(ud, d)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":
@@ -88,7 +95,7 @@ class Repo(FetchMethod):
     def unpack(self, ud, destdir, d):
         FetchMethod.unpack(self, ud, destdir, d)
         bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
-        runfetchcmd("repo sync", d, workdir=ud.repodir)
+        self.sync(ud, d)
 
     def supports_srcrev(self):
         return False
-- 
2.7.4