[oe-commits] [bitbake] 01/08: fetch: support multiple mirror tarball filenames
git at git.openembedded.org
git at git.openembedded.org
Fri May 26 21:00:02 UTC 2017
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
commit 02eebee6709e57b523862257f75929e64f16d6b0
Author: Christopher Larson <kergoth at gmail.com>
AuthorDate: Sat May 13 02:46:26 2017 +0500
fetch: support multiple mirror tarball filenames
Remove ud.mirrortarball in favor of ud.mirrortarballs. Each tarball will be
attempted, in order, and the first available will be used. This is needed for
git shallow mirror tarball support, as we want to be able to use either
a shallow or full mirror tarball.
Signed-off-by: Christopher Larson <chris_larson at mentor.com>
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/fetch2/__init__.py | 72 ++++++++++++++++++++++++-----------------------
lib/bb/fetch2/git.py | 5 ++--
lib/bb/fetch2/hg.py | 5 ++--
lib/bb/fetch2/npm.py | 7 +++--
4 files changed, 47 insertions(+), 42 deletions(-)
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 79a8906..1f63a04 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -425,7 +425,7 @@ def encodeurl(decoded):
return url
-def uri_replace(ud, uri_find, uri_replace, replacements, d):
+def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
if not ud.url or not uri_find or not uri_replace:
logger.error("uri_replace: passed an undefined value, not replacing")
return None
@@ -464,9 +464,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d):
if loc == 2:
# Handle path manipulations
basename = None
- if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball:
+ if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball:
# If the source and destination url types differ, must be a mirrortarball mapping
- basename = os.path.basename(ud.mirrortarball)
+ basename = os.path.basename(mirrortarball)
# Kill parameters, they make no sense for mirror tarballs
uri_decoded[5] = {}
elif ud.localpath and ud.method.supports_checksum(ud):
@@ -892,45 +892,47 @@ def build_mirroruris(origud, mirrors, ld):
replacements["BASENAME"] = origud.path.split("/")[-1]
replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.')
- def adduri(ud, uris, uds, mirrors):
+ def adduri(ud, uris, uds, mirrors, tarballs):
for line in mirrors:
try:
(find, replace) = line
except ValueError:
continue
- newuri = uri_replace(ud, find, replace, replacements, ld)
- if not newuri or newuri in uris or newuri == origud.url:
- continue
- if not trusted_network(ld, newuri):
- logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri))
- continue
+ for tarball in tarballs:
+ newuri = uri_replace(ud, find, replace, replacements, ld, tarball)
+ if not newuri or newuri in uris or newuri == origud.url:
+ continue
- # Create a local copy of the mirrors minus the current line
- # this will prevent us from recursively processing the same line
- # as well as indirect recursion A -> B -> C -> A
- localmirrors = list(mirrors)
- localmirrors.remove(line)
+ if not trusted_network(ld, newuri):
+ logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri))
+ continue
+
+ # Create a local copy of the mirrors minus the current line
+ # this will prevent us from recursively processing the same line
+ # as well as indirect recursion A -> B -> C -> A
+ localmirrors = list(mirrors)
+ localmirrors.remove(line)
- try:
- newud = FetchData(newuri, ld)
- newud.setup_localpath(ld)
- except bb.fetch2.BBFetchException as e:
- logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
- logger.debug(1, str(e))
try:
- # setup_localpath of file:// urls may fail, we should still see
- # if mirrors of the url exist
- adduri(newud, uris, uds, localmirrors)
- except UnboundLocalError:
- pass
- continue
- uris.append(newuri)
- uds.append(newud)
+ newud = FetchData(newuri, ld)
+ newud.setup_localpath(ld)
+ except bb.fetch2.BBFetchException as e:
+ logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+ logger.debug(1, str(e))
+ try:
+ # setup_localpath of file:// urls may fail, we should still see
+ # if mirrors of the url exist
+ adduri(newud, uris, uds, localmirrors, tarballs)
+ except UnboundLocalError:
+ pass
+ continue
+ uris.append(newuri)
+ uds.append(newud)
- adduri(newud, uris, uds, localmirrors)
+ adduri(newud, uris, uds, localmirrors, tarballs)
- adduri(origud, uris, uds, mirrors)
+ adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None])
return uris, uds
@@ -975,8 +977,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
# We may be obtaining a mirror tarball which needs further processing by the real fetcher
# If that tarball is a local file:// we need to provide a symlink to it
dldir = ld.getVar("DL_DIR")
- if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
- and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
+
+ if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
# Create donestamp in old format to avoid triggering a re-download
if ud.donestamp:
bb.utils.mkdirhier(os.path.dirname(ud.donestamp))
@@ -993,7 +995,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
pass
if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld):
origud.method.download(origud, ld)
- if hasattr(origud.method,"build_mirror_data"):
+ if hasattr(origud.method, "build_mirror_data"):
origud.method.build_mirror_data(origud, ld)
return origud.localpath
# Otherwise the result is a local file:// and we symlink to it
@@ -1190,7 +1192,7 @@ class FetchData(object):
self.localfile = ""
self.localpath = None
self.lockfile = None
- self.mirrortarball = None
+ self.mirrortarballs = []
self.basename = None
self.basepath = None
(self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))
diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index 2550bde..01d4bbd 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -205,8 +205,9 @@ class Git(FetchMethod):
if ud.rebaseable:
for name in ud.names:
gitsrcname = gitsrcname + '_' + ud.revisions[name]
- ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname
- ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
+ mirrortarball = 'git2_%s.tar.gz' % gitsrcname
+ ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
+ ud.mirrortarballs = [mirrortarball]
gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
ud.clonedir = os.path.join(gitdir, gitsrcname)
diff --git a/lib/bb/fetch2/hg.py b/lib/bb/fetch2/hg.py
index b5f2686..d0857e6 100644
--- a/lib/bb/fetch2/hg.py
+++ b/lib/bb/fetch2/hg.py
@@ -76,8 +76,9 @@ class Hg(FetchMethod):
# Create paths to mercurial checkouts
hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
ud.host, ud.path.replace('/', '.'))
- ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname
- ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
+ mirrortarball = 'hg_%s.tar.gz' % hgsrcname
+ ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
+ ud.mirrortarballs = [mirrortarball]
hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/")
ud.pkgdir = os.path.join(hgdir, hgsrcname)
diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
index 73a75fe..5f79972 100644
--- a/lib/bb/fetch2/npm.py
+++ b/lib/bb/fetch2/npm.py
@@ -91,9 +91,10 @@ class Npm(FetchMethod):
ud.prefixdir = prefixdir
ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
- ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
- ud.mirrortarball = ud.mirrortarball.replace('/', '-')
- ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
+ mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
+ mirrortarball = ud.mirrortarball.replace('/', '-')
+ ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
+ ud.mirrortarballs = [mirrortarball]
def need_update(self, ud, d):
if os.path.exists(ud.localpath):
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list