[bitbake-devel] [PATCH 7/9] fetch2: provide original url in addition to the mirrored url to FetchData.__init__ and FetchMethod.urldata_init
Urs Fässler
urs.fassler at bbv.ch
Mon Jul 23 15:42:57 UTC 2018
When a mirror rewrite rule is used, it can happen that the downloaded
file is named differently depending on the used mirror. In preparation
to a consistent naming, we provide the original url to the fetcher.
Signed-off-by: Urs Fässler <urs.fassler at bbv.ch>
Signed-off-by: Pascal Bach <pascal.bach at siemens.com>
---
lib/bb/fetch2/__init__.py | 12 +++++++-----
lib/bb/fetch2/bzr.py | 2 +-
lib/bb/fetch2/clearcase.py | 2 +-
lib/bb/fetch2/cvs.py | 2 +-
lib/bb/fetch2/git.py | 2 +-
lib/bb/fetch2/gitannex.py | 4 ++--
lib/bb/fetch2/hg.py | 2 +-
lib/bb/fetch2/local.py | 2 +-
lib/bb/fetch2/npm.py | 2 +-
lib/bb/fetch2/osc.py | 2 +-
lib/bb/fetch2/perforce.py | 2 +-
lib/bb/fetch2/repo.py | 2 +-
lib/bb/fetch2/s3.py | 2 +-
lib/bb/fetch2/sftp.py | 2 +-
lib/bb/fetch2/ssh.py | 2 +-
lib/bb/fetch2/svn.py | 2 +-
lib/bb/fetch2/wget.py | 2 +-
17 files changed, 24 insertions(+), 22 deletions(-)
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index c8653d62..61009e43 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -932,7 +932,7 @@ def build_mirroruris(origud, mirrors, ld):
localmirrors.remove(line)
try:
- newud = FetchData(newuri, ld)
+ newud = FetchData(newuri, ld, origud.url)
newud.setup_localpath(ld)
except bb.fetch2.BBFetchException as e:
logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
@@ -1202,7 +1202,7 @@ class FetchData(object):
"""
A class which represents the fetcher state for a given URI.
"""
- def __init__(self, url, d, localonly = False):
+ def __init__(self, url, d, original_url = None, localonly = False):
# localpath is the location of a downloaded result. If not set, the file is local.
self.donestamp = None
self.needdonestamp = True
@@ -1213,6 +1213,8 @@ class FetchData(object):
self.basename = None
self.basepath = None
(self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))
+ if not original_url:
+ original_url = url
self.date = self.getSRCDate(d)
self.url = url
if not self.user and "user" in self.parm:
@@ -1259,7 +1261,7 @@ class FetchData(object):
logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN'))
self.parm["protocol"] = self.parm.get("proto", None)
- self.method.urldata_init(self, d)
+ self.method.urldata_init(self, d, original_url)
if "localpath" in self.parm:
# if user sets localpath for file, use it instead.
@@ -1348,7 +1350,7 @@ class FetchMethod(object):
return True
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
init method specific variable within url data
"""
@@ -1594,7 +1596,7 @@ class Fetch(object):
for url in urls:
if url not in self.ud:
try:
- self.ud[url] = FetchData(url, d, localonly)
+ self.ud[url] = FetchData(url, d, localonly = localonly)
except NonLocalMethod:
if localonly:
self.ud[url] = None
diff --git a/lib/bb/fetch2/bzr.py b/lib/bb/fetch2/bzr.py
index 658502f9..4be73320 100644
--- a/lib/bb/fetch2/bzr.py
+++ b/lib/bb/fetch2/bzr.py
@@ -36,7 +36,7 @@ class Bzr(FetchMethod):
def supports(self, ud, d):
return ud.type in ['bzr']
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
init bzr specific variable within url data
"""
diff --git a/lib/bb/fetch2/clearcase.py b/lib/bb/fetch2/clearcase.py
index 3a6573d0..ef1db138 100644
--- a/lib/bb/fetch2/clearcase.py
+++ b/lib/bb/fetch2/clearcase.py
@@ -84,7 +84,7 @@ class ClearCase(FetchMethod):
def debug(self, msg):
logger.debug(1, "ClearCase: %s", msg)
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
init ClearCase specific variable within url data
"""
diff --git a/lib/bb/fetch2/cvs.py b/lib/bb/fetch2/cvs.py
index 0e0a3196..7c5296fb 100644
--- a/lib/bb/fetch2/cvs.py
+++ b/lib/bb/fetch2/cvs.py
@@ -42,7 +42,7 @@ class Cvs(FetchMethod):
"""
return ud.type in ['cvs']
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
if not "module" in ud.parm:
raise MissingParameterError("module", ud.url)
ud.module = ud.parm["module"]
diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index 13b9b9d8..7f7951f7 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -141,7 +141,7 @@ class Git(FetchMethod):
def supports_checksum(self, urldata):
return False
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
init git specific variable within url data
so that the git method like latest_revision() can work
diff --git a/lib/bb/fetch2/gitannex.py b/lib/bb/fetch2/gitannex.py
index a9b69caa..187e6b2e 100644
--- a/lib/bb/fetch2/gitannex.py
+++ b/lib/bb/fetch2/gitannex.py
@@ -33,8 +33,8 @@ class GitANNEX(Git):
"""
return ud.type in ['gitannex']
- def urldata_init(self, ud, d):
- super(GitANNEX, self).urldata_init(ud, d)
+ def urldata_init(self, ud, d, original_url):
+ super(GitANNEX, self).urldata_init(ud, d, original_url)
if ud.shallow:
ud.shallow_extra_refs += ['refs/heads/git-annex', 'refs/heads/synced/*']
diff --git a/lib/bb/fetch2/hg.py b/lib/bb/fetch2/hg.py
index 936d0431..5fe08e80 100644
--- a/lib/bb/fetch2/hg.py
+++ b/lib/bb/fetch2/hg.py
@@ -50,7 +50,7 @@ class Hg(FetchMethod):
"""
return False
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
init hg specific variable within url data
"""
diff --git a/lib/bb/fetch2/local.py b/lib/bb/fetch2/local.py
index a114ac12..9e1b918c 100644
--- a/lib/bb/fetch2/local.py
+++ b/lib/bb/fetch2/local.py
@@ -39,7 +39,7 @@ class Local(FetchMethod):
"""
return urldata.type in ['file']
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
# We don't set localfile as for this fetcher the file is already local!
ud.decodedurl = urllib.parse.unquote(ud.url.split("://")[1].split(";")[0])
ud.basename = os.path.basename(ud.decodedurl)
diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
index 408dfc3d..cab792d4 100644
--- a/lib/bb/fetch2/npm.py
+++ b/lib/bb/fetch2/npm.py
@@ -60,7 +60,7 @@ class Npm(FetchMethod):
bb.utils.remove(ud.pkgdatadir, True)
bb.utils.remove(ud.fullmirror, False)
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
init NPM specific variable within url data
"""
diff --git a/lib/bb/fetch2/osc.py b/lib/bb/fetch2/osc.py
index 6c60456b..18dd6047 100644
--- a/lib/bb/fetch2/osc.py
+++ b/lib/bb/fetch2/osc.py
@@ -25,7 +25,7 @@ class Osc(FetchMethod):
"""
return ud.type in ['osc']
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
if not "module" in ud.parm:
raise MissingParameterError('module', ud.url)
diff --git a/lib/bb/fetch2/perforce.py b/lib/bb/fetch2/perforce.py
index 903a8e61..3a15754a 100644
--- a/lib/bb/fetch2/perforce.py
+++ b/lib/bb/fetch2/perforce.py
@@ -37,7 +37,7 @@ class Perforce(FetchMethod):
""" Check to see if a given url can be fetched with perforce. """
return ud.type in ['p4']
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
Initialize perforce specific variables within url data. If P4CONFIG is
provided by the env, use it. If P4PORT is specified by the recipe, use
diff --git a/lib/bb/fetch2/repo.py b/lib/bb/fetch2/repo.py
index 8c7e8185..c56df7e1 100644
--- a/lib/bb/fetch2/repo.py
+++ b/lib/bb/fetch2/repo.py
@@ -37,7 +37,7 @@ class Repo(FetchMethod):
"""
return ud.type in ["repo"]
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
We don"t care about the git rev of the manifests repository, but
we do care about the manifest to use. The default is "default".
diff --git a/lib/bb/fetch2/s3.py b/lib/bb/fetch2/s3.py
index 16292886..e00f7072 100644
--- a/lib/bb/fetch2/s3.py
+++ b/lib/bb/fetch2/s3.py
@@ -47,7 +47,7 @@ class S3(FetchMethod):
def recommends_checksum(self, urldata):
return True
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
if 'downloadfilename' in ud.parm:
ud.basename = ud.parm['downloadfilename']
else:
diff --git a/lib/bb/fetch2/sftp.py b/lib/bb/fetch2/sftp.py
index 81884a6a..f68cbca1 100644
--- a/lib/bb/fetch2/sftp.py
+++ b/lib/bb/fetch2/sftp.py
@@ -78,7 +78,7 @@ class SFTP(FetchMethod):
def recommends_checksum(self, urldata):
return True
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
if 'protocol' in ud.parm and ud.parm['protocol'] == 'git':
raise bb.fetch2.ParameterError(
"Invalid protocol - if you wish to fetch from a " +
diff --git a/lib/bb/fetch2/ssh.py b/lib/bb/fetch2/ssh.py
index 6047ee41..4f9e0853 100644
--- a/lib/bb/fetch2/ssh.py
+++ b/lib/bb/fetch2/ssh.py
@@ -77,7 +77,7 @@ class SSH(FetchMethod):
def supports_checksum(self, urldata):
return False
- def urldata_init(self, urldata, d):
+ def urldata_init(self, urldata, d, original_url):
if 'protocol' in urldata.parm and urldata.parm['protocol'] == 'git':
raise bb.fetch2.ParameterError(
"Invalid protocol - if you wish to fetch from a git " +
diff --git a/lib/bb/fetch2/svn.py b/lib/bb/fetch2/svn.py
index ed70bcf8..be0e9eeb 100644
--- a/lib/bb/fetch2/svn.py
+++ b/lib/bb/fetch2/svn.py
@@ -42,7 +42,7 @@ class Svn(FetchMethod):
"""
return ud.type in ['svn']
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
"""
init svn specific variable within url data
"""
diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py
index 8f505b6d..bd2a0c6b 100644
--- a/lib/bb/fetch2/wget.py
+++ b/lib/bb/fetch2/wget.py
@@ -74,7 +74,7 @@ class Wget(FetchMethod):
def recommends_checksum(self, urldata):
return True
- def urldata_init(self, ud, d):
+ def urldata_init(self, ud, d, original_url):
if 'protocol' in ud.parm:
if ud.parm['protocol'] == 'git':
raise bb.fetch2.ParameterError("Invalid protocol - if you wish to fetch from a git repository using http, you need to instead use the git:// prefix with protocol=http", ud.url)
--
2.18.0
More information about the bitbake-devel
mailing list