[bitbake-devel] [PATCH] fetch2/hg.py: Several fixes
Robert Yang
liezhi.yang at windriver.com
Wed Jun 3 00:36:43 UTC 2015
On 06/03/2015 12:01 AM, Dan McGregor wrote:
> From: Daniel McGregor <daniel.mcgregor at vecima.com>
>
> Generate a full mirror bundle of the repository.
> Share the repository with the working directory checkout.
> Remove option to keep scm data; always keep it.
>
> Signed-off-by: Daniel McGregor <daniel.mcgregor at vecima.com>
> ---
> lib/bb/fetch2/hg.py | 181 +++++++++++++++++++++++++++++++++-------------------
> 1 file changed, 115 insertions(+), 66 deletions(-)
>
> diff --git a/lib/bb/fetch2/hg.py b/lib/bb/fetch2/hg.py
> index cdef4aa..4b476c4 100644
> --- a/lib/bb/fetch2/hg.py
> +++ b/lib/bb/fetch2/hg.py
> @@ -63,6 +63,10 @@ class Hg(FetchMethod):
> relpath = self._strip_leading_slashes(ud.path)
> ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath)
> ud.moddir = os.path.join(ud.pkgdir, ud.module)
> + ud.localfile = ud.moddir
> +
> + ud.mirrortarball = data.expand('hg_%s_%s_%s.bundle' % (ud.host, relpath.rstrip('/').replace('/', '.'), ud.module.replace('/', '.')), d)
Does the mirror tarball work when the mirror is not on the local disk ?
for example, when the mirror tarball is on the server via http, something
like yocto's mirror server:
PREMIRRORS ??= "\
bzr://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \
cvs://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \
git://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \
gitsm://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \
hg://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \
osc://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \
p4://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \
svn://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n"
You can setup something familiar server to test the mirror tarball, it
should be able to down from the mirror rather than SRC_URI.
// Robert
> + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
>
> ud.setup_revisons(d)
>
> @@ -71,15 +75,19 @@ class Hg(FetchMethod):
> elif not ud.revision:
> ud.revision = self.latest_revision(ud, d)
>
> - ud.localfile = ud.moddir
> -
> ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg"
> + ud.write_bundle = (data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0"
>
> def need_update(self, ud, d):
> revTag = ud.parm.get('rev', 'tip')
> if revTag == "tip":
> return True
> - if not os.path.exists(ud.localpath):
> + if ud.write_bundle and not os.path.exists(ud.fullmirror):
> + return True
> + if not os.path.exists(ud.moddir):
> + return True
> + os.chdir(ud.moddir)
> + if not self._contains_ref(ud, d):
> return True
> return False
>
> @@ -89,51 +97,33 @@ class Hg(FetchMethod):
> command is "fetch", "update", "info"
> """
>
> + repourl = self._get_repo_url(ud)
> proto = ud.parm.get('protocol', 'http')
>
> - host = ud.host
> - if proto == "file":
> - host = "/"
> - ud.host = "localhost"
> + options = [];
>
> - if not ud.user:
> - hgroot = host + ud.path
> - else:
> - if ud.pswd:
> - hgroot = ud.user + ":" + ud.pswd + "@" + host + ud.path
> - else:
> - hgroot = ud.user + "@" + host + ud.path
> + if ud.user and ud.pswd:
> + options.append(" --config auth.default.prefix=*")
> + options.append(" --config auth.default.username=%s" % (ud.user))
> + options.append(" --config auth.default.password=%s" % (ud.pswd))
> + options.append(" --config \"auth.default.schemes=%s\"" % (proto))
>
> if command == "info":
> - return "%s identify -i %s://%s/%s" % (ud.basecmd, proto, hgroot, ud.module)
> -
> - options = [];
> + return "%s identify %s -i %s%s" % (ud.basecmd, " ".join(options), repourl, ud.module)
>
> - # Don't specify revision for the fetch; clone the entire repo.
> + # Don't specify revision for fetch or pull; fetch the entire repo.
> # This avoids an issue if the specified revision is a tag, because
> # the tag actually exists in the specified revision + 1, so it won't
> # be available when used in any successive commands.
> - if ud.revision and command != "fetch":
> - options.append("-r %s" % ud.revision)
>
> if command == "fetch":
> - if ud.user and ud.pswd:
> - cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" clone %s %s://%s/%s %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options), proto, hgroot, ud.module, ud.module)
> - else:
> - cmd = "%s clone %s %s://%s/%s %s" % (ud.basecmd, " ".join(options), proto, hgroot, ud.module, ud.module)
> + cmd = "%s clone -U %s %s%s %s" % (ud.basecmd, " ".join(options), repourl, ud.module, ud.module)
> elif command == "pull":
> - # do not pass options list; limiting pull to rev causes the local
> - # repo not to contain it and immediately following "update" command
> - # will crash
> - if ud.user and ud.pswd:
> - cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto)
> - else:
> - cmd = "%s pull" % (ud.basecmd)
> + cmd = "%s pull %s %s%s" % (ud.basecmd, " ".join(options), repourl, ud.module)
> elif command == "update":
> - if ud.user and ud.pswd:
> - cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options))
> - else:
> - cmd = "%s update -C %s" % (ud.basecmd, " ".join(options))
> + if ud.revision:
> + options.append("-r %s" % ud.revision)
> + cmd = "%s update -C %s" % (ud.basecmd, " ".join(options))
> else:
> raise FetchError("Invalid hg command %s" % command, ud.url)
>
> @@ -142,38 +132,49 @@ class Hg(FetchMethod):
> def download(self, ud, d):
> """Fetch url"""
>
> + if not os.path.exists(os.path.join(ud.moddir, '.hg')):
> + logger.debug(2, "Fetch: creating module directory '" + ud.moddir + "'")
> + bb.utils.mkdirhier(ud.moddir)
> + runfetchcmd("%s init %s" % (ud.basecmd, ud.moddir), d)
> + self._generate_hgrc(ud, ud.moddir, d)
> +
> + if os.path.exists(ud.fullmirror):
> + logger.debug(2, "Fetch: unbundling module directory '" + ud.moddir + "'")
> + runfetchcmd("%s unbundle -R %s %s" % (ud.basecmd, ud.moddir, ud.fullmirror), d)
> +
> logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
>
> - if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
> + os.chdir(ud.moddir)
> + if not self._contains_ref(ud, d):
> + self._generate_hgrc(ud, ud.moddir, d)
> updatecmd = self._buildhgcommand(ud, d, "pull")
> logger.info("Update " + ud.url)
> # update sources there
> - os.chdir(ud.moddir)
> logger.debug(1, "Running %s", updatecmd)
> bb.fetch2.check_network_access(d, updatecmd, ud.url)
> runfetchcmd(updatecmd, d)
>
> - else:
> - fetchcmd = self._buildhgcommand(ud, d, "fetch")
> - logger.info("Fetch " + ud.url)
> - # check out sources there
> - bb.utils.mkdirhier(ud.pkgdir)
> - os.chdir(ud.pkgdir)
> - logger.debug(1, "Running %s", fetchcmd)
> - bb.fetch2.check_network_access(d, fetchcmd, ud.url)
> - runfetchcmd(fetchcmd, d)
> -
> - # Even when we clone (fetch), we still need to update as hg's clone
> - # won't checkout the specified revision if its on a branch
> - updatecmd = self._buildhgcommand(ud, d, "update")
> - os.chdir(ud.moddir)
> - logger.debug(1, "Running %s", updatecmd)
> - runfetchcmd(updatecmd, d)
> + # When we clone (fetch), we disable the working directory.
> + # We don't want a checkout at all.
> +
> + def build_mirror_data(self, ud, d):
> + # Generate a mirror tarball if needed
> + if ud.write_bundle and not os.path.exists(ud.fullmirror):
> + # it's possible that this symlink points to read-only filesystem with PREMIRROR
> + if os.path.islink(ud.fullmirror):
> + os.unlink(ud.fullmirror)
> +
> + os.chdir(ud.moddir)
> + logger.info("Creating bundle of mercurial repository")
> + runfetchcmd("%s bundle --base null --all %s" % (ud.basecmd, ud.fullmirror), d)
> + runfetchcmd("touch %s.done" % (ud.fullmirror), d)
>
> def clean(self, ud, d):
> """ Clean the hg dir """
>
> bb.utils.remove(ud.localpath, True)
> + bb.utils.remove(ud.fullmirror)
> + bb.utils.remove(ud.fullmirror + ".done")
>
> def supports_srcrev(self):
> return True
> @@ -200,23 +201,71 @@ class Hg(FetchMethod):
>
> def unpack(self, ud, destdir, d):
> """
> - Make a local clone or export for the url
> + Make a local clone
> """
>
> revflag = "-r %s" % ud.revision
> subdir = ud.parm.get("destsuffix", ud.module)
> codir = "%s/%s" % (destdir, subdir)
>
> - scmdata = ud.parm.get("scmdata", "")
> - if scmdata != "nokeep":
> - if not os.access(os.path.join(codir, '.hg'), os.R_OK):
> - logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
> - runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
> - logger.debug(2, "Unpack: updating source in '" + codir + "'")
> - os.chdir(codir)
> - runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d)
> - runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d)
> + if not os.access(os.path.join(codir, '.hg'), os.R_OK):
> + logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
> + runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
> + logger.debug(2, "Unpack: updating source in '" + codir + "'")
> + self._generate_hgrc(ud, codir, d)
> + os.chdir(codir)
> + runfetchcmd("printf %s/.hg > %s/.hg/sharedpath" % (ud.moddir, codir), d, quiet=True)
> + runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d)
> +
> + def _contains_ref(self, ud, d):
> + cmd = "%s log --limit 1 -r %s --template '{firstline(desc)}\\n' 2> /dev/null | wc -l" % (
> + ud.basecmd, ud.revision)
> + try:
> + output = runfetchcmd(cmd, d, quiet=True)
> + except bb.fetch2.FetchError:
> + return False
> + if len(output.split()) > 1:
> + raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
> + return output.split()[0] != "0"
> +
> + def _get_repo_url(self, ud):
> + """
> + Return the repository URL
> + """
> + proto = ud.parm.get('protocol', 'http')
> +
> + host = ud.host
> + if proto == "file":
> + host = "/"
> + ud.host = "localhost"
> + if not ud.user:
> + root = host
> else:
> - logger.debug(2, "Unpack: extracting source to '" + codir + "'")
> - os.chdir(ud.moddir)
> - runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d)
> + if ud.pswd:
> + root = ud.user + ":" + ud.pswd + "@" + host
> + else:
> + root = ud.user + "@" + host
> +
> + return "%s://%s%s" % (proto, root, ud.path)
> +
> + def _generate_hgrc(self, ud, destdir, d):
> + """
> + Generate hgrc for this repo
> + """
> + root = self._get_repo_url(ud)
> + proto = ud.parm.get('protocol', 'http')
> + config_path = os.path.join(destdir, '.hg/hgrc')
> +
> + f = open(config_path, 'w')
> + f.write("[paths]\n")
> + f.write("default = %s%s\n" % (root, ud.module))
> + f.write("\n")
> + f.write("[auth]\n")
> + f.write("default.prefix=*\n")
> + f.write("default.username=%s\n" % (ud.user))
> + f.write("default.password=%s\n" % (ud.pswd))
> + f.write("default.schemes=%s\n" % (proto))
> + f.write("\n")
> + f.write("[extensions]\n")
> + f.write("share =\n")
> + f.close()
>
More information about the bitbake-devel
mailing list