<div dir="ltr">Master has just landed the Python3 port, so can you rebase this to current git master?<div><br></div><div>Thanks,</div><div>Ross</div></div><div class="gmail_extra"><br><div class="gmail_quote">On 25 May 2016 at 20:48, Andrew Bradford <span dir="ltr"><<a href="mailto:andrew@bradfordembedded.com" target="_blank">andrew@bradfordembedded.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">From: Andrew Bradford <<a href="mailto:andrew.bradford@kodakalaris.com">andrew.bradford@kodakalaris.com</a>><br>
<br>
In recipes which use the perforce fetcher, enable use of SRCREV to<br>
specify any of: ${AUTOREV}, changeset number, p4date, or label. This is<br>
more in-line with how the other fetchers work for source control<br>
systems.<br>
<br>
Allow use of the P4CONFIG env variable to define the server URL,<br>
username, and password if not provided in a recipe.<br>
<br>
This does change existing perforce fetcher usage by recipes and will<br>
need those recipes which use the perforce fetcher to be updated. No<br>
recipes in oe-core use the perforce fetcher.<br>
<br>
References [YOCTO #6303]<br>
<br>
Signed-off-by: Andrew Bradford <<a href="mailto:andrew.bradford@kodakalaris.com">andrew.bradford@kodakalaris.com</a>><br>
---<br>
lib/bb/fetch2/__init__.py | 3 +-<br>
lib/bb/fetch2/perforce.py | 279 ++++++++++++++++++++++++----------------------<br>
2 files changed, 147 insertions(+), 135 deletions(-)<br>
<br>
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py<br>
index 7d2f350..b6e9f9a 100644<br>
--- a/lib/bb/fetch2/__init__.py<br>
+++ b/lib/bb/fetch2/__init__.py<br>
@@ -809,7 +809,8 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):<br>
'GIT_SMART_HTTP',<br>
'SSH_AUTH_SOCK', 'SSH_AGENT_PID',<br>
'SOCKS5_USER', 'SOCKS5_PASSWD',<br>
- 'DBUS_SESSION_BUS_ADDRESS']<br>
+ 'DBUS_SESSION_BUS_ADDRESS',<br>
+ 'P4CONFIG']<br>
<br>
if not cleanup:<br>
cleanup = []<br>
diff --git a/lib/bb/fetch2/perforce.py b/lib/bb/fetch2/perforce.py<br>
index 3a10c7c..b84489a 100644<br>
--- a/lib/bb/fetch2/perforce.py<br>
+++ b/lib/bb/fetch2/perforce.py<br>
@@ -1,14 +1,12 @@<br>
# ex:ts=4:sw=4:sts=4:et<br>
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-<br>
"""<br>
-BitBake 'Fetch' implementations<br>
-<br>
-Classes for obtaining upstream sources for the<br>
-BitBake build tools.<br>
+BitBake 'Fetch' implementation for perforce<br>
<br>
"""<br>
<br>
# Copyright (C) 2003, 2004 Chris Larson<br>
+# Copyright (C) 2016 Kodak Alaris, Inc.<br>
#<br>
# This program is free software; you can redistribute it and/or modify<br>
# it under the terms of the GNU General Public License version 2 as<br>
@@ -25,9 +23,8 @@ BitBake build tools.<br>
#<br>
# Based on functions from the base bb module, Copyright 2003 Holger Schurig<br>
<br>
-from future_builtins import zip<br>
import os<br>
-import subprocess<br>
+import sys<br>
import logging<br>
import bb<br>
from bb import data<br>
@@ -37,151 +34,165 @@ from bb.fetch2 import logger<br>
from bb.fetch2 import runfetchcmd<br>
<br>
class Perforce(FetchMethod):<br>
+ """ Class to fetch from perforce repositories """<br>
def supports(self, ud, d):<br>
+ """ Check to see if a given url can be fetched with perforce. """<br>
return ud.type in ['p4']<br>
<br>
- def doparse(url, d):<br>
- parm = {}<br>
- path = url.split("://")[1]<br>
+ def urldata_init(self, ud, d):<br>
+ """<br>
+ Initialize perforce specific variables within url data. If P4PORT is<br>
+ not provided by the recipe but P4CONFIG is provided by the env, let<br>
+ perforce use P4CONFIG as it naturally does to set P4USER, P4PASSWD, and<br>
+ P4PORT.<br>
+ """<br>
+ ud.basecmd = d.getVar('FETCHCMD_p4', True)<br>
+ ud.usingp4config = False<br>
+<br>
+ path = ud.url.split("://")[1]<br>
+ path = path.split(";")[0]<br>
delim = path.find("@");<br>
if delim != -1:<br>
- (user, pswd, host, port) = path.split('@')[0].split(":")<br>
- path = path.split('@')[1]<br>
+ (ud.user, ud.pswd) = path.split('@')[0].split(":")<br>
+ ud.path = path.split('@')[1]<br>
else:<br>
- (host, port) = d.getVar('P4PORT', False).split(':')<br>
- user = ""<br>
- pswd = ""<br>
-<br>
- if path.find(";") != -1:<br>
- keys=[]<br>
- values=[]<br>
- plist = path.split(';')<br>
- for item in plist:<br>
- if item.count('='):<br>
- (key, value) = item.split('=')<br>
- keys.append(key)<br>
- values.append(value)<br>
-<br>
- parm = dict(zip(keys, values))<br>
- path = "//" + path.split(';')[0]<br>
- host += ":%s" % (port)<br>
- parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm)<br>
-<br>
- return host, path, user, pswd, parm<br>
- doparse = staticmethod(doparse)<br>
-<br>
- def getcset(d, depot, host, user, pswd, parm):<br>
- p4opt = ""<br>
- if "cset" in parm:<br>
- return parm["cset"];<br>
- if user:<br>
- p4opt += " -u %s" % (user)<br>
- if pswd:<br>
- p4opt += " -P %s" % (pswd)<br>
- if host:<br>
- p4opt += " -p %s" % (host)<br>
-<br>
- p4date = d.getVar("P4DATE", True)<br>
- if "revision" in parm:<br>
- depot += "#%s" % (parm["revision"])<br>
- elif "label" in parm:<br>
- depot += "@%s" % (parm["label"])<br>
- elif p4date:<br>
- depot += "@%s" % (p4date)<br>
-<br>
- p4cmd = d.getVar('FETCHCMD_p4', True) or "p4"<br>
- logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot)<br>
- p4file, errors = bb.process.run("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot))<br>
- cset = p4file.strip()<br>
- logger.debug(1, "READ %s", cset)<br>
- if not cset:<br>
- return -1<br>
-<br>
- return cset.split(' ')[1]<br>
- getcset = staticmethod(getcset)<br>
-<br>
- def urldata_init(self, ud, d):<br>
- (host, path, user, pswd, parm) = Perforce.doparse(ud.url, d)<br>
+ ud.path = path<br>
<br>
- base_path = path.replace('/...', '')<br>
- base_path = self._strip_leading_slashes(base_path)<br>
-<br>
- if "label" in parm:<br>
- version = parm["label"]<br>
+ p4port = d.getVar('P4PORT', True)<br>
+ if p4port:<br>
+ ud.host = p4port<br>
else:<br>
- version = Perforce.getcset(d, path, host, user, pswd, parm)<br>
-<br>
- ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base_path.replace('/', '.'), version), d)<br>
-<br>
- def download(self, ud, d):<br>
+ p4config = d.getVar('P4CONFIG', True)<br>
+ if p4config:<br>
+ <a href="http://logger.info" rel="noreferrer" target="_blank">logger.info</a>("Using P4CONFIG to specify P4PORT")<br>
+ ud.usingp4config = True<br>
+ ud.host, errors = bb.process.run("p4 info | grep \"Server address\"")<br>
+ ud.host = ud.host.split(': ')[1].strip()<br>
+ <a href="http://logger.info" rel="noreferrer" target="_blank">logger.info</a>("Determined P4PORT to be: %s" % ud.host)<br>
+ if not ud.host:<br>
+ raise FetchError("Could not determine server name from P4CONFIG")<br>
+ else:<br>
+ raise FetchError("SRC_URI is not valid, did you forget to specify P4PORT or P4CONFIG?")<br>
+<br>
+ relpath = ud.path.replace('/...', '')<br>
+ relpath = self._strip_leading_slashes(relpath)<br>
+ relpath = relpath.replace('/', '.')<br>
+ cleanedhost = ud.host.replace(':', '.')<br>
+ ud.pkgdir = os.path.join(data.expand('${P4DIR}', d), cleanedhost, relpath)<br>
+<br>
+ ud.setup_revisons(d)<br>
+<br>
+ ud.localfile = data.expand('%s_%s_%s.tar.gz' % (cleanedhost, relpath, ud.revision), d)<br>
+<br>
+ def _buildp4command(self, ud, d, command, depot_filename=None):<br>
"""<br>
- Fetch urls<br>
+ Build a p4 commandline. Valid commands are "changes", "print", and<br>
+ "files". depot_filename is the full path to the file in the depot<br>
+ including the trailing '#rev' value.<br>
"""<br>
-<br>
- (host, depot, user, pswd, parm) = Perforce.doparse(ud.url, d)<br>
-<br>
- if depot.find('/...') != -1:<br>
- path = depot[:depot.find('/...')]<br>
- else:<br>
- path = depot[:depot.rfind('/')]<br>
-<br>
- module = parm.get('module', os.path.basename(path))<br>
-<br>
- # Get the p4 command<br>
p4opt = ""<br>
- if user:<br>
- p4opt += " -u %s" % (user)<br>
-<br>
- if pswd:<br>
- p4opt += " -P %s" % (pswd)<br>
-<br>
- if host:<br>
- p4opt += " -p %s" % (host)<br>
-<br>
- p4cmd = d.getVar('FETCHCMD_p4', True) or "p4"<br>
-<br>
- # create temp directory<br>
- logger.debug(2, "Fetch: creating temporary directory")<br>
- bb.utils.mkdirhier(d.expand('${WORKDIR}'))<br>
- mktemp = d.getVar("FETCHCMD_p4mktemp", True) or d.expand("mktemp -d -q '${WORKDIR}/oep4.XXXXXX'")<br>
- tmpfile, errors = bb.process.run(mktemp)<br>
- tmpfile = tmpfile.strip()<br>
- if not tmpfile:<br>
- raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", ud.url)<br>
-<br>
- if "label" in parm:<br>
- depot = "%s@%s" % (depot, parm["label"])<br>
+ if ud.user:<br>
+ p4opt += " -u \"%s\"" % (ud.user)<br>
+ if ud.pswd:<br>
+ p4opt += " -P \"%s\"" % (ud.pswd)<br>
+ if ud.host and not ud.usingp4config:<br>
+ p4opt += " -p %s" % (ud.host)<br>
+ if hasattr(ud, 'revision') and ud.revision:<br>
+ pathnrev = "%s@%s" % (ud.path, ud.revision)<br>
else:<br>
- cset = Perforce.getcset(d, depot, host, user, pswd, parm)<br>
- depot = "%s@%s" % (depot, cset)<br>
-<br>
- os.chdir(tmpfile)<br>
- <a href="http://logger.info" rel="noreferrer" target="_blank">logger.info</a>("Fetch " + ud.url)<br>
- <a href="http://logger.info" rel="noreferrer" target="_blank">logger.info</a>("%s%s files %s", p4cmd, p4opt, depot)<br>
- p4file, errors = bb.process.run("%s%s files %s" % (p4cmd, p4opt, depot))<br>
- p4file = [f.rstrip() for f in p4file.splitlines()]<br>
+ pathnrev = "%s" % (ud.path)<br>
+<br>
+ if depot_filename:<br>
+ # FIXME: handle when SRC_URI is a single file and not a dir<br>
+ filename = depot_filename[len(ud.path)-1:] # Remove leading path<br>
+ where = filename.find("#") # Remove trailing '#rev'<br>
+ filename = filename[:where]<br>
+<br>
+ if command == "changes":<br>
+ p4cmd = "%s%s changes -m 1 //%s" % (ud.basecmd, p4opt, pathnrev)<br>
+ elif command == "print":<br>
+ if depot_filename != None:<br>
+ p4cmd = "%s%s print -o \"p4/%s\" \"%s\"" % (ud.basecmd, p4opt, filename, depot_filename)<br>
+ else:<br>
+ raise FetchError("No depot file name provided to p4 %s" % command, ud.url)<br>
+ elif command == "files":<br>
+ p4cmd = "%s%s files //%s" % (ud.basecmd, p4opt, pathnrev)<br>
+ else:<br>
+ raise FetchError("Invalid p4 command %s" % command, ud.url)<br>
<br>
- if not p4file:<br>
- raise FetchError("Fetch: unable to get the P4 files from %s" % depot, ud.url)<br>
+ return p4cmd<br>
<br>
+ def _p4listfiles(self, ud, d):<br>
+ """<br>
+ Return a list of the file names which are present in the depot using the<br>
+ 'p4 files' command, including trailing '#rev' file revision indicator<br>
+ """<br>
+ p4cmd = self._buildp4command(ud, d, "files")<br>
+ bb.fetch2.check_network_access(d, p4cmd)<br>
+ p4fileslist = runfetchcmd(p4cmd, d, True)<br>
+ p4fileslist = [f.rstrip() for f in p4fileslist.splitlines()]<br>
+ if not p4fileslist:<br>
+ raise FetchError("Unable to fetch listing of p4 files from %s@%s" % ud.host, ud.path)<br>
count = 0<br>
-<br>
- for file in p4file:<br>
- list = file.split()<br>
-<br>
- if list[2] == "delete":<br>
+ filelist = []<br>
+ for filename in p4fileslist:<br>
+ item = filename.split(" - ")<br>
+ lastaction = item[1].split()<br>
+ <a href="http://logger.info" rel="noreferrer" target="_blank">logger.info</a>("File: " + item[0] + " Last Action: " + lastaction[0])<br>
+ if lastaction[0] == "delete":<br>
continue<br>
+ filelist.append(item[0])<br>
<br>
- dest = list[0][len(path)+1:]<br>
- where = dest.find("#")<br>
+ return filelist<br>
<br>
- subprocess.call("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0]), shell=True)<br>
- count = count + 1<br>
+ def download(self, ud, d):<br>
+ """ Get the list of files, fetch each one """<br>
+ filelist = self._p4listfiles(ud, d)<br>
+<br>
+ if not filelist:<br>
+ raise FetchError("No files found in depot %s@%s" % ud.host, ud.path)<br>
<br>
- if count == 0:<br>
- logger.error()<br>
- raise FetchError("Fetch: No files gathered from the P4 fetch", ud.url)<br>
+ <a href="http://logger.info" rel="noreferrer" target="_blank">logger.info</a>("Fetch " + ud.url)<br>
+ bb.utils.remove(ud.pkgdir, True)<br>
+ bb.utils.mkdirhier(ud.pkgdir)<br>
+ os.chdir(ud.pkgdir)<br>
+<br>
+ for afile in filelist:<br>
+ p4fetchcmd = self._buildp4command(ud, d, "print", afile)<br>
+ logger.debug(1, "Running %s", p4fetchcmd)<br>
+ bb.fetch2.check_network_access(d, p4fetchcmd)<br>
+ runfetchcmd(p4fetchcmd, d)<br>
+<br>
+ os.chdir(ud.pkgdir)<br>
+ runfetchcmd("tar -czf %s p4" % (ud.localpath), d, cleanup = [ud.localpath])<br>
+<br>
+ def clean(self, ud, d):<br>
+ """ Cleanup p4 specific files and dirs"""<br>
+ bb.utils.remove(ud.localpath)<br>
+ bb.utils.remove(ud.pkgdir, True)<br>
+<br>
+ def supports_srcrev(self):<br>
+ return True<br>
+<br>
+ def _revision_key(self, ud, d, name):<br>
+ """ Return a unique key for the url """<br>
+ return "p4:" + ud.pkgdir<br>
+<br>
+ def _latest_revision(self, ud, d, name):<br>
+ """ Return the latest upstream scm revision number """<br>
+ p4cmd = self._buildp4command(ud, d, "changes")<br>
+ bb.fetch2.check_network_access(d, p4cmd)<br>
+ tip = runfetchcmd(p4cmd, d, True)<br>
+ if not tip:<br>
+ raise FetchError("Could not determine the latest perforce changeset")<br>
+ tipcset = tip.split(' ')[1]<br>
+ logger.debug(1, "p4 tip found to be changeset %s", tipcset)<br>
+ return tipcset<br>
+<br>
+ def sortable_revision(self, ud, d, name):<br>
+ """ Return a sortable revision number """<br>
+ return False, self._build_revision(ud, d)<br>
+<br>
+ def _build_revision(self, ud, d):<br>
+ return ud.revision<br>
<br>
- runfetchcmd("tar -czf %s %s" % (ud.localpath, module), d, cleanup = [ud.localpath])<br>
- # cleanup<br>
- bb.utils.prunedir(tmpfile)<br>
<span class="HOEnZb"><font color="#888888">--<br>
2.8.1<br>
<br>
--<br>
_______________________________________________<br>
bitbake-devel mailing list<br>
<a href="mailto:bitbake-devel@lists.openembedded.org">bitbake-devel@lists.openembedded.org</a><br>
<a href="http://lists.openembedded.org/mailman/listinfo/bitbake-devel" rel="noreferrer" target="_blank">http://lists.openembedded.org/mailman/listinfo/bitbake-devel</a><br>
</font></span></blockquote></div><br></div>