[bitbake-devel] [PATCH] bitbake: fetch2/svn: Fix SVN repository concurrent update race
Ulf.Magnusson at bmw.de
Ulf.Magnusson at bmw.de
Wed May 25 11:32:56 UTC 2016
The ${DL_DIR}/svn directory is used by BitBake to keep checked-out SVN
repositories from which tarballs are generated. These repositories were
protected from concurrent update with a lock on the tarballs. However,
the tarballs are specific to the SRCREV and module checked out (many
tarballs can come from the same repository), meaning a repository could
be modified concurrently if two recipes checked out two different
SRCREVs or modules from it in parallel.
This caused errors like the following:
ERROR: Fetcher failure: Fetch command failed with exit code 1, output:
svn: E155004: Run 'svn cleanup' to remove locks (type 'svn help cleanup' for details)
svn: E155004: Working copy '/home/foo/downloads/svn/repo/trunk' locked.
svn: E155004: '/home/foo/downloads/svn/repo/trunk' is already locked.
Fix it by adding a per-repository lock that's independent of the module
and SRCREV.
---
bitbake/lib/bb/fetch2/svn.py | 69 +++++++++++++++++++++++++-------------------
1 file changed, 39 insertions(+), 30 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index 8a29193..f83b740 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -63,6 +63,9 @@ class Svn(FetchMethod):
relpath = self._strip_leading_slashes(ud.path)
ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath)
ud.moddir = os.path.join(ud.pkgdir, ud.module)
+ # Protects the repository from concurrent updates, e.g. from two
+ # recipes fetching different revisions at the same time
+ ud.svnlock = os.path.join(ud.pkgdir, "svn.lock")
ud.setup_revisons(d)
@@ -123,38 +126,44 @@ class Svn(FetchMethod):
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
- if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
- svnupdatecmd = self._buildsvncommand(ud, d, "update")
- logger.info("Update " + ud.url)
- # update sources there
- os.chdir(ud.moddir)
- # We need to attempt to run svn upgrade first in case its an older working format
- try:
- runfetchcmd(ud.basecmd + " upgrade", d)
- except FetchError:
- pass
- logger.debug(1, "Running %s", svnupdatecmd)
- bb.fetch2.check_network_access(d, svnupdatecmd, ud.url)
- runfetchcmd(svnupdatecmd, d)
- else:
- svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
- logger.info("Fetch " + ud.url)
- # check out sources there
- bb.utils.mkdirhier(ud.pkgdir)
- os.chdir(ud.pkgdir)
- logger.debug(1, "Running %s", svnfetchcmd)
- bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
- runfetchcmd(svnfetchcmd, d)
+ lf = bb.utils.lockfile(ud.svnlock)
+
+ try:
+ if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
+ svnupdatecmd = self._buildsvncommand(ud, d, "update")
+ logger.info("Update " + ud.url)
+ # update sources there
+ os.chdir(ud.moddir)
+ # We need to attempt to run svn upgrade first in case its an older working format
+ try:
+ runfetchcmd(ud.basecmd + " upgrade", d)
+ except FetchError:
+ pass
+ logger.debug(1, "Running %s", svnupdatecmd)
+ bb.fetch2.check_network_access(d, svnupdatecmd, ud.url)
+ runfetchcmd(svnupdatecmd, d)
+ else:
+ svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
+ logger.info("Fetch " + ud.url)
+ # check out sources there
+ bb.utils.mkdirhier(ud.pkgdir)
+ os.chdir(ud.pkgdir)
+ logger.debug(1, "Running %s", svnfetchcmd)
+ bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
+ runfetchcmd(svnfetchcmd, d)
+
+ scmdata = ud.parm.get("scmdata", "")
+ if scmdata == "keep":
+ tar_flags = ""
+ else:
+ tar_flags = "--exclude '.svn'"
- scmdata = ud.parm.get("scmdata", "")
- if scmdata == "keep":
- tar_flags = ""
- else:
- tar_flags = "--exclude '.svn'"
+ os.chdir(ud.pkgdir)
+ # tar them up to a defined filename
+ runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d, cleanup = [ud.localpath])
- os.chdir(ud.pkgdir)
- # tar them up to a defined filename
- runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d, cleanup = [ud.localpath])
+ finally:
+ bb.utils.unlockfile(lf)
def clean(self, ud, d):
""" Clean SVN specific files and dirs """
--
2.7.4
More information about the bitbake-devel
mailing list