[oe-commits] [bitbake] branch master-next updated: svn.py: Stop SVN from directly pulling from an external layer w/o fetcher
git at git.openembedded.org
git at git.openembedded.org
Thu May 16 20:45:20 UTC 2019
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
The following commit(s) were added to refs/heads/master-next by this push:
new dd2e7dd svn.py: Stop SVN from directly pulling from an external layer w/o fetcher
dd2e7dd is described below
commit dd2e7dd4e582302d7f51011ca2c7a3092c93e973
Author: Mark Hatle <mark.hatle at windriver.com>
AuthorDate: Thu May 16 06:46:16 2019 -0400
svn.py: Stop SVN from directly pulling from an external layer w/o fetcher
Add a new option to the svn fetcher url "externals=allowed". This will allow
a user to enable svn co w/ externals. However, this does avoid the fetcher,
network access and mirror systems.
By default we no longer allow externals in the checkout. This ensures a
deterministic download. The system does attempt to identify SVN repos that
have externals enabled, and will warn the user. It is up to the user to
determine if these are necessary for the recipe. They may disable the warning
by adding "externals=nowarn" to the url.
In the future we would like to parse this list and see if the items are already
in the SRC_URI for that recipe, but with SVN being in limited use these days
that extra work is likely not worth the trouble.
Add test cases that generated a local SVN tree, with an external source
set to github bitbake in svn format. One test case checks that externals are
ignored, and one checks that they in downloaded.
Signed-off-by: Mark Hatle <mark.hatle at windriver.com>
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/fetch2/svn.py | 19 +++++++++++++++
lib/bb/tests/fetch.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 86 insertions(+)
diff --git a/lib/bb/fetch2/svn.py b/lib/bb/fetch2/svn.py
index baeb0e7..59ce931 100644
--- a/lib/bb/fetch2/svn.py
+++ b/lib/bb/fetch2/svn.py
@@ -91,6 +91,13 @@ class Svn(FetchMethod):
svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
else:
suffix = ""
+
+ # externals may be either 'allowed' or 'nowarn', but not both. Allowed
+ # will not issue a warning, but will log to the debug buffer what has likely
+ # been downloaded by SVN.
+ if not ("externals" in ud.parm and ud.parm["externals"] == "allowed"):
+ options.append("--ignore-externals")
+
if ud.revision:
options.append("-r %s" % ud.revision)
suffix = "@%s" % (ud.revision)
@@ -136,6 +143,18 @@ class Svn(FetchMethod):
bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir)
+ if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"):
+ # Warn the user if this had externals (won't catch them all)
+ output = runfetchcmd("svn propget svn:externals", d, workdir=ud.moddir)
+ if output:
+ if "--ignore-externals" in svnfetchcmd.split():
+ bb.warn("%s contains svn:externals." % ud.url)
+ bb.warn("These should be added to the recipe SRC_URI as necessary.")
+ bb.warn("svn fetch has ignored externals:\n%s" % output)
+ bb.warn("To disable this warning add ';externals=nowarn' to the url.")
+ else:
+ bb.debug(1, "svn repository has externals:\n%s" % output)
+
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 6bdf041..803dc6b 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -991,6 +991,73 @@ class FetcherNetworkTest(FetcherTest):
self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
+class SVNTest(FetcherTest):
+
+ def setUp(self):
+ """ Create a local repository """
+
+ super(SVNTest, self).setUp()
+
+ # Create something we can fetch
+ src_dir = tempfile.mkdtemp(dir=self.tempdir,
+ prefix='svnfetch_srcdir_')
+ src_dir = os.path.abspath(src_dir)
+ bb.process.run("echo readme > README.md", cwd=src_dir)
+
+ # Store it in a local SVN repository
+ repo_dir = tempfile.mkdtemp(dir=self.tempdir,
+ prefix='svnfetch_localrepo_')
+ repo_dir = os.path.abspath(repo_dir)
+ bb.process.run("svnadmin create project", cwd=repo_dir)
+
+ self.repo_url = "file://%s/project" % repo_dir
+ bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url),
+ cwd=repo_dir)
+
+ bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
+ # Github will emulate SVN. Use this to check if we're downloding...
+ bb.process.run("svn propset svn:externals 'bitbake http://github.com/openembedded/bitbake' .",
+ cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
+ bb.process.run("svn commit --non-interactive -m 'Add external'",
+ cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
+
+ self.src_dir = src_dir
+ self.repo_dir = repo_dir
+
+ def tearDown(self):
+ os.chdir(self.origdir)
+ if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
+ print("Not cleaning up %s. Please remove manually." % self.tempdir)
+ else:
+ bb.utils.prunedir(self.tempdir)
+
+ @skipIfNoNetwork()
+ def test_noexternal_svn(self):
+ # Always match the rev count from setUp (currently rev 2)
+ url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '')
+ fetcher = bb.fetch.Fetch([url], self.d)
+ fetcher.download()
+ os.chdir(os.path.dirname(self.unpackdir))
+ fetcher.unpack(self.unpackdir)
+
+ self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
+ self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
+ self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
+ self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
+
+ def test_external_svn(self):
+ # Always match the rev count from setUp (currently rev 2)
+ url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '')
+ fetcher = bb.fetch.Fetch([url], self.d)
+ fetcher.download()
+ os.chdir(os.path.dirname(self.unpackdir))
+ fetcher.unpack(self.unpackdir)
+
+ self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
+ self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
+ self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
+ self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
+
class TrustedNetworksTest(FetcherTest):
def test_trusted_network(self):
# Ensure trusted_network returns False when the host IS in the list.
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list