[oe-commits] [bitbake] 01/01: fetch/wget: mitigate a wget race condition when listing FTP directories

[git at git.openembedded.org]

This is an automated email from the git hooks/post-receive script.

rpurdie pushed a commit to branch master-next
in repository bitbake.

commit 91d4ca93df092cf86ab84faaa94cc66ff9f43057
Author: Ross Burton <ross.burton at intel.com>
AuthorDate: Mon Jul 24 22:42:01 2017 +0100

    fetch/wget: mitigate a wget race condition when listing FTP directories
    
    When wget is fetching a listing for a directory over FTP it writes to a
    temporary file called .listing in the current directory.  If there are many such
    operations happening in parallel - for example during 'bitbake world -c
    checkpkg' - then up to BB_NUMBER_THREADS instances of wget will be racing to
    write to, read, and delete the same file.
    
    This results in various failures such as the file disappearing before wget has
    processed it or the file changing contents, which causes checkpkg to randomly
    fail.
    
    Mitigate the race condition by creating a temporary directory to run wget in
    when doing directory listings.
    
    [ YOCTO #11828 ]
    
    Signed-off-by: Ross Burton <ross.burton at intel.com>
    Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
 lib/bb/fetch2/wget.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py
index 208ee9b..8ee9769 100644
--- a/lib/bb/fetch2/wget.py
+++ b/lib/bb/fetch2/wget.py
@@ -90,13 +90,13 @@ class Wget(FetchMethod):
 
         self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
 
-    def _runwget(self, ud, d, command, quiet):
+    def _runwget(self, ud, d, command, quiet, workdir=None):
 
         progresshandler = WgetProgressHandler(d)
 
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command, ud.url)
-        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
+        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
 
     def download(self, ud, d):
         """Fetch urls"""
@@ -422,17 +422,16 @@ class Wget(FetchMethod):
         Run fetch checkstatus to get directory information
         """
         f = tempfile.NamedTemporaryFile()
+        with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
+            agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
+            fetchcmd = self.basecmd
+            fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
+            try:
+                self._runwget(ud, d, fetchcmd, True, workdir=workdir)
+                fetchresult = f.read()
+            except bb.fetch2.BBFetchException:
+                fetchresult = ""
 
-        agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
-        fetchcmd = self.basecmd
-        fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
-        try:
-            self._runwget(ud, d, fetchcmd, True)
-            fetchresult = f.read()
-        except bb.fetch2.BBFetchException:
-            fetchresult = ""
-
-        f.close()
         return fetchresult
 
     def _check_latest_version(self, url, package, package_regex, current_version, ud, d):

-- 
To stop receiving notification emails like this one, please contact
the administrator of this repository.