[oe-commits] [openembedded-core] 10/59: package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages

[git at git.openembedded.org]

This is an automated email from the git hooks/post-receive script.

rpurdie pushed a commit to branch master
in repository openembedded-core.

commit 11350a67ba137f560d04aa643ff500a7ff112c73
Author: Leonardo Sandoval <leonardo.sandoval.gonzalez at linux.intel.com>
AuthorDate: Mon Sep 4 14:35:48 2017 -0700

    package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages
    
    Current implementation does not handle possible exceptions coming from child
    processes, the latter responsible for creating packages. With the aim to have more
    control, use pipes to communicate exceptions and stop package creation in case
    of failure.
    
    Helps to debug [YOCTO #12012].
    
    Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez at linux.intel.com>
    Signed-off-by: Ross Burton <ross.burton at intel.com>
---
 meta/classes/package_deb.bbclass | 36 +++++++++++++++++++++++++++++++++---
 meta/classes/package_ipk.bbclass | 36 +++++++++++++++++++++++++++++++++---
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index 3060534..5d29793 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -41,7 +41,29 @@ def debian_arch_map(arch, tune):
     return arch
 
 python do_package_deb () {
-    from multiprocessing import Process
+
+    import multiprocessing
+    import traceback
+
+    class DebianWritePkgProcess(multiprocessing.Process):
+        def __init__(self, *args, **kwargs):
+            multiprocessing.Process.__init__(self, *args, **kwargs)
+            self._pconn, self._cconn = multiprocessing.Pipe()
+            self._exception = None
+
+        def run(self):
+            try:
+                multiprocessing.Process.run(self)
+                self._cconn.send(None)
+            except Exception as e:
+                tb = traceback.format_exc()
+                self._cconn.send((e, tb))
+
+        @property
+        def exception(self):
+            if self._pconn.poll():
+                self._exception = self._pconn.recv()
+            return self._exception
 
     oldcwd = os.getcwd()
 
@@ -56,20 +78,28 @@ python do_package_deb () {
 
     max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
     launched = []
+    error = None
     pkgs = packages.split()
-    while pkgs:
+    while not error and pkgs:
         if len(launched) < max_process:
-            p = Process(target=deb_write_pkg, args=(pkgs.pop(), d))
+            p = DebianWritePkgProcess(target=deb_write_pkg, args=(pkgs.pop(), d))
             p.start()
             launched.append(p)
         for q in launched:
             # The finished processes are joined when calling is_alive()
             if not q.is_alive():
                 launched.remove(q)
+            if q.exception:
+                error, traceback = q.exception
+                break
+
     for p in launched:
         p.join()
 
     os.chdir(oldcwd)
+
+    if error:
+        raise error
 }
 do_package_deb[vardeps] += "deb_write_pkg"
 do_package_deb[vardepsexclude] = "BB_NUMBER_THREADS"
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index ec90996..8439cda 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -17,7 +17,29 @@ OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKA
 OPKGLIBDIR = "${localstatedir}/lib"
 
 python do_package_ipk () {
-    from multiprocessing import Process
+    import multiprocessing
+    import traceback
+
+    class IPKWritePkgProcess(multiprocessing.Process):
+        def __init__(self, *args, **kwargs):
+            multiprocessing.Process.__init__(self, *args, **kwargs)
+            self._pconn, self._cconn = multiprocessing.Pipe()
+            self._exception = None
+
+        def run(self):
+            try:
+                multiprocessing.Process.run(self)
+                self._cconn.send(None)
+            except Exception as e:
+                tb = traceback.format_exc()
+                self._cconn.send((e, tb))
+
+        @property
+        def exception(self):
+            if self._pconn.poll():
+                self._exception = self._pconn.recv()
+            return self._exception
+
 
     oldcwd = os.getcwd()
 
@@ -41,20 +63,28 @@ python do_package_ipk () {
 
     max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
     launched = []
+    error = None
     pkgs = packages.split()
-    while pkgs:
+    while not error and pkgs:
         if len(launched) < max_process:
-            p = Process(target=ipk_write_pkg, args=(pkgs.pop(), d))
+            p = IPKWritePkgProcess(target=ipk_write_pkg, args=(pkgs.pop(), d))
             p.start()
             launched.append(p)
         for q in launched:
             # The finished processes are joined when calling is_alive()
             if not q.is_alive():
                 launched.remove(q)
+            if q.exception:
+                error, traceback = q.exception
+                break
+
     for p in launched:
         p.join()
 
     os.chdir(oldcwd)
+
+    if error:
+        raise error
 }
 do_package_ipk[vardeps] += "ipk_write_pkg"
 do_package_ipk[vardepsexclude] = "BB_NUMBER_THREADS"

-- 
To stop receiving notification emails like this one, please contact
the administrator of this repository.