[OE-core] [PATCH] runtime ptest: run each ptest separately

[Armin Kuster]

resulttool still working

Recipe  | Passed    | Failed | Skipped | Time(s)
------------------------------------------------
openssh | 0         | 0      | 1      | 1
openssl | 146       | 1      | 8      | 80
python  | 0         | 0      | 0      | 21

Signed-off-by: Armin Kuster <akuster808 at gmail.com>
---
 meta/lib/oeqa/runtime/cases/ptest.py | 52 +++++++++++++++++-----------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/meta/lib/oeqa/runtime/cases/ptest.py b/meta/lib/oeqa/runtime/cases/ptest.py
index fa95fefe01..520f65d450 100644
--- a/meta/lib/oeqa/runtime/cases/ptest.py
+++ b/meta/lib/oeqa/runtime/cases/ptest.py
@@ -26,6 +26,7 @@ class PTestBase(OERuntimeTestCase):
     @classmethod
     def ptest_startup(cls):
         cls.failmsg = ""
+        cls.ptests = []
 
         cls.test_log_dir = cls.td.get('TEST_LOG_DIR', '')
         # The TEST_LOG_DIR maybe NULL when testimage is added after
@@ -46,6 +47,8 @@ class PTestBase(OERuntimeTestCase):
             cls.tc.extraresults = {}
 
         cls.extras = cls.tc.extraresults
+        cls.extras['ptestresult.rawlogs'] = {'log': ""}
+        cls.extras['ptestresult.sections'] = {}
 
     @classmethod
     def ptest_finishup(cls):
@@ -60,24 +63,7 @@ class PTestBase(OERuntimeTestCase):
             cls.fail(cls.failmsg)
 
 class PtestRunnerTest(PTestBase):
-
-    @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
-    @OETestDepends(['ssh.SSHTest.test_ssh'])
-    @OEHasPackage(['ptest-runner'])
-    def test_ptestrunner_check(self):
-        status, output = self.target.run('which ptest-runner')
-        msg = 'ptest-runner not installed .  %s' % output
-        self.assertEqual(status, 0, msg=msg)
-
-    @OETestDepends(['ptest.PtestRunnerTest.test_ptestrunner_check'])
-    def test_ptests_installed(self):
-        status, output = self.target.run('ptest-runner -l')
-        msg = 'No ptests found.  %s' % output
-        self.assertEqual(status, 0, msg=msg)
-
-    @OETestDepends(['ptest.PtestRunnerTest.test_ptests_installed'])
-    @unittest.expectedFailure
-    def test_ptestrunner(self):
+    def run_ptest(self, ptest):
         status, output = self.target.run('ptest-runner', 0)
         with open(self.ptest_runner_log, 'w') as f:
             f.write(output)
@@ -87,8 +73,8 @@ class PtestRunnerTest(PTestBase):
         results, sections = parser.parse(self.ptest_runner_log)
         parser.results_as_files(self.ptest_log_dir)
 
-        self.extras['ptestresult.rawlogs'] = {'log': output}
-        self.extras['ptestresult.sections'] = sections
+        self.extras['ptestresult.rawlogs']['log']  = self.extras['ptestresult.rawlogs']['log'] + output
+        self.extras['ptestresult.sections'][ptest]  = sections[ptest]
 
         trans = str.maketrans("()", "__")
         for section in results:
@@ -110,3 +96,29 @@ class PtestRunnerTest(PTestBase):
         if failed_tests:
             self.failmsg = self.failmsg + "Failed ptests:\n%s" % pprint.pformat(failed_tests)
 
+
+    @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
+    @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['ptest-runner'])
+    def test_ptestrunner_check(self):
+        status, output = self.target.run('which ptest-runner')
+        msg = 'ptest-runner not installed .  %s' % output
+        self.assertEqual(status, 0, msg=msg)
+
+    @OETestDepends(['ptest.PtestRunnerTest.test_ptestrunner_check'])
+    def test_ptests_installed(self):
+        status, output = self.target.run('ptest-runner -l')
+        msg = 'No ptests found.  %s' % output
+        self.assertEqual(status, 0, msg=msg)
+
+        # built ptest list
+        for ptest in output.split("\n"):
+            if ptest.startswith("Available"):
+                continue
+            self.ptests.append(ptest.split()[0])
+
+    @OETestDepends(['ptest.PtestRunnerTest.test_ptests_installed'])
+    @unittest.expectedFailure
+    def test_ptestrunner(self):
+        for ptest in self.ptests:
+            self.run_ptest(ptest)
-- 
2.17.1