[OE-core] [PATCH v2 10/12] oeqa/selftest/binutils: Create selftest case for binutils test suite

[Nathan Rossi]

Create a oeqa selftest test case to execute the binutils test suites and
report the results. The results are populated into the extraresults
variable of the test case which are written to testresults.json for
resulttool to analyse.

Signed-off-by: Nathan Rossi <nathan at nathanrossi.com>
---
Changes in v2:
- Update to use binutils-cross-testsuite and CHECK_TARGETS
- Drop gold and libiberty test cases (and result parsing code)
- Populate test results as ptestresults
- Mark tests with 'machine' tag
---
 meta/lib/oeqa/selftest/cases/binutils.py | 64 ++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 meta/lib/oeqa/selftest/cases/binutils.py

diff --git a/meta/lib/oeqa/selftest/cases/binutils.py b/meta/lib/oeqa/selftest/cases/binutils.py
new file mode 100644
index 0000000000..4edee09390
--- /dev/null
+++ b/meta/lib/oeqa/selftest/cases/binutils.py
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: MIT
+import os
+import sys
+import re
+import logging
+from oeqa.core.decorator import OETestTag
+from oeqa.selftest.case import OESelftestTestCase
+from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars
+
+def parse_values(content):
+    for i in content:
+        for v in ["PASS", "FAIL", "XPASS", "XFAIL", "UNRESOLVED", "UNSUPPORTED", "UNTESTED", "ERROR", "WARNING"]:
+            if i.startswith(v + ": "):
+                yield i[len(v) + 2:].strip(), v
+                break
+
+ at OETestTag("machine")
+class BinutilsCrossSelfTest(OESelftestTestCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        if not hasattr(cls.tc, "extraresults"):
+            cls.tc.extraresults = {}
+
+        if "ptestresult.sections" not in cls.tc.extraresults:
+            cls.tc.extraresults["ptestresult.sections"] = {}
+
+    def test_binutils(self):
+        self.run_binutils("binutils")
+
+    def test_gas(self):
+        self.run_binutils("gas")
+
+    def test_ld(self):
+        self.run_binutils("ld")
+
+    def run_binutils(self, suite):
+        features = []
+        features.append('CHECK_TARGETS = "{0}"'.format(suite))
+        self.write_config("\n".join(features))
+
+        recipe = "binutils-cross-testsuite"
+        bb_vars = get_bb_vars(["B", "TARGET_SYS", "T"], recipe)
+        builddir, target_sys, tdir = bb_vars["B"], bb_vars["TARGET_SYS"], bb_vars["T"]
+
+        bitbake("{0} -c check".format(recipe))
+
+        ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite
+        self.tc.extraresults["ptestresult.sections"][ptestsuite] = {}
+
+        sumspath = os.path.join(builddir, suite, "{0}.sum".format(suite))
+        if not os.path.exists(sumspath):
+            sumspath = os.path.join(builddir, suite, "testsuite", "{0}.sum".format(suite))
+
+        failed = 0
+        with open(sumspath, "r") as f:
+            for test, result in parse_values(f):
+                self.tc.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result}
+                if result == "FAIL":
+                    self.logger.info("failed: '{}'".format(test))
+                    failed += 1
+
+        self.assertEqual(failed, 0)
+
---
2.23.0.rc1