[bitbake-devel] [PATCH 10/11] toaster: tts: execute tests in numeric order
Ed Bartosh
ed.bartosh at linux.intel.com
Thu Jul 30 16:25:18 UTC 2015
From: Alexandru DAMIAN <alexandru.damian at intel.com>
As the tests are verifying different type of the functionality,
it is usually the case that a failing early test will completely
make the subsequent tests failing, e.g. if the system cannot
start due to a bug, there is little point in testing other
functions.
In order to prevent uneeded test runs, and to generate repeatable
test patterns, the test cases have now a numeric order in the
class name (e.g. Test01XXX). The tests are executed in this order,
and the first test failing will stop the test run.
Signed-off-by: Alexandru DAMIAN <alexandru.damian at intel.com>
Signed-off-by: Ed Bartosh <ed.bartosh at linux.intel.com>
---
lib/toaster/contrib/tts/runner.py | 11 +++++++----
lib/toaster/contrib/tts/tests.py | 6 +++---
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/lib/toaster/contrib/tts/runner.py b/lib/toaster/contrib/tts/runner.py
index 6fc3e77..bed6651 100755
--- a/lib/toaster/contrib/tts/runner.py
+++ b/lib/toaster/contrib/tts/runner.py
@@ -31,7 +31,7 @@ from __future__ import print_function
import sys, os
import unittest, importlib
import logging, pprint, json
-
+import re
from shellutils import ShellCmdException, mkdirhier, run_shell_cmd
import config
@@ -121,13 +121,16 @@ def execute_tests(dir_under_test, testname):
# pylint: disable=broad-except
# we disable the broad-except because we want to actually catch all possible exceptions
try:
+ # sorting the tests by the numeric order in the class name
+ tests = sorted(tests, key=lambda x: int(re.search(r"[0-9]+", x[1]).group(0)))
config.logger.debug("Discovered test clases: %s", pprint.pformat(tests))
unittest.installHandler()
suite = unittest.TestSuite()
loader = unittest.TestLoader()
result = unittest.TestResult()
- for module_file, name in tests:
- suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, name)))
+ result.failfast = True
+ for module_file, test_name in tests:
+ suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, test_name)))
config.logger.info("Running %d test(s)", suite.countTestCases())
suite.run(result)
@@ -202,7 +205,7 @@ def main():
config.TESTDIR = testdir # we let tests know where to run
# ensure that the test dir only contains no *.pyc leftovers
- run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \;" % testdir)
+ run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \\;" % testdir)
no_failures = execute_tests(testdir, options.singletest)
diff --git a/lib/toaster/contrib/tts/tests.py b/lib/toaster/contrib/tts/tests.py
index 9acef46..3a4eed1 100644
--- a/lib/toaster/contrib/tts/tests.py
+++ b/lib/toaster/contrib/tts/tests.py
@@ -30,7 +30,7 @@ import config
import pexpect
import sys, os, signal, time
-class TestPyCompilable(unittest.TestCase):
+class Test00PyCompilable(unittest.TestCase):
''' Verifies that all Python files are syntactically correct '''
def test_compile_file(self):
try:
@@ -38,7 +38,7 @@ class TestPyCompilable(unittest.TestCase):
except ShellCmdException as exc:
self.fail("Error compiling python files: %s" % (exc))
-class TestPySystemStart(unittest.TestCase):
+class Test01PySystemStart(unittest.TestCase):
''' Attempts to start Toaster, verify that it is succesfull, and stop it '''
def setUp(self):
run_shell_cmd("bash -c 'rm -f build/*log'")
@@ -55,7 +55,7 @@ class TestPySystemStart(unittest.TestCase):
except ShellCmdException as exc:
self.fail("Failed starting managed mode: %s" % (exc))
-class TestHTML5Compliance(unittest.TestCase):
+class Test02HTML5Compliance(unittest.TestCase):
def setUp(self):
self.origdir = os.getcwd()
self.crtdir = os.path.dirname(config.TESTDIR)
--
2.1.4
More information about the bitbake-devel
mailing list