[oe-commits] [openembedded-core] 54/62: oeqa.buildperf: add BuildPerfTestResult class
git at git.openembedded.org
git at git.openembedded.org
Wed Aug 17 09:37:11 UTC 2016
rpurdie pushed a commit to branch master
in repository openembedded-core.
commit 89eb37ef1ef8d5deb87fd55c9ea7b2cfa2681b07
Author: Markus Lehtonen <markus.lehtonen at linux.intel.com>
AuthorDate: Wed Jun 29 19:28:31 2016 +0300
oeqa.buildperf: add BuildPerfTestResult class
The new class is derived from unittest.TextTestResult class. It is
actually implemented by modifying the old BuildPerfTestRunner class
which, in turn, is replaced by a totally new simple implementation
derived from unittest.TestRunner.
Signed-off-by: Markus Lehtonen <markus.lehtonen at linux.intel.com>
Signed-off-by: Ross Burton <ross.burton at intel.com>
---
meta/lib/oeqa/buildperf/__init__.py | 4 +-
meta/lib/oeqa/buildperf/base.py | 150 +++++++++++++++++++-----------------
scripts/oe-build-perf-test | 10 +++
3 files changed, 90 insertions(+), 74 deletions(-)
diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py
index 7e51726..85abf3a 100644
--- a/meta/lib/oeqa/buildperf/__init__.py
+++ b/meta/lib/oeqa/buildperf/__init__.py
@@ -10,9 +10,9 @@
# more details.
#
"""Build performance tests"""
-from .base import (perf_test_case,
- BuildPerfTestCase,
+from .base import (BuildPerfTestCase,
BuildPerfTestLoader,
+ BuildPerfTestResult,
BuildPerfTestRunner,
KernelDropCaches)
from .test_basic import *
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index 1ee546d..d592bd2 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -76,25 +76,26 @@ def time_cmd(cmd, **kwargs):
return ret, timedata
-class BuildPerfTestRunner(object):
+class BuildPerfTestResult(unittest.TextTestResult):
"""Runner class for executing the individual tests"""
# List of test cases to run
test_run_queue = []
- def __init__(self, out_dir):
- self.results = {}
- self.out_dir = os.path.abspath(out_dir)
- if not os.path.exists(self.out_dir):
- os.makedirs(self.out_dir)
+ def __init__(self, out_dir, *args, **kwargs):
+ super(BuildPerfTestResult, self).__init__(*args, **kwargs)
+ self.out_dir = out_dir
# Get Git parameters
try:
self.repo = GitRepo('.')
except GitError:
self.repo = None
- self.git_rev, self.git_branch = self.get_git_revision()
+ self.git_revision, self.git_branch = self.get_git_revision()
+ self.hostname = socket.gethostname()
+ self.start_time = self.elapsed_time = None
+ self.successes = []
log.info("Using Git branch:revision %s:%s", self.git_branch,
- self.git_rev)
+ self.git_revision)
def get_git_revision(self):
"""Get git branch and revision under testing"""
@@ -117,79 +118,71 @@ class BuildPerfTestRunner(object):
branch = None
return str(rev), str(branch)
- def run_tests(self):
- """Method that actually runs the tests"""
- self.results['schema_version'] = 1
- self.results['git_revision'] = self.git_rev
- self.results['git_branch'] = self.git_branch
- self.results['tester_host'] = socket.gethostname()
- start_time = datetime.utcnow()
- self.results['start_time'] = start_time
- self.results['tests'] = {}
-
- self.archive_build_conf()
- for test_class in self.test_run_queue:
- log.info("Executing test %s: %s", test_class.name,
- test_class.description)
-
- test = test_class(self.out_dir)
- try:
- test.run()
- except Exception:
- # Catch all exceptions. This way e.g buggy tests won't scrap
- # the whole test run
- sep = '-' * 5 + ' TRACEBACK ' + '-' * 60 + '\n'
- tb_msg = sep + traceback.format_exc() + sep
- log.error("Test execution failed with:\n" + tb_msg)
- self.results['tests'][test.name] = test.results
-
- self.results['elapsed_time'] = datetime.utcnow() - start_time
- return 0
-
- def archive_build_conf(self):
- """Archive build/conf to test results"""
- src_dir = os.path.join(os.environ['BUILDDIR'], 'conf')
- tgt_dir = os.path.join(self.out_dir, 'build', 'conf')
- os.makedirs(os.path.dirname(tgt_dir))
- shutil.copytree(src_dir, tgt_dir)
+ def addSuccess(self, test):
+ """Record results from successful tests"""
+ super(BuildPerfTestResult, self).addSuccess(test)
+ self.successes.append((test, None))
+
+ def startTest(self, test):
+ """Pre-test hook"""
+ test.out_dir = self.out_dir
+ log.info("Executing test %s: %s", test.name, test.shortDescription())
+ self.stream.write(datetime.now().strftime("[%Y-%m-%d %H:%M:%S] "))
+ super(BuildPerfTestResult, self).startTest(test)
+
+ def startTestRun(self):
+ """Pre-run hook"""
+ self.start_time = datetime.utcnow()
+
+ def stopTestRun(self):
+ """Pre-run hook"""
+ self.elapsed_time = datetime.utcnow() - self.start_time
+
+ def all_results(self):
+ result_map = {'SUCCESS': self.successes,
+ 'FAIL': self.failures,
+ 'ERROR': self.errors,
+ 'EXP_FAIL': self.expectedFailures,
+ 'UNEXP_SUCCESS': self.unexpectedSuccesses}
+ for status, tests in result_map.items():
+ for test in tests:
+ yield (status, test)
+
def update_globalres_file(self, filename):
"""Write results to globalres csv file"""
+ # Map test names to time and size columns in globalres
+ # The tuples represent index and length of times and sizes
+ # respectively
+ gr_map = {'test1': ((0, 1), (8, 1)),
+ 'test12': ((1, 1), (None, None)),
+ 'test13': ((2, 1), (9, 1)),
+ 'test2': ((3, 1), (None, None)),
+ 'test3': ((4, 3), (None, None)),
+ 'test4': ((7, 1), (10, 2))}
+
if self.repo:
- git_tag_rev = self.repo.run_cmd(['describe', self.git_rev])
+ git_tag_rev = self.repo.run_cmd(['describe', self.git_revision])
else:
- git_tag_rev = self.git_rev
- times = []
- sizes = []
- for test in self.results['tests'].values():
- for measurement in test['measurements']:
- res_type = measurement['type']
- values = measurement['values']
- if res_type == BuildPerfTest.SYSRES:
- e_sec = values['elapsed_time'].total_seconds()
- times.append('{:d}:{:02d}:{:.2f}'.format(
- int(e_sec / 3600),
- int((e_sec % 3600) / 60),
- e_sec % 60))
- elif res_type == BuildPerfTest.DISKUSAGE:
- sizes.append(str(values['size']))
- else:
- log.warning("Unable to handle '%s' values in "
- "globalres.log", res_type)
+ git_tag_rev = self.git_revision
+
+ values = ['0'] * 12
+ for status, test in self.all_results():
+ if status not in ['SUCCESS', 'FAILURE', 'EXP_SUCCESS']:
+ continue
+ (t_ind, t_len), (s_ind, s_len) = gr_map[test.name]
+ if t_ind is not None:
+ values[t_ind:t_ind + t_len] = test.times
+ if s_ind is not None:
+ values[s_ind:s_ind + s_len] = test.sizes
log.debug("Writing globalres log to %s", filename)
with open(filename, 'a') as fobj:
- fobj.write('{},{}:{},{},'.format(self.results['tester_host'],
- self.results['git_branch'],
- self.results['git_revision'],
+ fobj.write('{},{}:{},{},'.format(self.hostname,
+ self.git_branch,
+ self.git_revision,
git_tag_rev))
- fobj.write(','.join(times + sizes) + '\n')
-
-
-def perf_test_case(obj):
- """Decorator for adding test classes"""
- BuildPerfTestRunner.test_run_queue.append(obj)
- return obj
+ fobj.write(','.join(values) + '\n')
class BuildPerfTestCase(unittest.TestCase):
@@ -330,3 +323,16 @@ class BuildPerfTestCase(unittest.TestCase):
class BuildPerfTestLoader(unittest.TestLoader):
"""Test loader for build performance tests"""
sortTestMethodsUsing = None
+
+
+class BuildPerfTestRunner(unittest.TextTestRunner):
+ """Test loader for build performance tests"""
+ sortTestMethodsUsing = None
+
+ def __init__(self, out_dir, *args, **kwargs):
+ super(BuildPerfTestRunner, self).__init__(*args, **kwargs)
+ self.out_dir = out_dir
+
+ def _makeResult(self):
+ return BuildPerfTestResult(self.out_dir, self.stream, self.descriptions,
+ self.verbosity)
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
index 996996b..8142b03 100755
--- a/scripts/oe-build-perf-test
+++ b/scripts/oe-build-perf-test
@@ -19,6 +19,7 @@ import errno
import fcntl
import logging
import os
+import shutil
import sys
from datetime import datetime
@@ -78,6 +79,14 @@ def setup_file_logging(log_file):
log.addHandler(handler)
+def archive_build_conf(out_dir):
+ """Archive build/conf to test results"""
+ src_dir = os.path.join(os.environ['BUILDDIR'], 'conf')
+ tgt_dir = os.path.join(out_dir, 'build', 'conf')
+ os.makedirs(os.path.dirname(tgt_dir))
+ shutil.copytree(src_dir, tgt_dir)
+
+
def parse_args(argv):
"""Parse command line arguments"""
parser = argparse.ArgumentParser(
@@ -120,6 +129,7 @@ def main(argv=None):
# Run actual tests
runner = BuildPerfTestRunner(out_dir)
+ archive_build_conf(out_dir)
ret = runner.run_tests()
if not ret:
if args.globalres_file:
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list