diff options
Diffstat (limited to 'poky/scripts/lib/resulttool')
| -rw-r--r-- | poky/scripts/lib/resulttool/junit.py | 77 | ||||
| -rwxr-xr-x | poky/scripts/lib/resulttool/manualexecution.py | 2 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/report.py | 2 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/resultutils.py | 76 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/store.py | 26 |
5 files changed, 163 insertions, 20 deletions
diff --git a/poky/scripts/lib/resulttool/junit.py b/poky/scripts/lib/resulttool/junit.py new file mode 100644 index 0000000000..c7a53dc550 --- /dev/null +++ b/poky/scripts/lib/resulttool/junit.py @@ -0,0 +1,77 @@ +# resulttool - report test results in JUnit XML format +# +# Copyright (c) 2024, Siemens AG. +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import re +import xml.etree.ElementTree as ET +import resulttool.resultutils as resultutils + +def junit(args, logger): + testresults = resultutils.load_resultsdata(args.json_file, configmap=resultutils.store_map) + + total_time = 0 + skipped = 0 + failures = 0 + errors = 0 + + for tests in testresults.values(): + results = tests[next(reversed(tests))].get("result", {}) + + for result_id, result in results.items(): + # filter out ptestresult.rawlogs and ptestresult.sections + if re.search(r'\.test_', result_id): + total_time += result.get("duration", 0) + + if result['status'] == "FAILED": + failures += 1 + elif result['status'] == "ERROR": + errors += 1 + elif result['status'] == "SKIPPED": + skipped += 1 + + testsuites_node = ET.Element("testsuites") + testsuites_node.set("time", "%s" % total_time) + testsuite_node = ET.SubElement(testsuites_node, "testsuite") + testsuite_node.set("name", "Testimage") + testsuite_node.set("time", "%s" % total_time) + testsuite_node.set("tests", "%s" % len(results)) + testsuite_node.set("failures", "%s" % failures) + testsuite_node.set("errors", "%s" % errors) + testsuite_node.set("skipped", "%s" % skipped) + + for result_id, result in results.items(): + if re.search(r'\.test_', result_id): + testcase_node = ET.SubElement(testsuite_node, "testcase", { + "name": result_id, + "classname": "Testimage", + "time": str(result['duration']) + }) + if result['status'] == "SKIPPED": + ET.SubElement(testcase_node, "skipped", message=result['log']) + elif result['status'] == "FAILED": + ET.SubElement(testcase_node, "failure", message=result['log']) + elif result['status'] == "ERROR": + ET.SubElement(testcase_node, "error", message=result['log']) + + tree = ET.ElementTree(testsuites_node) + + if args.junit_xml_path is None: + args.junit_xml_path = os.environ['BUILDDIR'] + '/tmp/log/oeqa/junit.xml' + tree.write(args.junit_xml_path, encoding='UTF-8', xml_declaration=True) + + logger.info('Saved JUnit XML report as %s' % args.junit_xml_path) + +def register_commands(subparsers): + """Register subcommands from this plugin""" + parser_build = subparsers.add_parser('junit', help='create test report in JUnit XML format', + description='generate unit test report in JUnit XML format based on the latest test results in the testresults.json.', + group='analysis') + parser_build.set_defaults(func=junit) + parser_build.add_argument('json_file', + help='json file should point to the testresults.json') + parser_build.add_argument('-j', '--junit_xml_path', + help='junit xml path allows setting the path of the generated test report. The default location is <build_dir>/tmp/log/oeqa/junit.xml') diff --git a/poky/scripts/lib/resulttool/manualexecution.py b/poky/scripts/lib/resulttool/manualexecution.py index ecb27c5933..ae0861ac6b 100755 --- a/poky/scripts/lib/resulttool/manualexecution.py +++ b/poky/scripts/lib/resulttool/manualexecution.py @@ -22,7 +22,7 @@ def load_json_file(f): def write_json_file(f, json_data): os.makedirs(os.path.dirname(f), exist_ok=True) with open(f, 'w') as filedata: - filedata.write(json.dumps(json_data, sort_keys=True, indent=4)) + filedata.write(json.dumps(json_data, sort_keys=True, indent=1)) class ManualTestRunner(object): diff --git a/poky/scripts/lib/resulttool/report.py b/poky/scripts/lib/resulttool/report.py index a349510ab8..1c100b00ab 100644 --- a/poky/scripts/lib/resulttool/report.py +++ b/poky/scripts/lib/resulttool/report.py @@ -256,7 +256,7 @@ class ResultsTextReport(object): if selected_test_case_only: print_selected_testcase_result(raw_results, selected_test_case_only) else: - print(json.dumps(raw_results, sort_keys=True, indent=4)) + print(json.dumps(raw_results, sort_keys=True, indent=1)) else: print('Could not find raw test result for %s' % raw_test) return 0 diff --git a/poky/scripts/lib/resulttool/resultutils.py b/poky/scripts/lib/resulttool/resultutils.py index c5521d81bd..b8fc79a6ac 100644 --- a/poky/scripts/lib/resulttool/resultutils.py +++ b/poky/scripts/lib/resulttool/resultutils.py @@ -14,8 +14,11 @@ import scriptpath import copy import urllib.request import posixpath +import logging scriptpath.add_oe_lib_path() +logger = logging.getLogger('resulttool') + flatten_map = { "oeselftest": [], "runtime": [], @@ -31,13 +34,19 @@ regression_map = { "manual": ['TEST_TYPE', 'TEST_MODULE', 'IMAGE_BASENAME', 'MACHINE'] } store_map = { - "oeselftest": ['TEST_TYPE'], + "oeselftest": ['TEST_TYPE', 'TESTSERIES', 'MACHINE'], "runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'], "sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], "sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME'] } +rawlog_sections = { + "ptestresult.rawlogs": "ptest", + "ltpresult.rawlogs": "ltp", + "ltpposixresult.rawlogs": "ltpposix" +} + def is_url(p): """ Helper for determining if the given path is a URL @@ -108,21 +117,57 @@ def filter_resultsdata(results, resultid): newresults[r][i] = results[r][i] return newresults -def strip_ptestresults(results): +def strip_logs(results): newresults = copy.deepcopy(results) - #for a in newresults2: - # newresults = newresults2[a] for res in newresults: if 'result' not in newresults[res]: continue - if 'ptestresult.rawlogs' in newresults[res]['result']: - del newresults[res]['result']['ptestresult.rawlogs'] + for logtype in rawlog_sections: + if logtype in newresults[res]['result']: + del newresults[res]['result'][logtype] if 'ptestresult.sections' in newresults[res]['result']: for i in newresults[res]['result']['ptestresult.sections']: if 'log' in newresults[res]['result']['ptestresult.sections'][i]: del newresults[res]['result']['ptestresult.sections'][i]['log'] return newresults +# For timing numbers, crazy amounts of precision don't make sense and just confuse +# the logs. For numbers over 1, trim to 3 decimal places, for numbers less than 1, +# trim to 4 significant digits +def trim_durations(results): + for res in results: + if 'result' not in results[res]: + continue + for entry in results[res]['result']: + if 'duration' in results[res]['result'][entry]: + duration = results[res]['result'][entry]['duration'] + if duration > 1: + results[res]['result'][entry]['duration'] = float("%.3f" % duration) + elif duration < 1: + results[res]['result'][entry]['duration'] = float("%.4g" % duration) + return results + +def handle_cleanups(results): + # Remove pointless path duplication from old format reproducibility results + for res2 in results: + try: + section = results[res2]['result']['reproducible']['files'] + for pkgtype in section: + for filelist in section[pkgtype].copy(): + if section[pkgtype][filelist] and type(section[pkgtype][filelist][0]) == dict: + newlist = [] + for entry in section[pkgtype][filelist]: + newlist.append(entry["reference"].split("/./")[1]) + section[pkgtype][filelist] = newlist + + except KeyError: + pass + # Remove pointless duplicate rawlogs data + try: + del results[res2]['result']['reproducible.rawlogs'] + except KeyError: + pass + def decode_log(logdata): if isinstance(logdata, str): return logdata @@ -155,9 +200,6 @@ def generic_get_rawlogs(sectname, results): return None return decode_log(results[sectname]['log']) -def ptestresult_get_rawlogs(results): - return generic_get_rawlogs('ptestresult.rawlogs', results) - def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, ptestlogs=False): for res in results: if res: @@ -167,16 +209,20 @@ def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, p os.makedirs(os.path.dirname(dst), exist_ok=True) resultsout = results[res] if not ptestjson: - resultsout = strip_ptestresults(results[res]) + resultsout = strip_logs(results[res]) + trim_durations(resultsout) + handle_cleanups(resultsout) with open(dst, 'w') as f: - f.write(json.dumps(resultsout, sort_keys=True, indent=4)) + f.write(json.dumps(resultsout, sort_keys=True, indent=1)) for res2 in results[res]: if ptestlogs and 'result' in results[res][res2]: seriesresults = results[res][res2]['result'] - rawlogs = ptestresult_get_rawlogs(seriesresults) - if rawlogs is not None: - with open(dst.replace(fn, "ptest-raw.log"), "w+") as f: - f.write(rawlogs) + for logtype in rawlog_sections: + logdata = generic_get_rawlogs(logtype, seriesresults) + if logdata is not None: + logger.info("Extracting " + rawlog_sections[logtype] + "-raw.log") + with open(dst.replace(fn, rawlog_sections[logtype] + "-raw.log"), "w+") as f: + f.write(logdata) if 'ptestresult.sections' in seriesresults: for i in seriesresults['ptestresult.sections']: sectionlog = ptestresult_get_log(seriesresults, i) diff --git a/poky/scripts/lib/resulttool/store.py b/poky/scripts/lib/resulttool/store.py index e0951f0a8f..578910d234 100644 --- a/poky/scripts/lib/resulttool/store.py +++ b/poky/scripts/lib/resulttool/store.py @@ -65,18 +65,34 @@ def store(args, logger): for r in revisions: results = revisions[r] + if args.revision and r[0] != args.revision: + logger.info('skipping %s as non-matching' % r[0]) + continue keywords = {'commit': r[0], 'branch': r[1], "commit_count": r[2]} - subprocess.check_call(["find", tempdir, "!", "-path", "./.git/*", "-delete"]) + subprocess.check_call(["find", tempdir, "-name", "testresults.json", "!", "-path", "./.git/*", "-delete"]) resultutils.save_resultsdata(results, tempdir, ptestlogs=True) logger.info('Storing test result into git repository %s' % args.git_dir) - gitarchive.gitarchive(tempdir, args.git_dir, False, False, + excludes = [] + if args.logfile_archive: + excludes = ['*.log', "*.log.zst"] + + tagname = gitarchive.gitarchive(tempdir, args.git_dir, False, False, "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}", False, "{branch}/{commit_count}-g{commit}/{tag_number}", 'Test run #{tag_number} of {branch}:{commit}', '', - [], [], False, keywords, logger) + excludes, [], False, keywords, logger) + if args.logfile_archive: + logdir = args.logfile_archive + "/" + tagname + shutil.copytree(tempdir, logdir) + for root, dirs, files in os.walk(logdir): + for name in files: + if not name.endswith(".log"): + continue + f = os.path.join(root, name) + subprocess.run(["zstd", f, "--rm"], check=True, capture_output=True) finally: subprocess.check_call(["rm", "-rf", tempdir]) @@ -102,3 +118,7 @@ def register_commands(subparsers): help='add executed-by configuration to each result file') parser_build.add_argument('-t', '--extra-test-env', default='', help='add extra test environment data to each result file configuration') + parser_build.add_argument('-r', '--revision', default='', + help='only store data for the specified revision') + parser_build.add_argument('-l', '--logfile-archive', default='', + help='directory to separately archive log files along with a copy of the results') |