diff options
Diffstat (limited to 'poky/scripts')
| -rwxr-xr-x | poky/scripts/cve-json-to-text.py | 147 | ||||
| -rwxr-xr-x | poky/scripts/install-buildtools | 4 | ||||
| -rw-r--r-- | poky/scripts/lib/devtool/standard.py | 7 | ||||
| -rw-r--r-- | poky/scripts/lib/devtool/upgrade.py | 56 | ||||
| -rw-r--r-- | poky/scripts/lib/recipetool/create.py | 42 | ||||
| -rw-r--r-- | poky/scripts/lib/recipetool/create_go.py | 4 | ||||
| -rw-r--r-- | poky/scripts/lib/recipetool/create_npm.py | 95 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/junit.py | 77 | ||||
| -rwxr-xr-x | poky/scripts/lib/resulttool/manualexecution.py | 2 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/report.py | 2 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/resultutils.py | 76 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/store.py | 26 | ||||
| -rw-r--r-- | poky/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in | 2 | ||||
| -rw-r--r-- | poky/scripts/lib/wic/plugins/source/bootimg-partition.py | 39 | ||||
| -rwxr-xr-x | poky/scripts/oe-setup-build | 6 | ||||
| -rw-r--r-- | poky/scripts/patchtest.README | 60 | ||||
| -rwxr-xr-x | poky/scripts/pull-sdpx-licenses.py | 101 | ||||
| -rw-r--r-- | poky/scripts/pybootchartgui/pybootchartgui/parsing.py | 10 | ||||
| -rwxr-xr-x | poky/scripts/resulttool | 5 | ||||
| -rwxr-xr-x | poky/scripts/runqemu | 33 |
20 files changed, 589 insertions, 205 deletions
diff --git a/poky/scripts/cve-json-to-text.py b/poky/scripts/cve-json-to-text.py new file mode 100755 index 0000000000..87a5669987 --- /dev/null +++ b/poky/scripts/cve-json-to-text.py @@ -0,0 +1,147 @@ +#!/bin/env python3 +# SPDX-FileCopyrightText: OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT + +# CVE results conversion script: JSON format to text +# Derived from cve-report.py from Oniro (MIT, by Huawei Inc) + +import sys +import getopt + +infile = "in.json" +outfile = "out.txt" + + +def show_syntax_and_exit(code): + """ + Show the program syntax and exit with an errror + Arguments: + code: the error code to return + """ + print("Syntax: %s [-h] [-i inputJSONfile][-o outputfile]" % sys.argv[0]) + sys.exit(code) + + +def exit_error(code, message): + """ + Show the error message and exit with an errror + Arguments: + code: the error code to return + message: the message to show + """ + print("Error: %s" % message) + sys.exit(code) + + +def parse_args(argv): + """ + Parse the program arguments, put options in global variables + Arguments: + argv: program arguments + """ + global infile, outfile + try: + opts, args = getopt.getopt( + argv, "hi:o:", ["help", "input", "output"] + ) + except getopt.GetoptError: + show_syntax_and_exit(1) + for opt, arg in opts: + if opt in ("-h", "--help"): + show_syntax_and_exit(0) + elif opt in ("-a", "--all"): + show_all = True + show_unknown = True + elif opt in ("-i", "--input"): + infile = arg + +def load_json(filename): + """ + Load the JSON file, return the resulting dictionary + Arguments: + filename: the file to open + Returns: + Parsed file as a dictionary + """ + import json + + out = {} + try: + with open(filename, "r") as f: + out = json.load(f) + except FileNotFoundError: + exit_error(1, "Input file (%s) not found" % (filename)) + except json.decoder.JSONDecodeError as error: + exit_error(1, "Malformed JSON file: %s" % str(error)) + return out + + +def process_data(filename, data): + """ + Write the resulting CSV with one line for each package + Arguments: + filename: the file to write to + data: dictionary from parsing the JSON file + Returns: + None + """ + if not "version" in data or data["version"] != "1": + exit_error(1, "Unrecognized format version number") + if not "package" in data: + exit_error(1, "Mandatory 'package' key not found") + + lines = "" + total_issue_count = 0 + for package in data["package"]: + package_info = "" + keys_in_package = {"name", "layer", "version", "issue"} + if keys_in_package - package.keys(): + exit_error( + 1, + "Missing a mandatory key in package: %s" + % (keys_in_package - package.keys()), + ) + + package_info += "LAYER: %s\n" % package["layer"] + package_info += "PACKAGE NAME: %s\n" % package["name"] + package_info += "PACKAGE VERSION: %s\n" % package["version"] + + for issue in package["issue"]: + keys_in_issue = {"id", "status", "detail"} + if keys_in_issue - issue.keys(): + print("Warning: Missing keys %s in 'issue' for the package '%s'" + % (keys_in_issue - issue.keys(), package["name"])) + + lines += package_info + lines += "CVE: %s\n" % issue["id"] + lines += "CVE STATUS: %s\n" % issue["status"] + lines += "CVE DETAIL: %s\n" % issue["detail"] + if "description" in issue: + lines += "CVE DESCRIPTION: %s\n" % issue["description"] + if "summary" in issue: + lines += "CVE SUMMARY: %s\n" % issue["summary"] + if "scorev2" in issue: + lines += "CVSS v2 BASE SCORE: %s\n" % issue["scorev2"] + if "scorev3" in issue: + lines += "CVSS v3 BASE SCORE: %s\n" % issue["scorev3"] + if "scorev4" in issue: + lines += "CVSS v4 BASE SCORE: %s\n" % issue["scorev4"] + if "vector" in issue: + lines += "VECTOR: %s\n" % issue["vector"] + if "vectorString" in issue: + lines += "VECTORSTRING: %s\n" % issue["vectorString"] + lines += "MORE INFORMATION: https://nvd.nist.gov/vuln/detail/%s\n" % issue["id"] + lines += "\n" + + with open(filename, "w") as f: + f.write(lines) + +def main(argv): + parse_args(argv) + data = load_json(infile) + process_data(outfile, data) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/poky/scripts/install-buildtools b/poky/scripts/install-buildtools index 5b86c13077..6387287ade 100755 --- a/poky/scripts/install-buildtools +++ b/poky/scripts/install-buildtools @@ -57,8 +57,8 @@ logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools') DEFAULT_BASE_URL = 'https://downloads.yoctoproject.org/releases/yocto' -DEFAULT_RELEASE = 'yocto-5.0.1' -DEFAULT_INSTALLER_VERSION = '5.0.1' +DEFAULT_RELEASE = 'yocto-5.1' +DEFAULT_INSTALLER_VERSION = '5.1' DEFAULT_BUILDDATE = '202110XX' # Python version sanity check diff --git a/poky/scripts/lib/devtool/standard.py b/poky/scripts/lib/devtool/standard.py index 1d0fe13788..b2e1a6ca3a 100644 --- a/poky/scripts/lib/devtool/standard.py +++ b/poky/scripts/lib/devtool/standard.py @@ -952,13 +952,6 @@ def modify(args, config, basepath, workspace): f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) if bb.data.inherits_class('kernel', rd): - f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' - 'do_fetch do_unpack do_kernel_configcheck"\n') - f.write('\ndo_patch[noexec] = "1"\n') - f.write('\ndo_configure:append() {\n' - ' cp ${B}/.config ${S}/.config.baseline\n' - ' ln -sfT ${B}/.config ${S}/.config.new\n' - '}\n') f.write('\ndo_kernel_configme:prepend() {\n' ' if [ -e ${S}/.config ]; then\n' ' mv ${S}/.config ${S}/.config.old\n' diff --git a/poky/scripts/lib/devtool/upgrade.py b/poky/scripts/lib/devtool/upgrade.py index 8e13833b51..eed3a49e4b 100644 --- a/poky/scripts/lib/devtool/upgrade.py +++ b/poky/scripts/lib/devtool/upgrade.py @@ -76,19 +76,19 @@ def _rename_recipe_dirs(oldpv, newpv, path): bb.utils.rename(os.path.join(path, oldfile), os.path.join(path, newfile)) -def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path): +def _rename_recipe_file(oldrecipe, pn, oldpv, newpv, path): oldrecipe = os.path.basename(oldrecipe) if oldrecipe.endswith('_%s.bb' % oldpv): - newrecipe = '%s_%s.bb' % (bpn, newpv) + newrecipe = '%s_%s.bb' % (pn, newpv) if oldrecipe != newrecipe: shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe)) else: newrecipe = oldrecipe return os.path.join(path, newrecipe) -def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path): +def _rename_recipe_files(oldrecipe, pn, oldpv, newpv, path): _rename_recipe_dirs(oldpv, newpv, path) - return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path) + return _rename_recipe_file(oldrecipe, pn, oldpv, newpv, path) def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d): """Writes an append file""" @@ -335,19 +335,19 @@ def _add_license_diff_to_recipe(path, diff): def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): """Creates the new recipe under workspace""" - bpn = rd.getVar('BPN') - path = os.path.join(workspace, 'recipes', bpn) + pn = rd.getVar('PN') + path = os.path.join(workspace, 'recipes', pn) bb.utils.mkdirhier(path) copied, _ = oe.recipeutils.copy_recipe_files(rd, path, all_variants=True) if not copied: - raise DevtoolError('Internal error - no files were copied for recipe %s' % bpn) + raise DevtoolError('Internal error - no files were copied for recipe %s' % pn) logger.debug('Copied %s to %s' % (copied, path)) oldpv = rd.getVar('PV') if not newpv: newpv = oldpv origpath = rd.getVar('FILE') - fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path) + fullpath = _rename_recipe_files(origpath, pn, oldpv, newpv, path) logger.debug('Upgraded %s => %s' % (origpath, fullpath)) newvalues = {} @@ -534,14 +534,14 @@ def _generate_license_diff(old_licenses, new_licenses): diff = diff + line return diff -def _run_recipe_update_extra_tasks(pn, rd, tinfoil): +def _run_recipe_upgrade_extra_tasks(pn, rd, tinfoil): tasks = [] - for task in (rd.getVar('RECIPE_UPDATE_EXTRA_TASKS') or '').split(): - logger.info('Running extra recipe update task: %s' % task) + for task in (rd.getVar('RECIPE_UPGRADE_EXTRA_TASKS') or '').split(): + logger.info('Running extra recipe upgrade task: %s' % task) res = tinfoil.build_targets(pn, task, handle_events=True) if not res: - raise DevtoolError('Running extra recipe update task %s for %s failed' % (task, pn)) + raise DevtoolError('Running extra recipe upgrade task %s for %s failed' % (task, pn)) def upgrade(args, config, basepath, workspace): """Entry point for the devtool 'upgrade' subcommand""" @@ -610,7 +610,7 @@ def upgrade(args, config, basepath, workspace): license_diff = _generate_license_diff(old_licenses, new_licenses) rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) except (bb.process.CmdError, DevtoolError) as e: - recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('BPN')) + recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('PN')) _upgrade_error(e, recipedir, srctree, args.keep_failure) standard._add_md5(config, pn, os.path.dirname(rf)) @@ -618,7 +618,7 @@ def upgrade(args, config, basepath, workspace): copied, config.workspace_path, rd) standard._add_md5(config, pn, af) - _run_recipe_update_extra_tasks(pn, rd, tinfoil) + _run_recipe_upgrade_extra_tasks(pn, rd, tinfoil) update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) @@ -654,18 +654,28 @@ def latest_version(args, config, basepath, workspace): return 0 def check_upgrade_status(args, config, basepath, workspace): + def _print_status(recipe): + print("{:25} {:15} {:15} {} {} {}".format( recipe['pn'], + recipe['cur_ver'], + recipe['status'] if recipe['status'] != 'UPDATE' else (recipe['next_ver'] if not recipe['next_ver'].endswith("new-commits-available") else "new commits"), + recipe['maintainer'], + recipe['revision'] if recipe['revision'] != 'N/A' else "", + "cannot be updated due to: %s" %(recipe['no_upgrade_reason']) if recipe['no_upgrade_reason'] else "")) if not args.recipe: logger.info("Checking the upstream status for all recipes may take a few minutes") results = oe.recipeutils.get_recipe_upgrade_status(args.recipe) - for result in results: - # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason - if args.all or result[1] != 'MATCH': - print("{:25} {:15} {:15} {} {} {}".format( result[0], - result[2], - result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"), - result[4], - result[5] if result[5] != 'N/A' else "", - "cannot be updated due to: %s" %(result[6]) if result[6] else "")) + for recipegroup in results: + upgrades = [r for r in recipegroup if r['status'] != 'MATCH'] + currents = [r for r in recipegroup if r['status'] == 'MATCH'] + if len(upgrades) > 1: + print("These recipes need to be upgraded together {") + for r in upgrades: + _print_status(r) + if len(upgrades) > 1: + print("}") + for r in currents: + if args.all: + _print_status(r) def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" diff --git a/poky/scripts/lib/recipetool/create.py b/poky/scripts/lib/recipetool/create.py index 066366e34f..ea2ef5be63 100644 --- a/poky/scripts/lib/recipetool/create.py +++ b/poky/scripts/lib/recipetool/create.py @@ -960,7 +960,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): # Someone else has already handled the license vars, just return their value return lichandled[0][1] - licvalues = guess_license(srctree, d) + licvalues = find_licenses(srctree, d) licenses = [] lic_files_chksum = [] lic_unknown = [] @@ -1216,13 +1216,7 @@ def crunch_license(licfile): lictext = '' return md5val, lictext -def guess_license(srctree, d): - import bb - md5sums = get_license_md5sums(d) - - crunched_md5sums = crunch_known_licenses(d) - - licenses = [] +def find_license_files(srctree): licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10'] skip_extensions = (".html", ".js", ".json", ".svg", ".ts", ".go") licfiles = [] @@ -1235,11 +1229,22 @@ def guess_license(srctree, d): fullpath = os.path.join(root, fn) if not fullpath in licfiles: licfiles.append(fullpath) + + return licfiles + +def match_licenses(licfiles, srctree, d): + import bb + md5sums = get_license_md5sums(d) + + crunched_md5sums = crunch_known_licenses(d) + + licenses = [] for licfile in sorted(licfiles): - md5value = bb.utils.md5_file(licfile) + resolved_licfile = d.expand(licfile) + md5value = bb.utils.md5_file(resolved_licfile) license = md5sums.get(md5value, None) if not license: - crunched_md5, lictext = crunch_license(licfile) + crunched_md5, lictext = crunch_license(resolved_licfile) license = crunched_md5sums.get(crunched_md5, None) if lictext and not license: license = 'Unknown' @@ -1249,13 +1254,19 @@ def guess_license(srctree, d): if license: licenses.append((license, os.path.relpath(licfile, srctree), md5value)) + return licenses + +def find_licenses(srctree, d): + licfiles = find_license_files(srctree) + licenses = match_licenses(licfiles, srctree, d) + # FIXME should we grab at least one source file with a license header and add that too? return licenses def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'): """ - Given a list of (license, path, md5sum) as returned by guess_license(), + Given a list of (license, path, md5sum) as returned by match_licenses(), a dict of package name to path mappings, write out a set of package-specific LICENSE values. """ @@ -1284,6 +1295,14 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn outlicenses[pkgname] = licenses return outlicenses +def generate_common_licenses_chksums(common_licenses, d): + lic_files_chksums = [] + for license in tidy_licenses(common_licenses): + licfile = '${COMMON_LICENSE_DIR}/' + license + md5value = bb.utils.md5_file(d.expand(licfile)) + lic_files_chksums.append('file://%s;md5=%s' % (licfile, md5value)) + return lic_files_chksums + def read_pkgconfig_provides(d): pkgdatadir = d.getVar('PKGDATA_DIR') pkgmap = {} @@ -1418,4 +1437,3 @@ def register_commands(subparsers): parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS) parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).') parser_create.set_defaults(func=create_recipe) - diff --git a/poky/scripts/lib/recipetool/create_go.py b/poky/scripts/lib/recipetool/create_go.py index a85a2f2786..5cc53931f0 100644 --- a/poky/scripts/lib/recipetool/create_go.py +++ b/poky/scripts/lib/recipetool/create_go.py @@ -14,7 +14,7 @@ from collections import namedtuple from enum import Enum from html.parser import HTMLParser from recipetool.create import RecipeHandler, handle_license_vars -from recipetool.create import guess_license, tidy_licenses, fixup_license +from recipetool.create import find_licenses, tidy_licenses, fixup_license from recipetool.create import determine_from_url from urllib.error import URLError, HTTPError @@ -624,7 +624,7 @@ class GoRecipeHandler(RecipeHandler): licenses = [] lic_files_chksum = [] - licvalues = guess_license(tmp_vendor_dir, d) + licvalues = find_licenses(tmp_vendor_dir, d) shutil.rmtree(tmp_vendor_dir) if licvalues: diff --git a/poky/scripts/lib/recipetool/create_npm.py b/poky/scripts/lib/recipetool/create_npm.py index 113a89f6a6..3363a0e7ee 100644 --- a/poky/scripts/lib/recipetool/create_npm.py +++ b/poky/scripts/lib/recipetool/create_npm.py @@ -16,8 +16,7 @@ from bb.fetch2.npm import NpmEnvironment from bb.fetch2.npm import npm_package from bb.fetch2.npmsw import foreach_dependencies from recipetool.create import RecipeHandler -from recipetool.create import get_license_md5sums -from recipetool.create import guess_license +from recipetool.create import match_licenses, find_license_files, generate_common_licenses_chksums from recipetool.create import split_pkg_licenses logger = logging.getLogger('recipetool') @@ -112,40 +111,54 @@ class NpmRecipeHandler(RecipeHandler): """Return the extra license files and the list of packages""" licfiles = [] packages = {} + # Licenses from package.json will point to COMMON_LICENSE_DIR so we need + # to associate them explicitely to packages for split_pkg_licenses() + fallback_licenses = dict() + + def _find_package_licenses(destdir): + """Either find license files, or use package.json metadata""" + def _get_licenses_from_package_json(package_json): + with open(os.path.join(srctree, package_json), "r") as f: + data = json.load(f) + if "license" in data: + licenses = data["license"].split(" ") + licenses = [license.strip("()") for license in licenses if license != "OR" and license != "AND"] + return [], licenses + else: + return [package_json], None - # Handle the parent package - packages["${PN}"] = "" - - def _licfiles_append_fallback_readme_files(destdir): - """Append README files as fallback to license files if a license files is missing""" - - fallback = True - readmes = [] basedir = os.path.join(srctree, destdir) - for fn in os.listdir(basedir): - upper = fn.upper() - if upper.startswith("README"): - fullpath = os.path.join(basedir, fn) - readmes.append(fullpath) - if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper: - fallback = False - if fallback: - for readme in readmes: - licfiles.append(os.path.relpath(readme, srctree)) + licfiles = find_license_files(basedir) + if len(licfiles) > 0: + return licfiles, None + else: + # A license wasn't found in the package directory, so we'll use the package.json metadata + pkg_json = os.path.join(basedir, "package.json") + return _get_licenses_from_package_json(pkg_json) + + def _get_package_licenses(destdir, package): + (package_licfiles, package_licenses) = _find_package_licenses(destdir) + if package_licfiles: + licfiles.extend(package_licfiles) + else: + fallback_licenses[package] = package_licenses # Handle the dependencies def _handle_dependency(name, params, destdir): deptree = destdir.split('node_modules/') suffix = "-".join([npm_package(dep) for dep in deptree]) packages["${PN}" + suffix] = destdir - _licfiles_append_fallback_readme_files(destdir) + _get_package_licenses(destdir, "${PN}" + suffix) with open(shrinkwrap_file, "r") as f: shrinkwrap = json.load(f) - foreach_dependencies(shrinkwrap, _handle_dependency, dev) - return licfiles, packages + # Handle the parent package + packages["${PN}"] = "" + _get_package_licenses(srctree, "${PN}") + + return licfiles, packages, fallback_licenses # Handle the peer dependencies def _handle_peer_dependency(self, shrinkwrap_file): @@ -266,36 +279,12 @@ class NpmRecipeHandler(RecipeHandler): fetcher.unpack(srctree) bb.note("Handling licences ...") - (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev) - - def _guess_odd_license(licfiles): - import bb - - md5sums = get_license_md5sums(d, linenumbers=True) - - chksums = [] - licenses = [] - for licfile in licfiles: - f = os.path.join(srctree, licfile) - md5value = bb.utils.md5_file(f) - (license, beginline, endline, md5) = md5sums.get(md5value, - (None, "", "", "")) - if not license: - license = "Unknown" - logger.info("Please add the following line for '%s' to a " - "'lib/recipetool/licenses.csv' and replace `Unknown`, " - "`X`, `Y` and `MD5` with the license, begin line, " - "end line and partial MD5 checksum:\n" \ - "%s,Unknown,X,Y,MD5" % (licfile, md5value)) - chksums.append("file://%s%s%s;md5=%s" % (licfile, - ";beginline=%s" % (beginline) if beginline else "", - ";endline=%s" % (endline) if endline else "", - md5 if md5 else md5value)) - licenses.append((license, licfile, md5value)) - return (licenses, chksums) - - (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles) - split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after) + (licfiles, packages, fallback_licenses) = self._handle_licenses(srctree, shrinkwrap_file, dev) + licvalues = match_licenses(licfiles, srctree, d) + split_pkg_licenses(licvalues, packages, lines_after, fallback_licenses) + fallback_licenses_flat = [license for sublist in fallback_licenses.values() for license in sublist] + extravalues["LIC_FILES_CHKSUM"] = generate_common_licenses_chksums(fallback_licenses_flat, d) + extravalues["LICENSE"] = fallback_licenses_flat classes.append("npm") handled.append("buildsystem") diff --git a/poky/scripts/lib/resulttool/junit.py b/poky/scripts/lib/resulttool/junit.py new file mode 100644 index 0000000000..c7a53dc550 --- /dev/null +++ b/poky/scripts/lib/resulttool/junit.py @@ -0,0 +1,77 @@ +# resulttool - report test results in JUnit XML format +# +# Copyright (c) 2024, Siemens AG. +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import re +import xml.etree.ElementTree as ET +import resulttool.resultutils as resultutils + +def junit(args, logger): + testresults = resultutils.load_resultsdata(args.json_file, configmap=resultutils.store_map) + + total_time = 0 + skipped = 0 + failures = 0 + errors = 0 + + for tests in testresults.values(): + results = tests[next(reversed(tests))].get("result", {}) + + for result_id, result in results.items(): + # filter out ptestresult.rawlogs and ptestresult.sections + if re.search(r'\.test_', result_id): + total_time += result.get("duration", 0) + + if result['status'] == "FAILED": + failures += 1 + elif result['status'] == "ERROR": + errors += 1 + elif result['status'] == "SKIPPED": + skipped += 1 + + testsuites_node = ET.Element("testsuites") + testsuites_node.set("time", "%s" % total_time) + testsuite_node = ET.SubElement(testsuites_node, "testsuite") + testsuite_node.set("name", "Testimage") + testsuite_node.set("time", "%s" % total_time) + testsuite_node.set("tests", "%s" % len(results)) + testsuite_node.set("failures", "%s" % failures) + testsuite_node.set("errors", "%s" % errors) + testsuite_node.set("skipped", "%s" % skipped) + + for result_id, result in results.items(): + if re.search(r'\.test_', result_id): + testcase_node = ET.SubElement(testsuite_node, "testcase", { + "name": result_id, + "classname": "Testimage", + "time": str(result['duration']) + }) + if result['status'] == "SKIPPED": + ET.SubElement(testcase_node, "skipped", message=result['log']) + elif result['status'] == "FAILED": + ET.SubElement(testcase_node, "failure", message=result['log']) + elif result['status'] == "ERROR": + ET.SubElement(testcase_node, "error", message=result['log']) + + tree = ET.ElementTree(testsuites_node) + + if args.junit_xml_path is None: + args.junit_xml_path = os.environ['BUILDDIR'] + '/tmp/log/oeqa/junit.xml' + tree.write(args.junit_xml_path, encoding='UTF-8', xml_declaration=True) + + logger.info('Saved JUnit XML report as %s' % args.junit_xml_path) + +def register_commands(subparsers): + """Register subcommands from this plugin""" + parser_build = subparsers.add_parser('junit', help='create test report in JUnit XML format', + description='generate unit test report in JUnit XML format based on the latest test results in the testresults.json.', + group='analysis') + parser_build.set_defaults(func=junit) + parser_build.add_argument('json_file', + help='json file should point to the testresults.json') + parser_build.add_argument('-j', '--junit_xml_path', + help='junit xml path allows setting the path of the generated test report. The default location is <build_dir>/tmp/log/oeqa/junit.xml') diff --git a/poky/scripts/lib/resulttool/manualexecution.py b/poky/scripts/lib/resulttool/manualexecution.py index ecb27c5933..ae0861ac6b 100755 --- a/poky/scripts/lib/resulttool/manualexecution.py +++ b/poky/scripts/lib/resulttool/manualexecution.py @@ -22,7 +22,7 @@ def load_json_file(f): def write_json_file(f, json_data): os.makedirs(os.path.dirname(f), exist_ok=True) with open(f, 'w') as filedata: - filedata.write(json.dumps(json_data, sort_keys=True, indent=4)) + filedata.write(json.dumps(json_data, sort_keys=True, indent=1)) class ManualTestRunner(object): diff --git a/poky/scripts/lib/resulttool/report.py b/poky/scripts/lib/resulttool/report.py index a349510ab8..1c100b00ab 100644 --- a/poky/scripts/lib/resulttool/report.py +++ b/poky/scripts/lib/resulttool/report.py @@ -256,7 +256,7 @@ class ResultsTextReport(object): if selected_test_case_only: print_selected_testcase_result(raw_results, selected_test_case_only) else: - print(json.dumps(raw_results, sort_keys=True, indent=4)) + print(json.dumps(raw_results, sort_keys=True, indent=1)) else: print('Could not find raw test result for %s' % raw_test) return 0 diff --git a/poky/scripts/lib/resulttool/resultutils.py b/poky/scripts/lib/resulttool/resultutils.py index c5521d81bd..b8fc79a6ac 100644 --- a/poky/scripts/lib/resulttool/resultutils.py +++ b/poky/scripts/lib/resulttool/resultutils.py @@ -14,8 +14,11 @@ import scriptpath import copy import urllib.request import posixpath +import logging scriptpath.add_oe_lib_path() +logger = logging.getLogger('resulttool') + flatten_map = { "oeselftest": [], "runtime": [], @@ -31,13 +34,19 @@ regression_map = { "manual": ['TEST_TYPE', 'TEST_MODULE', 'IMAGE_BASENAME', 'MACHINE'] } store_map = { - "oeselftest": ['TEST_TYPE'], + "oeselftest": ['TEST_TYPE', 'TESTSERIES', 'MACHINE'], "runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'], "sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], "sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME'] } +rawlog_sections = { + "ptestresult.rawlogs": "ptest", + "ltpresult.rawlogs": "ltp", + "ltpposixresult.rawlogs": "ltpposix" +} + def is_url(p): """ Helper for determining if the given path is a URL @@ -108,21 +117,57 @@ def filter_resultsdata(results, resultid): newresults[r][i] = results[r][i] return newresults -def strip_ptestresults(results): +def strip_logs(results): newresults = copy.deepcopy(results) - #for a in newresults2: - # newresults = newresults2[a] for res in newresults: if 'result' not in newresults[res]: continue - if 'ptestresult.rawlogs' in newresults[res]['result']: - del newresults[res]['result']['ptestresult.rawlogs'] + for logtype in rawlog_sections: + if logtype in newresults[res]['result']: + del newresults[res]['result'][logtype] if 'ptestresult.sections' in newresults[res]['result']: for i in newresults[res]['result']['ptestresult.sections']: if 'log' in newresults[res]['result']['ptestresult.sections'][i]: del newresults[res]['result']['ptestresult.sections'][i]['log'] return newresults +# For timing numbers, crazy amounts of precision don't make sense and just confuse +# the logs. For numbers over 1, trim to 3 decimal places, for numbers less than 1, +# trim to 4 significant digits +def trim_durations(results): + for res in results: + if 'result' not in results[res]: + continue + for entry in results[res]['result']: + if 'duration' in results[res]['result'][entry]: + duration = results[res]['result'][entry]['duration'] + if duration > 1: + results[res]['result'][entry]['duration'] = float("%.3f" % duration) + elif duration < 1: + results[res]['result'][entry]['duration'] = float("%.4g" % duration) + return results + +def handle_cleanups(results): + # Remove pointless path duplication from old format reproducibility results + for res2 in results: + try: + section = results[res2]['result']['reproducible']['files'] + for pkgtype in section: + for filelist in section[pkgtype].copy(): + if section[pkgtype][filelist] and type(section[pkgtype][filelist][0]) == dict: + newlist = [] + for entry in section[pkgtype][filelist]: + newlist.append(entry["reference"].split("/./")[1]) + section[pkgtype][filelist] = newlist + + except KeyError: + pass + # Remove pointless duplicate rawlogs data + try: + del results[res2]['result']['reproducible.rawlogs'] + except KeyError: + pass + def decode_log(logdata): if isinstance(logdata, str): return logdata @@ -155,9 +200,6 @@ def generic_get_rawlogs(sectname, results): return None return decode_log(results[sectname]['log']) -def ptestresult_get_rawlogs(results): - return generic_get_rawlogs('ptestresult.rawlogs', results) - def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, ptestlogs=False): for res in results: if res: @@ -167,16 +209,20 @@ def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, p os.makedirs(os.path.dirname(dst), exist_ok=True) resultsout = results[res] if not ptestjson: - resultsout = strip_ptestresults(results[res]) + resultsout = strip_logs(results[res]) + trim_durations(resultsout) + handle_cleanups(resultsout) with open(dst, 'w') as f: - f.write(json.dumps(resultsout, sort_keys=True, indent=4)) + f.write(json.dumps(resultsout, sort_keys=True, indent=1)) for res2 in results[res]: if ptestlogs and 'result' in results[res][res2]: seriesresults = results[res][res2]['result'] - rawlogs = ptestresult_get_rawlogs(seriesresults) - if rawlogs is not None: - with open(dst.replace(fn, "ptest-raw.log"), "w+") as f: - f.write(rawlogs) + for logtype in rawlog_sections: + logdata = generic_get_rawlogs(logtype, seriesresults) + if logdata is not None: + logger.info("Extracting " + rawlog_sections[logtype] + "-raw.log") + with open(dst.replace(fn, rawlog_sections[logtype] + "-raw.log"), "w+") as f: + f.write(logdata) if 'ptestresult.sections' in seriesresults: for i in seriesresults['ptestresult.sections']: sectionlog = ptestresult_get_log(seriesresults, i) diff --git a/poky/scripts/lib/resulttool/store.py b/poky/scripts/lib/resulttool/store.py index e0951f0a8f..578910d234 100644 --- a/poky/scripts/lib/resulttool/store.py +++ b/poky/scripts/lib/resulttool/store.py @@ -65,18 +65,34 @@ def store(args, logger): for r in revisions: results = revisions[r] + if args.revision and r[0] != args.revision: + logger.info('skipping %s as non-matching' % r[0]) + continue keywords = {'commit': r[0], 'branch': r[1], "commit_count": r[2]} - subprocess.check_call(["find", tempdir, "!", "-path", "./.git/*", "-delete"]) + subprocess.check_call(["find", tempdir, "-name", "testresults.json", "!", "-path", "./.git/*", "-delete"]) resultutils.save_resultsdata(results, tempdir, ptestlogs=True) logger.info('Storing test result into git repository %s' % args.git_dir) - gitarchive.gitarchive(tempdir, args.git_dir, False, False, + excludes = [] + if args.logfile_archive: + excludes = ['*.log', "*.log.zst"] + + tagname = gitarchive.gitarchive(tempdir, args.git_dir, False, False, "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}", False, "{branch}/{commit_count}-g{commit}/{tag_number}", 'Test run #{tag_number} of {branch}:{commit}', '', - [], [], False, keywords, logger) + excludes, [], False, keywords, logger) + if args.logfile_archive: + logdir = args.logfile_archive + "/" + tagname + shutil.copytree(tempdir, logdir) + for root, dirs, files in os.walk(logdir): + for name in files: + if not name.endswith(".log"): + continue + f = os.path.join(root, name) + subprocess.run(["zstd", f, "--rm"], check=True, capture_output=True) finally: subprocess.check_call(["rm", "-rf", tempdir]) @@ -102,3 +118,7 @@ def register_commands(subparsers): help='add executed-by configuration to each result file') parser_build.add_argument('-t', '--extra-test-env', default='', help='add extra test environment data to each result file configuration') + parser_build.add_argument('-r', '--revision', default='', + help='only store data for the specified revision') + parser_build.add_argument('-l', '--logfile-archive', default='', + help='directory to separately archive log files along with a copy of the results') diff --git a/poky/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in b/poky/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in index 2fd286ff98..5211972955 100644 --- a/poky/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in +++ b/poky/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in @@ -1,3 +1,3 @@ bootloader --ptable gpt -part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.1 +part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.2 part / --source rootfs --fstype=ext4 --label root --align 1024 --exclude-path boot/ diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-partition.py b/poky/scripts/lib/wic/plugins/source/bootimg-partition.py index 1071d1af3f..589853a439 100644 --- a/poky/scripts/lib/wic/plugins/source/bootimg-partition.py +++ b/poky/scripts/lib/wic/plugins/source/bootimg-partition.py @@ -16,7 +16,7 @@ import logging import os import re -from glob import glob +from oe.bootfiles import get_boot_files from wic import WicError from wic.engine import get_custom_config @@ -66,42 +66,7 @@ class BootimgPartitionPlugin(SourcePlugin): logger.debug('Boot files: %s', boot_files) - # list of tuples (src_name, dst_name) - deploy_files = [] - for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files): - if ';' in src_entry: - dst_entry = tuple(src_entry.split(';')) - if not dst_entry[0] or not dst_entry[1]: - raise WicError('Malformed boot file entry: %s' % src_entry) - else: - dst_entry = (src_entry, src_entry) - - logger.debug('Destination entry: %r', dst_entry) - deploy_files.append(dst_entry) - - cls.install_task = []; - for deploy_entry in deploy_files: - src, dst = deploy_entry - if '*' in src: - # by default install files under their basename - entry_name_fn = os.path.basename - if dst != src: - # unless a target name was given, then treat name - # as a directory and append a basename - entry_name_fn = lambda name: \ - os.path.join(dst, - os.path.basename(name)) - - srcs = glob(os.path.join(kernel_dir, src)) - - logger.debug('Globbed sources: %s', ', '.join(srcs)) - for entry in srcs: - src = os.path.relpath(entry, kernel_dir) - entry_dst_name = entry_name_fn(entry) - cls.install_task.append((src, entry_dst_name)) - else: - cls.install_task.append((src, dst)) - + cls.install_task = get_boot_files(kernel_dir, boot_files) if source_params.get('loader') != "u-boot": return diff --git a/poky/scripts/oe-setup-build b/poky/scripts/oe-setup-build index c0476992a2..80d8c70bac 100755 --- a/poky/scripts/oe-setup-build +++ b/poky/scripts/oe-setup-build @@ -77,7 +77,7 @@ def find_template(template_name, templates): for t in templates: if t["templatename"] == template_name: return t - print("Configuration {} is not one of {}, please try again.".format(tempalte_name, [t["templatename"] for t in templates])) + print("Configuration {} is not one of {}, please try again.".format(template_name, [t["templatename"] for t in templates])) return None def setup_build_env(args): @@ -102,9 +102,9 @@ def setup_build_env(args): cmd = "TEMPLATECONF={} {}".format(template["templatepath"], cmd_base) if not no_shell: - cmd = cmd + " && {}".format(os.environ['SHELL']) + cmd = cmd + " && {}".format(os.environ.get('SHELL','bash')) print("Running:", cmd) - subprocess.run(cmd, shell=True, executable=os.environ['SHELL']) + subprocess.run(cmd, shell=True, executable=os.environ.get('SHELL','bash')) parser = argparse.ArgumentParser(description="A script that discovers available build configurations and sets up a build environment based on one of them. Run without arguments to choose one interactively.") parser.add_argument("--layerlist", default=defaultlayers(), help='Where to look for available layers (as written out by setup-layers script) (default is {}).'.format(defaultlayers())) diff --git a/poky/scripts/patchtest.README b/poky/scripts/patchtest.README index 76b5fcdb6d..816406ff54 100644 --- a/poky/scripts/patchtest.README +++ b/poky/scripts/patchtest.README @@ -3,40 +3,35 @@ ## Introduction Patchtest is a test framework for community patches based on the standard -unittest python module. As input, it needs tree elements to work properly: -a patch in mbox format (either created with `git format-patch` or fetched -from 'patchwork'), a test suite and a target repository. +unittest python module. As input, it needs three elements to work properly: + +- a patch in mbox format (either created with `git format-patch` or fetched +from 'patchwork') +- a test suite +- a target repository The first test suite intended to be used with patchtest is found in the -openembedded-core repository [1] targeted for patches that get into the +openembedded-core repository [1], targeted for patches that get into the openembedded-core mailing list [2]. This suite is also intended as a baseline for development of similar suites for other layers as needed. -Patchtest can either run on a host or a guest machine, depending on which -environment the execution needs to be done. If you plan to test your own patches -(a good practice before these are sent to the mailing list), the easiest way is -to install and execute on your local host; in the other hand, if automatic -testing is intended, the guest method is strongly recommended. The guest -method requires the use of the patchtest layer, in addition to the tools -available in oe-core: https://git.yoctoproject.org/patchtest/ +Patchtest can either run on a host or a guest machine, depending on +which environment you prefer. If you plan to test your own patches (a +good practice before these are sent to the mailing list), the easiest +way is to install and execute on your local host; in the other hand, if +automatic testing is intended, the guest method is strongly recommended. +The guest method requires the use of the patchtest layer, in addition to +the tools available in oe-core: https://git.yoctoproject.org/patchtest/ ## Installation -As a tool for use with the Yocto Project, the [quick start guide](https://docs.yoctoproject.org/brief-yoctoprojectqs/index.html) -contains the necessary prerequisites for a basic project. In addition, -patchtest relies on the following Python modules: - -- boto3 (for sending automated results emails only) -- git-pw>=2.5.0 -- jinja2 -- pylint -- pyparsing>=3.0.9 -- unidiff - -These can be installed by running `pip install -r -meta/lib/patchtest/requirements.txt`. Note that git-pw is not -automatically added to the user's PATH; by default, it is installed at -~/.local/bin/git-pw. +As a tool for use with the Yocto Project, the [quick start +guide](https://docs.yoctoproject.org/brief-yoctoprojectqs/index.html) +contains the necessary prerequisites. In addition, patchtest relies on +several Python modules for parsing and analysis, which can be installed +by running `pip install -r meta/lib/patchtest/requirements.txt`. Note +that git-pw is not automatically added to the user's PATH; by default, +it is installed at ~/.local/bin/git-pw. For git-pw (and therefore scripts such as patchtest-get--series) to work, you need to provide a Patchwork instance in your user's .gitconfig, like so (the project @@ -123,7 +118,7 @@ The general flow of guest mode is: -device virtio-9p-pci,fsdev=test_mount,mount_tag=test_mount -smp 4 -m 2048"` -Patchtest runs as an initscript for the core-image-patchtest image and +Patchtest is run by an initscript for the core-image-patchtest image and shuts down after completion, so there is no input required from a user during operation. Unlike in host mode, the guest is designed to automatically generate test result files, in the same directory as the @@ -131,6 +126,17 @@ targeted patch files but with .testresult as an extension. These contain the entire output of the patchtest run for each respective pass, including the PASS, FAIL, and SKIP indicators for each test run. +### Running Patchtest Selftests + +Patchtest also includes selftests, which are currently in the form of +several contrived patch files and a runner script found in +`meta/lib/patchtest/selftest/`. In order to run these, the +`meta-selftest` layer must be added to bblayers.conf. It is also +recommended to set BB_SERVER_TIMEOUT (and thus enable memory-resident +bitbake) in local.conf to reduce runtime, as the bitbake startup process +will otherwise add to it significantly when restarted for each test +patch. + ## Contributing The yocto mailing list (openembedded-core@lists.openembedded.org) is used for questions, diff --git a/poky/scripts/pull-sdpx-licenses.py b/poky/scripts/pull-sdpx-licenses.py new file mode 100755 index 0000000000..597a62133f --- /dev/null +++ b/poky/scripts/pull-sdpx-licenses.py @@ -0,0 +1,101 @@ +#! /usr/bin/env python3 +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only + +import argparse +import json +import sys +import urllib.request +from pathlib import Path + +TOP_DIR = Path(__file__).parent.parent + + +def main(): + parser = argparse.ArgumentParser( + description="Update SPDX License files from upstream" + ) + parser.add_argument( + "-v", + "--version", + metavar="MAJOR.MINOR[.MICRO]", + help="Pull specific version of License list instead of latest", + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Update existing license file text with upstream text", + ) + parser.add_argument( + "--deprecated", + action="store_true", + help="Update deprecated licenses", + ) + parser.add_argument( + "--dest", + type=Path, + default=TOP_DIR / "meta" / "files" / "common-licenses", + help="Write licenses to directory DEST. Default is %(default)s", + ) + + args = parser.parse_args() + + if args.version: + version = f"v{args.version}" + else: + # Fetch the latest release + req = urllib.request.Request( + "https://api.github.com/repos/spdx/license-list-data/releases/latest" + ) + req.add_header("X-GitHub-Api-Version", "2022-11-28") + req.add_header("Accept", "application/vnd.github+json") + with urllib.request.urlopen(req) as response: + data = json.load(response) + version = data["tag_name"] + + print(f"Pulling SPDX license list version {version}") + req = urllib.request.Request( + f"https://raw.githubusercontent.com/spdx/license-list-data/{version}/json/licenses.json" + ) + with urllib.request.urlopen(req) as response: + spdx_licenses = json.load(response) + + with (TOP_DIR / "meta" / "files" / "spdx-licenses.json").open("w") as f: + json.dump(spdx_licenses, f, sort_keys=True, indent=2) + + total_count = len(spdx_licenses["licenses"]) + updated = 0 + for idx, lic in enumerate(spdx_licenses["licenses"]): + lic_id = lic["licenseId"] + + print(f"[{idx + 1} of {total_count}] ", end="") + + dest_license_file = args.dest / lic_id + if dest_license_file.is_file() and not args.overwrite: + print(f"Skipping {lic_id} since it already exists") + continue + + print(f"Fetching {lic_id}... ", end="", flush=True) + + req = urllib.request.Request(lic["detailsUrl"]) + with urllib.request.urlopen(req) as response: + lic_data = json.load(response) + + if lic_data["isDeprecatedLicenseId"] and not args.deprecated: + print("Skipping (deprecated)") + continue + + with dest_license_file.open("w") as f: + f.write(lic_data["licenseText"]) + updated += 1 + print("done") + + print(f"Updated {updated} licenses") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/poky/scripts/pybootchartgui/pybootchartgui/parsing.py b/poky/scripts/pybootchartgui/pybootchartgui/parsing.py index 63a53b6b88..144a16c723 100644 --- a/poky/scripts/pybootchartgui/pybootchartgui/parsing.py +++ b/poky/scripts/pybootchartgui/pybootchartgui/parsing.py @@ -457,7 +457,7 @@ def _parse_proc_disk_stat_log(file): not sda1, sda2 etc. The format of relevant lines should be: {major minor name rio rmerge rsect ruse wio wmerge wsect wuse running use aveq} """ - disk_regex_re = re.compile ('^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$') + disk_regex_re = re.compile (r'^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$') # this gets called an awful lot. def is_relevant_line(linetokens): @@ -594,8 +594,8 @@ def _parse_pressure_logs(file, filename): # [ 0.039993] calling migration_init+0x0/0x6b @ 1 # [ 0.039993] initcall migration_init+0x0/0x6b returned 1 after 0 usecs def _parse_dmesg(writer, file): - timestamp_re = re.compile ("^\[\s*(\d+\.\d+)\s*]\s+(.*)$") - split_re = re.compile ("^(\S+)\s+([\S\+_-]+) (.*)$") + timestamp_re = re.compile (r"^\[\s*(\d+\.\d+)\s*]\s+(.*)$") + split_re = re.compile (r"^(\S+)\s+([\S\+_-]+) (.*)$") processMap = {} idx = 0 inc = 1.0 / 1000000 @@ -640,7 +640,7 @@ def _parse_dmesg(writer, file): # print "foo: '%s' '%s' '%s'" % (type, func, rest) if type == "calling": ppid = kernel.pid - p = re.match ("\@ (\d+)", rest) + p = re.match (r"\@ (\d+)", rest) if p is not None: ppid = float (p.group(1)) // 1000 # print "match: '%s' ('%g') at '%s'" % (func, ppid, time_ms) @@ -742,7 +742,7 @@ def get_num_cpus(headers): cpu_model = headers.get("system.cpu") if cpu_model is None: return 1 - mat = re.match(".*\\((\\d+)\\)", cpu_model) + mat = re.match(r".*\\((\\d+)\\)", cpu_model) if mat is None: return 1 return max (int(mat.group(1)), 1) diff --git a/poky/scripts/resulttool b/poky/scripts/resulttool index fc282bda6c..66a6af9959 100755 --- a/poky/scripts/resulttool +++ b/poky/scripts/resulttool @@ -15,6 +15,9 @@ # To report test report, execute the below # $ resulttool report <source_dir> # +# To create a unit test report in JUnit XML format, execute the below +# $ resulttool junit <json_file> +# # To perform regression file analysis, execute the below # $ resulttool regression-file <base_result_file> <target_result_file> # @@ -43,6 +46,7 @@ import resulttool.regression import resulttool.report import resulttool.manualexecution import resulttool.log +import resulttool.junit logger = scriptutils.logger_create('resulttool') def main(): @@ -61,6 +65,7 @@ def main(): resulttool.regression.register_commands(subparsers) resulttool.report.register_commands(subparsers) resulttool.log.register_commands(subparsers) + resulttool.junit.register_commands(subparsers) args = parser.parse_args() if args.debug: diff --git a/poky/scripts/runqemu b/poky/scripts/runqemu index 69cd44864e..14eb939b3e 100755 --- a/poky/scripts/runqemu +++ b/poky/scripts/runqemu @@ -1192,19 +1192,22 @@ to your build configuration. raise RunQemuError("a new one with sudo.") gid = os.getgid() - uid = os.getuid() logger.info("Setting up tap interface under sudo") cmd = ('sudo', self.qemuifup, str(gid)) - try: - tap = subprocess.check_output(cmd).decode('utf-8').strip() - except subprocess.CalledProcessError as e: - logger.error('Setting up tap device failed:\n%s\nRun runqemu-gen-tapdevs to manually create one.' % str(e)) - sys.exit(1) - lockfile = os.path.join(lockdir, tap) - self.taplock = lockfile + '.lock' - self.acquire_taplock() - self.cleantap = True - logger.debug('Created tap: %s' % tap) + for _ in range(5): + try: + tap = subprocess.check_output(cmd).decode('utf-8').strip() + except subprocess.CalledProcessError as e: + logger.error('Setting up tap device failed:\n%s\nRun runqemu-gen-tapdevs to manually create one.' % str(e)) + sys.exit(1) + lockfile = os.path.join(lockdir, tap) + self.taplock = lockfile + '.lock' + if self.acquire_taplock(): + self.cleantap = True + logger.debug('Created tap: %s' % tap) + break + else: + tap = None if not tap: logger.error("Failed to setup tap device. Run runqemu-gen-tapdevs to manually create.") @@ -1295,6 +1298,10 @@ to your build configuration. elif drive_type.startswith("/dev/hd"): logger.info('Using ide drive') vm_drive = "-drive file=%s,format=%s" % (self.rootfs, rootfs_format) + elif drive_type.startswith("/dev/mmcblk"): + logger.info('Using sdcard drive') + vm_drive = '-drive id=sdcard0,if=none,file=%s,format=%s -device sdhci-pci -device sd-card,drive=sdcard0' \ + % (self.rootfs, rootfs_format) elif drive_type.startswith("/dev/vdb"): logger.info('Using block virtio drive'); vm_drive = '-drive id=disk0,file=%s,if=none,format=%s -device virtio-blk-device,drive=disk0%s' \ @@ -1483,7 +1490,7 @@ to your build configuration. # If no serial or serialtcp options were specified, only ttyS0 is created # and sysvinit shows an error trying to enable ttyS1: # INIT: Id "S1" respawning too fast: disabled for 5 minutes - serial_num = len(re.findall("-serial", self.qemu_opt)) + serial_num = len(re.findall("(^| )-serial ", self.qemu_opt)) # Assume if the user passed serial options, they know what they want # and pad to two devices @@ -1503,7 +1510,7 @@ to your build configuration. self.qemu_opt += " %s" % self.get("QB_SERIAL_OPT") - serial_num = len(re.findall("-serial", self.qemu_opt)) + serial_num = len(re.findall("(^| )-serial ", self.qemu_opt)) if serial_num < 2: self.qemu_opt += " -serial null" |