diff options
Diffstat (limited to 'poky/scripts/lib')
| -rwxr-xr-x | poky/scripts/lib/devtool/ide_sdk.py | 63 | ||||
| -rw-r--r-- | poky/scripts/lib/devtool/standard.py | 3 | ||||
| -rwxr-xr-x | poky/scripts/lib/resulttool/manualexecution.py | 2 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/report.py | 2 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/resultutils.py | 76 | ||||
| -rw-r--r-- | poky/scripts/lib/resulttool/store.py | 26 | ||||
| -rw-r--r-- | poky/scripts/lib/wic/engine.py | 2 | ||||
| -rw-r--r-- | poky/scripts/lib/wic/plugins/source/bootimg-efi.py | 6 |
8 files changed, 113 insertions, 67 deletions
diff --git a/poky/scripts/lib/devtool/ide_sdk.py b/poky/scripts/lib/devtool/ide_sdk.py index 65873b088d..0b50165a12 100755 --- a/poky/scripts/lib/devtool/ide_sdk.py +++ b/poky/scripts/lib/devtool/ide_sdk.py @@ -288,6 +288,7 @@ class RecipeModified: self.bblayers = None self.bpn = None self.d = None + self.debug_build = None self.fakerootcmd = None self.fakerootenv = None self.libdir = None @@ -348,6 +349,7 @@ class RecipeModified: self.bpn = recipe_d.getVar('BPN') self.cxx = recipe_d.getVar('CXX') self.d = recipe_d.getVar('D') + self.debug_build = recipe_d.getVar('DEBUG_BUILD') self.fakerootcmd = recipe_d.getVar('FAKEROOTCMD') self.fakerootenv = recipe_d.getVar('FAKEROOTENV') self.libdir = recipe_d.getVar('libdir') @@ -389,17 +391,6 @@ class RecipeModified: self.recipe_id = self.bpn + "-" + self.package_arch self.recipe_id_pretty = self.bpn + ": " + self.package_arch - def append_to_bbappend(self, append_text): - with open(self.bbappend, 'a') as bbap: - bbap.write(append_text) - - def remove_from_bbappend(self, append_text): - with open(self.bbappend, 'r') as bbap: - text = bbap.read() - new_text = text.replace(append_text, '') - with open(self.bbappend, 'w') as bbap: - bbap.write(new_text) - @staticmethod def is_valid_shell_variable(var): """Skip strange shell variables like systemd @@ -412,34 +403,6 @@ class RecipeModified: return True return False - def debug_build_config(self, args): - """Explicitely set for example CMAKE_BUILD_TYPE to Debug if not defined otherwise""" - if self.build_tool is BuildTool.CMAKE: - append_text = os.linesep + \ - 'OECMAKE_ARGS:append = " -DCMAKE_BUILD_TYPE:STRING=Debug"' + os.linesep - if args.debug_build_config and not 'CMAKE_BUILD_TYPE' in self.cmake_cache_vars: - self.cmake_cache_vars['CMAKE_BUILD_TYPE'] = { - "type": "STRING", - "value": "Debug", - } - self.append_to_bbappend(append_text) - elif 'CMAKE_BUILD_TYPE' in self.cmake_cache_vars: - del self.cmake_cache_vars['CMAKE_BUILD_TYPE'] - self.remove_from_bbappend(append_text) - elif self.build_tool is BuildTool.MESON: - append_text = os.linesep + 'MESON_BUILDTYPE = "debug"' + os.linesep - if args.debug_build_config and self.meson_buildtype != "debug": - self.mesonopts.replace( - '--buildtype ' + self.meson_buildtype, '--buildtype debug') - self.append_to_bbappend(append_text) - elif self.meson_buildtype == "debug": - self.mesonopts.replace( - '--buildtype debug', '--buildtype plain') - self.remove_from_bbappend(append_text) - elif args.debug_build_config: - logger.warn( - "--debug-build-config is not implemented for this build tool yet.") - def solib_search_path(self, image): """Search for debug symbols in the rootfs and rootfs-dbg @@ -493,7 +456,7 @@ class RecipeModified: vars = (key for key in d.keys() if not key.startswith( "__") and not d.getVarFlag(key, "func", False)) - for var in vars: + for var in sorted(vars): func = d.getVarFlag(var, "func", False) if d.getVarFlag(var, 'python', False) and func: continue @@ -545,7 +508,7 @@ class RecipeModified: cache_vars = {} oecmake_args = d.getVar('OECMAKE_ARGS').split() extra_oecmake = d.getVar('EXTRA_OECMAKE').split() - for param in oecmake_args + extra_oecmake: + for param in sorted(oecmake_args + extra_oecmake): d_pref = "-D" if param.startswith(d_pref): param = param[len(d_pref):] @@ -988,6 +951,13 @@ def ide_setup(args, config, basepath, workspace): recipe_modified.gen_meson_wrapper() ide.setup_modified_recipe( args, recipe_image, recipe_modified) + + if recipe_modified.debug_build != '1': + logger.warn( + 'Recipe %s is compiled with release build configuration. ' + 'You might want to add DEBUG_BUILD = "1" to %s. ' + 'Note that devtool modify --debug-build can do this automatically.', + recipe_modified.name, recipe_modified.bbappend) else: raise DevtoolError("Must not end up here.") @@ -995,6 +965,15 @@ def ide_setup(args, config, basepath, workspace): def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" + # The ide-sdk command bootstraps the SDK from the bitbake environment before the IDE + # configuration is generated. In the case of the eSDK, the bootstrapping is performed + # during the installation of the eSDK installer. Running the ide-sdk plugin from an + # eSDK installer-based setup would require skipping the bootstrapping and probably + # taking some other differences into account when generating the IDE configurations. + # This would be possible. But it is not implemented. + if context.fixed_setup: + return + global ide_plugins # Search for IDE plugins in all sub-folders named ide_plugins where devtool seraches for plugins. @@ -1065,6 +1044,4 @@ def register_commands(subparsers, context): '-p', '--no-preserve', help='Do not preserve existing files', action='store_true') parser_ide_sdk.add_argument( '--no-check-space', help='Do not check for available space before deploying', action='store_true') - parser_ide_sdk.add_argument( - '--debug-build-config', help='Use debug build flags, for example set CMAKE_BUILD_TYPE=Debug', action='store_true') parser_ide_sdk.set_defaults(func=ide_setup) diff --git a/poky/scripts/lib/devtool/standard.py b/poky/scripts/lib/devtool/standard.py index 05161942b7..908869cc4f 100644 --- a/poky/scripts/lib/devtool/standard.py +++ b/poky/scripts/lib/devtool/standard.py @@ -1031,6 +1031,8 @@ def modify(args, config, basepath, workspace): if branch == args.branch: continue f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) + if args.debug_build: + f.write('\nDEBUG_BUILD = "1"\n') update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) @@ -2396,6 +2398,7 @@ def register_commands(subparsers, context): parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") + parser_modify.add_argument('--debug-build', action="store_true", help='Add DEBUG_BUILD = "1" to the modified recipe') parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', diff --git a/poky/scripts/lib/resulttool/manualexecution.py b/poky/scripts/lib/resulttool/manualexecution.py index ecb27c5933..ae0861ac6b 100755 --- a/poky/scripts/lib/resulttool/manualexecution.py +++ b/poky/scripts/lib/resulttool/manualexecution.py @@ -22,7 +22,7 @@ def load_json_file(f): def write_json_file(f, json_data): os.makedirs(os.path.dirname(f), exist_ok=True) with open(f, 'w') as filedata: - filedata.write(json.dumps(json_data, sort_keys=True, indent=4)) + filedata.write(json.dumps(json_data, sort_keys=True, indent=1)) class ManualTestRunner(object): diff --git a/poky/scripts/lib/resulttool/report.py b/poky/scripts/lib/resulttool/report.py index a349510ab8..1c100b00ab 100644 --- a/poky/scripts/lib/resulttool/report.py +++ b/poky/scripts/lib/resulttool/report.py @@ -256,7 +256,7 @@ class ResultsTextReport(object): if selected_test_case_only: print_selected_testcase_result(raw_results, selected_test_case_only) else: - print(json.dumps(raw_results, sort_keys=True, indent=4)) + print(json.dumps(raw_results, sort_keys=True, indent=1)) else: print('Could not find raw test result for %s' % raw_test) return 0 diff --git a/poky/scripts/lib/resulttool/resultutils.py b/poky/scripts/lib/resulttool/resultutils.py index c5521d81bd..b8fc79a6ac 100644 --- a/poky/scripts/lib/resulttool/resultutils.py +++ b/poky/scripts/lib/resulttool/resultutils.py @@ -14,8 +14,11 @@ import scriptpath import copy import urllib.request import posixpath +import logging scriptpath.add_oe_lib_path() +logger = logging.getLogger('resulttool') + flatten_map = { "oeselftest": [], "runtime": [], @@ -31,13 +34,19 @@ regression_map = { "manual": ['TEST_TYPE', 'TEST_MODULE', 'IMAGE_BASENAME', 'MACHINE'] } store_map = { - "oeselftest": ['TEST_TYPE'], + "oeselftest": ['TEST_TYPE', 'TESTSERIES', 'MACHINE'], "runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'], "sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], "sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME'] } +rawlog_sections = { + "ptestresult.rawlogs": "ptest", + "ltpresult.rawlogs": "ltp", + "ltpposixresult.rawlogs": "ltpposix" +} + def is_url(p): """ Helper for determining if the given path is a URL @@ -108,21 +117,57 @@ def filter_resultsdata(results, resultid): newresults[r][i] = results[r][i] return newresults -def strip_ptestresults(results): +def strip_logs(results): newresults = copy.deepcopy(results) - #for a in newresults2: - # newresults = newresults2[a] for res in newresults: if 'result' not in newresults[res]: continue - if 'ptestresult.rawlogs' in newresults[res]['result']: - del newresults[res]['result']['ptestresult.rawlogs'] + for logtype in rawlog_sections: + if logtype in newresults[res]['result']: + del newresults[res]['result'][logtype] if 'ptestresult.sections' in newresults[res]['result']: for i in newresults[res]['result']['ptestresult.sections']: if 'log' in newresults[res]['result']['ptestresult.sections'][i]: del newresults[res]['result']['ptestresult.sections'][i]['log'] return newresults +# For timing numbers, crazy amounts of precision don't make sense and just confuse +# the logs. For numbers over 1, trim to 3 decimal places, for numbers less than 1, +# trim to 4 significant digits +def trim_durations(results): + for res in results: + if 'result' not in results[res]: + continue + for entry in results[res]['result']: + if 'duration' in results[res]['result'][entry]: + duration = results[res]['result'][entry]['duration'] + if duration > 1: + results[res]['result'][entry]['duration'] = float("%.3f" % duration) + elif duration < 1: + results[res]['result'][entry]['duration'] = float("%.4g" % duration) + return results + +def handle_cleanups(results): + # Remove pointless path duplication from old format reproducibility results + for res2 in results: + try: + section = results[res2]['result']['reproducible']['files'] + for pkgtype in section: + for filelist in section[pkgtype].copy(): + if section[pkgtype][filelist] and type(section[pkgtype][filelist][0]) == dict: + newlist = [] + for entry in section[pkgtype][filelist]: + newlist.append(entry["reference"].split("/./")[1]) + section[pkgtype][filelist] = newlist + + except KeyError: + pass + # Remove pointless duplicate rawlogs data + try: + del results[res2]['result']['reproducible.rawlogs'] + except KeyError: + pass + def decode_log(logdata): if isinstance(logdata, str): return logdata @@ -155,9 +200,6 @@ def generic_get_rawlogs(sectname, results): return None return decode_log(results[sectname]['log']) -def ptestresult_get_rawlogs(results): - return generic_get_rawlogs('ptestresult.rawlogs', results) - def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, ptestlogs=False): for res in results: if res: @@ -167,16 +209,20 @@ def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, p os.makedirs(os.path.dirname(dst), exist_ok=True) resultsout = results[res] if not ptestjson: - resultsout = strip_ptestresults(results[res]) + resultsout = strip_logs(results[res]) + trim_durations(resultsout) + handle_cleanups(resultsout) with open(dst, 'w') as f: - f.write(json.dumps(resultsout, sort_keys=True, indent=4)) + f.write(json.dumps(resultsout, sort_keys=True, indent=1)) for res2 in results[res]: if ptestlogs and 'result' in results[res][res2]: seriesresults = results[res][res2]['result'] - rawlogs = ptestresult_get_rawlogs(seriesresults) - if rawlogs is not None: - with open(dst.replace(fn, "ptest-raw.log"), "w+") as f: - f.write(rawlogs) + for logtype in rawlog_sections: + logdata = generic_get_rawlogs(logtype, seriesresults) + if logdata is not None: + logger.info("Extracting " + rawlog_sections[logtype] + "-raw.log") + with open(dst.replace(fn, rawlog_sections[logtype] + "-raw.log"), "w+") as f: + f.write(logdata) if 'ptestresult.sections' in seriesresults: for i in seriesresults['ptestresult.sections']: sectionlog = ptestresult_get_log(seriesresults, i) diff --git a/poky/scripts/lib/resulttool/store.py b/poky/scripts/lib/resulttool/store.py index e0951f0a8f..578910d234 100644 --- a/poky/scripts/lib/resulttool/store.py +++ b/poky/scripts/lib/resulttool/store.py @@ -65,18 +65,34 @@ def store(args, logger): for r in revisions: results = revisions[r] + if args.revision and r[0] != args.revision: + logger.info('skipping %s as non-matching' % r[0]) + continue keywords = {'commit': r[0], 'branch': r[1], "commit_count": r[2]} - subprocess.check_call(["find", tempdir, "!", "-path", "./.git/*", "-delete"]) + subprocess.check_call(["find", tempdir, "-name", "testresults.json", "!", "-path", "./.git/*", "-delete"]) resultutils.save_resultsdata(results, tempdir, ptestlogs=True) logger.info('Storing test result into git repository %s' % args.git_dir) - gitarchive.gitarchive(tempdir, args.git_dir, False, False, + excludes = [] + if args.logfile_archive: + excludes = ['*.log', "*.log.zst"] + + tagname = gitarchive.gitarchive(tempdir, args.git_dir, False, False, "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}", False, "{branch}/{commit_count}-g{commit}/{tag_number}", 'Test run #{tag_number} of {branch}:{commit}', '', - [], [], False, keywords, logger) + excludes, [], False, keywords, logger) + if args.logfile_archive: + logdir = args.logfile_archive + "/" + tagname + shutil.copytree(tempdir, logdir) + for root, dirs, files in os.walk(logdir): + for name in files: + if not name.endswith(".log"): + continue + f = os.path.join(root, name) + subprocess.run(["zstd", f, "--rm"], check=True, capture_output=True) finally: subprocess.check_call(["rm", "-rf", tempdir]) @@ -102,3 +118,7 @@ def register_commands(subparsers): help='add executed-by configuration to each result file') parser_build.add_argument('-t', '--extra-test-env', default='', help='add extra test environment data to each result file configuration') + parser_build.add_argument('-r', '--revision', default='', + help='only store data for the specified revision') + parser_build.add_argument('-l', '--logfile-archive', default='', + help='directory to separately archive log files along with a copy of the results') diff --git a/poky/scripts/lib/wic/engine.py b/poky/scripts/lib/wic/engine.py index 674ccfc244..ce7e6c5d75 100644 --- a/poky/scripts/lib/wic/engine.py +++ b/poky/scripts/lib/wic/engine.py @@ -359,7 +359,7 @@ class Disk: Remove files/dirs and their contents from the partition. This only applies to ext* partition. """ - abs_path = re.sub('\/\/+', '/', path) + abs_path = re.sub(r'\/\/+', '/', path) cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs, self._get_part_image(pnum), abs_path) diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-efi.py b/poky/scripts/lib/wic/plugins/source/bootimg-efi.py index 13a9cddf4e..37d07093f5 100644 --- a/poky/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/poky/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -245,7 +245,7 @@ class BootimgEFIPlugin(SourcePlugin): # list of tuples (src_name, dst_name) deploy_files = [] - for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files): + for src_entry in re.findall(r'[\w;\-\.\+/\*]+', boot_files): if ';' in src_entry: dst_entry = tuple(src_entry.split(';')) if not dst_entry[0] or not dst_entry[1]: @@ -428,10 +428,10 @@ class BootimgEFIPlugin(SourcePlugin): elif source_params['loader'] == 'uefi-kernel': kernel = get_bitbake_var("KERNEL_IMAGETYPE") if not kernel: - raise WicError("Empty KERNEL_IMAGETYPE %s\n" % target) + raise WicError("Empty KERNEL_IMAGETYPE") target = get_bitbake_var("TARGET_SYS") if not target: - raise WicError("Unknown arch (TARGET_SYS) %s\n" % target) + raise WicError("Empty TARGET_SYS") if re.match("x86_64", target): kernel_efi_image = "bootx64.efi" |