diff options
Diffstat (limited to 'BaseTools/Plugin')
31 files changed, 4640 insertions, 4640 deletions
diff --git a/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator.py b/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator.py index 9f86b1c358..9d94c25273 100644 --- a/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator.py +++ b/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator.py @@ -1,69 +1,69 @@ -##
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-import os
-import logging
-import json
-
-try:
- from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin
-
- class BuildToolsReportGenerator(IUefiBuildPlugin):
- def do_report(self, thebuilder):
- try:
- from edk2toolext.environment import version_aggregator
- except ImportError:
- logging.critical("Loading BuildToolsReportGenerator failed, please update your Edk2-PyTool-Extensions")
- return 0
-
- OutputReport = os.path.join(thebuilder.env.GetValue("BUILD_OUTPUT_BASE"), "BUILD_TOOLS_REPORT")
- OutputReport = os.path.normpath(OutputReport)
- if not os.path.isdir(os.path.dirname(OutputReport)):
- os.makedirs(os.path.dirname(OutputReport))
-
- Report = BuildToolsReport()
- Report.MakeReport(version_aggregator.GetVersionAggregator().GetAggregatedVersionInformation(), OutputReport=OutputReport)
-
- def do_pre_build(self, thebuilder):
- self.do_report(thebuilder)
- return 0
-
- def do_post_build(self, thebuilder):
- self.do_report(thebuilder)
- return 0
-
-except ImportError:
- pass
-
-
-class BuildToolsReport(object):
- MY_FOLDER = os.path.dirname(os.path.realpath(__file__))
- VERSION = "1.00"
-
- def __init__(self):
- pass
-
- def MakeReport(self, BuildTools, OutputReport="BuildToolsReport"):
- logging.info("Writing BuildToolsReports to {0}".format(OutputReport))
- versions_list = []
- for key, value in BuildTools.items():
- versions_list.append(value)
- versions_list = sorted(versions_list, key=lambda k: k['type'])
- json_dict = {"modules": versions_list,
- "PluginVersion": BuildToolsReport.VERSION}
-
- htmlfile = open(OutputReport + ".html", "w")
- jsonfile = open(OutputReport + ".json", "w")
- template = open(os.path.join(BuildToolsReport.MY_FOLDER, "BuildToolsReport_Template.html"), "r")
-
- for line in template.readlines():
- if "%TO_BE_FILLED_IN_BY_PYTHON_SCRIPT%" in line:
- line = line.replace("%TO_BE_FILLED_IN_BY_PYTHON_SCRIPT%", json.dumps(json_dict))
- htmlfile.write(line)
-
- jsonfile.write(json.dumps(versions_list, indent=4))
-
- jsonfile.close()
- template.close()
- htmlfile.close()
+## +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +import logging +import json + +try: + from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin + + class BuildToolsReportGenerator(IUefiBuildPlugin): + def do_report(self, thebuilder): + try: + from edk2toolext.environment import version_aggregator + except ImportError: + logging.critical("Loading BuildToolsReportGenerator failed, please update your Edk2-PyTool-Extensions") + return 0 + + OutputReport = os.path.join(thebuilder.env.GetValue("BUILD_OUTPUT_BASE"), "BUILD_TOOLS_REPORT") + OutputReport = os.path.normpath(OutputReport) + if not os.path.isdir(os.path.dirname(OutputReport)): + os.makedirs(os.path.dirname(OutputReport)) + + Report = BuildToolsReport() + Report.MakeReport(version_aggregator.GetVersionAggregator().GetAggregatedVersionInformation(), OutputReport=OutputReport) + + def do_pre_build(self, thebuilder): + self.do_report(thebuilder) + return 0 + + def do_post_build(self, thebuilder): + self.do_report(thebuilder) + return 0 + +except ImportError: + pass + + +class BuildToolsReport(object): + MY_FOLDER = os.path.dirname(os.path.realpath(__file__)) + VERSION = "1.00" + + def __init__(self): + pass + + def MakeReport(self, BuildTools, OutputReport="BuildToolsReport"): + logging.info("Writing BuildToolsReports to {0}".format(OutputReport)) + versions_list = [] + for key, value in BuildTools.items(): + versions_list.append(value) + versions_list = sorted(versions_list, key=lambda k: k['type']) + json_dict = {"modules": versions_list, + "PluginVersion": BuildToolsReport.VERSION} + + htmlfile = open(OutputReport + ".html", "w") + jsonfile = open(OutputReport + ".json", "w") + template = open(os.path.join(BuildToolsReport.MY_FOLDER, "BuildToolsReport_Template.html"), "r") + + for line in template.readlines(): + if "%TO_BE_FILLED_IN_BY_PYTHON_SCRIPT%" in line: + line = line.replace("%TO_BE_FILLED_IN_BY_PYTHON_SCRIPT%", json.dumps(json_dict)) + htmlfile.write(line) + + jsonfile.write(json.dumps(versions_list, indent=4)) + + jsonfile.close() + template.close() + htmlfile.close() diff --git a/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator_plug_in.yaml b/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator_plug_in.yaml index 8933535729..8656a49708 100644 --- a/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator_plug_in.yaml +++ b/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator_plug_in.yaml @@ -1,12 +1,12 @@ -## @file
-# Build Plugin used to output html report of all versions collected
-# during the build
-#
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-{
- "scope": "global",
- "name": "Build Tools Report Generator",
- "module": "BuildToolsReportGenerator"
-}
+## @file +# Build Plugin used to output html report of all versions collected +# during the build +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global", + "name": "Build Tools Report Generator", + "module": "BuildToolsReportGenerator" +} diff --git a/BaseTools/Plugin/BuildToolsReport/BuildToolsReport_Template.html b/BaseTools/Plugin/BuildToolsReport/BuildToolsReport_Template.html index 8273fdee49..a93c960ce3 100644 --- a/BaseTools/Plugin/BuildToolsReport/BuildToolsReport_Template.html +++ b/BaseTools/Plugin/BuildToolsReport/BuildToolsReport_Template.html @@ -1,126 +1,126 @@ -<!doctype html>
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible">
- <title>Build Tools Report</title>
- <meta name="viewport" content="width=device-width, initial-scale=1">
- <link rel="stylesheet" type="text/css" href="https://ajax.aspnetcdn.com/ajax/bootstrap/3.3.7/css/bootstrap.min.css" />
- <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.15/css/dataTables.bootstrap.min.css" />
- <style>
- div.attribution {
- border: 1px solid #ddd;
- background-color: #bbb;
- padding-left: 20px;
- }
- </style>
-</head>
-<body>
- <div class="container-fluid">
- <h1>Build Tools Report</h1>
- <ul class="nav nav-tabs">
- <li class="active"><a data-toggle="tab" href="#tabs-1">Tools</a></li>
- <li><a data-toggle="tab" href="#tabs-2">About</a></li>
- </ul>
- <div class="tab-content">
- <div id="tabs-1" class="tab-pane fade in active">
- <table id="modinfo" class="table table-striped table-bordered table-hover" cellspacing="0">
- <thead>
- <tr>
- <th>Key</th>
- <th>Value</th>
- <th>Type</th>
- </tr>
- </thead>
- <tbody></tbody>
- </table>
- </div>
- <div id="tabs-2" class="tab-pane">
- <div class="row">
- <div class="col-xs-7">
- <p></p>
- <p>
- Build Tools Report Template Version: <span id="ReportTemplateVersion">1.00</span><br />
- Build Tools Report Plugin Version: <span id='ReportToolVersion'></span><br />
- </p>
- <h3>License</h3>
- <hr />
- <div id="ToolLicenseContent">
- <p>
- <span class="copyright">Copyright (c) Microsoft Corporation.</span><br />
- <span class="license">
- SPDX-License-Identifier: BSD-2-Clause-Patent
- </span>
- </p>
- </div>
- </div>
- <div id="AttributionListWrapper" class="col-xs-5">
- <h3>External Licenses</h3>
- </div>
- </div>
- </div>
- </div>
- </div>
-
- <!-- Javascript libraries -->
- <script type="text/javascript" charset="utf8" src="https://ajax.aspnetcdn.com/ajax/jQuery/jquery-3.2.1.min.js"></script>
- <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.15/js/jquery.dataTables.min.js"></script>
- <script type="text/javascript" charset="utf8" src="https://ajax.aspnetcdn.com/ajax/bootstrap/3.3.7/bootstrap.min.js"></script>
- <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.15/js/dataTables.bootstrap.min.js"></script>
-
- <script>
- var EmbeddedJd = %TO_BE_FILLED_IN_BY_PYTHON_SCRIPT%;
- </script>
- <!-- Add javascript here -->
- <script>
- var MODULE_TABLE_OFFSET = 350; //Space needed for other stuff besides the Table
- $(document).ready(function () {
- $('span#ReportToolVersion').text(EmbeddedJd.PluginVersion);
- //To support tabs and correct column width we need this change
- $('a[data-toggle="tab"][href="#tabs-1"]').on('shown.bs.tab', function (e) {
- $.fn.dataTable.tables({ visible: true, api: true }).columns.adjust();
- });
- //table for modules
- var mTable = $('table#modinfo').dataTable({
- "aaData": EmbeddedJd.modules,
- "paginate": false,
- "autoWidth": false,
- "scrollY": ($(window).height() - MODULE_TABLE_OFFSET) + "px",
- "aaSorting": [[2, "asc"]],
- "aoColumnDefs": [
- {
- "mData": "name",
- "aTargets": [0]
- },
-
- {
- "mData": "version",
- "aTargets": [1]
- },
- {
- "mData": "type",
- "aTargets": [2],
- }
- ] //end of column def
- }); //end of modules table
-
- //
- // Create Attribution List for all external libraries used
- //
- [
- { Title: "JQuery", Copyright: "Copyright 2017 The jQuery Foundation", Version: $.fn.jquery, LicenseType: "MIT", LicenseLink: "https://jquery.org/license/" },
- { Title: "DataTables", Copyright: "DataTables designed and created by SpryMedia Ltd Copyright 2007-2017", Version: $.fn.dataTable.version, LicenseType: "MIT", LicenseLink: "https://datatables.net/license/mit" },
- { Title: "BootStrap", Copyright: "Code and documentation copyright 2011-2017 the Bootstrap Authors and Twitter, Inc.", Version: "3.3.7", LicenseType: "MIT", LicenseLink: "https://github.com/twbs/bootstrap/blob/master/LICENSE" }
- ].forEach(function (element) {
- $("<div class='attribution'><h4>" + element.Title + "</h4><p>Version: <span class='version'>" + element.Version + "</span><br /><span class='copyright'>" +
- element.Copyright + "</span><br />License: <a class='license' href='" + element.LicenseLink + "'>" + element.LicenseType + "</a></p></div>").appendTo("div#AttributionListWrapper");
- });
- });
- $(window).resize(function() {
- $.fn.dataTable.tables({ visible: true, api: true }).columns.adjust();
- });
-
-
- </script>
-</body>
-</html>
+<!doctype html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible"> + <title>Build Tools Report</title> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link rel="stylesheet" type="text/css" href="https://ajax.aspnetcdn.com/ajax/bootstrap/3.3.7/css/bootstrap.min.css" /> + <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.15/css/dataTables.bootstrap.min.css" /> + <style> + div.attribution { + border: 1px solid #ddd; + background-color: #bbb; + padding-left: 20px; + } + </style> +</head> +<body> + <div class="container-fluid"> + <h1>Build Tools Report</h1> + <ul class="nav nav-tabs"> + <li class="active"><a data-toggle="tab" href="#tabs-1">Tools</a></li> + <li><a data-toggle="tab" href="#tabs-2">About</a></li> + </ul> + <div class="tab-content"> + <div id="tabs-1" class="tab-pane fade in active"> + <table id="modinfo" class="table table-striped table-bordered table-hover" cellspacing="0"> + <thead> + <tr> + <th>Key</th> + <th>Value</th> + <th>Type</th> + </tr> + </thead> + <tbody></tbody> + </table> + </div> + <div id="tabs-2" class="tab-pane"> + <div class="row"> + <div class="col-xs-7"> + <p></p> + <p> + Build Tools Report Template Version: <span id="ReportTemplateVersion">1.00</span><br /> + Build Tools Report Plugin Version: <span id='ReportToolVersion'></span><br /> + </p> + <h3>License</h3> + <hr /> + <div id="ToolLicenseContent"> + <p> + <span class="copyright">Copyright (c) Microsoft Corporation.</span><br /> + <span class="license"> + SPDX-License-Identifier: BSD-2-Clause-Patent + </span> + </p> + </div> + </div> + <div id="AttributionListWrapper" class="col-xs-5"> + <h3>External Licenses</h3> + </div> + </div> + </div> + </div> + </div> + + <!-- Javascript libraries --> + <script type="text/javascript" charset="utf8" src="https://ajax.aspnetcdn.com/ajax/jQuery/jquery-3.2.1.min.js"></script> + <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.15/js/jquery.dataTables.min.js"></script> + <script type="text/javascript" charset="utf8" src="https://ajax.aspnetcdn.com/ajax/bootstrap/3.3.7/bootstrap.min.js"></script> + <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.15/js/dataTables.bootstrap.min.js"></script> + + <script> + var EmbeddedJd = %TO_BE_FILLED_IN_BY_PYTHON_SCRIPT%; + </script> + <!-- Add javascript here --> + <script> + var MODULE_TABLE_OFFSET = 350; //Space needed for other stuff besides the Table + $(document).ready(function () { + $('span#ReportToolVersion').text(EmbeddedJd.PluginVersion); + //To support tabs and correct column width we need this change + $('a[data-toggle="tab"][href="#tabs-1"]').on('shown.bs.tab', function (e) { + $.fn.dataTable.tables({ visible: true, api: true }).columns.adjust(); + }); + //table for modules + var mTable = $('table#modinfo').dataTable({ + "aaData": EmbeddedJd.modules, + "paginate": false, + "autoWidth": false, + "scrollY": ($(window).height() - MODULE_TABLE_OFFSET) + "px", + "aaSorting": [[2, "asc"]], + "aoColumnDefs": [ + { + "mData": "name", + "aTargets": [0] + }, + + { + "mData": "version", + "aTargets": [1] + }, + { + "mData": "type", + "aTargets": [2], + } + ] //end of column def + }); //end of modules table + + // + // Create Attribution List for all external libraries used + // + [ + { Title: "JQuery", Copyright: "Copyright 2017 The jQuery Foundation", Version: $.fn.jquery, LicenseType: "MIT", LicenseLink: "https://jquery.org/license/" }, + { Title: "DataTables", Copyright: "DataTables designed and created by SpryMedia Ltd Copyright 2007-2017", Version: $.fn.dataTable.version, LicenseType: "MIT", LicenseLink: "https://datatables.net/license/mit" }, + { Title: "BootStrap", Copyright: "Code and documentation copyright 2011-2017 the Bootstrap Authors and Twitter, Inc.", Version: "3.3.7", LicenseType: "MIT", LicenseLink: "https://github.com/twbs/bootstrap/blob/master/LICENSE" } + ].forEach(function (element) { + $("<div class='attribution'><h4>" + element.Title + "</h4><p>Version: <span class='version'>" + element.Version + "</span><br /><span class='copyright'>" + + element.Copyright + "</span><br />License: <a class='license' href='" + element.LicenseLink + "'>" + element.LicenseType + "</a></p></div>").appendTo("div#AttributionListWrapper"); + }); + }); + $(window).resize(function() { + $.fn.dataTable.tables({ visible: true, api: true }).columns.adjust(); + }); + + + </script> +</body> +</html> diff --git a/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py b/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py index 9734478f8b..761b964e58 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py +++ b/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py @@ -1,222 +1,222 @@ -# @file CodeQAnalyzePlugin.py
-#
-# A build plugin that analyzes a CodeQL database.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-import json
-import logging
-import os
-import yaml
-
-from analyze import analyze_filter
-from common import codeql_plugin
-
-from edk2toolext import edk2_logging
-from edk2toolext.environment.plugintypes.uefi_build_plugin import \
- IUefiBuildPlugin
-from edk2toolext.environment.uefi_build import UefiBuilder
-from edk2toollib.uefi.edk2.path_utilities import Edk2Path
-from edk2toollib.utility_functions import RunCmd
-from pathlib import Path
-
-
-class CodeQlAnalyzePlugin(IUefiBuildPlugin):
-
- def do_post_build(self, builder: UefiBuilder) -> int:
- """CodeQL analysis post-build functionality.
-
- Args:
- builder (UefiBuilder): A UEFI builder object for this build.
-
- Returns:
- int: The number of CodeQL errors found. Zero indicates that
- AuditOnly mode is enabled or no failures were found.
- """
- self.builder = builder
- self.package = builder.edk2path.GetContainingPackage(
- builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
- builder.env.GetValue("ACTIVE_PLATFORM")
- )
- )
-
- self.package_path = Path(
- builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
- self.package
- )
- )
- self.target = builder.env.GetValue("TARGET")
-
- self.codeql_db_path = codeql_plugin.get_codeql_db_path(
- builder.ws, self.package, self.target,
- new_path=False)
-
- self.codeql_path = codeql_plugin.get_codeql_cli_path()
- if not self.codeql_path:
- logging.critical("CodeQL build enabled but CodeQL CLI application "
- "not found.")
- return -1
-
- codeql_sarif_dir_path = self.codeql_db_path[
- :self.codeql_db_path.rindex('-')]
- codeql_sarif_dir_path = codeql_sarif_dir_path.replace(
- "-db-", "-analysis-")
- self.codeql_sarif_path = os.path.join(
- codeql_sarif_dir_path,
- (os.path.basename(
- self.codeql_db_path) +
- ".sarif"))
-
- edk2_logging.log_progress(f"Analyzing {self.package} ({self.target}) "
- f"CodeQL database at:\n"
- f" {self.codeql_db_path}")
- edk2_logging.log_progress(f"Results will be written to:\n"
- f" {self.codeql_sarif_path}")
-
- # Packages are allowed to specify package-specific query specifiers
- # in the package CI YAML file that override the global query specifier.
- audit_only = False
- query_specifiers = None
- package_config_file = Path(os.path.join(
- self.package_path, self.package + ".ci.yaml"))
- plugin_data = None
- if package_config_file.is_file():
- with open(package_config_file, 'r') as cf:
- package_config_file_data = yaml.safe_load(cf)
- if "CodeQlAnalyze" in package_config_file_data:
- plugin_data = package_config_file_data["CodeQlAnalyze"]
- if "AuditOnly" in plugin_data:
- audit_only = plugin_data["AuditOnly"]
- if "QuerySpecifiers" in plugin_data:
- logging.debug(f"Loading CodeQL query specifiers in "
- f"{str(package_config_file)}")
- query_specifiers = plugin_data["QuerySpecifiers"]
-
- global_audit_only = builder.env.GetValue("STUART_CODEQL_AUDIT_ONLY")
- if global_audit_only:
- if global_audit_only.strip().lower() == "true":
- audit_only = True
-
- if audit_only:
- logging.info(f"CodeQL Analyze plugin is in audit only mode for "
- f"{self.package} ({self.target}).")
-
- # Builds can override the query specifiers defined in this plugin
- # by setting the value in the STUART_CODEQL_QUERY_SPECIFIERS
- # environment variable.
- if not query_specifiers:
- query_specifiers = builder.env.GetValue(
- "STUART_CODEQL_QUERY_SPECIFIERS")
-
- # Use this plugins query set file as the default fallback if it is
- # not overridden. It is possible the file is not present if modified
- # locally. In that case, skip the plugin.
- plugin_query_set = Path(Path(__file__).parent, "CodeQlQueries.qls")
-
- if not query_specifiers and plugin_query_set.is_file():
- query_specifiers = str(plugin_query_set.resolve())
-
- if not query_specifiers:
- logging.warning("Skipping CodeQL analysis since no CodeQL query "
- "specifiers were provided.")
- return 0
-
- codeql_params = (f'database analyze {self.codeql_db_path} '
- f'{query_specifiers} --format=sarifv2.1.0 '
- f'--output={self.codeql_sarif_path} --download '
- f'--threads=0')
-
- # CodeQL requires the sarif file parent directory to exist already.
- Path(self.codeql_sarif_path).parent.mkdir(exist_ok=True, parents=True)
-
- cmd_ret = RunCmd(self.codeql_path, codeql_params)
- if cmd_ret != 0:
- logging.critical(f"CodeQL CLI analysis failed with return code "
- f"{cmd_ret}.")
-
- if not os.path.isfile(self.codeql_sarif_path):
- logging.critical(f"The sarif file {self.codeql_sarif_path} was "
- f"not created. Analysis cannot continue.")
- return -1
-
- filter_pattern_data = []
- global_filter_file_value = builder.env.GetValue(
- "STUART_CODEQL_FILTER_FILES")
- if global_filter_file_value:
- global_filter_files = global_filter_file_value.strip().split(',')
- global_filter_files = [Path(f) for f in global_filter_files]
-
- for global_filter_file in global_filter_files:
- if global_filter_file.is_file():
- with open(global_filter_file, 'r') as ff:
- global_filter_file_data = yaml.safe_load(ff)
- if "Filters" in global_filter_file_data:
- current_pattern_data = \
- global_filter_file_data["Filters"]
- if type(current_pattern_data) is not list:
- logging.critical(
- f"CodeQL pattern data must be a list of "
- f"strings. Data in "
- f"{str(global_filter_file.resolve())} is "
- f"invalid. CodeQL analysis is incomplete.")
- return -1
- filter_pattern_data += current_pattern_data
- else:
- logging.critical(
- f"CodeQL global filter file "
- f"{str(global_filter_file.resolve())} is "
- f"malformed. Missing Filters section. CodeQL "
- f"analysis is incomplete.")
- return -1
- else:
- logging.critical(
- f"CodeQL global filter file "
- f"{str(global_filter_file.resolve())} was not found. "
- f"CodeQL analysis is incomplete.")
- return -1
-
- if plugin_data and "Filters" in plugin_data:
- if type(plugin_data["Filters"]) is not list:
- logging.critical(
- "CodeQL pattern data must be a list of strings. "
- "CodeQL analysis is incomplete.")
- return -1
- filter_pattern_data.extend(plugin_data["Filters"])
-
- if filter_pattern_data:
- logging.info("Applying CodeQL SARIF result filters.")
- analyze_filter.filter_sarif(
- self.codeql_sarif_path,
- self.codeql_sarif_path,
- filter_pattern_data,
- split_lines=False)
-
- with open(self.codeql_sarif_path, 'r') as sf:
- sarif_file_data = json.load(sf)
-
- try:
- # Perform minimal JSON parsing to find the number of errors.
- total_errors = 0
- for run in sarif_file_data['runs']:
- total_errors += len(run['results'])
- except KeyError:
- logging.critical("Sarif file does not contain expected data. "
- "Analysis cannot continue.")
- return -1
-
- if total_errors > 0:
- if audit_only:
- # Show a warning message so CodeQL analysis is not forgotten.
- # If the repo owners truly do not want to fix CodeQL issues,
- # analysis should be disabled entirely.
- logging.warning(f"{self.package} ({self.target}) CodeQL "
- f"analysis ignored {total_errors} errors due "
- f"to audit mode being enabled.")
- return 0
- else:
- logging.error(f"{self.package} ({self.target}) CodeQL "
- f"analysis failed with {total_errors} errors.")
-
- return total_errors
+# @file CodeQAnalyzePlugin.py +# +# A build plugin that analyzes a CodeQL database. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import json +import logging +import os +import yaml + +from analyze import analyze_filter +from common import codeql_plugin + +from edk2toolext import edk2_logging +from edk2toolext.environment.plugintypes.uefi_build_plugin import \ + IUefiBuildPlugin +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toollib.uefi.edk2.path_utilities import Edk2Path +from edk2toollib.utility_functions import RunCmd +from pathlib import Path + + +class CodeQlAnalyzePlugin(IUefiBuildPlugin): + + def do_post_build(self, builder: UefiBuilder) -> int: + """CodeQL analysis post-build functionality. + + Args: + builder (UefiBuilder): A UEFI builder object for this build. + + Returns: + int: The number of CodeQL errors found. Zero indicates that + AuditOnly mode is enabled or no failures were found. + """ + self.builder = builder + self.package = builder.edk2path.GetContainingPackage( + builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath( + builder.env.GetValue("ACTIVE_PLATFORM") + ) + ) + + self.package_path = Path( + builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath( + self.package + ) + ) + self.target = builder.env.GetValue("TARGET") + + self.codeql_db_path = codeql_plugin.get_codeql_db_path( + builder.ws, self.package, self.target, + new_path=False) + + self.codeql_path = codeql_plugin.get_codeql_cli_path() + if not self.codeql_path: + logging.critical("CodeQL build enabled but CodeQL CLI application " + "not found.") + return -1 + + codeql_sarif_dir_path = self.codeql_db_path[ + :self.codeql_db_path.rindex('-')] + codeql_sarif_dir_path = codeql_sarif_dir_path.replace( + "-db-", "-analysis-") + self.codeql_sarif_path = os.path.join( + codeql_sarif_dir_path, + (os.path.basename( + self.codeql_db_path) + + ".sarif")) + + edk2_logging.log_progress(f"Analyzing {self.package} ({self.target}) " + f"CodeQL database at:\n" + f" {self.codeql_db_path}") + edk2_logging.log_progress(f"Results will be written to:\n" + f" {self.codeql_sarif_path}") + + # Packages are allowed to specify package-specific query specifiers + # in the package CI YAML file that override the global query specifier. + audit_only = False + query_specifiers = None + package_config_file = Path(os.path.join( + self.package_path, self.package + ".ci.yaml")) + plugin_data = None + if package_config_file.is_file(): + with open(package_config_file, 'r') as cf: + package_config_file_data = yaml.safe_load(cf) + if "CodeQlAnalyze" in package_config_file_data: + plugin_data = package_config_file_data["CodeQlAnalyze"] + if "AuditOnly" in plugin_data: + audit_only = plugin_data["AuditOnly"] + if "QuerySpecifiers" in plugin_data: + logging.debug(f"Loading CodeQL query specifiers in " + f"{str(package_config_file)}") + query_specifiers = plugin_data["QuerySpecifiers"] + + global_audit_only = builder.env.GetValue("STUART_CODEQL_AUDIT_ONLY") + if global_audit_only: + if global_audit_only.strip().lower() == "true": + audit_only = True + + if audit_only: + logging.info(f"CodeQL Analyze plugin is in audit only mode for " + f"{self.package} ({self.target}).") + + # Builds can override the query specifiers defined in this plugin + # by setting the value in the STUART_CODEQL_QUERY_SPECIFIERS + # environment variable. + if not query_specifiers: + query_specifiers = builder.env.GetValue( + "STUART_CODEQL_QUERY_SPECIFIERS") + + # Use this plugins query set file as the default fallback if it is + # not overridden. It is possible the file is not present if modified + # locally. In that case, skip the plugin. + plugin_query_set = Path(Path(__file__).parent, "CodeQlQueries.qls") + + if not query_specifiers and plugin_query_set.is_file(): + query_specifiers = str(plugin_query_set.resolve()) + + if not query_specifiers: + logging.warning("Skipping CodeQL analysis since no CodeQL query " + "specifiers were provided.") + return 0 + + codeql_params = (f'database analyze {self.codeql_db_path} ' + f'{query_specifiers} --format=sarifv2.1.0 ' + f'--output={self.codeql_sarif_path} --download ' + f'--threads=0') + + # CodeQL requires the sarif file parent directory to exist already. + Path(self.codeql_sarif_path).parent.mkdir(exist_ok=True, parents=True) + + cmd_ret = RunCmd(self.codeql_path, codeql_params) + if cmd_ret != 0: + logging.critical(f"CodeQL CLI analysis failed with return code " + f"{cmd_ret}.") + + if not os.path.isfile(self.codeql_sarif_path): + logging.critical(f"The sarif file {self.codeql_sarif_path} was " + f"not created. Analysis cannot continue.") + return -1 + + filter_pattern_data = [] + global_filter_file_value = builder.env.GetValue( + "STUART_CODEQL_FILTER_FILES") + if global_filter_file_value: + global_filter_files = global_filter_file_value.strip().split(',') + global_filter_files = [Path(f) for f in global_filter_files] + + for global_filter_file in global_filter_files: + if global_filter_file.is_file(): + with open(global_filter_file, 'r') as ff: + global_filter_file_data = yaml.safe_load(ff) + if "Filters" in global_filter_file_data: + current_pattern_data = \ + global_filter_file_data["Filters"] + if type(current_pattern_data) is not list: + logging.critical( + f"CodeQL pattern data must be a list of " + f"strings. Data in " + f"{str(global_filter_file.resolve())} is " + f"invalid. CodeQL analysis is incomplete.") + return -1 + filter_pattern_data += current_pattern_data + else: + logging.critical( + f"CodeQL global filter file " + f"{str(global_filter_file.resolve())} is " + f"malformed. Missing Filters section. CodeQL " + f"analysis is incomplete.") + return -1 + else: + logging.critical( + f"CodeQL global filter file " + f"{str(global_filter_file.resolve())} was not found. " + f"CodeQL analysis is incomplete.") + return -1 + + if plugin_data and "Filters" in plugin_data: + if type(plugin_data["Filters"]) is not list: + logging.critical( + "CodeQL pattern data must be a list of strings. " + "CodeQL analysis is incomplete.") + return -1 + filter_pattern_data.extend(plugin_data["Filters"]) + + if filter_pattern_data: + logging.info("Applying CodeQL SARIF result filters.") + analyze_filter.filter_sarif( + self.codeql_sarif_path, + self.codeql_sarif_path, + filter_pattern_data, + split_lines=False) + + with open(self.codeql_sarif_path, 'r') as sf: + sarif_file_data = json.load(sf) + + try: + # Perform minimal JSON parsing to find the number of errors. + total_errors = 0 + for run in sarif_file_data['runs']: + total_errors += len(run['results']) + except KeyError: + logging.critical("Sarif file does not contain expected data. " + "Analysis cannot continue.") + return -1 + + if total_errors > 0: + if audit_only: + # Show a warning message so CodeQL analysis is not forgotten. + # If the repo owners truly do not want to fix CodeQL issues, + # analysis should be disabled entirely. + logging.warning(f"{self.package} ({self.target}) CodeQL " + f"analysis ignored {total_errors} errors due " + f"to audit mode being enabled.") + return 0 + else: + logging.error(f"{self.package} ({self.target}) CodeQL " + f"analysis failed with {total_errors} errors.") + + return total_errors diff --git a/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml b/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml index ec01e55c53..e3cac3a2e5 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml +++ b/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml @@ -1,13 +1,13 @@ -## @file CodeQlAnalyze_plug_in.py
-#
-# Build plugin used to analyze CodeQL results.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-analyze",
- "name": "CodeQL Analyze Plugin",
- "module": "CodeQlAnalyzePlugin"
-}
+## @file CodeQlAnalyze_plug_in.py +# +# Build plugin used to analyze CodeQL results. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-analyze", + "name": "CodeQL Analyze Plugin", + "module": "CodeQlAnalyzePlugin" +} diff --git a/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py b/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py index 2f6c928c21..06898dfec6 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py +++ b/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py @@ -1,169 +1,169 @@ -# @file CodeQlBuildPlugin.py
-#
-# A build plugin that produces CodeQL results for the present build.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-import glob
-import logging
-import os
-import stat
-from common import codeql_plugin
-from pathlib import Path
-
-from edk2toolext import edk2_logging
-from edk2toolext.environment.plugintypes.uefi_build_plugin import \
- IUefiBuildPlugin
-from edk2toolext.environment.uefi_build import UefiBuilder
-from edk2toollib.uefi.edk2.path_utilities import Edk2Path
-from edk2toollib.utility_functions import GetHostInfo, RemoveTree
-
-
-class CodeQlBuildPlugin(IUefiBuildPlugin):
-
- def do_pre_build(self, builder: UefiBuilder) -> int:
- """CodeQL pre-build functionality.
-
- Args:
- builder (UefiBuilder): A UEFI builder object for this build.
-
- Returns:
- int: The plugin return code. Zero indicates the plugin ran
- successfully. A non-zero value indicates an unexpected error
- occurred during plugin execution.
- """
-
- if not builder.SkipBuild:
- self.builder = builder
- self.package = builder.edk2path.GetContainingPackage(
- builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
- builder.env.GetValue("ACTIVE_PLATFORM")
- )
- )
-
- self.target = builder.env.GetValue("TARGET")
-
- self.build_output_dir = builder.env.GetValue("BUILD_OUTPUT_BASE")
-
- self.codeql_db_path = codeql_plugin.get_codeql_db_path(
- builder.ws, self.package, self.target)
-
- edk2_logging.log_progress(f"{self.package} will be built for CodeQL")
- edk2_logging.log_progress(f" CodeQL database will be written to "
- f"{self.codeql_db_path}")
-
- self.codeql_path = codeql_plugin.get_codeql_cli_path()
- if not self.codeql_path:
- logging.critical("CodeQL build enabled but CodeQL CLI application "
- "not found.")
- return -1
-
- # CodeQL can only generate a database on clean build
- #
- # Note: builder.CleanTree() cannot be used here as some platforms
- # have build steps that run before this plugin that store
- # files in the build output directory.
- #
- # CodeQL does not care about with those files or many others such
- # as the FV directory, build logs, etc. so instead focus on
- # removing only the directories with compilation/linker output
- # for the architectures being built (that need clean runs for
- # CodeQL to work).
- targets = self.builder.env.GetValue("TARGET_ARCH").split(" ")
- for target in targets:
- directory_to_delete = Path(self.build_output_dir, target)
-
- if directory_to_delete.is_dir():
- logging.debug(f"Removing {str(directory_to_delete)} to have a "
- f"clean build for CodeQL.")
- RemoveTree(str(directory_to_delete))
-
- # CodeQL CLI does not handle spaces passed in CLI commands well
- # (perhaps at all) as discussed here:
- # 1. https://github.com/github/codeql-cli-binaries/issues/73
- # 2. https://github.com/github/codeql/issues/4910
- #
- # Since it's unclear how quotes are handled and may change in the
- # future, this code is going to use the workaround to place the
- # command in an executable file that is instead passed to CodeQL.
- self.codeql_cmd_path = Path(self.build_output_dir, "codeql_build_command")
-
- build_params = self._get_build_params()
-
- codeql_build_cmd = ""
- if GetHostInfo().os == "Windows":
- self.codeql_cmd_path = self.codeql_cmd_path.parent / (
- self.codeql_cmd_path.name + '.bat')
- elif GetHostInfo().os == "Linux":
- self.codeql_cmd_path = self.codeql_cmd_path.parent / (
- self.codeql_cmd_path.name + '.sh')
- codeql_build_cmd += f"#!/bin/bash{os.linesep * 2}"
- codeql_build_cmd += "build " + build_params
-
- self.codeql_cmd_path.parent.mkdir(exist_ok=True, parents=True)
- self.codeql_cmd_path.write_text(encoding='utf8', data=codeql_build_cmd)
-
- if GetHostInfo().os == "Linux":
- os.chmod(self.codeql_cmd_path,
- os.stat(self.codeql_cmd_path).st_mode | stat.S_IEXEC)
- for f in glob.glob(os.path.join(
- os.path.dirname(self.codeql_path), '**/*'), recursive=True):
- os.chmod(f, os.stat(f).st_mode | stat.S_IEXEC)
-
- codeql_params = (f'database create {self.codeql_db_path} '
- f'--language=cpp '
- f'--source-root={builder.ws} '
- f'--command={self.codeql_cmd_path}')
-
- # Set environment variables so the CodeQL build command is picked up
- # as the active build command.
- #
- # Note: Requires recent changes in edk2-pytool-extensions (0.20.0)
- # to support reading these variables.
- builder.env.SetValue(
- "EDK_BUILD_CMD", self.codeql_path, "Set in CodeQL Build Plugin")
- builder.env.SetValue(
- "EDK_BUILD_PARAMS", codeql_params, "Set in CodeQL Build Plugin")
-
- return 0
-
- def _get_build_params(self) -> str:
- """Returns the build command parameters for this build.
-
- Based on the well-defined `build` command-line parameters.
-
- Returns:
- str: A string representing the parameters for the build command.
- """
- build_params = f"-p {self.builder.env.GetValue('ACTIVE_PLATFORM')}"
- build_params += f" -b {self.target}"
- build_params += f" -t {self.builder.env.GetValue('TOOL_CHAIN_TAG')}"
-
- max_threads = self.builder.env.GetValue('MAX_CONCURRENT_THREAD_NUMBER')
- if max_threads is not None:
- build_params += f" -n {max_threads}"
-
- rt = self.builder.env.GetValue("TARGET_ARCH").split(" ")
- for t in rt:
- build_params += " -a " + t
-
- if (self.builder.env.GetValue("BUILDREPORTING") == "TRUE"):
- build_params += (" -y " +
- self.builder.env.GetValue("BUILDREPORT_FILE"))
- rt = self.builder.env.GetValue("BUILDREPORT_TYPES").split(" ")
- for t in rt:
- build_params += " -Y " + t
-
- # add special processing to handle building a single module
- mod = self.builder.env.GetValue("BUILDMODULE")
- if (mod is not None and len(mod.strip()) > 0):
- build_params += " -m " + mod
- edk2_logging.log_progress("Single Module Build: " + mod)
-
- build_vars = self.builder.env.GetAllBuildKeyValues(self.target)
- for key, value in build_vars.items():
- build_params += " -D " + key + "=" + value
-
- return build_params
+# @file CodeQlBuildPlugin.py +# +# A build plugin that produces CodeQL results for the present build. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import glob +import logging +import os +import stat +from common import codeql_plugin +from pathlib import Path + +from edk2toolext import edk2_logging +from edk2toolext.environment.plugintypes.uefi_build_plugin import \ + IUefiBuildPlugin +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toollib.uefi.edk2.path_utilities import Edk2Path +from edk2toollib.utility_functions import GetHostInfo, RemoveTree + + +class CodeQlBuildPlugin(IUefiBuildPlugin): + + def do_pre_build(self, builder: UefiBuilder) -> int: + """CodeQL pre-build functionality. + + Args: + builder (UefiBuilder): A UEFI builder object for this build. + + Returns: + int: The plugin return code. Zero indicates the plugin ran + successfully. A non-zero value indicates an unexpected error + occurred during plugin execution. + """ + + if not builder.SkipBuild: + self.builder = builder + self.package = builder.edk2path.GetContainingPackage( + builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath( + builder.env.GetValue("ACTIVE_PLATFORM") + ) + ) + + self.target = builder.env.GetValue("TARGET") + + self.build_output_dir = builder.env.GetValue("BUILD_OUTPUT_BASE") + + self.codeql_db_path = codeql_plugin.get_codeql_db_path( + builder.ws, self.package, self.target) + + edk2_logging.log_progress(f"{self.package} will be built for CodeQL") + edk2_logging.log_progress(f" CodeQL database will be written to " + f"{self.codeql_db_path}") + + self.codeql_path = codeql_plugin.get_codeql_cli_path() + if not self.codeql_path: + logging.critical("CodeQL build enabled but CodeQL CLI application " + "not found.") + return -1 + + # CodeQL can only generate a database on clean build + # + # Note: builder.CleanTree() cannot be used here as some platforms + # have build steps that run before this plugin that store + # files in the build output directory. + # + # CodeQL does not care about with those files or many others such + # as the FV directory, build logs, etc. so instead focus on + # removing only the directories with compilation/linker output + # for the architectures being built (that need clean runs for + # CodeQL to work). + targets = self.builder.env.GetValue("TARGET_ARCH").split(" ") + for target in targets: + directory_to_delete = Path(self.build_output_dir, target) + + if directory_to_delete.is_dir(): + logging.debug(f"Removing {str(directory_to_delete)} to have a " + f"clean build for CodeQL.") + RemoveTree(str(directory_to_delete)) + + # CodeQL CLI does not handle spaces passed in CLI commands well + # (perhaps at all) as discussed here: + # 1. https://github.com/github/codeql-cli-binaries/issues/73 + # 2. https://github.com/github/codeql/issues/4910 + # + # Since it's unclear how quotes are handled and may change in the + # future, this code is going to use the workaround to place the + # command in an executable file that is instead passed to CodeQL. + self.codeql_cmd_path = Path(self.build_output_dir, "codeql_build_command") + + build_params = self._get_build_params() + + codeql_build_cmd = "" + if GetHostInfo().os == "Windows": + self.codeql_cmd_path = self.codeql_cmd_path.parent / ( + self.codeql_cmd_path.name + '.bat') + elif GetHostInfo().os == "Linux": + self.codeql_cmd_path = self.codeql_cmd_path.parent / ( + self.codeql_cmd_path.name + '.sh') + codeql_build_cmd += f"#!/bin/bash{os.linesep * 2}" + codeql_build_cmd += "build " + build_params + + self.codeql_cmd_path.parent.mkdir(exist_ok=True, parents=True) + self.codeql_cmd_path.write_text(encoding='utf8', data=codeql_build_cmd) + + if GetHostInfo().os == "Linux": + os.chmod(self.codeql_cmd_path, + os.stat(self.codeql_cmd_path).st_mode | stat.S_IEXEC) + for f in glob.glob(os.path.join( + os.path.dirname(self.codeql_path), '**/*'), recursive=True): + os.chmod(f, os.stat(f).st_mode | stat.S_IEXEC) + + codeql_params = (f'database create {self.codeql_db_path} ' + f'--language=cpp ' + f'--source-root={builder.ws} ' + f'--command={self.codeql_cmd_path}') + + # Set environment variables so the CodeQL build command is picked up + # as the active build command. + # + # Note: Requires recent changes in edk2-pytool-extensions (0.20.0) + # to support reading these variables. + builder.env.SetValue( + "EDK_BUILD_CMD", self.codeql_path, "Set in CodeQL Build Plugin") + builder.env.SetValue( + "EDK_BUILD_PARAMS", codeql_params, "Set in CodeQL Build Plugin") + + return 0 + + def _get_build_params(self) -> str: + """Returns the build command parameters for this build. + + Based on the well-defined `build` command-line parameters. + + Returns: + str: A string representing the parameters for the build command. + """ + build_params = f"-p {self.builder.env.GetValue('ACTIVE_PLATFORM')}" + build_params += f" -b {self.target}" + build_params += f" -t {self.builder.env.GetValue('TOOL_CHAIN_TAG')}" + + max_threads = self.builder.env.GetValue('MAX_CONCURRENT_THREAD_NUMBER') + if max_threads is not None: + build_params += f" -n {max_threads}" + + rt = self.builder.env.GetValue("TARGET_ARCH").split(" ") + for t in rt: + build_params += " -a " + t + + if (self.builder.env.GetValue("BUILDREPORTING") == "TRUE"): + build_params += (" -y " + + self.builder.env.GetValue("BUILDREPORT_FILE")) + rt = self.builder.env.GetValue("BUILDREPORT_TYPES").split(" ") + for t in rt: + build_params += " -Y " + t + + # add special processing to handle building a single module + mod = self.builder.env.GetValue("BUILDMODULE") + if (mod is not None and len(mod.strip()) > 0): + build_params += " -m " + mod + edk2_logging.log_progress("Single Module Build: " + mod) + + build_vars = self.builder.env.GetAllBuildKeyValues(self.target) + for key, value in build_vars.items(): + build_params += " -D " + key + "=" + value + + return build_params diff --git a/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml b/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml index 13baa58d0c..25a35ad27a 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml +++ b/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml @@ -1,13 +1,13 @@ -## @file CodeQlBuild_plug_in.py
-#
-# Build plugin used to produce a CodeQL database from a build.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-build",
- "name": "CodeQL Build Plugin",
- "module": "CodeQlBuildPlugin"
-}
+## @file CodeQlBuild_plug_in.py +# +# Build plugin used to produce a CodeQL database from a build. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-build", + "name": "CodeQL Build Plugin", + "module": "CodeQlBuildPlugin" +} diff --git a/BaseTools/Plugin/CodeQL/CodeQlQueries.qls b/BaseTools/Plugin/CodeQL/CodeQlQueries.qls index 1a50983221..84db69f86a 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlQueries.qls +++ b/BaseTools/Plugin/CodeQL/CodeQlQueries.qls @@ -1,118 +1,118 @@ ----
-- description: C++ queries
-
-- queries: '.'
- from: codeql/cpp-queries
-
-##########################################################################################
-# Queries
-##########################################################################################
-
-## Errors
-- include:
- id: cpp/badoverflowguard
-- include:
- id: cpp/infiniteloop
-- include:
- id: cpp/likely-bugs/memory-management/v2/conditionally-uninitialized-variable
-- include:
- id: cpp/missing-null-test
-- include:
- id: cpp/missing-return
-- include:
- id: cpp/no-space-for-terminator
-- include:
- id: cpp/pointer-overflow-check
-- include:
- id: cpp/redundant-null-check-simple
-- include:
- id: cpp/sizeof/const-int-argument
-- include:
- id: cpp/sizeof/sizeof-or-operation-as-argument
-- include:
- id: cpp/unguardednullreturndereferenc
-- include:
- id: cpp/very-likely-overrunning-write
-
-## Warnings
-- include:
- id: cpp/comparison-with-wider-type
-- include:
- id: cpp/conditionallyuninitializedvariable
-- include:
- id: cpp/comparison-precedence
-- include:
- id: cpp/implicit-bitfield-downcast
-- include:
- id: cpp/infinite-loop-with-unsatisfiable-exit-condition
-- include:
- id: cpp/offset-use-before-range-check
-- include:
- id: cpp/overflow-buffer
-- include:
- id: cpp/overflow-calculated
-- include:
- id: cpp/overflow-destination
-- include:
- id: cpp/paddingbyteinformationdisclosure
-- include:
- id: cpp/return-stack-allocated-memory
-- include:
- id: cpp/static-buffer-overflow
-- include:
- id: cpp/unsigned-comparison-zero
-- include:
- id: cpp/uselesstest
-
-## Recommendations
-- include:
- id: cpp/missing-header-guard
-- include:
- id: cpp/unused-local-variable
-- include:
- id: cpp/unused-static-function
-- include:
- id: cpp/unused-static-variable
-
-# Note: Some queries above are not active by default with the below filter.
-# Update the filter and run the queries again to get all results.
-- include:
- tags:
- - "security"
- - "correctness"
- severity:
- - "error"
- - "warning"
- - "recommendation"
-
-# Specifically hide the results of these.
-#
-# The following rules have been evaluated and explicitly not included for the following reasons:
-# - `cpp/allocation-too-small` - Appears to be hardcoded for C standard library functions `malloc`, `calloc`,
-# `realloc`, so it consumes time without much value with custom allocation functions in the codebase.
-# - `cpp/commented-out-code` - Triggers often. Needs further review.
-# - `cpp/duplicate-include-guard` - The <Phase>EntryPoint.h files includes a common include guard value
-# `__MODULE_ENTRY_POINT_H__`. This was the only occurrence found. So not very useful.
-# - `cpp/invalid-pointer-deref` - Very limited results with what appear to be false positives.
-# - `cpp/use-of-goto` - Goto is valid and allowed in the codebase.
-# - `cpp/useless-expression` - Triggers too often on cases where a NULL lib implementation is provided for a function.
-# Because the implementation simply returns, the check considers it useless.
-# - `cpp/weak-crypto/*` - Crypto algorithms are tracked outside CodeQL.
-- exclude:
- id: cpp/allocation-too-small
-- exclude:
- id: cpp/commented-out-code
-- exclude:
- id: cpp/duplicate-include-guard
-- exclude:
- id: cpp/invalid-pointer-deref
-- exclude:
- id: cpp/use-of-goto
-- exclude:
- id: cpp/useless-expression
-- exclude:
- id: cpp/weak-crypto/banned-hash-algorithms
-- exclude:
- id: cpp/weak-crypto/capi/banned-modes
-- exclude:
- id: cpp/weak-crypto/openssl/banned-hash-algorithms
+--- +- description: C++ queries + +- queries: '.' + from: codeql/cpp-queries + +########################################################################################## +# Queries +########################################################################################## + +## Errors +- include: + id: cpp/badoverflowguard +- include: + id: cpp/infiniteloop +- include: + id: cpp/likely-bugs/memory-management/v2/conditionally-uninitialized-variable +- include: + id: cpp/missing-null-test +- include: + id: cpp/missing-return +- include: + id: cpp/no-space-for-terminator +- include: + id: cpp/pointer-overflow-check +- include: + id: cpp/redundant-null-check-simple +- include: + id: cpp/sizeof/const-int-argument +- include: + id: cpp/sizeof/sizeof-or-operation-as-argument +- include: + id: cpp/unguardednullreturndereferenc +- include: + id: cpp/very-likely-overrunning-write + +## Warnings +- include: + id: cpp/comparison-with-wider-type +- include: + id: cpp/conditionallyuninitializedvariable +- include: + id: cpp/comparison-precedence +- include: + id: cpp/implicit-bitfield-downcast +- include: + id: cpp/infinite-loop-with-unsatisfiable-exit-condition +- include: + id: cpp/offset-use-before-range-check +- include: + id: cpp/overflow-buffer +- include: + id: cpp/overflow-calculated +- include: + id: cpp/overflow-destination +- include: + id: cpp/paddingbyteinformationdisclosure +- include: + id: cpp/return-stack-allocated-memory +- include: + id: cpp/static-buffer-overflow +- include: + id: cpp/unsigned-comparison-zero +- include: + id: cpp/uselesstest + +## Recommendations +- include: + id: cpp/missing-header-guard +- include: + id: cpp/unused-local-variable +- include: + id: cpp/unused-static-function +- include: + id: cpp/unused-static-variable + +# Note: Some queries above are not active by default with the below filter. +# Update the filter and run the queries again to get all results. +- include: + tags: + - "security" + - "correctness" + severity: + - "error" + - "warning" + - "recommendation" + +# Specifically hide the results of these. +# +# The following rules have been evaluated and explicitly not included for the following reasons: +# - `cpp/allocation-too-small` - Appears to be hardcoded for C standard library functions `malloc`, `calloc`, +# `realloc`, so it consumes time without much value with custom allocation functions in the codebase. +# - `cpp/commented-out-code` - Triggers often. Needs further review. +# - `cpp/duplicate-include-guard` - The <Phase>EntryPoint.h files includes a common include guard value +# `__MODULE_ENTRY_POINT_H__`. This was the only occurrence found. So not very useful. +# - `cpp/invalid-pointer-deref` - Very limited results with what appear to be false positives. +# - `cpp/use-of-goto` - Goto is valid and allowed in the codebase. +# - `cpp/useless-expression` - Triggers too often on cases where a NULL lib implementation is provided for a function. +# Because the implementation simply returns, the check considers it useless. +# - `cpp/weak-crypto/*` - Crypto algorithms are tracked outside CodeQL. +- exclude: + id: cpp/allocation-too-small +- exclude: + id: cpp/commented-out-code +- exclude: + id: cpp/duplicate-include-guard +- exclude: + id: cpp/invalid-pointer-deref +- exclude: + id: cpp/use-of-goto +- exclude: + id: cpp/useless-expression +- exclude: + id: cpp/weak-crypto/banned-hash-algorithms +- exclude: + id: cpp/weak-crypto/capi/banned-modes +- exclude: + id: cpp/weak-crypto/openssl/banned-hash-algorithms diff --git a/BaseTools/Plugin/CodeQL/Readme.md b/BaseTools/Plugin/CodeQL/Readme.md index 18587e2b25..046c28e057 100644 --- a/BaseTools/Plugin/CodeQL/Readme.md +++ b/BaseTools/Plugin/CodeQL/Readme.md @@ -1,388 +1,388 @@ -# CodeQL Plugin
-
-The set of CodeQL plugins provided include two main plugins that seamlessly integrate into a Stuart build environment:
-
-1. `CodeQlBuildPlugin` - Used to produce a CodeQL database from a build.
-2. `CodeQlAnalyzePlugin` - Used to analyze a CodeQL database.
-
-While CodeQL can be run in a CI environment with other approaches. This plugin offers the following advantages:
-
-1. Provides exactly the same results locally as on a CI server.
-2. Integrates very well into VS Code.
-3. Very simple to use - just use normal Stuart update and build commands.
-4. Very simple to understand - minimally wraps the official CodeQL CLI.
-5. Very simple to integrate - works like any other Stuart build plugin.
- - Integration is usually just a few lines of code.
-6. Portable - not tied to Azure DevOps specific, GitHub specific, or other host infrastructure.
-7. Versioned - the query and filters are versioned in source control so easy to find and track.
-
-It is very important to read the Integration Instructions in this file and determine how to best integrate the
-CodeQL plugin into your environment.
-
-Due to the total size of dependencies required to run CodeQL and the flexibility needed by a platform to determine what
-CodeQL queries to run and how to interpret results, a number of configuration options are provided to allow a high
-degree of flexibility during platform integration.
-
-This document is focused on those setting up the CodeQL plugin in their environment. Once setup, end users simply need
-to use their normal build commands and process and CodeQL will be integrated with it. The most relevant section for
-such users is [Local Development Tips](#local-development-tips).
-
-## Table of Contents
-
-1. [Database and Analysis Result Locations](#database-and-analysis-result-locations)
-2. [Global Configuration](#global-configuration)
-3. [Package-Specific Configuration](#package-specific-configuration)
-4. [Filter Patterns](#filter-patterns)
-5. [Integration Instructions](#integration-instructions)
- - [Integration Step 1 - Choose Scopes](#integration-step-1---choose-scopes)
- - [Scopes Available](#scopes-available)
- - [Integration Step 2 - Choose CodeQL Queries](#integration-step-2---choose-codeql-queries)
- - [Integration Step 3 - Determine Global Configuration Values](#integration-step-3---determine-global-configuration-values)
- - [Integration Step 4 - Determine Package-Specific Configuration Values](#integration-step-4---determine-package-specific-configuration-values)
- - [Integration Step 5 - Testing](#integration-step-5---testing)
- - [Integration Step 6 - Define Inclusion and Exclusion Filter Patterns](#integration-step-6---define-inclusion-and-exclusion-filter-patterns)
-6. [High-Level Operation](#high-level-operation)
- - [CodeQlBuildPlugin](#codeqlbuildplugin)
- - [CodeQlAnalyzePlugin](#codeqlanalyzeplugin)
-7. [Local Development Tips](#local-development-tips)
-8. [Resolution Guidelines](#resolution-guidelines)
-
-## Database and Analysis Result Locations
-
-The CodeQL database is written to a directory unique to the package and target being built:
-
- `Build/codeql-db-<package>-<target>-<instance>`
-
-For example: `Build/codeql-db-mdemodulepkg-debug-0`
-
-The plugin does not delete or overwrite existing databases, the instance value is simply increased. This is
-because databases are large, take a long time to generate, and are important for reproducing analysis results. The user
-is responsible for deleting database directories when they are no longer needed.
-
-Similarly, analysis results are written to a directory unique to the package and target. For analysis, results are
-stored in individual files so those files are stored in a single directory.
-
-For example, all analysis results for the above package and target will be stored in:
- `codeql-analysis-mdemodulepkg-debug`
-
-CodeQL results are stored in [SARIF](https://sarifweb.azurewebsites.net/) (Static Analysis Results Interchange Format)
-([CodeQL SARIF documentation](https://codeql.github.com/docs/codeql-cli/sarif-output/)) files. Each SARIF file
-corresponding to a database will be stored in a file with an instance matching the database instance.
-
-For example, the analysis result file for the above database would be stored in this file:
- `codeql-analysis-mdemodulepkg-debug/codeql-db-mdemodulepkg-debug-0.sarif`
-
-Result files are overwritten. This is because result files are quick to generate and need to represent the latest
-results for the last analysis operation performed. The user is responsible for backing up SARIF result files if they
-need to saved.
-
-## Global Configuration
-
-Global configuration values are specified with build environment variables.
-
-These values are all optional. They provide a convenient mechanism for a build script to set the value for all packages
-built by the script.
-
-- `STUART_CODEQL_AUDIT_ONLY` - If `true` (case insensitive), `CodeQlAnalyzePlugin` will be in audit-only mode. In this
- mode all CodeQL failures are ignored.
-- `STUART_CODEQL_PATH` - The path to the CodeQL CLI application to use.
-- `STUART_CODEQL_QUERY_SPECIFIERS` - The CodeQL CLI query specifiers to use. See [Running codeql database analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-with-the-codeql-cli/#running-codeql-database-analyze)
- for possible options.
-- `STUART_CODEQL_FILTER_FILES` - The path to "filter" files that contains filter patterns as described in
- [Filter Patterns](#filter-patterns).
- - More than one file may be specified by separating each absolute file path with a comma.
- - This might be useful to reference a global filter file from an upstream repo and also include a global filter
- file for the local repo.
- - Filters are concatenated in the order of files in the variable. Patterns in later files can override patterns
- in earlier files.
- - The file only needs to contain a list of filter pattern strings under a `"Filters"` key. For example:
-
- ```yaml
- {
- "Filters": [
- "<pattern-line-1>",
- "<pattern-line-2>"
- ]
- }
- ...
- ```
-
- Comments are allowed in the filter files and begin with `#` (like a normal YAML file).
-
-## Package-Specific Configuration
-
-Package-specific configuration values reuse existing package-level configuration approaches to simplify adjusting
-CodeQL plugin behavior per package.
-
-These values are all optional. They provide a convenient mechanism for a package owner to adjust settings specific to
-the package.
-
-``` yaml
- "CodeQlAnalyze": {
- "AuditOnly": False, # Don't fail the build if there are errors. Just log them.
- "QuerySpecifiers": "" # Query specifiers to pass to CodeQL CLI.
- "Filters": "" # Inclusion/exclusion filters
- }
-```
-
-> _NOTE:_ If a global filter set is provided via `STUART_CODEQL_FILTER_FILES` and a package has a package-specific
-> list, then the package-specific filter list (in a package CI YAML file) is appended onto the global filter list and
-> may be used to override settings in the global list.
-
-The format used to specify items in `"Filters"` is specified in [Filter Patterns](#filter-patterns).
-
-## Filter Patterns
-
-As you inspect results, you may want to include or exclude certain sets of results. For example, exclude some files by
-file path entirely or adjust the CodeQL rule applied to a certain file. This plugin reuses logic from a popular
-GitHub Action called [`filter-sarif`](https://github.com/advanced-security/filter-sarif) to allow filtering as part of
-the plugin analysis process.
-
-If any results are excluded using filters, the results are removed from the SARIF file. This allows the exclude results
-seen locally to exactly match the results on the CI server.
-
-Read the ["Patterns"](https://github.com/advanced-security/filter-sarif#patterns) section there for more details. The
-patterns section is also copied below with some updates to make the information more relevant for an edk2 codebase
-for convenience.
-
-Each pattern line is of the form:
-
-```plaintext
-[+/-]<file pattern>[:<rule pattern>]
-```
-
-For example:
-
-```yaml
--**/*Test*.c:** # exclusion pattern: remove all alerts from all test files
--**/*Test*.c # ditto, short form of the line above
-+**/*.c:cpp/infiniteloop # inclusion pattern: This line has precedence over the first two
- # and thus "allow lists" alerts of type "cpp/infiniteloop"
-**/*.c:cpp/infiniteloop # ditto, the "+" in inclusion patterns is optional
-** # allow all alerts in all files (reverses all previous lines)
-```
-
-- The path separator character in patterns is always `/`, independent of the platform the code is running on and
- independent of the paths in the SARIF file.
-- `*` matches any character, except a path separator
-- `**` matches any character and is only allowed between path separators, e.g. `/**/file.txt`, `**/file.txt` or `**`.
- NOT allowed: `**.txt`, `/etc**`
-- The rule pattern is optional. If omitted, it will apply to alerts of all types.
-- Subsequent lines override earlier ones. By default all alerts are included.
-- If you need to use the literals `+`, `-`, `\` or `:` in your pattern, you can escape them with `\`, e.g.
- `\-this/is/an/inclusion/file/pattern\:with-a-semicolon:and/a/rule/pattern/with/a/\\/backslash`. For `+` and `-`, this
- is only necessary if they appear at the beginning of the pattern line.
-
-## Integration Instructions
-
-First, note that most CodeQL CLI operations will take a long time the first time they are run. This is due to:
-
-1. Downloads - Downloading the CodeQL CLI binary (during `stuart_update`) and downloading CodeQL queries during
- CodeQL plugin execution
-2. Cache not established - CodeQL CLI caches data as it performs analysis. The first time analysis is performed will
- take more time than in the future.
-
-Second, these are build plugins. This means a build needs to take place for the plugins to run. This typically happens
-in the following two scenarios:
-
-1. `stuart_build` - A single package is built and the build process is started by the stuart tools.
-2. `stuart_ci_build` - A number of packages may be built and the build process is started by the `CompilerPlugin`.
-
-In any case, each time a package is built, the CodeQL plugins will be run if their scopes are active.
-
-### Integration Step 1 - Choose Scopes
-
-Decide which scopes need to be enabled in your platform, see [Scopes Available](#scopes-available).
-
-Consider using a build profile to enable CodeQL so developers and pipelines can use the profile when they are
-interested in CodeQL results but in other cases they can easily work without CodeQL in the way.
-
-Furthermore, build-script specific command-line parameters might be useful to control CodeQL scopes and other
-behavior.
-
-#### Scopes Available
-
-This CodeQL plugin leverages scopes to control major pieces of functionality. Any combination of scopes can be
-returned from the `GetActiveScopes()` function in the platform settings manager to add and remove functionality.
-
-Plugin scopes:
-
-- `codeql-analyze` - Activate `CodeQlAnalyzePlugin` to perform post-build analysis of the last generated database for
- the package and target specified.
-- `codeql-build` - Activate `CodeQlBuildPlugin` to hook the firmware build in pre-build such that the build will
- generate a CodeQL database during build.
-
-In most cases, to perform a full CodeQL run, `codeql-build` should be enabled so a new CodeQL database is generated
-during build and `codeql-analyze` should be be enabled so analysis of that database is performed after the build is
-completed.
-
-External dependency scopes:
-
-- `codeql-ext-dep` - Downloads the cross-platform CodeQL CLI as an external dependency.
-- `codeql-linux-ext-dep` - Downloads the Linux CodeQL CLI as an external dependency.
-- `codeql-windows-ext-dep` - Downloads the Windows CodeQL CLI as an external dependency.
-
-Note, that the CodeQL CLI is large in size. Sizes as of the [v2.11.2 release](https://github.com/github/codeql-cli-binaries/releases/tag/v2.11.2).
-
-| Cross-platform | Linux | Windows |
-|:--------------:|:------:|:-------:|
-| 934 MB | 415 MB | 290 MB |
-
-Therefore, the following is recommended:
-
-1. **Ideal** - Create container images for build agents and install the CodeQL CLI for the container OS into the
- container.
-2. Leverage host-OS detection (e.g. [`GetHostInfo()`](https://github.com/tianocore/edk2-pytool-library/blob/42ad6561af73ba34564f1577f64f7dbaf1d0a5a2/edk2toollib/utility_functions.py#L112))
-to set the scope for the appropriate operating system. This will download the much smaller OS-specific application.
-
-> _NOTE:_ You should never have more than one CodeQL external dependency scope enabled at a time.
-
-### Integration Step 2 - Choose CodeQL Queries
-
-Determine which queries need to be run against packages in your repo. In most cases, the same set of queries will be
-run against all packages. It is also possible to customize the queries run at the package level.
-
-The default set of Project Mu CodeQL queries is specified in the `MuCodeQlQueries.qls` file in this plugin.
-
-> _NOTE:_ The queries in `MuCodeQlQueries.qls` may change at any time. If you do not want these changes to impact
-> your platform, do not relay on option (3).
-
-The plugin decides what queries to run based on the following, in order of preference:
-
-1. Package CI YAML file query specifier
-2. Build environment variable query specifier
-3. Plugin default query set file
-
-For details on how to set (1) and (2), see the Package CI Configuration and Environment Variable sections respectively.
-
-> _NOTE:_ The value specified is directly passed as a `query specifier` to CodeQL CLI. Therefore, the arguments
-> allowed by the `<query-specifiers>` argument of CodeQL CLI are allowed here. See
-> [Running codeql database analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-with-the-codeql-cli/#running-codeql-database-analyze).
-
-A likely scenario is that a platform needs to run local/closed source queries in addition to the open-source queries.
-There's various ways to handle that:
-
-1. Create a query specifier that includes all the queries needed, both public and private and use that query specifier,
- either globally or at package-level.
-
- For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` = _"Absolute_path_to_AllMyQueries.qls"_
-
-2. Specify a query specifier that includes the closed sources queries and reuse the public query list provided by
- this plugin.
-
- For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` = _"Absolute_path_to_MuCodeQlQueries.qls
- Absolute_path_to_ClosedSourceQueries.qls"_
-
-Refer to the CodeQL documentation noted above on query specifiers to devise other options.
-
-### Integration Step 3 - Determine Global Configuration Values
-
-Review the Environment Variable section to determine which, if any, global values need to be set in your build script.
-
-### Integration Step 4 - Determine Package-Specific Configuration Values
-
-Review the Package CI Configuration section to determine which, if any, global values need to be set in your
-package's CI YAML file.
-
-### Integration Step 5 - Testing
-
-Verify a `stuart_update` and `stuart_build` (or `stuart_ci_build`) command work.
-
-### Integration Step 6 - Define Inclusion and Exclusion Filter Patterns
-
-After reviewing the test results from Step 5, determine if you need to apply any filters as described in
-[Filter Patterns](#filter-patterns).
-
-## High-Level Operation
-
-This section summarizes the complete CodeQL plugin flow. This is to help developers understand basic theory of
-operation behind the plugin and can be skipped by anyone not interested in those details.
-
-### CodeQlBuildPlugin
-
-1. Register a pre-build hook
-2. Determine the package and target being built
-3. Determine the best CodeQL CLI path to use
- - First choice, the `STUART_CODEQL_PATH` environment variable
- - Note: This is set by the CodeQL CLI external dependency if that is used
- - Second choice, `codeql` as found on the system path
-4. Determine the directory name for the CodeQL database
- - Format: `Build/codeql-db-<package>-<target>-<instance>`
-5. Clean the build directory of the active platform and target
- - CodeQL database generation only works on clean builds
-6. Ensure the "build" step is not skipped as a build is needed to generate a CodeQL database
-7. Build a CodeQL file that wraps around the edk2 build
- - Written to the package build directory
- - Example: `Build/MdeModulePkg/VS2022/codeql_build_command.bat`
-8. Set the variables necessary for stuart to call CodeQL CLI during the build phase
- - Sets `EDK_BUILD_CMD` and `EDK_BUILD_PARAMS`
-
-### CodeQlAnalyzePlugin
-
-1. Register a post-build hook
-2. Determine the package and target being built
-3. Determine the best CodeQL CLI path to use
- - First choice, the `STUART_CODEQL_PATH` environment variable
- - Note: This is set by the CodeQL CLI external dependency if that is used
- - Second choice, `codeql` as found on the system path
-4. Determine the directory name for the most recent CodeQL database
- - Format: `Build/codeql-db-<package>-<target>-<instance>`
-5. Determine plugin audit status for the given package and target
- - Check if `AuditOnly` is enabled either globally or for the package
-6. Determine the CodeQL query specifiers to use for the given package and target
- - First choice, the package CI YAML file value
- - Second choice, the `STUART_CODEQL_QUERY_SPECIFIERS`
- - Third choice, use `CodeQlQueries.qls` (in the plugin directory)
-7. Run CodeQL CLI to perform database analysis
-8. Parse the analysis SARIF file to determine the number of CodeQL failures
-9. Return the number of failures (or zero if `AuditOnly` is enabled)
-
-## Local Development Tips
-
-This section contains helpful tips to expedite common scenarios when working with CodeQL locally.
-
-1. Pre-build, Build, and Post-Build
-
- Generating a database requires the pre-build and build steps. Analyzing a database requires the post-build step.
-
- Therefore, if you are making tweaks that don't affect the build, such as modifying the CodeQL queries used or level
- of severity reported, you can save time by skipping pre-build and post-build (e.g. `--skipprebuild` and
- `--skipbuild`).
-
-2. Scopes
-
- Similar to (1), add/remove `codeql-build` and `codeql-analyze` from the active scopes to save time depending on what
- you are trying to do.
-
- If you are focusing on coding, remove the code CodeQL scopes if they are active. If you are ready to check your
- changes against CodeQL, simply add the scopes back. It is recommended to use build profiles to do this more
- conveniently.
-
- If you already have CodeQL CLI enabled, you can remove the `codeql-ext-dep` scope locally. The build will use the
- `codeql` command on your path.
-
-3. CodeQL Output is in the CI Build Log
-
- To see exactly which queries CodeQL ran or why it might be taking longer than expected, look in the CI build log
- (i.e. `Build/CI_BUILDLOG.txt`) where the CodeQL CLI application output is written.
-
- Search for the text you see in the progress output (e.g. "Analyzing _MdeModulePkg_ (_DEBUG_) CodeQL database at")
- to jump to the section of the log just before the CodeQL CLI is invoked.
-
-4. Use a SARIF Viewer to Read Results
-
-The [SARIF Viewer extension for VS Code](https://marketplace.visualstudio.com/items?itemName=MS-SarifVSCode.sarif-viewer)
-can open the .sarif file generated by this plugin and allow you to click links directly to the problem area in source
-files.
-
-## Resolution Guidelines
-
-This section captures brief guidelines to keep in mind while resolving CodeQL issues.
-
-1. Look at surrounding code. Changes should always take into account the context of nearby code. The new logic may
- need to account conditions not immediately obvious based on the issue alone. It is easy to focus only on the line
- of code highlighted by CodeQL and miss the code's role in the big picture.
-2. A CodeQL alert may be benign but the code can be refactored to prevent the alert. Often refactoring the code makes
- the code intention clearer and avoids an unnecessary exception.
-3. Consider adding unit tests while making CodeQL fixes especially for commonly used code and code with a high volume
- of CodeQL alerts.
+# CodeQL Plugin + +The set of CodeQL plugins provided include two main plugins that seamlessly integrate into a Stuart build environment: + +1. `CodeQlBuildPlugin` - Used to produce a CodeQL database from a build. +2. `CodeQlAnalyzePlugin` - Used to analyze a CodeQL database. + +While CodeQL can be run in a CI environment with other approaches. This plugin offers the following advantages: + +1. Provides exactly the same results locally as on a CI server. +2. Integrates very well into VS Code. +3. Very simple to use - just use normal Stuart update and build commands. +4. Very simple to understand - minimally wraps the official CodeQL CLI. +5. Very simple to integrate - works like any other Stuart build plugin. + - Integration is usually just a few lines of code. +6. Portable - not tied to Azure DevOps specific, GitHub specific, or other host infrastructure. +7. Versioned - the query and filters are versioned in source control so easy to find and track. + +It is very important to read the Integration Instructions in this file and determine how to best integrate the +CodeQL plugin into your environment. + +Due to the total size of dependencies required to run CodeQL and the flexibility needed by a platform to determine what +CodeQL queries to run and how to interpret results, a number of configuration options are provided to allow a high +degree of flexibility during platform integration. + +This document is focused on those setting up the CodeQL plugin in their environment. Once setup, end users simply need +to use their normal build commands and process and CodeQL will be integrated with it. The most relevant section for +such users is [Local Development Tips](#local-development-tips). + +## Table of Contents + +1. [Database and Analysis Result Locations](#database-and-analysis-result-locations) +2. [Global Configuration](#global-configuration) +3. [Package-Specific Configuration](#package-specific-configuration) +4. [Filter Patterns](#filter-patterns) +5. [Integration Instructions](#integration-instructions) + - [Integration Step 1 - Choose Scopes](#integration-step-1---choose-scopes) + - [Scopes Available](#scopes-available) + - [Integration Step 2 - Choose CodeQL Queries](#integration-step-2---choose-codeql-queries) + - [Integration Step 3 - Determine Global Configuration Values](#integration-step-3---determine-global-configuration-values) + - [Integration Step 4 - Determine Package-Specific Configuration Values](#integration-step-4---determine-package-specific-configuration-values) + - [Integration Step 5 - Testing](#integration-step-5---testing) + - [Integration Step 6 - Define Inclusion and Exclusion Filter Patterns](#integration-step-6---define-inclusion-and-exclusion-filter-patterns) +6. [High-Level Operation](#high-level-operation) + - [CodeQlBuildPlugin](#codeqlbuildplugin) + - [CodeQlAnalyzePlugin](#codeqlanalyzeplugin) +7. [Local Development Tips](#local-development-tips) +8. [Resolution Guidelines](#resolution-guidelines) + +## Database and Analysis Result Locations + +The CodeQL database is written to a directory unique to the package and target being built: + + `Build/codeql-db-<package>-<target>-<instance>` + +For example: `Build/codeql-db-mdemodulepkg-debug-0` + +The plugin does not delete or overwrite existing databases, the instance value is simply increased. This is +because databases are large, take a long time to generate, and are important for reproducing analysis results. The user +is responsible for deleting database directories when they are no longer needed. + +Similarly, analysis results are written to a directory unique to the package and target. For analysis, results are +stored in individual files so those files are stored in a single directory. + +For example, all analysis results for the above package and target will be stored in: + `codeql-analysis-mdemodulepkg-debug` + +CodeQL results are stored in [SARIF](https://sarifweb.azurewebsites.net/) (Static Analysis Results Interchange Format) +([CodeQL SARIF documentation](https://codeql.github.com/docs/codeql-cli/sarif-output/)) files. Each SARIF file +corresponding to a database will be stored in a file with an instance matching the database instance. + +For example, the analysis result file for the above database would be stored in this file: + `codeql-analysis-mdemodulepkg-debug/codeql-db-mdemodulepkg-debug-0.sarif` + +Result files are overwritten. This is because result files are quick to generate and need to represent the latest +results for the last analysis operation performed. The user is responsible for backing up SARIF result files if they +need to saved. + +## Global Configuration + +Global configuration values are specified with build environment variables. + +These values are all optional. They provide a convenient mechanism for a build script to set the value for all packages +built by the script. + +- `STUART_CODEQL_AUDIT_ONLY` - If `true` (case insensitive), `CodeQlAnalyzePlugin` will be in audit-only mode. In this + mode all CodeQL failures are ignored. +- `STUART_CODEQL_PATH` - The path to the CodeQL CLI application to use. +- `STUART_CODEQL_QUERY_SPECIFIERS` - The CodeQL CLI query specifiers to use. See [Running codeql database analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-with-the-codeql-cli/#running-codeql-database-analyze) + for possible options. +- `STUART_CODEQL_FILTER_FILES` - The path to "filter" files that contains filter patterns as described in + [Filter Patterns](#filter-patterns). + - More than one file may be specified by separating each absolute file path with a comma. + - This might be useful to reference a global filter file from an upstream repo and also include a global filter + file for the local repo. + - Filters are concatenated in the order of files in the variable. Patterns in later files can override patterns + in earlier files. + - The file only needs to contain a list of filter pattern strings under a `"Filters"` key. For example: + + ```yaml + { + "Filters": [ + "<pattern-line-1>", + "<pattern-line-2>" + ] + } + ... + ``` + + Comments are allowed in the filter files and begin with `#` (like a normal YAML file). + +## Package-Specific Configuration + +Package-specific configuration values reuse existing package-level configuration approaches to simplify adjusting +CodeQL plugin behavior per package. + +These values are all optional. They provide a convenient mechanism for a package owner to adjust settings specific to +the package. + +``` yaml + "CodeQlAnalyze": { + "AuditOnly": False, # Don't fail the build if there are errors. Just log them. + "QuerySpecifiers": "" # Query specifiers to pass to CodeQL CLI. + "Filters": "" # Inclusion/exclusion filters + } +``` + +> _NOTE:_ If a global filter set is provided via `STUART_CODEQL_FILTER_FILES` and a package has a package-specific +> list, then the package-specific filter list (in a package CI YAML file) is appended onto the global filter list and +> may be used to override settings in the global list. + +The format used to specify items in `"Filters"` is specified in [Filter Patterns](#filter-patterns). + +## Filter Patterns + +As you inspect results, you may want to include or exclude certain sets of results. For example, exclude some files by +file path entirely or adjust the CodeQL rule applied to a certain file. This plugin reuses logic from a popular +GitHub Action called [`filter-sarif`](https://github.com/advanced-security/filter-sarif) to allow filtering as part of +the plugin analysis process. + +If any results are excluded using filters, the results are removed from the SARIF file. This allows the exclude results +seen locally to exactly match the results on the CI server. + +Read the ["Patterns"](https://github.com/advanced-security/filter-sarif#patterns) section there for more details. The +patterns section is also copied below with some updates to make the information more relevant for an edk2 codebase +for convenience. + +Each pattern line is of the form: + +```plaintext +[+/-]<file pattern>[:<rule pattern>] +``` + +For example: + +```yaml +-**/*Test*.c:** # exclusion pattern: remove all alerts from all test files +-**/*Test*.c # ditto, short form of the line above ++**/*.c:cpp/infiniteloop # inclusion pattern: This line has precedence over the first two + # and thus "allow lists" alerts of type "cpp/infiniteloop" +**/*.c:cpp/infiniteloop # ditto, the "+" in inclusion patterns is optional +** # allow all alerts in all files (reverses all previous lines) +``` + +- The path separator character in patterns is always `/`, independent of the platform the code is running on and + independent of the paths in the SARIF file. +- `*` matches any character, except a path separator +- `**` matches any character and is only allowed between path separators, e.g. `/**/file.txt`, `**/file.txt` or `**`. + NOT allowed: `**.txt`, `/etc**` +- The rule pattern is optional. If omitted, it will apply to alerts of all types. +- Subsequent lines override earlier ones. By default all alerts are included. +- If you need to use the literals `+`, `-`, `\` or `:` in your pattern, you can escape them with `\`, e.g. + `\-this/is/an/inclusion/file/pattern\:with-a-semicolon:and/a/rule/pattern/with/a/\\/backslash`. For `+` and `-`, this + is only necessary if they appear at the beginning of the pattern line. + +## Integration Instructions + +First, note that most CodeQL CLI operations will take a long time the first time they are run. This is due to: + +1. Downloads - Downloading the CodeQL CLI binary (during `stuart_update`) and downloading CodeQL queries during + CodeQL plugin execution +2. Cache not established - CodeQL CLI caches data as it performs analysis. The first time analysis is performed will + take more time than in the future. + +Second, these are build plugins. This means a build needs to take place for the plugins to run. This typically happens +in the following two scenarios: + +1. `stuart_build` - A single package is built and the build process is started by the stuart tools. +2. `stuart_ci_build` - A number of packages may be built and the build process is started by the `CompilerPlugin`. + +In any case, each time a package is built, the CodeQL plugins will be run if their scopes are active. + +### Integration Step 1 - Choose Scopes + +Decide which scopes need to be enabled in your platform, see [Scopes Available](#scopes-available). + +Consider using a build profile to enable CodeQL so developers and pipelines can use the profile when they are +interested in CodeQL results but in other cases they can easily work without CodeQL in the way. + +Furthermore, build-script specific command-line parameters might be useful to control CodeQL scopes and other +behavior. + +#### Scopes Available + +This CodeQL plugin leverages scopes to control major pieces of functionality. Any combination of scopes can be +returned from the `GetActiveScopes()` function in the platform settings manager to add and remove functionality. + +Plugin scopes: + +- `codeql-analyze` - Activate `CodeQlAnalyzePlugin` to perform post-build analysis of the last generated database for + the package and target specified. +- `codeql-build` - Activate `CodeQlBuildPlugin` to hook the firmware build in pre-build such that the build will + generate a CodeQL database during build. + +In most cases, to perform a full CodeQL run, `codeql-build` should be enabled so a new CodeQL database is generated +during build and `codeql-analyze` should be be enabled so analysis of that database is performed after the build is +completed. + +External dependency scopes: + +- `codeql-ext-dep` - Downloads the cross-platform CodeQL CLI as an external dependency. +- `codeql-linux-ext-dep` - Downloads the Linux CodeQL CLI as an external dependency. +- `codeql-windows-ext-dep` - Downloads the Windows CodeQL CLI as an external dependency. + +Note, that the CodeQL CLI is large in size. Sizes as of the [v2.11.2 release](https://github.com/github/codeql-cli-binaries/releases/tag/v2.11.2). + +| Cross-platform | Linux | Windows | +|:--------------:|:------:|:-------:| +| 934 MB | 415 MB | 290 MB | + +Therefore, the following is recommended: + +1. **Ideal** - Create container images for build agents and install the CodeQL CLI for the container OS into the + container. +2. Leverage host-OS detection (e.g. [`GetHostInfo()`](https://github.com/tianocore/edk2-pytool-library/blob/42ad6561af73ba34564f1577f64f7dbaf1d0a5a2/edk2toollib/utility_functions.py#L112)) +to set the scope for the appropriate operating system. This will download the much smaller OS-specific application. + +> _NOTE:_ You should never have more than one CodeQL external dependency scope enabled at a time. + +### Integration Step 2 - Choose CodeQL Queries + +Determine which queries need to be run against packages in your repo. In most cases, the same set of queries will be +run against all packages. It is also possible to customize the queries run at the package level. + +The default set of Project Mu CodeQL queries is specified in the `MuCodeQlQueries.qls` file in this plugin. + +> _NOTE:_ The queries in `MuCodeQlQueries.qls` may change at any time. If you do not want these changes to impact +> your platform, do not relay on option (3). + +The plugin decides what queries to run based on the following, in order of preference: + +1. Package CI YAML file query specifier +2. Build environment variable query specifier +3. Plugin default query set file + +For details on how to set (1) and (2), see the Package CI Configuration and Environment Variable sections respectively. + +> _NOTE:_ The value specified is directly passed as a `query specifier` to CodeQL CLI. Therefore, the arguments +> allowed by the `<query-specifiers>` argument of CodeQL CLI are allowed here. See +> [Running codeql database analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-with-the-codeql-cli/#running-codeql-database-analyze). + +A likely scenario is that a platform needs to run local/closed source queries in addition to the open-source queries. +There's various ways to handle that: + +1. Create a query specifier that includes all the queries needed, both public and private and use that query specifier, + either globally or at package-level. + + For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` = _"Absolute_path_to_AllMyQueries.qls"_ + +2. Specify a query specifier that includes the closed sources queries and reuse the public query list provided by + this plugin. + + For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` = _"Absolute_path_to_MuCodeQlQueries.qls + Absolute_path_to_ClosedSourceQueries.qls"_ + +Refer to the CodeQL documentation noted above on query specifiers to devise other options. + +### Integration Step 3 - Determine Global Configuration Values + +Review the Environment Variable section to determine which, if any, global values need to be set in your build script. + +### Integration Step 4 - Determine Package-Specific Configuration Values + +Review the Package CI Configuration section to determine which, if any, global values need to be set in your +package's CI YAML file. + +### Integration Step 5 - Testing + +Verify a `stuart_update` and `stuart_build` (or `stuart_ci_build`) command work. + +### Integration Step 6 - Define Inclusion and Exclusion Filter Patterns + +After reviewing the test results from Step 5, determine if you need to apply any filters as described in +[Filter Patterns](#filter-patterns). + +## High-Level Operation + +This section summarizes the complete CodeQL plugin flow. This is to help developers understand basic theory of +operation behind the plugin and can be skipped by anyone not interested in those details. + +### CodeQlBuildPlugin + +1. Register a pre-build hook +2. Determine the package and target being built +3. Determine the best CodeQL CLI path to use + - First choice, the `STUART_CODEQL_PATH` environment variable + - Note: This is set by the CodeQL CLI external dependency if that is used + - Second choice, `codeql` as found on the system path +4. Determine the directory name for the CodeQL database + - Format: `Build/codeql-db-<package>-<target>-<instance>` +5. Clean the build directory of the active platform and target + - CodeQL database generation only works on clean builds +6. Ensure the "build" step is not skipped as a build is needed to generate a CodeQL database +7. Build a CodeQL file that wraps around the edk2 build + - Written to the package build directory + - Example: `Build/MdeModulePkg/VS2022/codeql_build_command.bat` +8. Set the variables necessary for stuart to call CodeQL CLI during the build phase + - Sets `EDK_BUILD_CMD` and `EDK_BUILD_PARAMS` + +### CodeQlAnalyzePlugin + +1. Register a post-build hook +2. Determine the package and target being built +3. Determine the best CodeQL CLI path to use + - First choice, the `STUART_CODEQL_PATH` environment variable + - Note: This is set by the CodeQL CLI external dependency if that is used + - Second choice, `codeql` as found on the system path +4. Determine the directory name for the most recent CodeQL database + - Format: `Build/codeql-db-<package>-<target>-<instance>` +5. Determine plugin audit status for the given package and target + - Check if `AuditOnly` is enabled either globally or for the package +6. Determine the CodeQL query specifiers to use for the given package and target + - First choice, the package CI YAML file value + - Second choice, the `STUART_CODEQL_QUERY_SPECIFIERS` + - Third choice, use `CodeQlQueries.qls` (in the plugin directory) +7. Run CodeQL CLI to perform database analysis +8. Parse the analysis SARIF file to determine the number of CodeQL failures +9. Return the number of failures (or zero if `AuditOnly` is enabled) + +## Local Development Tips + +This section contains helpful tips to expedite common scenarios when working with CodeQL locally. + +1. Pre-build, Build, and Post-Build + + Generating a database requires the pre-build and build steps. Analyzing a database requires the post-build step. + + Therefore, if you are making tweaks that don't affect the build, such as modifying the CodeQL queries used or level + of severity reported, you can save time by skipping pre-build and post-build (e.g. `--skipprebuild` and + `--skipbuild`). + +2. Scopes + + Similar to (1), add/remove `codeql-build` and `codeql-analyze` from the active scopes to save time depending on what + you are trying to do. + + If you are focusing on coding, remove the code CodeQL scopes if they are active. If you are ready to check your + changes against CodeQL, simply add the scopes back. It is recommended to use build profiles to do this more + conveniently. + + If you already have CodeQL CLI enabled, you can remove the `codeql-ext-dep` scope locally. The build will use the + `codeql` command on your path. + +3. CodeQL Output is in the CI Build Log + + To see exactly which queries CodeQL ran or why it might be taking longer than expected, look in the CI build log + (i.e. `Build/CI_BUILDLOG.txt`) where the CodeQL CLI application output is written. + + Search for the text you see in the progress output (e.g. "Analyzing _MdeModulePkg_ (_DEBUG_) CodeQL database at") + to jump to the section of the log just before the CodeQL CLI is invoked. + +4. Use a SARIF Viewer to Read Results + +The [SARIF Viewer extension for VS Code](https://marketplace.visualstudio.com/items?itemName=MS-SarifVSCode.sarif-viewer) +can open the .sarif file generated by this plugin and allow you to click links directly to the problem area in source +files. + +## Resolution Guidelines + +This section captures brief guidelines to keep in mind while resolving CodeQL issues. + +1. Look at surrounding code. Changes should always take into account the context of nearby code. The new logic may + need to account conditions not immediately obvious based on the issue alone. It is easy to focus only on the line + of code highlighted by CodeQL and miss the code's role in the big picture. +2. A CodeQL alert may be benign but the code can be refactored to prevent the alert. Often refactoring the code makes + the code intention clearer and avoids an unnecessary exception. +3. Consider adding unit tests while making CodeQL fixes especially for commonly used code and code with a high volume + of CodeQL alerts. diff --git a/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py b/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py index f363dd378f..e0e9dfbc55 100644 --- a/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py +++ b/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py @@ -1,184 +1,184 @@ -# @file analyze_filter.py
-#
-# Filters results in a SARIF file.
-#
-# Apache License
-# Version 2.0, January 2004
-# http://www.apache.org/licenses/
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This file has been altered from its original form. Based on code in:
-# https://github.com/advanced-security/filter-sarif
-#
-# It primarily contains modifications made to integrate with the CodeQL plugin.
-#
-# Specifically:
-# https://github.com/advanced-security/filter-sarif/blob/main/filter_sarif.py
-#
-# View the full and complete license as provided by that repository here:
-# https://github.com/advanced-security/filter-sarif/blob/main/LICENSE
-#
-# SPDX-License-Identifier: Apache-2.0
-##
-
-import json
-import logging
-import re
-from os import PathLike
-from typing import Iterable, List, Tuple
-
-from analyze.globber import match
-
-
-def _match_path_and_rule(
- path: str, rule: str, patterns: Iterable[str]) -> bool:
- """Returns whether a given path matches a given rule.
-
- Args:
- path (str): A file path string.
- rule (str): A rule file path string.
- patterns (Iterable[str]): An iterable of pattern strings.
-
- Returns:
- bool: True if the path matches a rule. Otherwise, False.
- """
- result = True
- for s, fp, rp in patterns:
- if match(rp, rule) and match(fp, path):
- result = s
- return result
-
-
-def _parse_pattern(line: str) -> Tuple[str]:
- """Parses a given pattern line.
-
- Args:
- line (str): The line string that contains the rule.
-
- Returns:
- Tuple[str]: The parsed sign, file pattern, and rule pattern from the
- line.
- """
- sep_char = ':'
- esc_char = '\\'
- file_pattern = ''
- rule_pattern = ''
- seen_separator = False
- sign = True
-
- # inclusion or exclusion pattern?
- u_line = line
- if line:
- if line[0] == '-':
- sign = False
- u_line = line[1:]
- elif line[0] == '+':
- u_line = line[1:]
-
- i = 0
- while i < len(u_line):
- c = u_line[i]
- i = i + 1
- if c == sep_char:
- if seen_separator:
- raise Exception(
- 'Invalid pattern: "' + line + '" Contains more than one '
- 'separator!')
- seen_separator = True
- continue
- elif c == esc_char:
- next_c = u_line[i] if (i < len(u_line)) else None
- if next_c in ['+' , '-', esc_char, sep_char]:
- i = i + 1
- c = next_c
- if seen_separator:
- rule_pattern = rule_pattern + c
- else:
- file_pattern = file_pattern + c
-
- if not rule_pattern:
- rule_pattern = '**'
-
- return sign, file_pattern, rule_pattern
-
-
-def filter_sarif(input_sarif: PathLike,
- output_sarif: PathLike,
- patterns: List[str],
- split_lines: bool) -> None:
- """Filters a SARIF file with a given set of filter patterns.
-
- Args:
- input_sarif (PathLike): Input SARIF file path.
- output_sarif (PathLike): Output SARIF file path.
- patterns (PathLike): List of filter pattern strings.
- split_lines (PathLike): Whether to split lines in individual patterns.
- """
- if split_lines:
- tmp = []
- for p in patterns:
- tmp = tmp + re.split('\r?\n', p)
- patterns = tmp
-
- patterns = [_parse_pattern(p) for p in patterns if p]
-
- logging.debug('Given patterns:')
- for s, fp, rp in patterns:
- logging.debug(
- 'files: {file_pattern} rules: {rule_pattern} ({sign})'.format(
- file_pattern=fp,
- rule_pattern=rp,
- sign='positive' if s else 'negative'))
-
- with open(input_sarif, 'r') as f:
- s = json.load(f)
-
- for run in s.get('runs', []):
- if run.get('results', []):
- new_results = []
- for r in run['results']:
- if r.get('locations', []):
- new_locations = []
- for l in r['locations']:
- # TODO: The uri field is optional. We might have to
- # fetch the actual uri from "artifacts" via
- # "index"
- # (see https://github.com/microsoft/sarif-tutorials/blob/main/docs/2-Basics.md#-linking-results-to-artifacts)
- uri = l.get(
- 'physicalLocation', {}).get(
- 'artifactLocation', {}).get(
- 'uri', None)
-
- # TODO: The ruleId field is optional and potentially
- # ambiguous. We might have to fetch the actual
- # ruleId from the rule metadata via the ruleIndex
- # field.
- # (see https://github.com/microsoft/sarif-tutorials/blob/main/docs/2-Basics.md#rule-metadata)
- ruleId = r['ruleId']
-
- if (uri is None or
- _match_path_and_rule(uri, ruleId, patterns)):
- new_locations.append(l)
- r['locations'] = new_locations
- if new_locations:
- new_results.append(r)
- else:
- # locations array doesn't exist or is empty, so we can't
- # match on anything. Therefore, we include the result in
- # the output.
- new_results.append(r)
- run['results'] = new_results
-
- with open(output_sarif, 'w') as f:
- json.dump(s, f, indent=2)
+# @file analyze_filter.py +# +# Filters results in a SARIF file. +# +# Apache License +# Version 2.0, January 2004 +# http://www.apache.org/licenses/ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been altered from its original form. Based on code in: +# https://github.com/advanced-security/filter-sarif +# +# It primarily contains modifications made to integrate with the CodeQL plugin. +# +# Specifically: +# https://github.com/advanced-security/filter-sarif/blob/main/filter_sarif.py +# +# View the full and complete license as provided by that repository here: +# https://github.com/advanced-security/filter-sarif/blob/main/LICENSE +# +# SPDX-License-Identifier: Apache-2.0 +## + +import json +import logging +import re +from os import PathLike +from typing import Iterable, List, Tuple + +from analyze.globber import match + + +def _match_path_and_rule( + path: str, rule: str, patterns: Iterable[str]) -> bool: + """Returns whether a given path matches a given rule. + + Args: + path (str): A file path string. + rule (str): A rule file path string. + patterns (Iterable[str]): An iterable of pattern strings. + + Returns: + bool: True if the path matches a rule. Otherwise, False. + """ + result = True + for s, fp, rp in patterns: + if match(rp, rule) and match(fp, path): + result = s + return result + + +def _parse_pattern(line: str) -> Tuple[str]: + """Parses a given pattern line. + + Args: + line (str): The line string that contains the rule. + + Returns: + Tuple[str]: The parsed sign, file pattern, and rule pattern from the + line. + """ + sep_char = ':' + esc_char = '\\' + file_pattern = '' + rule_pattern = '' + seen_separator = False + sign = True + + # inclusion or exclusion pattern? + u_line = line + if line: + if line[0] == '-': + sign = False + u_line = line[1:] + elif line[0] == '+': + u_line = line[1:] + + i = 0 + while i < len(u_line): + c = u_line[i] + i = i + 1 + if c == sep_char: + if seen_separator: + raise Exception( + 'Invalid pattern: "' + line + '" Contains more than one ' + 'separator!') + seen_separator = True + continue + elif c == esc_char: + next_c = u_line[i] if (i < len(u_line)) else None + if next_c in ['+' , '-', esc_char, sep_char]: + i = i + 1 + c = next_c + if seen_separator: + rule_pattern = rule_pattern + c + else: + file_pattern = file_pattern + c + + if not rule_pattern: + rule_pattern = '**' + + return sign, file_pattern, rule_pattern + + +def filter_sarif(input_sarif: PathLike, + output_sarif: PathLike, + patterns: List[str], + split_lines: bool) -> None: + """Filters a SARIF file with a given set of filter patterns. + + Args: + input_sarif (PathLike): Input SARIF file path. + output_sarif (PathLike): Output SARIF file path. + patterns (PathLike): List of filter pattern strings. + split_lines (PathLike): Whether to split lines in individual patterns. + """ + if split_lines: + tmp = [] + for p in patterns: + tmp = tmp + re.split('\r?\n', p) + patterns = tmp + + patterns = [_parse_pattern(p) for p in patterns if p] + + logging.debug('Given patterns:') + for s, fp, rp in patterns: + logging.debug( + 'files: {file_pattern} rules: {rule_pattern} ({sign})'.format( + file_pattern=fp, + rule_pattern=rp, + sign='positive' if s else 'negative')) + + with open(input_sarif, 'r') as f: + s = json.load(f) + + for run in s.get('runs', []): + if run.get('results', []): + new_results = [] + for r in run['results']: + if r.get('locations', []): + new_locations = [] + for l in r['locations']: + # TODO: The uri field is optional. We might have to + # fetch the actual uri from "artifacts" via + # "index" + # (see https://github.com/microsoft/sarif-tutorials/blob/main/docs/2-Basics.md#-linking-results-to-artifacts) + uri = l.get( + 'physicalLocation', {}).get( + 'artifactLocation', {}).get( + 'uri', None) + + # TODO: The ruleId field is optional and potentially + # ambiguous. We might have to fetch the actual + # ruleId from the rule metadata via the ruleIndex + # field. + # (see https://github.com/microsoft/sarif-tutorials/blob/main/docs/2-Basics.md#rule-metadata) + ruleId = r['ruleId'] + + if (uri is None or + _match_path_and_rule(uri, ruleId, patterns)): + new_locations.append(l) + r['locations'] = new_locations + if new_locations: + new_results.append(r) + else: + # locations array doesn't exist or is empty, so we can't + # match on anything. Therefore, we include the result in + # the output. + new_results.append(r) + run['results'] = new_results + + with open(output_sarif, 'w') as f: + json.dump(s, f, indent=2) diff --git a/BaseTools/Plugin/CodeQL/analyze/globber.py b/BaseTools/Plugin/CodeQL/analyze/globber.py index 5d45abaa1f..11ef9d6cef 100644 --- a/BaseTools/Plugin/CodeQL/analyze/globber.py +++ b/BaseTools/Plugin/CodeQL/analyze/globber.py @@ -1,127 +1,127 @@ -# @file globber.py
-#
-# Provides global functionality for use by the CodeQL plugin.
-#
-# Copyright 2019 Jaakko Kangasharju
-#
-# Apache License
-# Version 2.0, January 2004
-# http://www.apache.org/licenses/
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This file has been altered from its original form. Based on code in:
-# https://github.com/advanced-security/filter-sarif
-#
-# Specifically:
-# https://github.com/advanced-security/filter-sarif/blob/main/filter_sarif.py
-#
-# It primarily contains modifications made to integrate with the CodeQL plugin.
-#
-# SPDX-License-Identifier: Apache-2.0
-##
-
-import re
-
-_double_star_after_invalid_regex = re.compile(r'[^/\\]\*\*')
-_double_star_first_before_invalid_regex = re.compile('^\\*\\*[^/]')
-_double_star_middle_before_invalid_regex = re.compile(r'[^\\]\*\*[^/]')
-
-
-def _match_component(pattern_component, file_name_component):
- if len(pattern_component) == 0 and len(file_name_component) == 0:
- return True
- elif len(pattern_component) == 0:
- return False
- elif len(file_name_component) == 0:
- return pattern_component == '*'
- elif pattern_component[0] == '*':
- return (_match_component(pattern_component, file_name_component[1:]) or
- _match_component(pattern_component[1:], file_name_component))
- elif pattern_component[0] == '?':
- return _match_component(pattern_component[1:], file_name_component[1:])
- elif pattern_component[0] == '\\':
- return (len(pattern_component) >= 2 and
- pattern_component[1] == file_name_component[0] and
- _match_component(
- pattern_component[2:], file_name_component[1:]))
- elif pattern_component[0] != file_name_component[0]:
- return False
- else:
- return _match_component(pattern_component[1:], file_name_component[1:])
-
-
-def _match_components(pattern_components, file_name_components):
- if len(pattern_components) == 0 and len(file_name_components) == 0:
- return True
- if len(pattern_components) == 0:
- return False
- if len(file_name_components) == 0:
- return len(pattern_components) == 1 and pattern_components[0] == '**'
- if pattern_components[0] == '**':
- return (_match_components(pattern_components, file_name_components[1:])
- or _match_components(
- pattern_components[1:], file_name_components))
- else:
- return (
- _match_component(
- pattern_components[0], file_name_components[0]) and
- _match_components(
- pattern_components[1:], file_name_components[1:]))
-
-
-def match(pattern: str, file_name: str):
- """Match a glob pattern against a file name.
-
- Glob pattern matching is for file names, which do not need to exist as
- files on the file system.
-
- A file name is a sequence of directory names, possibly followed by the name
- of a file, with the components separated by a path separator. A glob
- pattern is similar, except it may contain special characters: A '?' matches
- any character in a name. A '*' matches any sequence of characters (possibly
- empty) in a name. Both of these match only within a single component, i.e.,
- they will not match a path separator. A component in a pattern may also be
- a literal '**', which matches zero or more components in the complete file
- name. A backslash '\\' in a pattern acts as an escape character, and
- indicates that the following character is to be matched literally, even if
- it is a special character.
-
- Args:
- pattern (str): The pattern to match. The path separator in patterns is
- always '/'.
- file_name (str): The file name to match against. The path separator in
- file names is the platform separator
-
- Returns:
- bool: True if the pattern matches, False otherwise.
- """
- if (_double_star_after_invalid_regex.search(pattern) is not None or
- _double_star_first_before_invalid_regex.search(
- pattern) is not None or
- _double_star_middle_before_invalid_regex.search(pattern) is not None):
- raise ValueError(
- '** in {} not alone between path separators'.format(pattern))
-
- pattern = pattern.rstrip('/')
- file_name = file_name.rstrip('/')
-
- while '**/**' in pattern:
- pattern = pattern.replace('**/**', '**')
-
- pattern_components = pattern.split('/')
-
- # We split on '\' as well as '/' to support unix and windows-style paths
- file_name_components = re.split(r'[\\/]', file_name)
-
- return _match_components(pattern_components, file_name_components)
+# @file globber.py +# +# Provides global functionality for use by the CodeQL plugin. +# +# Copyright 2019 Jaakko Kangasharju +# +# Apache License +# Version 2.0, January 2004 +# http://www.apache.org/licenses/ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been altered from its original form. Based on code in: +# https://github.com/advanced-security/filter-sarif +# +# Specifically: +# https://github.com/advanced-security/filter-sarif/blob/main/filter_sarif.py +# +# It primarily contains modifications made to integrate with the CodeQL plugin. +# +# SPDX-License-Identifier: Apache-2.0 +## + +import re + +_double_star_after_invalid_regex = re.compile(r'[^/\\]\*\*') +_double_star_first_before_invalid_regex = re.compile('^\\*\\*[^/]') +_double_star_middle_before_invalid_regex = re.compile(r'[^\\]\*\*[^/]') + + +def _match_component(pattern_component, file_name_component): + if len(pattern_component) == 0 and len(file_name_component) == 0: + return True + elif len(pattern_component) == 0: + return False + elif len(file_name_component) == 0: + return pattern_component == '*' + elif pattern_component[0] == '*': + return (_match_component(pattern_component, file_name_component[1:]) or + _match_component(pattern_component[1:], file_name_component)) + elif pattern_component[0] == '?': + return _match_component(pattern_component[1:], file_name_component[1:]) + elif pattern_component[0] == '\\': + return (len(pattern_component) >= 2 and + pattern_component[1] == file_name_component[0] and + _match_component( + pattern_component[2:], file_name_component[1:])) + elif pattern_component[0] != file_name_component[0]: + return False + else: + return _match_component(pattern_component[1:], file_name_component[1:]) + + +def _match_components(pattern_components, file_name_components): + if len(pattern_components) == 0 and len(file_name_components) == 0: + return True + if len(pattern_components) == 0: + return False + if len(file_name_components) == 0: + return len(pattern_components) == 1 and pattern_components[0] == '**' + if pattern_components[0] == '**': + return (_match_components(pattern_components, file_name_components[1:]) + or _match_components( + pattern_components[1:], file_name_components)) + else: + return ( + _match_component( + pattern_components[0], file_name_components[0]) and + _match_components( + pattern_components[1:], file_name_components[1:])) + + +def match(pattern: str, file_name: str): + """Match a glob pattern against a file name. + + Glob pattern matching is for file names, which do not need to exist as + files on the file system. + + A file name is a sequence of directory names, possibly followed by the name + of a file, with the components separated by a path separator. A glob + pattern is similar, except it may contain special characters: A '?' matches + any character in a name. A '*' matches any sequence of characters (possibly + empty) in a name. Both of these match only within a single component, i.e., + they will not match a path separator. A component in a pattern may also be + a literal '**', which matches zero or more components in the complete file + name. A backslash '\\' in a pattern acts as an escape character, and + indicates that the following character is to be matched literally, even if + it is a special character. + + Args: + pattern (str): The pattern to match. The path separator in patterns is + always '/'. + file_name (str): The file name to match against. The path separator in + file names is the platform separator + + Returns: + bool: True if the pattern matches, False otherwise. + """ + if (_double_star_after_invalid_regex.search(pattern) is not None or + _double_star_first_before_invalid_regex.search( + pattern) is not None or + _double_star_middle_before_invalid_regex.search(pattern) is not None): + raise ValueError( + '** in {} not alone between path separators'.format(pattern)) + + pattern = pattern.rstrip('/') + file_name = file_name.rstrip('/') + + while '**/**' in pattern: + pattern = pattern.replace('**/**', '**') + + pattern_components = pattern.split('/') + + # We split on '\' as well as '/' to support unix and windows-style paths + file_name_components = re.split(r'[\\/]', file_name) + + return _match_components(pattern_components, file_name_components) diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml b/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml index dbc9c2ba02..a6e711b78b 100644 --- a/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml +++ b/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml @@ -1,26 +1,26 @@ -## @file codeqlcli_ext_dep.yaml
-#
-# Downloads the CodeQL Command-Line Interface (CLI) application that support Linux, Windows, and Mac OS X.
-#
-# This download is very large but conveniently provides support for all operating systems. Use it if you
-# need CodeQL CLI support without concern for the host operating system.
-#
-# In an environment where a platform might build in different operating systems, it is recommended to set
-# the scope for the appropriate CodeQL external dependency based on the host operating system being used.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-ext-dep",
- "type": "web",
- "name": "codeql_cli",
- "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql.zip",
- "version": "2.17.3",
- "sha256": "e5ac1d87ab38e405c9af5db234a338b10dffabc98a648903f1664dd2a566dfd5",
- "compression_type": "zip",
- "internal_path": "/codeql/",
- "flags": ["set_shell_var", ],
- "var_name": "STUART_CODEQL_PATH"
-}
+## @file codeqlcli_ext_dep.yaml +# +# Downloads the CodeQL Command-Line Interface (CLI) application that support Linux, Windows, and Mac OS X. +# +# This download is very large but conveniently provides support for all operating systems. Use it if you +# need CodeQL CLI support without concern for the host operating system. +# +# In an environment where a platform might build in different operating systems, it is recommended to set +# the scope for the appropriate CodeQL external dependency based on the host operating system being used. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-ext-dep", + "type": "web", + "name": "codeql_cli", + "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql.zip", + "version": "2.17.3", + "sha256": "e5ac1d87ab38e405c9af5db234a338b10dffabc98a648903f1664dd2a566dfd5", + "compression_type": "zip", + "internal_path": "/codeql/", + "flags": ["set_shell_var", ], + "var_name": "STUART_CODEQL_PATH" +} diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml b/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml index 536322f2b3..16f1f5643d 100644 --- a/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml +++ b/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml @@ -1,24 +1,24 @@ -## @file codeqlcli_linux_ext_dep.yaml
-#
-# Downloads the Linux CodeQL Command-Line Interface (CLI) application.
-#
-# This download only supports Linux. In an environment where a platform might build in different operating
-# systems, it is recommended to set the scope for the appropriate CodeQL external dependency based on the
-# host operating system being used.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-linux-ext-dep",
- "type": "web",
- "name": "codeql_linux_cli",
- "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql-linux64.zip",
- "version": "2.17.3",
- "sha256": "9fba000c4b821534d354bc16821aa066fdb1304446226ea449870e64a8ad3c7a",
- "compression_type": "zip",
- "internal_path": "/codeql/",
- "flags": ["set_shell_var", ],
- "var_name": "STUART_CODEQL_PATH"
-}
+## @file codeqlcli_linux_ext_dep.yaml +# +# Downloads the Linux CodeQL Command-Line Interface (CLI) application. +# +# This download only supports Linux. In an environment where a platform might build in different operating +# systems, it is recommended to set the scope for the appropriate CodeQL external dependency based on the +# host operating system being used. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-linux-ext-dep", + "type": "web", + "name": "codeql_linux_cli", + "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql-linux64.zip", + "version": "2.17.3", + "sha256": "9fba000c4b821534d354bc16821aa066fdb1304446226ea449870e64a8ad3c7a", + "compression_type": "zip", + "internal_path": "/codeql/", + "flags": ["set_shell_var", ], + "var_name": "STUART_CODEQL_PATH" +} diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml b/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml index 93a81ffd50..e997a9c7ef 100644 --- a/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml +++ b/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml @@ -1,24 +1,24 @@ -## @file codeqlcli_windows_ext_dep.yaml
-#
-# Downloads the Windows CodeQL Command-Line Interface (CLI) application.
-#
-# This download only supports Windows. In an environment where a platform might build in different operating
-# systems, it is recommended to set the scope for the appropriate CodeQL external dependency based on the
-# host operating system being used.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-windows-ext-dep",
- "type": "web",
- "name": "codeql_windows_cli",
- "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql-win64.zip",
- "version": "2.17.3",
- "sha256": "4c6fbf2ea2eaf0f47bf0347eacf54c6b9d6bdf7acb6b63e17f9e6f2dd83b34e7",
- "compression_type": "zip",
- "internal_path": "/codeql/",
- "flags": ["set_shell_var", ],
- "var_name": "STUART_CODEQL_PATH"
-}
+## @file codeqlcli_windows_ext_dep.yaml +# +# Downloads the Windows CodeQL Command-Line Interface (CLI) application. +# +# This download only supports Windows. In an environment where a platform might build in different operating +# systems, it is recommended to set the scope for the appropriate CodeQL external dependency based on the +# host operating system being used. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-windows-ext-dep", + "type": "web", + "name": "codeql_windows_cli", + "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql-win64.zip", + "version": "2.17.3", + "sha256": "4c6fbf2ea2eaf0f47bf0347eacf54c6b9d6bdf7acb6b63e17f9e6f2dd83b34e7", + "compression_type": "zip", + "internal_path": "/codeql/", + "flags": ["set_shell_var", ], + "var_name": "STUART_CODEQL_PATH" +} diff --git a/BaseTools/Plugin/CodeQL/common/codeql_plugin.py b/BaseTools/Plugin/CodeQL/common/codeql_plugin.py index c827cc30ae..72347ec13e 100644 --- a/BaseTools/Plugin/CodeQL/common/codeql_plugin.py +++ b/BaseTools/Plugin/CodeQL/common/codeql_plugin.py @@ -1,74 +1,74 @@ -# @file codeql_plugin.py
-#
-# Common logic shared across the CodeQL plugin.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-import os
-import shutil
-from os import PathLike
-
-from edk2toollib.utility_functions import GetHostInfo
-
-
-def get_codeql_db_path(workspace: PathLike, package: str, target: str,
- new_path: bool = True) -> str:
- """Return the CodeQL database path for this build.
-
- Args:
- workspace (PathLike): The workspace path.
- package (str): The package name (e.g. "MdeModulePkg")
- target (str): The target (e.g. "DEBUG")
- new_path (bool, optional): Whether to create a new database path or
- return an existing path. Defaults to True.
-
- Returns:
- str: The absolute path to the CodeQL database directory.
- """
- codeql_db_dir_name = "codeql-db-" + package + "-" + target
- codeql_db_dir_name = codeql_db_dir_name.lower()
- codeql_db_path = os.path.join("Build", codeql_db_dir_name)
- codeql_db_path = os.path.join(workspace, codeql_db_path)
-
- i = 0
- while os.path.isdir(f"{codeql_db_path + '-%s' % i}"):
- i += 1
-
- if not new_path:
- if i == 0:
- return None
- else:
- i -= 1
-
- return codeql_db_path + f"-{i}"
-
-
-def get_codeql_cli_path() -> str:
- """Return the current CodeQL CLI path.
-
- Returns:
- str: The absolute path to the CodeQL CLI application to use for
- this build.
- """
- # The CodeQL executable path can be passed via the
- # STUART_CODEQL_PATH environment variable (to override with a
- # custom value for this run) or read from the system path.
- codeql_path = None
-
- if "STUART_CODEQL_PATH" in os.environ:
- codeql_path = os.environ["STUART_CODEQL_PATH"]
-
- if GetHostInfo().os == "Windows":
- codeql_path = os.path.join(codeql_path, "codeql.exe")
- else:
- codeql_path = os.path.join(codeql_path, "codeql")
-
- if not os.path.isfile(codeql_path):
- codeql_path = None
-
- if not codeql_path:
- codeql_path = shutil.which("codeql")
-
- return codeql_path
+# @file codeql_plugin.py +# +# Common logic shared across the CodeQL plugin. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import os +import shutil +from os import PathLike + +from edk2toollib.utility_functions import GetHostInfo + + +def get_codeql_db_path(workspace: PathLike, package: str, target: str, + new_path: bool = True) -> str: + """Return the CodeQL database path for this build. + + Args: + workspace (PathLike): The workspace path. + package (str): The package name (e.g. "MdeModulePkg") + target (str): The target (e.g. "DEBUG") + new_path (bool, optional): Whether to create a new database path or + return an existing path. Defaults to True. + + Returns: + str: The absolute path to the CodeQL database directory. + """ + codeql_db_dir_name = "codeql-db-" + package + "-" + target + codeql_db_dir_name = codeql_db_dir_name.lower() + codeql_db_path = os.path.join("Build", codeql_db_dir_name) + codeql_db_path = os.path.join(workspace, codeql_db_path) + + i = 0 + while os.path.isdir(f"{codeql_db_path + '-%s' % i}"): + i += 1 + + if not new_path: + if i == 0: + return None + else: + i -= 1 + + return codeql_db_path + f"-{i}" + + +def get_codeql_cli_path() -> str: + """Return the current CodeQL CLI path. + + Returns: + str: The absolute path to the CodeQL CLI application to use for + this build. + """ + # The CodeQL executable path can be passed via the + # STUART_CODEQL_PATH environment variable (to override with a + # custom value for this run) or read from the system path. + codeql_path = None + + if "STUART_CODEQL_PATH" in os.environ: + codeql_path = os.environ["STUART_CODEQL_PATH"] + + if GetHostInfo().os == "Windows": + codeql_path = os.path.join(codeql_path, "codeql.exe") + else: + codeql_path = os.path.join(codeql_path, "codeql") + + if not os.path.isfile(codeql_path): + codeql_path = None + + if not codeql_path: + codeql_path = shutil.which("codeql") + + return codeql_path diff --git a/BaseTools/Plugin/CodeQL/integration/stuart_codeql.py b/BaseTools/Plugin/CodeQL/integration/stuart_codeql.py index a3941d1315..d778d7d949 100644 --- a/BaseTools/Plugin/CodeQL/integration/stuart_codeql.py +++ b/BaseTools/Plugin/CodeQL/integration/stuart_codeql.py @@ -1,79 +1,79 @@ -# @file stuart_codeql.py
-#
-# Exports functions commonly needed for Stuart-based platforms to easily
-# enable CodeQL in their platform build.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-from edk2toolext.environment.uefi_build import UefiBuilder
-from edk2toollib.utility_functions import GetHostInfo
-from argparse import ArgumentParser, Namespace
-from typing import Tuple
-
-
-def add_command_line_option(parser: ArgumentParser) -> None:
- """Adds the CodeQL command to the platform command line options.
-
- Args:
- parser (ArgumentParser): The argument parser used in this build.
-
- """
- parser.add_argument(
- '--codeql',
- dest='codeql',
- action='store_true',
- default=False,
- help="Optional - Produces CodeQL results from the build. See "
- "BaseTools/Plugin/CodeQL/Readme.md for more info.")
-
-
-def get_scopes(codeql_enabled: bool) -> Tuple[str]:
- """Returns the active CodeQL scopes for this build.
-
- Args:
- codeql_enabled (bool): Whether CodeQL is enabled.
-
- Returns:
- Tuple[str]: A tuple of strings containing scopes that enable the
- CodeQL plugin.
- """
- active_scopes = ()
-
- if codeql_enabled:
- if GetHostInfo().os == "Linux":
- active_scopes += ("codeql-linux-ext-dep",)
- else:
- active_scopes += ("codeql-windows-ext-dep",)
- active_scopes += ("codeql-build", "codeql-analyze")
-
- return active_scopes
-
-
-def is_codeql_enabled_on_command_line(args: Namespace) -> bool:
- """Returns whether CodeQL was enabled on the command line.
-
- Args:
- args (Namespace): Object holding a string representation of command
- line arguments.
-
- Returns:
- bool: True if CodeQL is enabled on the command line. Otherwise, false.
- """
- return args.codeql
-
-
-def set_audit_only_mode(uefi_builder: UefiBuilder) -> None:
- """Configures the CodeQL plugin to run in audit only mode.
-
- Args:
- uefi_builder (UefiBuilder): The UefiBuilder object for this platform
- build.
-
- """
-
- uefi_builder.env.SetValue(
- "STUART_CODEQL_AUDIT_ONLY",
- "true",
- "Platform Defined")
+# @file stuart_codeql.py +# +# Exports functions commonly needed for Stuart-based platforms to easily +# enable CodeQL in their platform build. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toollib.utility_functions import GetHostInfo +from argparse import ArgumentParser, Namespace +from typing import Tuple + + +def add_command_line_option(parser: ArgumentParser) -> None: + """Adds the CodeQL command to the platform command line options. + + Args: + parser (ArgumentParser): The argument parser used in this build. + + """ + parser.add_argument( + '--codeql', + dest='codeql', + action='store_true', + default=False, + help="Optional - Produces CodeQL results from the build. See " + "BaseTools/Plugin/CodeQL/Readme.md for more info.") + + +def get_scopes(codeql_enabled: bool) -> Tuple[str]: + """Returns the active CodeQL scopes for this build. + + Args: + codeql_enabled (bool): Whether CodeQL is enabled. + + Returns: + Tuple[str]: A tuple of strings containing scopes that enable the + CodeQL plugin. + """ + active_scopes = () + + if codeql_enabled: + if GetHostInfo().os == "Linux": + active_scopes += ("codeql-linux-ext-dep",) + else: + active_scopes += ("codeql-windows-ext-dep",) + active_scopes += ("codeql-build", "codeql-analyze") + + return active_scopes + + +def is_codeql_enabled_on_command_line(args: Namespace) -> bool: + """Returns whether CodeQL was enabled on the command line. + + Args: + args (Namespace): Object holding a string representation of command + line arguments. + + Returns: + bool: True if CodeQL is enabled on the command line. Otherwise, false. + """ + return args.codeql + + +def set_audit_only_mode(uefi_builder: UefiBuilder) -> None: + """Configures the CodeQL plugin to run in audit only mode. + + Args: + uefi_builder (UefiBuilder): The UefiBuilder object for this platform + build. + + """ + + uefi_builder.env.SetValue( + "STUART_CODEQL_AUDIT_ONLY", + "true", + "Platform Defined") diff --git a/BaseTools/Plugin/DebugMacroCheck/BuildPlugin/DebugMacroCheckBuildPlugin.py b/BaseTools/Plugin/DebugMacroCheck/BuildPlugin/DebugMacroCheckBuildPlugin.py index aa3a2bbcab..e86c0e23ad 100644 --- a/BaseTools/Plugin/DebugMacroCheck/BuildPlugin/DebugMacroCheckBuildPlugin.py +++ b/BaseTools/Plugin/DebugMacroCheck/BuildPlugin/DebugMacroCheckBuildPlugin.py @@ -1,127 +1,127 @@ -# @file DebugMacroCheckBuildPlugin.py
-#
-# A build plugin that checks if DEBUG macros are formatted properly.
-#
-# In particular, that print format specifiers are defined
-# with the expected number of arguments in the variable
-# argument list.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-import logging
-import os
-import pathlib
-import sys
-import yaml
-
-# Import the build plugin
-plugin_file = pathlib.Path(__file__)
-sys.path.append(str(plugin_file.parent.parent))
-
-# flake8 (E402): Ignore flake8 module level import not at top of file
-import DebugMacroCheck # noqa: E402
-
-from edk2toolext import edk2_logging # noqa: E402
-from edk2toolext.environment.plugintypes.uefi_build_plugin import \
- IUefiBuildPlugin # noqa: E402
-from edk2toolext.environment.uefi_build import UefiBuilder # noqa: E402
-from edk2toollib.uefi.edk2.path_utilities import Edk2Path # noqa: E402
-from pathlib import Path # noqa: E402
-
-
-class DebugMacroCheckBuildPlugin(IUefiBuildPlugin):
-
- def do_pre_build(self, builder: UefiBuilder) -> int:
- """Debug Macro Check pre-build functionality.
-
- The plugin is invoked in pre-build since it can operate independently
- of build tools and to notify the user of any errors earlier in the
- build process to reduce feedback time.
-
- Args:
- builder (UefiBuilder): A UEFI builder object for this build.
-
- Returns:
- int: The number of debug macro errors found. Zero indicates the
- check either did not run or no errors were found.
- """
-
- # Check if disabled in the environment
- env_disable = builder.env.GetValue("DISABLE_DEBUG_MACRO_CHECK")
- if env_disable:
- return 0
-
- # Only run on targets with compilation
- build_target = builder.env.GetValue("TARGET").lower()
- if "no-target" in build_target:
- return 0
-
- edk2 = builder.edk2path
- package = edk2.GetContainingPackage(
- builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
- builder.env.GetValue("ACTIVE_PLATFORM")
- )
- )
- package_path = Path(
- edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
- package))
-
- # Every debug macro is printed at DEBUG logging level.
- # Ensure the level is above DEBUG while executing the macro check
- # plugin to avoid flooding the log handler.
- handler_level_context = []
- for h in logging.getLogger().handlers:
- if h.level < logging.INFO:
- handler_level_context.append((h, h.level))
- h.setLevel(logging.INFO)
-
- edk2_logging.log_progress("Checking DEBUG Macros")
-
- # There are two ways to specify macro substitution data for this
- # plugin. If multiple options are present, data is appended from
- # each option.
- #
- # 1. Specify the substitution data in the package CI YAML file.
- # 2. Specify a standalone substitution data YAML file.
- ##
- sub_data = {}
-
- # 1. Allow substitution data to be specified in a "DebugMacroCheck" of
- # the package CI YAML file. This is used to provide a familiar per-
- # package customization flow for a package maintainer.
- package_config_file = Path(
- os.path.join(
- package_path, package + ".ci.yaml"))
- if package_config_file.is_file():
- with open(package_config_file, 'r') as cf:
- package_config_file_data = yaml.safe_load(cf)
- if "DebugMacroCheck" in package_config_file_data and \
- "StringSubstitutions" in \
- package_config_file_data["DebugMacroCheck"]:
- logging.info(f"Loading substitution data in "
- f"{str(package_config_file)}")
- sub_data |= package_config_file_data["DebugMacroCheck"]["StringSubstitutions"] # noqa
-
- # 2. Allow a substitution file to be specified as an environment
- # variable. This is used to provide flexibility in how to specify a
- # substitution file. The value can be set anywhere prior to this plugin
- # getting called such as pre-existing build script.
- sub_file = builder.env.GetValue("DEBUG_MACRO_CHECK_SUB_FILE")
- if sub_file:
- logging.info(f"Loading substitution file {sub_file}")
- with open(sub_file, 'r') as sf:
- sub_data |= yaml.safe_load(sf)
-
- try:
- error_count = DebugMacroCheck.check_macros_in_directory(
- package_path,
- ignore_git_submodules=False,
- show_progress_bar=False,
- **sub_data)
- finally:
- for h, l in handler_level_context:
- h.setLevel(l)
-
- return error_count
+# @file DebugMacroCheckBuildPlugin.py +# +# A build plugin that checks if DEBUG macros are formatted properly. +# +# In particular, that print format specifiers are defined +# with the expected number of arguments in the variable +# argument list. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import logging +import os +import pathlib +import sys +import yaml + +# Import the build plugin +plugin_file = pathlib.Path(__file__) +sys.path.append(str(plugin_file.parent.parent)) + +# flake8 (E402): Ignore flake8 module level import not at top of file +import DebugMacroCheck # noqa: E402 + +from edk2toolext import edk2_logging # noqa: E402 +from edk2toolext.environment.plugintypes.uefi_build_plugin import \ + IUefiBuildPlugin # noqa: E402 +from edk2toolext.environment.uefi_build import UefiBuilder # noqa: E402 +from edk2toollib.uefi.edk2.path_utilities import Edk2Path # noqa: E402 +from pathlib import Path # noqa: E402 + + +class DebugMacroCheckBuildPlugin(IUefiBuildPlugin): + + def do_pre_build(self, builder: UefiBuilder) -> int: + """Debug Macro Check pre-build functionality. + + The plugin is invoked in pre-build since it can operate independently + of build tools and to notify the user of any errors earlier in the + build process to reduce feedback time. + + Args: + builder (UefiBuilder): A UEFI builder object for this build. + + Returns: + int: The number of debug macro errors found. Zero indicates the + check either did not run or no errors were found. + """ + + # Check if disabled in the environment + env_disable = builder.env.GetValue("DISABLE_DEBUG_MACRO_CHECK") + if env_disable: + return 0 + + # Only run on targets with compilation + build_target = builder.env.GetValue("TARGET").lower() + if "no-target" in build_target: + return 0 + + edk2 = builder.edk2path + package = edk2.GetContainingPackage( + builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath( + builder.env.GetValue("ACTIVE_PLATFORM") + ) + ) + package_path = Path( + edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath( + package)) + + # Every debug macro is printed at DEBUG logging level. + # Ensure the level is above DEBUG while executing the macro check + # plugin to avoid flooding the log handler. + handler_level_context = [] + for h in logging.getLogger().handlers: + if h.level < logging.INFO: + handler_level_context.append((h, h.level)) + h.setLevel(logging.INFO) + + edk2_logging.log_progress("Checking DEBUG Macros") + + # There are two ways to specify macro substitution data for this + # plugin. If multiple options are present, data is appended from + # each option. + # + # 1. Specify the substitution data in the package CI YAML file. + # 2. Specify a standalone substitution data YAML file. + ## + sub_data = {} + + # 1. Allow substitution data to be specified in a "DebugMacroCheck" of + # the package CI YAML file. This is used to provide a familiar per- + # package customization flow for a package maintainer. + package_config_file = Path( + os.path.join( + package_path, package + ".ci.yaml")) + if package_config_file.is_file(): + with open(package_config_file, 'r') as cf: + package_config_file_data = yaml.safe_load(cf) + if "DebugMacroCheck" in package_config_file_data and \ + "StringSubstitutions" in \ + package_config_file_data["DebugMacroCheck"]: + logging.info(f"Loading substitution data in " + f"{str(package_config_file)}") + sub_data |= package_config_file_data["DebugMacroCheck"]["StringSubstitutions"] # noqa + + # 2. Allow a substitution file to be specified as an environment + # variable. This is used to provide flexibility in how to specify a + # substitution file. The value can be set anywhere prior to this plugin + # getting called such as pre-existing build script. + sub_file = builder.env.GetValue("DEBUG_MACRO_CHECK_SUB_FILE") + if sub_file: + logging.info(f"Loading substitution file {sub_file}") + with open(sub_file, 'r') as sf: + sub_data |= yaml.safe_load(sf) + + try: + error_count = DebugMacroCheck.check_macros_in_directory( + package_path, + ignore_git_submodules=False, + show_progress_bar=False, + **sub_data) + finally: + for h, l in handler_level_context: + h.setLevel(l) + + return error_count diff --git a/BaseTools/Plugin/DebugMacroCheck/BuildPlugin/DebugMacroCheck_plug_in.yaml b/BaseTools/Plugin/DebugMacroCheck/BuildPlugin/DebugMacroCheck_plug_in.yaml index 50f97cbd39..97320f0251 100644 --- a/BaseTools/Plugin/DebugMacroCheck/BuildPlugin/DebugMacroCheck_plug_in.yaml +++ b/BaseTools/Plugin/DebugMacroCheck/BuildPlugin/DebugMacroCheck_plug_in.yaml @@ -1,11 +1,11 @@ -## @file
-# Build plugin used to check that debug macros are formatted properly.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-{
- "scope": "global",
- "name": "Debug Macro Check Plugin",
- "module": "DebugMacroCheckBuildPlugin"
-}
+## @file +# Build plugin used to check that debug macros are formatted properly. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global", + "name": "Debug Macro Check Plugin", + "module": "DebugMacroCheckBuildPlugin" +} diff --git a/BaseTools/Plugin/DebugMacroCheck/DebugMacroCheck.py b/BaseTools/Plugin/DebugMacroCheck/DebugMacroCheck.py index ffabcdf91b..40f2083680 100644 --- a/BaseTools/Plugin/DebugMacroCheck/DebugMacroCheck.py +++ b/BaseTools/Plugin/DebugMacroCheck/DebugMacroCheck.py @@ -1,859 +1,859 @@ -# @file DebugMacroCheck.py
-#
-# A script that checks if DEBUG macros are formatted properly.
-#
-# In particular, that print format specifiers are defined
-# with the expected number of arguments in the variable
-# argument list.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-from argparse import RawTextHelpFormatter
-import logging
-import os
-import re
-import regex
-import sys
-import shutil
-import timeit
-import yaml
-
-from edk2toollib.utility_functions import RunCmd
-from io import StringIO
-from pathlib import Path, PurePath
-from typing import Dict, Iterable, List, Optional, Tuple
-
-
-PROGRAM_NAME = "Debug Macro Checker"
-
-
-class GitHelpers:
- """
- Collection of Git helpers.
-
- Will be moved to a more generic module and imported in the future.
- """
-
- @staticmethod
- def get_git_ignored_paths(directory_path: PurePath) -> List[Path]:
- """Returns ignored files in this git repository.
-
- Args:
- directory_path (PurePath): Path to the git directory.
-
- Returns:
- List[Path]: List of file absolute paths to all files ignored
- in this git repository. If git is not found, an empty
- list will be returned.
- """
- if not shutil.which("git"):
- logging.warn(
- "Git is not found on this system. Git submodule paths will "
- "not be considered.")
- return []
-
- out_stream_buffer = StringIO()
- exit_code = RunCmd("git", "ls-files --other",
- workingdir=str(directory_path),
- outstream=out_stream_buffer,
- logging_level=logging.NOTSET)
- if exit_code != 0:
- return []
-
- rel_paths = out_stream_buffer.getvalue().strip().splitlines()
- abs_paths = []
- for path in rel_paths:
- abs_paths.append(Path(directory_path, path))
- return abs_paths
-
- @staticmethod
- def get_git_submodule_paths(directory_path: PurePath) -> List[Path]:
- """Returns submodules in the given workspace directory.
-
- Args:
- directory_path (PurePath): Path to the git directory.
-
- Returns:
- List[Path]: List of directory absolute paths to the root of
- each submodule found from this folder. If submodules are not
- found, an empty list will be returned.
- """
- if not shutil.which("git"):
- return []
-
- if os.path.isfile(directory_path.joinpath(".gitmodules")):
- out_stream_buffer = StringIO()
- exit_code = RunCmd(
- "git", "config --file .gitmodules --get-regexp path",
- workingdir=str(directory_path),
- outstream=out_stream_buffer,
- logging_level=logging.NOTSET)
- if exit_code != 0:
- return []
-
- submodule_paths = []
- for line in out_stream_buffer.getvalue().strip().splitlines():
- submodule_paths.append(
- Path(directory_path, line.split()[1]))
-
- return submodule_paths
- else:
- return []
-
-
-class QuietFilter(logging.Filter):
- """A logging filter that temporarily suppresses message output."""
-
- def __init__(self, quiet: bool = False):
- """Class constructor method.
-
- Args:
- quiet (bool, optional): Indicates if messages are currently being
- printed (False) or not (True). Defaults to False.
- """
-
- self._quiet = quiet
-
- def filter(self, record: logging.LogRecord) -> bool:
- """Quiet filter method.
-
- Args:
- record (logging.LogRecord): A log record object that the filter is
- applied to.
-
- Returns:
- bool: True if messages are being suppressed. Otherwise, False.
- """
- return not self._quiet
-
-
-class ProgressFilter(logging.Filter):
- """A logging filter that suppresses 'Progress' messages."""
-
- def filter(self, record: logging.LogRecord) -> bool:
- """Progress filter method.
-
- Args:
- record (logging.LogRecord): A log record object that the filter is
- applied to.
-
- Returns:
- bool: True if the message is not a 'Progress' message. Otherwise,
- False.
- """
- return not record.getMessage().startswith("\rProgress")
-
-
-class CacheDuringProgressFilter(logging.Filter):
- """A logging filter that suppresses messages during progress operations."""
-
- _message_cache = []
-
- @property
- def message_cache(self) -> List[logging.LogRecord]:
- """Contains a cache of messages accumulated during time of operation.
-
- Returns:
- List[logging.LogRecord]: List of log records stored while the
- filter was active.
- """
- return self._message_cache
-
- def filter(self, record: logging.LogRecord):
- """Cache progress filter that suppresses messages during progress
- display output.
-
- Args:
- record (logging.LogRecord): A log record to cache.
- """
- self._message_cache.append(record)
-
-
-def check_debug_macros(macros: Iterable[Dict[str, str]],
- file_dbg_path: str,
- **macro_subs: str
- ) -> Tuple[int, int, int]:
- """Checks if debug macros contain formatting errors.
-
- Args:
- macros (Iterable[Dict[str, str]]): : A groupdict of macro matches.
- This is an iterable of dictionaries with group names from the regex
- match as the key and the matched string as the value for the key.
-
- file_dbg_path (str): The file path (or other custom string) to display
- in debug messages.
-
- macro_subs (Dict[str,str]): Variable-length keyword and replacement
- value string pairs to substitute during debug macro checks.
-
- Returns:
- Tuple[int, int, int]: A tuple of the number of formatting errors,
- number of print specifiers, and number of arguments for the macros
- given.
- """
-
- macro_subs = {k.lower(): v for k, v in macro_subs.items()}
-
- arg_cnt, failure_cnt, print_spec_cnt = 0, 0, 0
- for macro in macros:
- # Special Specifier Handling
- processed_dbg_str = macro['dbg_str'].strip().lower()
-
- logging.debug(f"Inspecting macro: {macro}")
-
- # Make any macro substitutions so further processing is applied
- # to the substituted value.
- for k in macro_subs.keys():
- processed_dbg_str = processed_dbg_str.replace(k, macro_subs[k])
-
- logging.debug("Debug macro string after replacements: "
- f"{processed_dbg_str}")
-
- # These are very rarely used in debug strings. They are somewhat
- # more common in HII code to control text displayed on the
- # console. Due to the rarity and likelihood usage is a mistake,
- # a warning is shown if found.
- specifier_display_replacements = ['%n', '%h', '%e', '%b', '%v']
- for s in specifier_display_replacements:
- if s in processed_dbg_str:
- logging.warning(f"File: {file_dbg_path}")
- logging.warning(f" {s} found in string and ignored:")
- logging.warning(f" \"{processed_dbg_str}\"")
- processed_dbg_str = processed_dbg_str.replace(s, '')
-
- # These are miscellaneous print specifiers that do not require
- # special parsing and simply need to be replaced since they do
- # have a corresponding argument associated with them.
- specifier_other_replacements = ['%%', '\r', '\n']
- for s in specifier_other_replacements:
- if s in processed_dbg_str:
- processed_dbg_str = processed_dbg_str.replace(s, '')
-
- processed_dbg_str = re.sub(
- r'%[.\-+ ,Ll0-9]*\*[.\-+ ,Ll0-9]*[a-zA-Z]', '%_%_',
- processed_dbg_str)
- logging.debug(f"Final macro before print specifier scan: "
- f"{processed_dbg_str}")
-
- print_spec_cnt = processed_dbg_str.count('%')
-
- # Need to take into account parentheses between args in function
- # calls that might be in the args list. Use regex module for
- # this one since the recursive pattern match helps simplify
- # only matching commas outside nested call groups.
- if macro['dbg_args'] is None:
- processed_arg_str = ""
- else:
- processed_arg_str = macro['dbg_args'].strip()
-
- argument_other_replacements = ['\r', '\n']
- for r in argument_other_replacements:
- if s in processed_arg_str:
- processed_arg_str = processed_arg_str.replace(s, '')
- processed_arg_str = re.sub(r' +', ' ', processed_arg_str)
-
- # Handle special case of commas in arg strings - remove them for
- # final count to pick up correct number of argument separating
- # commas.
- processed_arg_str = re.sub(
- r'([\"\'])(?:|\\.|[^\\])*?(\1)',
- '',
- processed_arg_str)
-
- arg_matches = regex.findall(
- r'(?:\((?:[^)(]+|(?R))*+\))|(,)',
- processed_arg_str,
- regex.MULTILINE)
-
- arg_cnt = 0
- if processed_arg_str != '':
- arg_cnt = arg_matches.count(',')
-
- if print_spec_cnt != arg_cnt:
- logging.error(f"File: {file_dbg_path}")
- logging.error(f" Message = {macro['dbg_str']}")
- logging.error(f" Arguments = \"{processed_arg_str}\"")
- logging.error(f" Specifier Count = {print_spec_cnt}")
- logging.error(f" Argument Count = {arg_cnt}")
-
- failure_cnt += 1
-
- return failure_cnt, print_spec_cnt, arg_cnt
-
-
-def get_debug_macros(file_contents: str) -> List[Dict[str, str]]:
- """Extract debug macros from the given file contents.
-
- Args:
- file_contents (str): A string of source file contents that may
- contain debug macros.
-
- Returns:
- List[Dict[str, str]]: A groupdict of debug macro regex matches
- within the file contents provided.
- """
-
- # This is the main regular expression that is responsible for identifying
- # DEBUG macros within source files and grouping the macro message string
- # and macro arguments strings so they can be further processed.
- r = regex.compile(
- r'(?>(?P<prologue>DEBUG\s*\(\s*\((?:.*?,))(?:\s*))(?P<dbg_str>.*?(?:\"'
- r'(?:[^\"\\]|\\.)*\".*?)*)(?:(?(?=,)(?<dbg_args>.*?(?=(?:\s*\)){2}\s*;'
- r'))))(?:\s*\)){2,};?',
- regex.MULTILINE | regex.DOTALL)
- return [m.groupdict() for m in r.finditer(file_contents)]
-
-
-def check_macros_in_string(src_str: str,
- file_dbg_path: str,
- **macro_subs: str) -> Tuple[int, int, int]:
- """Checks for debug macro formatting errors in a string.
-
- Args:
- src_str (str): Contents of the string with debug macros.
-
- file_dbg_path (str): The file path (or other custom string) to display
- in debug messages.
-
- macro_subs (Dict[str,str]): Variable-length keyword and replacement
- value string pairs to substitute during debug macro checks.
-
- Returns:
- Tuple[int, int, int]: A tuple of the number of formatting errors,
- number of print specifiers, and number of arguments for the macros
- in the string given.
- """
- return check_debug_macros(
- get_debug_macros(src_str), file_dbg_path, **macro_subs)
-
-
-def check_macros_in_file(file: PurePath,
- file_dbg_path: str,
- show_utf8_decode_warning: bool = False,
- **macro_subs: str) -> Tuple[int, int, int]:
- """Checks for debug macro formatting errors in a file.
-
- Args:
- file (PurePath): The file path to check.
-
- file_dbg_path (str): The file path (or other custom string) to display
- in debug messages.
-
- show_utf8_decode_warning (bool, optional): Indicates whether to show
- warnings if UTF-8 files fail to decode. Defaults to False.
-
- macro_subs (Dict[str,str]): Variable-length keyword and replacement
- value string pairs to substitute during debug macro checks.
-
- Returns:
- Tuple[int, int, int]: A tuple of the number of formatting errors,
- number of print specifiers, and number of arguments for the macros
- in the file given.
- """
- try:
- return check_macros_in_string(
- file.read_text(encoding='utf-8'), file_dbg_path,
- **macro_subs)
- except UnicodeDecodeError as e:
- if show_utf8_decode_warning:
- logging.warning(
- f"{file_dbg_path} UTF-8 decode error.\n"
- " Debug macro code check skipped!\n"
- f" -> {str(e)}")
- return 0, 0, 0
-
-
-def check_macros_in_directory(directory: PurePath,
- file_extensions: Iterable[str] = ('.c',),
- ignore_git_ignore_files: Optional[bool] = True,
- ignore_git_submodules: Optional[bool] = True,
- show_progress_bar: Optional[bool] = True,
- show_utf8_decode_warning: bool = False,
- **macro_subs: str
- ) -> int:
- """Checks files with the given extension in the given directory for debug
- macro formatting errors.
-
- Args:
- directory (PurePath): The path to the directory to check.
- file_extensions (Iterable[str], optional): An iterable of strings
- representing file extensions to check. Defaults to ('.c',).
-
- ignore_git_ignore_files (Optional[bool], optional): Indicates whether
- files ignored by git should be ignored for the debug macro check.
- Defaults to True.
-
- ignore_git_submodules (Optional[bool], optional): Indicates whether
- files located in git submodules should not be checked. Defaults to
- True.
-
- show_progress_bar (Optional[bool], optional): Indicates whether to
- show a progress bar to show progress status while checking macros.
- This is more useful on a very large directories. Defaults to True.
-
- show_utf8_decode_warning (bool, optional): Indicates whether to show
- warnings if UTF-8 files fail to decode. Defaults to False.
-
- macro_subs (Dict[str,str]): Variable-length keyword and replacement
- value string pairs to substitute during debug macro checks.
-
- Returns:
- int: Count of debug macro errors in the directory.
- """
- def _get_file_list(root_directory: PurePath,
- extensions: Iterable[str]) -> List[Path]:
- """Returns a list of files recursively located within the path.
-
- Args:
- root_directory (PurePath): A directory Path object to the root
- folder.
-
- extensions (Iterable[str]): An iterable of strings that
- represent file extensions to recursively search for within
- root_directory.
-
- Returns:
- List[Path]: List of file Path objects to files found in the
- given directory with the given extensions.
- """
- def _show_file_discovered_message(file_count: int,
- elapsed_time: float) -> None:
- print(f"\rDiscovered {file_count:,} files in",
- f"{current_start_delta:-.0f}s"
- f"{'.' * min(int(current_start_delta), 40)}", end="\r")
-
- start_time = timeit.default_timer()
- previous_indicator_time = start_time
-
- files = []
- for file in root_directory.rglob('*'):
- if file.suffix in extensions:
- files.append(Path(file))
-
- # Give an indicator progress is being made
- # This has a negligible impact on overall performance
- # with print emission limited to half second intervals.
- current_time = timeit.default_timer()
- current_start_delta = current_time - start_time
-
- if current_time - previous_indicator_time >= 0.5:
- # Since this rewrites the line, it can be considered a form
- # of progress bar
- if show_progress_bar:
- _show_file_discovered_message(len(files),
- current_start_delta)
- previous_indicator_time = current_time
-
- if show_progress_bar:
- _show_file_discovered_message(len(files), current_start_delta)
- print()
-
- return files
-
- logging.info(f"Checking Debug Macros in directory: "
- f"{directory.resolve()}\n")
-
- logging.info("Gathering the overall file list. This might take a"
- "while.\n")
-
- start_time = timeit.default_timer()
- file_list = set(_get_file_list(directory, file_extensions))
- end_time = timeit.default_timer() - start_time
-
- logging.debug(f"[PERF] File search found {len(file_list):,} files in "
- f"{end_time:.2f} seconds.")
-
- if ignore_git_ignore_files:
- logging.info("Getting git ignore files...")
- start_time = timeit.default_timer()
- ignored_file_paths = GitHelpers.get_git_ignored_paths(directory)
- end_time = timeit.default_timer() - start_time
-
- logging.debug(f"[PERF] File ignore gathering took {end_time:.2f} "
- f"seconds.")
-
- logging.info("Ignoring git ignore files...")
- logging.debug(f"File list count before git ignore {len(file_list):,}")
- start_time = timeit.default_timer()
- file_list = file_list.difference(ignored_file_paths)
- end_time = timeit.default_timer() - start_time
- logging.info(f" {len(ignored_file_paths):,} files are ignored by git")
- logging.info(f" {len(file_list):,} files after removing "
- f"ignored files")
-
- logging.debug(f"[PERF] File ignore calculation took {end_time:.2f} "
- f"seconds.")
-
- if ignore_git_submodules:
- logging.info("Ignoring git submodules...")
- submodule_paths = GitHelpers.get_git_submodule_paths(directory)
- if submodule_paths:
- logging.debug(f"File list count before git submodule exclusion "
- f"{len(file_list):,}")
- start_time = timeit.default_timer()
- file_list = [f for f in file_list
- if not f.is_relative_to(*submodule_paths)]
- end_time = timeit.default_timer() - start_time
-
- for path in enumerate(submodule_paths):
- logging.debug(" {0}. {1}".format(*path))
-
- logging.info(f" {len(submodule_paths):,} submodules found")
- logging.info(f" {len(file_list):,} files will be examined after "
- f"excluding files in submodules")
-
- logging.debug(f"[PERF] Submodule exclusion calculation took "
- f"{end_time:.2f} seconds.")
- else:
- logging.warning("No submodules found")
-
- logging.info(f"\nStarting macro check on {len(file_list):,} files.")
-
- cache_progress_filter = CacheDuringProgressFilter()
- handler = next((h for h in logging.getLogger().handlers if h.get_name() ==
- 'stdout_logger_handler'), None)
-
- if handler is not None:
- handler.addFilter(cache_progress_filter)
-
- start_time = timeit.default_timer()
-
- failure_cnt, file_cnt = 0, 0
- for file_cnt, file in enumerate(file_list):
- file_rel_path = str(file.relative_to(directory))
- failure_cnt += check_macros_in_file(
- file, file_rel_path, show_utf8_decode_warning,
- **macro_subs)[0]
- if show_progress_bar:
- _show_progress(file_cnt, len(file_list),
- f" {failure_cnt} errors" if failure_cnt > 0 else "")
-
- if show_progress_bar:
- _show_progress(len(file_list), len(file_list),
- f" {failure_cnt} errors" if failure_cnt > 0 else "")
- print("\n", flush=True)
-
- end_time = timeit.default_timer() - start_time
-
- if handler is not None:
- handler.removeFilter(cache_progress_filter)
-
- for record in cache_progress_filter.message_cache:
- handler.emit(record)
-
- logging.debug(f"[PERF] The macro check operation took {end_time:.2f} "
- f"seconds.")
-
- _log_failure_count(failure_cnt, file_cnt)
-
- return failure_cnt
-
-
-def _log_failure_count(failure_count: int, file_count: int) -> None:
- """Logs the failure count.
-
- Args:
- failure_count (int): Count of failures to log.
-
- file_count (int): Count of files with failures.
- """
- if failure_count > 0:
- logging.error("\n")
- logging.error(f"{failure_count:,} debug macro errors in "
- f"{file_count:,} files")
-
-
-def _show_progress(step: int, total: int, suffix: str = '') -> None:
- """Print progress of tick to total.
-
- Args:
- step (int): The current step count.
-
- total (int): The total step count.
-
- suffix (str): String to print at the end of the progress bar.
- """
- global _progress_start_time
-
- if step == 0:
- _progress_start_time = timeit.default_timer()
-
- terminal_col = shutil.get_terminal_size().columns
- var_consume_len = (len("Progress|\u2588| 000.0% Complete 000s") +
- len(suffix))
- avail_len = terminal_col - var_consume_len
-
- percent = f"{100 * (step / float(total)):3.1f}"
- filled = int(avail_len * step // total)
- bar = '\u2588' * filled + '-' * (avail_len - filled)
- step_time = timeit.default_timer() - _progress_start_time
-
- print(f'\rProgress|{bar}| {percent}% Complete {step_time:-3.0f}s'
- f'{suffix}', end='\r')
-
-
-def _module_invocation_check_macros_in_directory_wrapper() -> int:
- """Provides an command-line argument wrapper for checking debug macros.
-
- Returns:
- int: The system exit code value.
- """
- import argparse
- import builtins
-
- def _check_dir_path(dir_path: str) -> bool:
- """Returns the absolute path if the path is a directory."
-
- Args:
- dir_path (str): A directory file system path.
-
- Raises:
- NotADirectoryError: The directory path given is not a directory.
-
- Returns:
- bool: True if the path is a directory else False.
- """
- abs_dir_path = os.path.abspath(dir_path)
- if os.path.isdir(dir_path):
- return abs_dir_path
- else:
- raise NotADirectoryError(abs_dir_path)
-
- def _check_file_path(file_path: str) -> bool:
- """Returns the absolute path if the path is a file."
-
- Args:
- file_path (str): A file path.
-
- Raises:
- FileExistsError: The path is not a valid file.
-
- Returns:
- bool: True if the path is a valid file else False.
- """
- abs_file_path = os.path.abspath(file_path)
- if os.path.isfile(file_path):
- return abs_file_path
- else:
- raise FileExistsError(file_path)
-
- def _quiet_print(*args, **kwargs):
- """Replaces print when quiet is requested to prevent printing messages.
- """
- pass
-
- root_logger = logging.getLogger()
- root_logger.setLevel(logging.DEBUG)
-
- stdout_logger_handler = logging.StreamHandler(sys.stdout)
- stdout_logger_handler.set_name('stdout_logger_handler')
- stdout_logger_handler.setLevel(logging.INFO)
- stdout_logger_handler.setFormatter(logging.Formatter('%(message)s'))
- root_logger.addHandler(stdout_logger_handler)
-
- parser = argparse.ArgumentParser(
- prog=PROGRAM_NAME,
- description=(
- "Checks for debug macro formatting "
- "errors within files recursively located within "
- "a given directory."),
- formatter_class=RawTextHelpFormatter)
-
- io_req_group = parser.add_mutually_exclusive_group(required=True)
- io_opt_group = parser.add_argument_group(
- "Optional input and output")
- git_group = parser.add_argument_group("Optional git control")
-
- io_req_group.add_argument('-w', '--workspace-directory',
- type=_check_dir_path,
- help="Directory of source files to check.\n\n")
-
- io_req_group.add_argument('-i', '--input-file', nargs='?',
- type=_check_file_path,
- help="File path for an input file to check.\n\n"
- "Note that some other options do not apply "
- "if a single file is specified such as "
- "the\ngit options and file extensions.\n\n")
-
- io_opt_group.add_argument('-l', '--log-file',
- nargs='?',
- default=None,
- const='debug_macro_check.log',
- help="File path for log output.\n"
- "(default: if the flag is given with no "
- "file path then a file called\n"
- "debug_macro_check.log is created and used "
- "in the current directory)\n\n")
-
- io_opt_group.add_argument('-s', '--substitution-file',
- type=_check_file_path,
- help="A substitution YAML file specifies string "
- "substitutions to perform within the debug "
- "macro.\n\nThis is intended to be a simple "
- "mechanism to expand the rare cases of pre-"
- "processor\nmacros without directly "
- "involving the pre-processor. The file "
- "consists of one or more\nstring value "
- "pairs where the key is the identifier to "
- "replace and the value is the value\nto "
- "replace it with.\n\nThis can also be used "
- "as a method to ignore results by "
- "replacing the problematic string\nwith a "
- "different string.\n\n")
-
- io_opt_group.add_argument('-v', '--verbose-log-file',
- action='count',
- default=0,
- help="Set file logging verbosity level.\n"
- " - None: Info & > level messages\n"
- " - '-v': + Debug level messages\n"
- " - '-vv': + File name and function\n"
- " - '-vvv': + Line number\n"
- " - '-vvvv': + Timestamp\n"
- "(default: verbose logging is not enabled)"
- "\n\n")
-
- io_opt_group.add_argument('-n', '--no-progress-bar', action='store_true',
- help="Disables progress bars.\n"
- "(default: progress bars are used in some"
- "places to show progress)\n\n")
-
- io_opt_group.add_argument('-q', '--quiet', action='store_true',
- help="Disables console output.\n"
- "(default: console output is enabled)\n\n")
-
- io_opt_group.add_argument('-u', '--utf8w', action='store_true',
- help="Shows warnings for file UTF-8 decode "
- "errors.\n"
- "(default: UTF-8 decode errors are not "
- "shown)\n\n")
-
- git_group.add_argument('-df', '--do-not-ignore-git-ignore-files',
- action='store_true',
- help="Do not ignore git ignored files.\n"
- "(default: files in git ignore files are "
- "ignored)\n\n")
-
- git_group.add_argument('-ds', '--do-not-ignore-git_submodules',
- action='store_true',
- help="Do not ignore files in git submodules.\n"
- "(default: files in git submodules are "
- "ignored)\n\n")
-
- parser.add_argument('-e', '--extensions', nargs='*', default=['.c'],
- help="List of file extensions to include.\n"
- "(default: %(default)s)")
-
- args = parser.parse_args()
-
- if args.quiet:
- # Don't print in the few places that directly print
- builtins.print = _quiet_print
- stdout_logger_handler.addFilter(QuietFilter(args.quiet))
-
- if args.log_file:
- file_logger_handler = logging.FileHandler(filename=args.log_file,
- mode='w', encoding='utf-8')
-
- # In an ideal world, everyone would update to the latest Python
- # minor version (3.10) after a few weeks/months. Since that's not the
- # case, resist from using structural pattern matching in Python 3.10.
- # https://peps.python.org/pep-0636/
-
- if args.verbose_log_file == 0:
- file_logger_handler.setLevel(logging.INFO)
- file_logger_formatter = logging.Formatter(
- '%(levelname)-8s %(message)s')
- elif args.verbose_log_file == 1:
- file_logger_handler.setLevel(logging.DEBUG)
- file_logger_formatter = logging.Formatter(
- '%(levelname)-8s %(message)s')
- elif args.verbose_log_file == 2:
- file_logger_handler.setLevel(logging.DEBUG)
- file_logger_formatter = logging.Formatter(
- '[%(filename)s - %(funcName)20s() ] %(levelname)-8s '
- '%(message)s')
- elif args.verbose_log_file == 3:
- file_logger_handler.setLevel(logging.DEBUG)
- file_logger_formatter = logging.Formatter(
- '[%(filename)s:%(lineno)s - %(funcName)20s() ] '
- '%(levelname)-8s %(message)s')
- elif args.verbose_log_file == 4:
- file_logger_handler.setLevel(logging.DEBUG)
- file_logger_formatter = logging.Formatter(
- '%(asctime)s [%(filename)s:%(lineno)s - %(funcName)20s() ]'
- ' %(levelname)-8s %(message)s')
- else:
- file_logger_handler.setLevel(logging.DEBUG)
- file_logger_formatter = logging.Formatter(
- '%(asctime)s [%(filename)s:%(lineno)s - %(funcName)20s() ]'
- ' %(levelname)-8s %(message)s')
-
- file_logger_handler.addFilter(ProgressFilter())
- file_logger_handler.setFormatter(file_logger_formatter)
- root_logger.addHandler(file_logger_handler)
-
- logging.info(PROGRAM_NAME + "\n")
-
- substitution_data = {}
- if args.substitution_file:
- logging.info(f"Loading substitution file {args.substitution_file}")
- with open(args.substitution_file, 'r') as sf:
- substitution_data = yaml.safe_load(sf)
-
- if args.workspace_directory:
- return check_macros_in_directory(
- Path(args.workspace_directory),
- args.extensions,
- not args.do_not_ignore_git_ignore_files,
- not args.do_not_ignore_git_submodules,
- not args.no_progress_bar,
- args.utf8w,
- **substitution_data)
- else:
- curr_dir = Path(__file__).parent
- input_file = Path(args.input_file)
-
- rel_path = str(input_file)
- if input_file.is_relative_to(curr_dir):
- rel_path = str(input_file.relative_to(curr_dir))
-
- logging.info(f"Checking Debug Macros in File: "
- f"{input_file.resolve()}\n")
-
- start_time = timeit.default_timer()
- failure_cnt = check_macros_in_file(
- input_file,
- rel_path,
- args.utf8w,
- **substitution_data)[0]
- end_time = timeit.default_timer() - start_time
-
- logging.debug(f"[PERF] The file macro check operation took "
- f"{end_time:.2f} seconds.")
-
- _log_failure_count(failure_cnt, 1)
-
- return failure_cnt
-
-
-if __name__ == '__main__':
- # The exit status value is the number of macro formatting errors found.
- # Therefore, if no macro formatting errors are found, 0 is returned.
- # Some systems require the return value to be in the range 0-127, so
- # a lower maximum of 100 is enforced to allow a wide range of potential
- # values with a reasonably large maximum.
- try:
- sys.exit(max(_module_invocation_check_macros_in_directory_wrapper(),
- 100))
- except KeyboardInterrupt:
- logging.warning("Exiting due to keyboard interrupt.")
- # Actual formatting errors are only allowed to reach 100.
- # 101 signals a keyboard interrupt.
- sys.exit(101)
- except FileExistsError as e:
- # 102 signals a file not found error.
- logging.critical(f"Input file {e.args[0]} does not exist.")
- sys.exit(102)
+# @file DebugMacroCheck.py +# +# A script that checks if DEBUG macros are formatted properly. +# +# In particular, that print format specifiers are defined +# with the expected number of arguments in the variable +# argument list. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +from argparse import RawTextHelpFormatter +import logging +import os +import re +import regex +import sys +import shutil +import timeit +import yaml + +from edk2toollib.utility_functions import RunCmd +from io import StringIO +from pathlib import Path, PurePath +from typing import Dict, Iterable, List, Optional, Tuple + + +PROGRAM_NAME = "Debug Macro Checker" + + +class GitHelpers: + """ + Collection of Git helpers. + + Will be moved to a more generic module and imported in the future. + """ + + @staticmethod + def get_git_ignored_paths(directory_path: PurePath) -> List[Path]: + """Returns ignored files in this git repository. + + Args: + directory_path (PurePath): Path to the git directory. + + Returns: + List[Path]: List of file absolute paths to all files ignored + in this git repository. If git is not found, an empty + list will be returned. + """ + if not shutil.which("git"): + logging.warn( + "Git is not found on this system. Git submodule paths will " + "not be considered.") + return [] + + out_stream_buffer = StringIO() + exit_code = RunCmd("git", "ls-files --other", + workingdir=str(directory_path), + outstream=out_stream_buffer, + logging_level=logging.NOTSET) + if exit_code != 0: + return [] + + rel_paths = out_stream_buffer.getvalue().strip().splitlines() + abs_paths = [] + for path in rel_paths: + abs_paths.append(Path(directory_path, path)) + return abs_paths + + @staticmethod + def get_git_submodule_paths(directory_path: PurePath) -> List[Path]: + """Returns submodules in the given workspace directory. + + Args: + directory_path (PurePath): Path to the git directory. + + Returns: + List[Path]: List of directory absolute paths to the root of + each submodule found from this folder. If submodules are not + found, an empty list will be returned. + """ + if not shutil.which("git"): + return [] + + if os.path.isfile(directory_path.joinpath(".gitmodules")): + out_stream_buffer = StringIO() + exit_code = RunCmd( + "git", "config --file .gitmodules --get-regexp path", + workingdir=str(directory_path), + outstream=out_stream_buffer, + logging_level=logging.NOTSET) + if exit_code != 0: + return [] + + submodule_paths = [] + for line in out_stream_buffer.getvalue().strip().splitlines(): + submodule_paths.append( + Path(directory_path, line.split()[1])) + + return submodule_paths + else: + return [] + + +class QuietFilter(logging.Filter): + """A logging filter that temporarily suppresses message output.""" + + def __init__(self, quiet: bool = False): + """Class constructor method. + + Args: + quiet (bool, optional): Indicates if messages are currently being + printed (False) or not (True). Defaults to False. + """ + + self._quiet = quiet + + def filter(self, record: logging.LogRecord) -> bool: + """Quiet filter method. + + Args: + record (logging.LogRecord): A log record object that the filter is + applied to. + + Returns: + bool: True if messages are being suppressed. Otherwise, False. + """ + return not self._quiet + + +class ProgressFilter(logging.Filter): + """A logging filter that suppresses 'Progress' messages.""" + + def filter(self, record: logging.LogRecord) -> bool: + """Progress filter method. + + Args: + record (logging.LogRecord): A log record object that the filter is + applied to. + + Returns: + bool: True if the message is not a 'Progress' message. Otherwise, + False. + """ + return not record.getMessage().startswith("\rProgress") + + +class CacheDuringProgressFilter(logging.Filter): + """A logging filter that suppresses messages during progress operations.""" + + _message_cache = [] + + @property + def message_cache(self) -> List[logging.LogRecord]: + """Contains a cache of messages accumulated during time of operation. + + Returns: + List[logging.LogRecord]: List of log records stored while the + filter was active. + """ + return self._message_cache + + def filter(self, record: logging.LogRecord): + """Cache progress filter that suppresses messages during progress + display output. + + Args: + record (logging.LogRecord): A log record to cache. + """ + self._message_cache.append(record) + + +def check_debug_macros(macros: Iterable[Dict[str, str]], + file_dbg_path: str, + **macro_subs: str + ) -> Tuple[int, int, int]: + """Checks if debug macros contain formatting errors. + + Args: + macros (Iterable[Dict[str, str]]): : A groupdict of macro matches. + This is an iterable of dictionaries with group names from the regex + match as the key and the matched string as the value for the key. + + file_dbg_path (str): The file path (or other custom string) to display + in debug messages. + + macro_subs (Dict[str,str]): Variable-length keyword and replacement + value string pairs to substitute during debug macro checks. + + Returns: + Tuple[int, int, int]: A tuple of the number of formatting errors, + number of print specifiers, and number of arguments for the macros + given. + """ + + macro_subs = {k.lower(): v for k, v in macro_subs.items()} + + arg_cnt, failure_cnt, print_spec_cnt = 0, 0, 0 + for macro in macros: + # Special Specifier Handling + processed_dbg_str = macro['dbg_str'].strip().lower() + + logging.debug(f"Inspecting macro: {macro}") + + # Make any macro substitutions so further processing is applied + # to the substituted value. + for k in macro_subs.keys(): + processed_dbg_str = processed_dbg_str.replace(k, macro_subs[k]) + + logging.debug("Debug macro string after replacements: " + f"{processed_dbg_str}") + + # These are very rarely used in debug strings. They are somewhat + # more common in HII code to control text displayed on the + # console. Due to the rarity and likelihood usage is a mistake, + # a warning is shown if found. + specifier_display_replacements = ['%n', '%h', '%e', '%b', '%v'] + for s in specifier_display_replacements: + if s in processed_dbg_str: + logging.warning(f"File: {file_dbg_path}") + logging.warning(f" {s} found in string and ignored:") + logging.warning(f" \"{processed_dbg_str}\"") + processed_dbg_str = processed_dbg_str.replace(s, '') + + # These are miscellaneous print specifiers that do not require + # special parsing and simply need to be replaced since they do + # have a corresponding argument associated with them. + specifier_other_replacements = ['%%', '\r', '\n'] + for s in specifier_other_replacements: + if s in processed_dbg_str: + processed_dbg_str = processed_dbg_str.replace(s, '') + + processed_dbg_str = re.sub( + r'%[.\-+ ,Ll0-9]*\*[.\-+ ,Ll0-9]*[a-zA-Z]', '%_%_', + processed_dbg_str) + logging.debug(f"Final macro before print specifier scan: " + f"{processed_dbg_str}") + + print_spec_cnt = processed_dbg_str.count('%') + + # Need to take into account parentheses between args in function + # calls that might be in the args list. Use regex module for + # this one since the recursive pattern match helps simplify + # only matching commas outside nested call groups. + if macro['dbg_args'] is None: + processed_arg_str = "" + else: + processed_arg_str = macro['dbg_args'].strip() + + argument_other_replacements = ['\r', '\n'] + for r in argument_other_replacements: + if s in processed_arg_str: + processed_arg_str = processed_arg_str.replace(s, '') + processed_arg_str = re.sub(r' +', ' ', processed_arg_str) + + # Handle special case of commas in arg strings - remove them for + # final count to pick up correct number of argument separating + # commas. + processed_arg_str = re.sub( + r'([\"\'])(?:|\\.|[^\\])*?(\1)', + '', + processed_arg_str) + + arg_matches = regex.findall( + r'(?:\((?:[^)(]+|(?R))*+\))|(,)', + processed_arg_str, + regex.MULTILINE) + + arg_cnt = 0 + if processed_arg_str != '': + arg_cnt = arg_matches.count(',') + + if print_spec_cnt != arg_cnt: + logging.error(f"File: {file_dbg_path}") + logging.error(f" Message = {macro['dbg_str']}") + logging.error(f" Arguments = \"{processed_arg_str}\"") + logging.error(f" Specifier Count = {print_spec_cnt}") + logging.error(f" Argument Count = {arg_cnt}") + + failure_cnt += 1 + + return failure_cnt, print_spec_cnt, arg_cnt + + +def get_debug_macros(file_contents: str) -> List[Dict[str, str]]: + """Extract debug macros from the given file contents. + + Args: + file_contents (str): A string of source file contents that may + contain debug macros. + + Returns: + List[Dict[str, str]]: A groupdict of debug macro regex matches + within the file contents provided. + """ + + # This is the main regular expression that is responsible for identifying + # DEBUG macros within source files and grouping the macro message string + # and macro arguments strings so they can be further processed. + r = regex.compile( + r'(?>(?P<prologue>DEBUG\s*\(\s*\((?:.*?,))(?:\s*))(?P<dbg_str>.*?(?:\"' + r'(?:[^\"\\]|\\.)*\".*?)*)(?:(?(?=,)(?<dbg_args>.*?(?=(?:\s*\)){2}\s*;' + r'))))(?:\s*\)){2,};?', + regex.MULTILINE | regex.DOTALL) + return [m.groupdict() for m in r.finditer(file_contents)] + + +def check_macros_in_string(src_str: str, + file_dbg_path: str, + **macro_subs: str) -> Tuple[int, int, int]: + """Checks for debug macro formatting errors in a string. + + Args: + src_str (str): Contents of the string with debug macros. + + file_dbg_path (str): The file path (or other custom string) to display + in debug messages. + + macro_subs (Dict[str,str]): Variable-length keyword and replacement + value string pairs to substitute during debug macro checks. + + Returns: + Tuple[int, int, int]: A tuple of the number of formatting errors, + number of print specifiers, and number of arguments for the macros + in the string given. + """ + return check_debug_macros( + get_debug_macros(src_str), file_dbg_path, **macro_subs) + + +def check_macros_in_file(file: PurePath, + file_dbg_path: str, + show_utf8_decode_warning: bool = False, + **macro_subs: str) -> Tuple[int, int, int]: + """Checks for debug macro formatting errors in a file. + + Args: + file (PurePath): The file path to check. + + file_dbg_path (str): The file path (or other custom string) to display + in debug messages. + + show_utf8_decode_warning (bool, optional): Indicates whether to show + warnings if UTF-8 files fail to decode. Defaults to False. + + macro_subs (Dict[str,str]): Variable-length keyword and replacement + value string pairs to substitute during debug macro checks. + + Returns: + Tuple[int, int, int]: A tuple of the number of formatting errors, + number of print specifiers, and number of arguments for the macros + in the file given. + """ + try: + return check_macros_in_string( + file.read_text(encoding='utf-8'), file_dbg_path, + **macro_subs) + except UnicodeDecodeError as e: + if show_utf8_decode_warning: + logging.warning( + f"{file_dbg_path} UTF-8 decode error.\n" + " Debug macro code check skipped!\n" + f" -> {str(e)}") + return 0, 0, 0 + + +def check_macros_in_directory(directory: PurePath, + file_extensions: Iterable[str] = ('.c',), + ignore_git_ignore_files: Optional[bool] = True, + ignore_git_submodules: Optional[bool] = True, + show_progress_bar: Optional[bool] = True, + show_utf8_decode_warning: bool = False, + **macro_subs: str + ) -> int: + """Checks files with the given extension in the given directory for debug + macro formatting errors. + + Args: + directory (PurePath): The path to the directory to check. + file_extensions (Iterable[str], optional): An iterable of strings + representing file extensions to check. Defaults to ('.c',). + + ignore_git_ignore_files (Optional[bool], optional): Indicates whether + files ignored by git should be ignored for the debug macro check. + Defaults to True. + + ignore_git_submodules (Optional[bool], optional): Indicates whether + files located in git submodules should not be checked. Defaults to + True. + + show_progress_bar (Optional[bool], optional): Indicates whether to + show a progress bar to show progress status while checking macros. + This is more useful on a very large directories. Defaults to True. + + show_utf8_decode_warning (bool, optional): Indicates whether to show + warnings if UTF-8 files fail to decode. Defaults to False. + + macro_subs (Dict[str,str]): Variable-length keyword and replacement + value string pairs to substitute during debug macro checks. + + Returns: + int: Count of debug macro errors in the directory. + """ + def _get_file_list(root_directory: PurePath, + extensions: Iterable[str]) -> List[Path]: + """Returns a list of files recursively located within the path. + + Args: + root_directory (PurePath): A directory Path object to the root + folder. + + extensions (Iterable[str]): An iterable of strings that + represent file extensions to recursively search for within + root_directory. + + Returns: + List[Path]: List of file Path objects to files found in the + given directory with the given extensions. + """ + def _show_file_discovered_message(file_count: int, + elapsed_time: float) -> None: + print(f"\rDiscovered {file_count:,} files in", + f"{current_start_delta:-.0f}s" + f"{'.' * min(int(current_start_delta), 40)}", end="\r") + + start_time = timeit.default_timer() + previous_indicator_time = start_time + + files = [] + for file in root_directory.rglob('*'): + if file.suffix in extensions: + files.append(Path(file)) + + # Give an indicator progress is being made + # This has a negligible impact on overall performance + # with print emission limited to half second intervals. + current_time = timeit.default_timer() + current_start_delta = current_time - start_time + + if current_time - previous_indicator_time >= 0.5: + # Since this rewrites the line, it can be considered a form + # of progress bar + if show_progress_bar: + _show_file_discovered_message(len(files), + current_start_delta) + previous_indicator_time = current_time + + if show_progress_bar: + _show_file_discovered_message(len(files), current_start_delta) + print() + + return files + + logging.info(f"Checking Debug Macros in directory: " + f"{directory.resolve()}\n") + + logging.info("Gathering the overall file list. This might take a" + "while.\n") + + start_time = timeit.default_timer() + file_list = set(_get_file_list(directory, file_extensions)) + end_time = timeit.default_timer() - start_time + + logging.debug(f"[PERF] File search found {len(file_list):,} files in " + f"{end_time:.2f} seconds.") + + if ignore_git_ignore_files: + logging.info("Getting git ignore files...") + start_time = timeit.default_timer() + ignored_file_paths = GitHelpers.get_git_ignored_paths(directory) + end_time = timeit.default_timer() - start_time + + logging.debug(f"[PERF] File ignore gathering took {end_time:.2f} " + f"seconds.") + + logging.info("Ignoring git ignore files...") + logging.debug(f"File list count before git ignore {len(file_list):,}") + start_time = timeit.default_timer() + file_list = file_list.difference(ignored_file_paths) + end_time = timeit.default_timer() - start_time + logging.info(f" {len(ignored_file_paths):,} files are ignored by git") + logging.info(f" {len(file_list):,} files after removing " + f"ignored files") + + logging.debug(f"[PERF] File ignore calculation took {end_time:.2f} " + f"seconds.") + + if ignore_git_submodules: + logging.info("Ignoring git submodules...") + submodule_paths = GitHelpers.get_git_submodule_paths(directory) + if submodule_paths: + logging.debug(f"File list count before git submodule exclusion " + f"{len(file_list):,}") + start_time = timeit.default_timer() + file_list = [f for f in file_list + if not f.is_relative_to(*submodule_paths)] + end_time = timeit.default_timer() - start_time + + for path in enumerate(submodule_paths): + logging.debug(" {0}. {1}".format(*path)) + + logging.info(f" {len(submodule_paths):,} submodules found") + logging.info(f" {len(file_list):,} files will be examined after " + f"excluding files in submodules") + + logging.debug(f"[PERF] Submodule exclusion calculation took " + f"{end_time:.2f} seconds.") + else: + logging.warning("No submodules found") + + logging.info(f"\nStarting macro check on {len(file_list):,} files.") + + cache_progress_filter = CacheDuringProgressFilter() + handler = next((h for h in logging.getLogger().handlers if h.get_name() == + 'stdout_logger_handler'), None) + + if handler is not None: + handler.addFilter(cache_progress_filter) + + start_time = timeit.default_timer() + + failure_cnt, file_cnt = 0, 0 + for file_cnt, file in enumerate(file_list): + file_rel_path = str(file.relative_to(directory)) + failure_cnt += check_macros_in_file( + file, file_rel_path, show_utf8_decode_warning, + **macro_subs)[0] + if show_progress_bar: + _show_progress(file_cnt, len(file_list), + f" {failure_cnt} errors" if failure_cnt > 0 else "") + + if show_progress_bar: + _show_progress(len(file_list), len(file_list), + f" {failure_cnt} errors" if failure_cnt > 0 else "") + print("\n", flush=True) + + end_time = timeit.default_timer() - start_time + + if handler is not None: + handler.removeFilter(cache_progress_filter) + + for record in cache_progress_filter.message_cache: + handler.emit(record) + + logging.debug(f"[PERF] The macro check operation took {end_time:.2f} " + f"seconds.") + + _log_failure_count(failure_cnt, file_cnt) + + return failure_cnt + + +def _log_failure_count(failure_count: int, file_count: int) -> None: + """Logs the failure count. + + Args: + failure_count (int): Count of failures to log. + + file_count (int): Count of files with failures. + """ + if failure_count > 0: + logging.error("\n") + logging.error(f"{failure_count:,} debug macro errors in " + f"{file_count:,} files") + + +def _show_progress(step: int, total: int, suffix: str = '') -> None: + """Print progress of tick to total. + + Args: + step (int): The current step count. + + total (int): The total step count. + + suffix (str): String to print at the end of the progress bar. + """ + global _progress_start_time + + if step == 0: + _progress_start_time = timeit.default_timer() + + terminal_col = shutil.get_terminal_size().columns + var_consume_len = (len("Progress|\u2588| 000.0% Complete 000s") + + len(suffix)) + avail_len = terminal_col - var_consume_len + + percent = f"{100 * (step / float(total)):3.1f}" + filled = int(avail_len * step // total) + bar = '\u2588' * filled + '-' * (avail_len - filled) + step_time = timeit.default_timer() - _progress_start_time + + print(f'\rProgress|{bar}| {percent}% Complete {step_time:-3.0f}s' + f'{suffix}', end='\r') + + +def _module_invocation_check_macros_in_directory_wrapper() -> int: + """Provides an command-line argument wrapper for checking debug macros. + + Returns: + int: The system exit code value. + """ + import argparse + import builtins + + def _check_dir_path(dir_path: str) -> bool: + """Returns the absolute path if the path is a directory." + + Args: + dir_path (str): A directory file system path. + + Raises: + NotADirectoryError: The directory path given is not a directory. + + Returns: + bool: True if the path is a directory else False. + """ + abs_dir_path = os.path.abspath(dir_path) + if os.path.isdir(dir_path): + return abs_dir_path + else: + raise NotADirectoryError(abs_dir_path) + + def _check_file_path(file_path: str) -> bool: + """Returns the absolute path if the path is a file." + + Args: + file_path (str): A file path. + + Raises: + FileExistsError: The path is not a valid file. + + Returns: + bool: True if the path is a valid file else False. + """ + abs_file_path = os.path.abspath(file_path) + if os.path.isfile(file_path): + return abs_file_path + else: + raise FileExistsError(file_path) + + def _quiet_print(*args, **kwargs): + """Replaces print when quiet is requested to prevent printing messages. + """ + pass + + root_logger = logging.getLogger() + root_logger.setLevel(logging.DEBUG) + + stdout_logger_handler = logging.StreamHandler(sys.stdout) + stdout_logger_handler.set_name('stdout_logger_handler') + stdout_logger_handler.setLevel(logging.INFO) + stdout_logger_handler.setFormatter(logging.Formatter('%(message)s')) + root_logger.addHandler(stdout_logger_handler) + + parser = argparse.ArgumentParser( + prog=PROGRAM_NAME, + description=( + "Checks for debug macro formatting " + "errors within files recursively located within " + "a given directory."), + formatter_class=RawTextHelpFormatter) + + io_req_group = parser.add_mutually_exclusive_group(required=True) + io_opt_group = parser.add_argument_group( + "Optional input and output") + git_group = parser.add_argument_group("Optional git control") + + io_req_group.add_argument('-w', '--workspace-directory', + type=_check_dir_path, + help="Directory of source files to check.\n\n") + + io_req_group.add_argument('-i', '--input-file', nargs='?', + type=_check_file_path, + help="File path for an input file to check.\n\n" + "Note that some other options do not apply " + "if a single file is specified such as " + "the\ngit options and file extensions.\n\n") + + io_opt_group.add_argument('-l', '--log-file', + nargs='?', + default=None, + const='debug_macro_check.log', + help="File path for log output.\n" + "(default: if the flag is given with no " + "file path then a file called\n" + "debug_macro_check.log is created and used " + "in the current directory)\n\n") + + io_opt_group.add_argument('-s', '--substitution-file', + type=_check_file_path, + help="A substitution YAML file specifies string " + "substitutions to perform within the debug " + "macro.\n\nThis is intended to be a simple " + "mechanism to expand the rare cases of pre-" + "processor\nmacros without directly " + "involving the pre-processor. The file " + "consists of one or more\nstring value " + "pairs where the key is the identifier to " + "replace and the value is the value\nto " + "replace it with.\n\nThis can also be used " + "as a method to ignore results by " + "replacing the problematic string\nwith a " + "different string.\n\n") + + io_opt_group.add_argument('-v', '--verbose-log-file', + action='count', + default=0, + help="Set file logging verbosity level.\n" + " - None: Info & > level messages\n" + " - '-v': + Debug level messages\n" + " - '-vv': + File name and function\n" + " - '-vvv': + Line number\n" + " - '-vvvv': + Timestamp\n" + "(default: verbose logging is not enabled)" + "\n\n") + + io_opt_group.add_argument('-n', '--no-progress-bar', action='store_true', + help="Disables progress bars.\n" + "(default: progress bars are used in some" + "places to show progress)\n\n") + + io_opt_group.add_argument('-q', '--quiet', action='store_true', + help="Disables console output.\n" + "(default: console output is enabled)\n\n") + + io_opt_group.add_argument('-u', '--utf8w', action='store_true', + help="Shows warnings for file UTF-8 decode " + "errors.\n" + "(default: UTF-8 decode errors are not " + "shown)\n\n") + + git_group.add_argument('-df', '--do-not-ignore-git-ignore-files', + action='store_true', + help="Do not ignore git ignored files.\n" + "(default: files in git ignore files are " + "ignored)\n\n") + + git_group.add_argument('-ds', '--do-not-ignore-git_submodules', + action='store_true', + help="Do not ignore files in git submodules.\n" + "(default: files in git submodules are " + "ignored)\n\n") + + parser.add_argument('-e', '--extensions', nargs='*', default=['.c'], + help="List of file extensions to include.\n" + "(default: %(default)s)") + + args = parser.parse_args() + + if args.quiet: + # Don't print in the few places that directly print + builtins.print = _quiet_print + stdout_logger_handler.addFilter(QuietFilter(args.quiet)) + + if args.log_file: + file_logger_handler = logging.FileHandler(filename=args.log_file, + mode='w', encoding='utf-8') + + # In an ideal world, everyone would update to the latest Python + # minor version (3.10) after a few weeks/months. Since that's not the + # case, resist from using structural pattern matching in Python 3.10. + # https://peps.python.org/pep-0636/ + + if args.verbose_log_file == 0: + file_logger_handler.setLevel(logging.INFO) + file_logger_formatter = logging.Formatter( + '%(levelname)-8s %(message)s') + elif args.verbose_log_file == 1: + file_logger_handler.setLevel(logging.DEBUG) + file_logger_formatter = logging.Formatter( + '%(levelname)-8s %(message)s') + elif args.verbose_log_file == 2: + file_logger_handler.setLevel(logging.DEBUG) + file_logger_formatter = logging.Formatter( + '[%(filename)s - %(funcName)20s() ] %(levelname)-8s ' + '%(message)s') + elif args.verbose_log_file == 3: + file_logger_handler.setLevel(logging.DEBUG) + file_logger_formatter = logging.Formatter( + '[%(filename)s:%(lineno)s - %(funcName)20s() ] ' + '%(levelname)-8s %(message)s') + elif args.verbose_log_file == 4: + file_logger_handler.setLevel(logging.DEBUG) + file_logger_formatter = logging.Formatter( + '%(asctime)s [%(filename)s:%(lineno)s - %(funcName)20s() ]' + ' %(levelname)-8s %(message)s') + else: + file_logger_handler.setLevel(logging.DEBUG) + file_logger_formatter = logging.Formatter( + '%(asctime)s [%(filename)s:%(lineno)s - %(funcName)20s() ]' + ' %(levelname)-8s %(message)s') + + file_logger_handler.addFilter(ProgressFilter()) + file_logger_handler.setFormatter(file_logger_formatter) + root_logger.addHandler(file_logger_handler) + + logging.info(PROGRAM_NAME + "\n") + + substitution_data = {} + if args.substitution_file: + logging.info(f"Loading substitution file {args.substitution_file}") + with open(args.substitution_file, 'r') as sf: + substitution_data = yaml.safe_load(sf) + + if args.workspace_directory: + return check_macros_in_directory( + Path(args.workspace_directory), + args.extensions, + not args.do_not_ignore_git_ignore_files, + not args.do_not_ignore_git_submodules, + not args.no_progress_bar, + args.utf8w, + **substitution_data) + else: + curr_dir = Path(__file__).parent + input_file = Path(args.input_file) + + rel_path = str(input_file) + if input_file.is_relative_to(curr_dir): + rel_path = str(input_file.relative_to(curr_dir)) + + logging.info(f"Checking Debug Macros in File: " + f"{input_file.resolve()}\n") + + start_time = timeit.default_timer() + failure_cnt = check_macros_in_file( + input_file, + rel_path, + args.utf8w, + **substitution_data)[0] + end_time = timeit.default_timer() - start_time + + logging.debug(f"[PERF] The file macro check operation took " + f"{end_time:.2f} seconds.") + + _log_failure_count(failure_cnt, 1) + + return failure_cnt + + +if __name__ == '__main__': + # The exit status value is the number of macro formatting errors found. + # Therefore, if no macro formatting errors are found, 0 is returned. + # Some systems require the return value to be in the range 0-127, so + # a lower maximum of 100 is enforced to allow a wide range of potential + # values with a reasonably large maximum. + try: + sys.exit(max(_module_invocation_check_macros_in_directory_wrapper(), + 100)) + except KeyboardInterrupt: + logging.warning("Exiting due to keyboard interrupt.") + # Actual formatting errors are only allowed to reach 100. + # 101 signals a keyboard interrupt. + sys.exit(101) + except FileExistsError as e: + # 102 signals a file not found error. + logging.critical(f"Input file {e.args[0]} does not exist.") + sys.exit(102) diff --git a/BaseTools/Plugin/DebugMacroCheck/Readme.md b/BaseTools/Plugin/DebugMacroCheck/Readme.md index 33f1ad9790..ad51fbbbfc 100644 --- a/BaseTools/Plugin/DebugMacroCheck/Readme.md +++ b/BaseTools/Plugin/DebugMacroCheck/Readme.md @@ -1,253 +1,253 @@ -# Debug Macro Check
-
-This Python application scans all files in a build package for debug macro formatting issues. It is intended to be a
-fundamental build-time check that is part of a normal developer build process to catch errors right away.
-
-As a build plugin, it is capable of finding these errors early in the development process after code is initially
-written to ensure that all code tested is free of debug macro formatting errors. These errors often creep into debug
-prints in error conditions that are not frequently executed making debug even more difficult and confusing when they
-are encountered. In other cases, debug macros with these errors in the main code path can lead to unexpected behavior
-when executed. As a standalone script, it can be easily run manually or integrated into other CI processes.
-
-The plugin is part of a set of debug macro check scripts meant to be relatively portable so they can be applied to
-additional code bases with minimal effort.
-
-## 1. BuildPlugin/DebugMacroCheckBuildPlugin.py
-
-This is the build plugin. It is discovered within the Stuart Self-Describing Environment (SDE) due to the accompanying
-file `DebugMacroCheck_plugin_in.yaml`.
-
-Since macro errors are considered a coding bug that should be found and fixed during the build phase of the developer
-process (before debug and testing), this plugin is run in pre-build. It will run within the scope of the package
-being compiled. For a platform build, this means it will run against the package being built. In a CI build, it will
-run in pre-build for each package as each package is built.
-
-The build plugin has the following attributes:
-
- 1. Registered at `global` scope. This means it will always run.
-
- 2. Called only on compilable build targets (i.e. does nothing on `"NO-TARGET"`).
-
- 3. Runs as a pre-build step. This means it gives results right away to ensure compilation follows on a clean slate.
- This also means it runs in platform build and CI. It is run in CI as a pre-build step when the `CompilerPlugin`
- compiles code. This ensures even if the plugin was not run locally, all code submissions have been checked.
-
- 4. Reports any errors in the build log and fails the build upon error making it easy to discover problems.
-
- 5. Supports two methods of configuration via "substitution strings":
-
- 1. By setting a build variable called `DEBUG_MACRO_CHECK_SUB_FILE` with the name of a substitution YAML file to
- use.
-
- **Example:**
-
- ```python
- shell_environment.GetBuildVars().SetValue(
- "DEBUG_MACRO_CHECK_SUB_FILE",
- os.path.join(self.GetWorkspaceRoot(), "DebugMacroCheckSub.yaml"),
- "Set in CISettings.py")
- ```
-
- **Substitution File Content Example:**
-
- ```yaml
- ---
- # OvmfPkg/CpuHotplugSmm/ApicId.h
- # Reason: Substitute with macro value
- FMT_APIC_ID: 0x%08x
-
- # DynamicTablesPkg/Include/ConfigurationManagerObject.h
- # Reason: Substitute with macro value
- FMT_CM_OBJECT_ID: 0x%lx
-
- # OvmfPkg/IntelTdx/TdTcg2Dxe/TdTcg2Dxe.c
- # Reason: Acknowledging use of two format specifiers in string with one argument
- # Replace ternary operator in debug string with single specifier
- 'Index == COLUME_SIZE/2 ? " | %02x" : " %02x"': "%d"
-
- # DynamicTablesPkg/Library/Common/TableHelperLib/ConfigurationManagerObjectParser.c
- # ShellPkg/Library/UefiShellAcpiViewCommandLib/AcpiParser.c
- # Reason: Acknowledge that string *should* expand to one specifier
- # Replace variable with expected number of specifiers (1)
- Parser[Index].Format: "%d"
- ```
-
- 2. By entering the string substitutions directory into a dictionary called `StringSubstitutions` in a
- `DebugMacroCheck` section of the package CI YAML file.
-
- **Example:**
-
- ```yaml
- "DebugMacroCheck": {
- "StringSubstitutions": {
- "SUB_A": "%Lx"
- }
- }
- ```
-
-### Debug Macro Check Build Plugin: Simple Disable
-
-The build plugin can simply be disabled by setting an environment variable named `"DISABLE_DEBUG_MACRO_CHECK"`. The
-plugin is disabled on existence of the variable. The contents of the variable are not inspected at this time.
-
-## 2. DebugMacroCheck.py
-
-This is the main Python module containing the implementation logic. The build plugin simply wraps around it.
-
-When first running debug macro check against a new, large code base, it is recommended to first run this standalone
-script and address all of the issues and then enable the build plugin.
-
-The module supports a number of configuration parameters to ease debug of errors and to provide flexibility for
-different build environments.
-
-### EDK 2 PyTool Library Dependency
-
-This script has minimal library dependencies. However, it has one dependency you might not be familiar with on the
-Tianocore EDK 2 PyTool Library (edk2toollib):
-
-```py
-from edk2toollib.utility_functions import RunCmd
-```
-
-You simply need to install the following pip module to use this library: `edk2-pytool-library`
-(e.g. `pip install edk2-pytool-library`)
-
-More information is available here:
-
-- PyPI page: [edk2-pytool-library](https://pypi.org/project/edk2-pytool-library/)
-- GitHub repo: [tianocore/edk2-pytool-library](https://github.com/tianocore/edk2-pytool-library)
-
-If you strongly prefer not including this additional dependency, the functionality imported here is relatively
-simple to substitute with the Python [`subprocess`](https://docs.python.org/3/library/subprocess.html) built-in
-module.
-
-### Examples
-
-Simple run against current directory:
-
-`> python DebugMacroCheck.py -w .`
-
-Simple run against a single file:
-
-`> python DebugMacroCheck.py -i filename.c`
-
-Run against a directory with output placed into a file called "debug_macro_check.log":
-
-`> python DebugMacroCheck.py -w . -l`
-
-Run against a directory with output placed into a file called "custom.log" and debug log messages enabled:
-
-`> python DebugMacroCheck.py -w . -l custom.log -v`
-
-Run against a directory with output placed into a file called "custom.log", with debug log messages enabled including
-python script function and line number, use a substitution file called "file_sub.yaml", do not show the progress bar,
-and run against .c and .h files:
-
-`> python DebugMacroCheck.py -w . -l custom.log -vv -s file_sub.yaml -n -e .c .h`
-
-> **Note**: It is normally not recommended to run against .h files as they and many other non-.c files normally do
- not have full `DEBUG` macro prints.
-
-```plaintext
-usage: Debug Macro Checker [-h] (-w WORKSPACE_DIRECTORY | -i [INPUT_FILE]) [-l [LOG_FILE]] [-s SUBSTITUTION_FILE] [-v] [-n] [-q] [-u]
- [-df] [-ds] [-e [EXTENSIONS ...]]
-
-Checks for debug macro formatting errors within files recursively located within a given directory.
-
-options:
- -h, --help show this help message and exit
- -w WORKSPACE_DIRECTORY, --workspace-directory WORKSPACE_DIRECTORY
- Directory of source files to check.
-
- -i [INPUT_FILE], --input-file [INPUT_FILE]
- File path for an input file to check.
-
- Note that some other options do not apply if a single file is specified such as the
- git options and file extensions.
-
- -e [EXTENSIONS ...], --extensions [EXTENSIONS ...]
- List of file extensions to include.
- (default: ['.c'])
-
-Optional input and output:
- -l [LOG_FILE], --log-file [LOG_FILE]
- File path for log output.
- (default: if the flag is given with no file path then a file called
- debug_macro_check.log is created and used in the current directory)
-
- -s SUBSTITUTION_FILE, --substitution-file SUBSTITUTION_FILE
- A substitution YAML file specifies string substitutions to perform within the debug macro.
-
- This is intended to be a simple mechanism to expand the rare cases of pre-processor
- macros without directly involving the pre-processor. The file consists of one or more
- string value pairs where the key is the identifier to replace and the value is the value
- to replace it with.
-
- This can also be used as a method to ignore results by replacing the problematic string
- with a different string.
-
- -v, --verbose-log-file
- Set file logging verbosity level.
- - None: Info & > level messages
- - '-v': + Debug level messages
- - '-vv': + File name and function
- - '-vvv': + Line number
- - '-vvvv': + Timestamp
- (default: verbose logging is not enabled)
-
- -n, --no-progress-bar
- Disables progress bars.
- (default: progress bars are used in some places to show progress)
-
- -q, --quiet Disables console output.
- (default: console output is enabled)
-
- -u, --utf8w Shows warnings for file UTF-8 decode errors.
- (default: UTF-8 decode errors are not shown)
-
-
-Optional git control:
- -df, --do-not-ignore-git-ignore-files
- Do not ignore git ignored files.
- (default: files in git ignore files are ignored)
-
- -ds, --do-not-ignore-git_submodules
- Do not ignore files in git submodules.
- (default: files in git submodules are ignored)
-```
-
-## String Substitutions
-
-`DebugMacroCheck` currently runs separate from the compiler toolchain. This has the advantage that it is very portable
-and can run early in the build process, but it also means pre-processor macro expansion does not happen when it is
-invoked.
-
-In practice, it has been very rare that this is an issue for how most debug macros are written. In case it is, a
-substitution file can be used to inform `DebugMacroCheck` about the string substitution the pre-processor would
-perform.
-
-This pattern should be taken as a warning. It is just as difficult for humans to keep debug macro specifiers and
-arguments balanced as it is for `DebugMacroCheck` pre-processor macro substitution is used. By separating the string
-from the actual arguments provided, it is more likely for developers to make mistakes matching print specifiers in
-the string to the arguments. If usage is reasonable, a string substitution can be used as needed.
-
-### Ignoring Errors
-
-Since substitution files perform a straight textual substitution in macros discovered, it can be used to replace
-problematic text with text that passes allowing errors to be ignored.
-
-## Python Version Required (3.10)
-
-This script is written to take advantage of new Python language features in Python 3.10. If you are not using Python
-3.10 or later, you can:
-
- 1. Upgrade to Python 3.10 or greater
- 2. Run this script in a [virtual environment](https://docs.python.org/3/tutorial/venv.html) with Python 3.10
- or greater
- 3. Customize the script for compatibility with your Python version
-
-These are listed in order of recommendation. **(1)** is the simplest option and will upgrade your environment to a
-newer, safer, and better Python experience. **(2)** is the simplest approach to isolate dependencies to what is needed
-to run this script without impacting the rest of your system environment. **(3)** creates a one-off fork of the script
-that, by nature, has a limited lifespan and will make accepting future updates difficult but can be done with relatively
-minimal effort back to recent Python 3 releases.
+# Debug Macro Check + +This Python application scans all files in a build package for debug macro formatting issues. It is intended to be a +fundamental build-time check that is part of a normal developer build process to catch errors right away. + +As a build plugin, it is capable of finding these errors early in the development process after code is initially +written to ensure that all code tested is free of debug macro formatting errors. These errors often creep into debug +prints in error conditions that are not frequently executed making debug even more difficult and confusing when they +are encountered. In other cases, debug macros with these errors in the main code path can lead to unexpected behavior +when executed. As a standalone script, it can be easily run manually or integrated into other CI processes. + +The plugin is part of a set of debug macro check scripts meant to be relatively portable so they can be applied to +additional code bases with minimal effort. + +## 1. BuildPlugin/DebugMacroCheckBuildPlugin.py + +This is the build plugin. It is discovered within the Stuart Self-Describing Environment (SDE) due to the accompanying +file `DebugMacroCheck_plugin_in.yaml`. + +Since macro errors are considered a coding bug that should be found and fixed during the build phase of the developer +process (before debug and testing), this plugin is run in pre-build. It will run within the scope of the package +being compiled. For a platform build, this means it will run against the package being built. In a CI build, it will +run in pre-build for each package as each package is built. + +The build plugin has the following attributes: + + 1. Registered at `global` scope. This means it will always run. + + 2. Called only on compilable build targets (i.e. does nothing on `"NO-TARGET"`). + + 3. Runs as a pre-build step. This means it gives results right away to ensure compilation follows on a clean slate. + This also means it runs in platform build and CI. It is run in CI as a pre-build step when the `CompilerPlugin` + compiles code. This ensures even if the plugin was not run locally, all code submissions have been checked. + + 4. Reports any errors in the build log and fails the build upon error making it easy to discover problems. + + 5. Supports two methods of configuration via "substitution strings": + + 1. By setting a build variable called `DEBUG_MACRO_CHECK_SUB_FILE` with the name of a substitution YAML file to + use. + + **Example:** + + ```python + shell_environment.GetBuildVars().SetValue( + "DEBUG_MACRO_CHECK_SUB_FILE", + os.path.join(self.GetWorkspaceRoot(), "DebugMacroCheckSub.yaml"), + "Set in CISettings.py") + ``` + + **Substitution File Content Example:** + + ```yaml + --- + # OvmfPkg/CpuHotplugSmm/ApicId.h + # Reason: Substitute with macro value + FMT_APIC_ID: 0x%08x + + # DynamicTablesPkg/Include/ConfigurationManagerObject.h + # Reason: Substitute with macro value + FMT_CM_OBJECT_ID: 0x%lx + + # OvmfPkg/IntelTdx/TdTcg2Dxe/TdTcg2Dxe.c + # Reason: Acknowledging use of two format specifiers in string with one argument + # Replace ternary operator in debug string with single specifier + 'Index == COLUME_SIZE/2 ? " | %02x" : " %02x"': "%d" + + # DynamicTablesPkg/Library/Common/TableHelperLib/ConfigurationManagerObjectParser.c + # ShellPkg/Library/UefiShellAcpiViewCommandLib/AcpiParser.c + # Reason: Acknowledge that string *should* expand to one specifier + # Replace variable with expected number of specifiers (1) + Parser[Index].Format: "%d" + ``` + + 2. By entering the string substitutions directory into a dictionary called `StringSubstitutions` in a + `DebugMacroCheck` section of the package CI YAML file. + + **Example:** + + ```yaml + "DebugMacroCheck": { + "StringSubstitutions": { + "SUB_A": "%Lx" + } + } + ``` + +### Debug Macro Check Build Plugin: Simple Disable + +The build plugin can simply be disabled by setting an environment variable named `"DISABLE_DEBUG_MACRO_CHECK"`. The +plugin is disabled on existence of the variable. The contents of the variable are not inspected at this time. + +## 2. DebugMacroCheck.py + +This is the main Python module containing the implementation logic. The build plugin simply wraps around it. + +When first running debug macro check against a new, large code base, it is recommended to first run this standalone +script and address all of the issues and then enable the build plugin. + +The module supports a number of configuration parameters to ease debug of errors and to provide flexibility for +different build environments. + +### EDK 2 PyTool Library Dependency + +This script has minimal library dependencies. However, it has one dependency you might not be familiar with on the +Tianocore EDK 2 PyTool Library (edk2toollib): + +```py +from edk2toollib.utility_functions import RunCmd +``` + +You simply need to install the following pip module to use this library: `edk2-pytool-library` +(e.g. `pip install edk2-pytool-library`) + +More information is available here: + +- PyPI page: [edk2-pytool-library](https://pypi.org/project/edk2-pytool-library/) +- GitHub repo: [tianocore/edk2-pytool-library](https://github.com/tianocore/edk2-pytool-library) + +If you strongly prefer not including this additional dependency, the functionality imported here is relatively +simple to substitute with the Python [`subprocess`](https://docs.python.org/3/library/subprocess.html) built-in +module. + +### Examples + +Simple run against current directory: + +`> python DebugMacroCheck.py -w .` + +Simple run against a single file: + +`> python DebugMacroCheck.py -i filename.c` + +Run against a directory with output placed into a file called "debug_macro_check.log": + +`> python DebugMacroCheck.py -w . -l` + +Run against a directory with output placed into a file called "custom.log" and debug log messages enabled: + +`> python DebugMacroCheck.py -w . -l custom.log -v` + +Run against a directory with output placed into a file called "custom.log", with debug log messages enabled including +python script function and line number, use a substitution file called "file_sub.yaml", do not show the progress bar, +and run against .c and .h files: + +`> python DebugMacroCheck.py -w . -l custom.log -vv -s file_sub.yaml -n -e .c .h` + +> **Note**: It is normally not recommended to run against .h files as they and many other non-.c files normally do + not have full `DEBUG` macro prints. + +```plaintext +usage: Debug Macro Checker [-h] (-w WORKSPACE_DIRECTORY | -i [INPUT_FILE]) [-l [LOG_FILE]] [-s SUBSTITUTION_FILE] [-v] [-n] [-q] [-u] + [-df] [-ds] [-e [EXTENSIONS ...]] + +Checks for debug macro formatting errors within files recursively located within a given directory. + +options: + -h, --help show this help message and exit + -w WORKSPACE_DIRECTORY, --workspace-directory WORKSPACE_DIRECTORY + Directory of source files to check. + + -i [INPUT_FILE], --input-file [INPUT_FILE] + File path for an input file to check. + + Note that some other options do not apply if a single file is specified such as the + git options and file extensions. + + -e [EXTENSIONS ...], --extensions [EXTENSIONS ...] + List of file extensions to include. + (default: ['.c']) + +Optional input and output: + -l [LOG_FILE], --log-file [LOG_FILE] + File path for log output. + (default: if the flag is given with no file path then a file called + debug_macro_check.log is created and used in the current directory) + + -s SUBSTITUTION_FILE, --substitution-file SUBSTITUTION_FILE + A substitution YAML file specifies string substitutions to perform within the debug macro. + + This is intended to be a simple mechanism to expand the rare cases of pre-processor + macros without directly involving the pre-processor. The file consists of one or more + string value pairs where the key is the identifier to replace and the value is the value + to replace it with. + + This can also be used as a method to ignore results by replacing the problematic string + with a different string. + + -v, --verbose-log-file + Set file logging verbosity level. + - None: Info & > level messages + - '-v': + Debug level messages + - '-vv': + File name and function + - '-vvv': + Line number + - '-vvvv': + Timestamp + (default: verbose logging is not enabled) + + -n, --no-progress-bar + Disables progress bars. + (default: progress bars are used in some places to show progress) + + -q, --quiet Disables console output. + (default: console output is enabled) + + -u, --utf8w Shows warnings for file UTF-8 decode errors. + (default: UTF-8 decode errors are not shown) + + +Optional git control: + -df, --do-not-ignore-git-ignore-files + Do not ignore git ignored files. + (default: files in git ignore files are ignored) + + -ds, --do-not-ignore-git_submodules + Do not ignore files in git submodules. + (default: files in git submodules are ignored) +``` + +## String Substitutions + +`DebugMacroCheck` currently runs separate from the compiler toolchain. This has the advantage that it is very portable +and can run early in the build process, but it also means pre-processor macro expansion does not happen when it is +invoked. + +In practice, it has been very rare that this is an issue for how most debug macros are written. In case it is, a +substitution file can be used to inform `DebugMacroCheck` about the string substitution the pre-processor would +perform. + +This pattern should be taken as a warning. It is just as difficult for humans to keep debug macro specifiers and +arguments balanced as it is for `DebugMacroCheck` pre-processor macro substitution is used. By separating the string +from the actual arguments provided, it is more likely for developers to make mistakes matching print specifiers in +the string to the arguments. If usage is reasonable, a string substitution can be used as needed. + +### Ignoring Errors + +Since substitution files perform a straight textual substitution in macros discovered, it can be used to replace +problematic text with text that passes allowing errors to be ignored. + +## Python Version Required (3.10) + +This script is written to take advantage of new Python language features in Python 3.10. If you are not using Python +3.10 or later, you can: + + 1. Upgrade to Python 3.10 or greater + 2. Run this script in a [virtual environment](https://docs.python.org/3/tutorial/venv.html) with Python 3.10 + or greater + 3. Customize the script for compatibility with your Python version + +These are listed in order of recommendation. **(1)** is the simplest option and will upgrade your environment to a +newer, safer, and better Python experience. **(2)** is the simplest approach to isolate dependencies to what is needed +to run this script without impacting the rest of your system environment. **(3)** creates a one-off fork of the script +that, by nature, has a limited lifespan and will make accepting future updates difficult but can be done with relatively +minimal effort back to recent Python 3 releases. diff --git a/BaseTools/Plugin/DebugMacroCheck/tests/DebugMacroDataSet.py b/BaseTools/Plugin/DebugMacroCheck/tests/DebugMacroDataSet.py index 98629bb233..c3ce2faca8 100644 --- a/BaseTools/Plugin/DebugMacroCheck/tests/DebugMacroDataSet.py +++ b/BaseTools/Plugin/DebugMacroCheck/tests/DebugMacroDataSet.py @@ -1,674 +1,674 @@ -# @file DebugMacroDataSet.py
-#
-# Contains a debug macro test data set for verifying debug macros are
-# recognized and parsed properly.
-#
-# This data is automatically converted into test cases. Just add the new
-# data object here and run the tests.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-from .MacroTest import (NoSpecifierNoArgumentMacroTest,
- EqualSpecifierEqualArgumentMacroTest,
- MoreSpecifiersThanArgumentsMacroTest,
- LessSpecifiersThanArgumentsMacroTest,
- IgnoredSpecifiersMacroTest,
- SpecialParsingMacroTest,
- CodeSnippetMacroTest)
-
-
-# Ignore flake8 linter errors for lines that are too long (E501)
-# flake8: noqa: E501
-
-# Data Set of DEBUG macros and expected results.
-# macro: A string representing a DEBUG macro.
-# result: A tuple with the following value representations.
-# [0]: Count of total formatting errors
-# [1]: Count of print specifiers found
-# [2]: Count of macro arguments found
-DEBUG_MACROS = [
- #####################################################################
- # Section: No Print Specifiers No Arguments
- #####################################################################
- NoSpecifierNoArgumentMacroTest(
- r'',
- (0, 0, 0)
- ),
- NoSpecifierNoArgumentMacroTest(
- r'DEBUG ((DEBUG_ERROR, "\\"));',
- (0, 0, 0)
- ),
- NoSpecifierNoArgumentMacroTest(
- r'DEBUG ((DEBUG_EVENT, ""));',
- (0, 0, 0)
- ),
- NoSpecifierNoArgumentMacroTest(
- r'DEBUG ((DEBUG_EVENT, "\n"));',
- (0, 0, 0)
- ),
- NoSpecifierNoArgumentMacroTest(
- r'DEBUG ((DEBUG_EVENT, "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"));',
- (0, 0, 0)
- ),
- NoSpecifierNoArgumentMacroTest(
- r'DEBUG ((DEBUG_EVENT, "GCD:Initial GCD Memory Space Map\n"));',
- (0, 0, 0)
- ),
- NoSpecifierNoArgumentMacroTest(
- r'DEBUG ((DEBUG_GCD, "GCD:Initial GCD Memory Space Map\n"));',
- (0, 0, 0)
- ),
- NoSpecifierNoArgumentMacroTest(
- r'DEBUG ((DEBUG_INFO, " Retuning TimerCnt Disabled\n"));',
- (0, 0, 0)
- ),
-
- #####################################################################
- # Section: Equal Print Specifiers to Arguments
- #####################################################################
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_INFO, "%d", Number));',
- (0, 1, 1)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_BLKIO, "NorFlashBlockIoReset(MediaId=0x%x)\n", This->Media->MediaId));',
- (0, 1, 1)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_INFO, " Retuning TimerCnt %dseconds\n", 2 * (Capability->TimerCount - 1)));',
- (0, 1, 1)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_ERROR, "UsbEnumerateNewDev: failed to reset port %d - %r\n", Port, Status));',
- (0, 2, 2)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_ERROR, "UsbEnumerateNewDev: failed to reset port %d - %r\n", Port, Status));',
- (0, 2, 2)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_INFO, "Find GPT Partition [0x%lx", PartitionEntryBuffer[Index].StartingLBA));',
- (0, 1, 1)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_ERROR, "Failed to locate gEdkiiBootLogo2ProtocolGuid Status = %r. No Progress bar support. \n", Status));',
- (0, 1, 1)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_LOAD, " (%s)", Image->ExitData));',
- (0, 1, 1)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_DISPATCH, "%a%r%s%lx%p%c%g", Ascii, Status, Unicode, Hex, Pointer, Character, Guid));',
- (0, 7, 7)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_INFO, "LoadCapsuleOnDisk - LoadRecoveryCapsule (%d) - %r\n", CapsuleInstance, Status));',
- (0, 2, 2)
- ),
- EqualSpecifierEqualArgumentMacroTest(
- r'DEBUG ((DEBUG_DISPATCH, "%a%r%s%lx%p%c%g%a%r%s%lx%p%c%g%a%r%s%lx%p%c%g%a%r%s%lx%p%c%g", Ascii, Status, Unicode, Hex, Pointer, Character, Guid, Ascii, Status, Unicode, Hex, Pointer, Character, Guid, Ascii, Status, Unicode, Hex, Pointer, Character, Guid, Ascii, Status, Unicode, Hex, Pointer, Character, Guid));',
- (0, 28, 28)
- ),
-
- #####################################################################
- # Section: More Print Specifiers Than Arguments
- #####################################################################
- MoreSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_BLKIO, "NorFlashBlockIoReadBlocks(MediaId=0x%x, Lba=%ld, BufferSize=0x%x bytes (%d kB), BufferPtr @ 0x%08x)\n", MediaId, Lba, BufferSizeInBytes, Buffer));',
- (1, 5, 4)
- ),
- MoreSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_INFO, "%a: Request=%s\n", __func__));',
- (1, 2, 1)
- ),
- MoreSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_ERROR, "%a: Invalid request format %d for %d\n", CertFormat, CertRequest));',
- (1, 3, 2)
- ),
-
- #####################################################################
- # Section: Less Print Specifiers Than Arguments
- #####################################################################
- LessSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_INFO, "Find GPT Partition [0x%lx", PartitionEntryBuffer[Index].StartingLBA, BlockDevPtr->LastBlock));',
- (1, 1, 2)
- ),
- LessSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_INFO, " Retuning TimerCnt Disabled\n", 2 * (Capability->TimerCount - 1)));',
- (1, 0, 1)
- ),
- LessSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_ERROR, "Failed to locate gEdkiiBootLogo2ProtocolGuid. No Progress bar support. \n", Status));',
- (1, 0, 1)
- ),
- LessSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_ERROR, "UsbEnumeratePort: Critical Over Current\n", Port));',
- (1, 0, 1)
- ),
- LessSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_ERROR, "[TPM2] Submit PP Request failure! Sync PPRQ/PPRM with PP variable.\n", Status));',
- (1, 0, 1)
- ),
- LessSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_ERROR, ": Failed to update debug log index file: %r !\n", __func__, Status));',
- (1, 1, 2)
- ),
- LessSpecifiersThanArgumentsMacroTest(
- r'DEBUG ((DEBUG_ERROR, "%a - Failed to extract nonce from policy blob with return status %r\n", __func__, gPolicyBlobFieldName[MFCI_POLICY_TARGET_NONCE], Status));',
- (1, 2, 3)
- ),
-
- #####################################################################
- # Section: Macros with Ignored Specifiers
- #####################################################################
- IgnoredSpecifiersMacroTest(
- r'DEBUG ((DEBUG_INIT, "%HEmuOpenBlock: opened %a%N\n", Private->Filename));',
- (0, 1, 1)
- ),
- IgnoredSpecifiersMacroTest(
- r'DEBUG ((DEBUG_LOAD, " (%hs)", Image->ExitData));',
- (0, 1, 1)
- ),
- IgnoredSpecifiersMacroTest(
- r'DEBUG ((DEBUG_LOAD, "%H%s%N: Unknown flag - ''%H%s%N''\r\n", String1, String2));',
- (0, 2, 2)
- ),
-
- #####################################################################
- # Section: Macros with Special Parsing Scenarios
- #####################################################################
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_INFO, " File Name: %a\n", "Document.txt"))',
- (0, 1, 1),
- "Malformatted Macro - Missing Semicolon"
- ),
- SpecialParsingMacroTest(
- r'DEBUG (DEBUG_INFO, " File Name: %a\n", "Document.txt");',
- (0, 0, 0),
- "Malformatted Macro - Missing Two Parentheses"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_INFO, "%a\n", "Removable Slot"));',
- (0, 1, 1),
- "Single String Argument in Quotes"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_INFO, " SDR50 Tuning %a\n", Capability->TuningSDR50 ? "TRUE" : "FALSE"));',
- (0, 1, 1),
- "Ternary Operator Present"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_INFO, " SDR50 Tuning %a\n", Capability->TuningSDR50 ? "TRUE" : "FALSE"));',
- (0, 1, 1),
- "Ternary Operator Present"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((DEBUG_ERROR, "\\"));
- DEBUG ((DEBUG_ERROR, "\\"));
- DEBUG ((DEBUG_ERROR, "\\"));
- DEBUG ((DEBUG_ERROR, "\\"));
- ''',
- (0, 0, 0),
- "Multiple Macros with an Escaped Character"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_INFO,
- "UsbEnumerateNewDev: device uses translator (%d, %d)\n",
- Child->Translator.TranslatorHubAddress,
- Child->Translator.TranslatorPortNumber
- ));
- ''',
- (0, 2, 2),
- "Multi-line Macro"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_INFO,
- "UsbEnumeratePort: port %d state - %02x, change - %02x on %p\n",
- Port,
- PortState.PortStatus,
- PortState.PortChangeStatus,
- HubIf
- ));
- ''',
- (0, 4, 4),
- "Multi-line Macro"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_ERROR,
- "%a:%a: failed to allocate reserved pages: "
- "BufferSize=%Lu LoadFile=\"%s\" FilePath=\"%s\"\n",
- gEfiCallerBaseName,
- __func__,
- (UINT64)BufferSize,
- LoadFileText,
- FileText
- ));
- ''',
- (0, 5, 5),
- "Multi-line Macro with Compiler String Concatenation"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_ERROR,
- "ERROR: GTDT: GT Block Frame Info Structures %d and %d have the same " \
- "frame number: 0x%x.\n",
- Index1,
- Index2,
- FrameNumber1
- ));
- ''',
- (0, 3, 3),
- "Multi-line Macro with Backslash String Concatenation"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_ERROR,
- "ERROR: PPTT: Too many private resources. Count = %d. " \
- "Maximum supported Processor Node size exceeded. " \
- "Token = %p. Status = %r\n",
- ProcInfoNode->NoOfPrivateResources,
- ProcInfoNode->ParentToken,
- Status
- ));
- ''',
- (0, 3, 3),
- "Multi-line Macro with Backslash String Concatenation"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_VERBOSE,
- "% 20a % 20a % 20a % 20a\n",
- "PhysicalStart(0x)",
- "PhysicalSize(0x)",
- "CpuStart(0x)",
- "RegionState(0x)"
- ));
- ''',
- (0, 4, 4),
- "Multi-line Macro with Quoted String Arguments"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_ERROR,
- "XenPvBlk: "
- "%a error %d on %a at sector %Lx, num bytes %Lx\n",
- Response->operation == BLKIF_OP_READ ? "read" : "write",
- Status,
- IoData->Dev->NodeName,
- (UINT64)IoData->Sector,
- (UINT64)IoData->Size
- ));
- ''',
- (0, 5, 5),
- "Multi-line Macro with Ternary Operator and Quoted String Arguments"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_ERROR,
- "%a: Label=\"%s\" OldParentNodeId=%Lu OldName=\"%a\" "
- "NewParentNodeId=%Lu NewName=\"%a\" Errno=%d\n",
- __func__,
- VirtioFs->Label,
- OldParentNodeId,
- OldName,
- NewParentNodeId,
- NewName,
- CommonResp.Error
- ));
- ''',
- (0, 7, 7),
- "Multi-line Macro with Escaped Quotes and String Concatenation"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((DEBUG_WARN, "Failed to retrieve Variable:\"MebxData\", Status = %r\n", Status));
- ''',
- (0, 1, 1),
- "Escaped Parentheses in Debug Message"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG((DEBUG_INFO, "%0d %s", XbB_ddr4[1][bankBit][xorBit], xorBit == (XaB_NUM_OF_BITS-1) ? "]": ", "));
- ''',
- (0, 2, 2),
- "Parentheses in Ternary Operator Expression"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_INFO | DEBUG_EVENT | DEBUG_WARN, " %u\n", &Structure->Block.Value));',
- (0, 1, 1),
- "Multiple Print Specifier Levels Present"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString()));',
- (0, 1, 1),
- "Function Call Argument No Params"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1)));',
- (0, 1, 1),
- "Function Call Argument 1 Param"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, Param2)));',
- (0, 1, 1),
- "Function Call Argument Multiple Params"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam())));',
- (0, 1, 1),
- "Function Call Argument 2-Level Depth No 2nd-Level Param"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam(*Param))));',
- (0, 1, 1),
- "Function Call Argument 2-Level Depth 1 2nd-Level Param"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam(*Param, &ParamNext))));',
- (0, 1, 1),
- "Function Call Argument 2-Level Depth Multiple 2nd-Level Param"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam(*Param, GetParam(1, 2, 3)))));',
- (0, 1, 1),
- "Function Call Argument 3-Level Depth Multiple Params"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam(*Param, GetParam(1, 2, 3), NextParam))));',
- (0, 1, 1),
- "Function Call Argument 3-Level Depth Multiple Params with Param After Function Call"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s-%a\n", ReturnString(&Param1), ReturnString2(&ParamN)));',
- (0, 2, 2),
- "Multiple Function Call Arguments"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1), ReturnString2(&ParamN)));',
- (1, 1, 2),
- "Multiple Function Call Arguments with Imbalance"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s%s\n", (ReturnString(&Param1)), (ReturnString2(&ParamN))));',
- (0, 2, 2),
- "Multiple Function Call Arguments Surrounded with Parentheses"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, " %s\n", ((((ReturnString(&Param1)))))));',
- (0, 1, 1),
- "Multiple Function Call Arguments Surrounded with Many Parentheses"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, ""%B%08X%N: %-48a %V*%a*%N"", HexNumber, ReturnString(Array[Index]), &AsciiString[0]));',
- (0, 3, 3),
- "Complex String Print Specifier 1"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, "0x%-8x:%H%s%N % -64s(%73-.73s){%g}<%H% -70s%N>\n. Size: 0x%-16x (%-,d) bytes.\n\n", HexNumber, GetUnicodeString (), &UnicodeString[4], UnicodeString2, &Guid, AnotherUnicodeString, Struct.SomeSize, CommaDecimalValue));',
- (0, 8, 8),
- "Multiple Complex Print Specifiers 1"
- ),
- SpecialParsingMacroTest(
- r'DEBUG ((DEBUG_WARN, "0x%-8x:%H%s%N % -64s(%73-.73s){%g}<%H% -70s%N%r>\n. Size: 0x%-16x (%-,d) bytes.\n\n", HexNumber, GetUnicodeString (), &UnicodeString[4], UnicodeString2, &Guid, AnotherUnicodeString, Struct.SomeSize, CommaDecimalValue));',
- (1, 9, 8),
- "Multiple Complex Print Specifiers Imbalance 1"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_ERROR,
- ("%a: Label=\"%s\" CanonicalPathname=\"%a\" FileName=\"%s\" "
- "OpenMode=0x%Lx Attributes=0x%Lx: nonsensical request to possibly "
- "create a file marked read-only, for read-write access\n"),
- __func__,
- VirtioFs->Label,
- VirtioFsFile->CanonicalPathname,
- FileName,
- OpenMode,
- Attributes
- ));
- ''',
- (0, 6, 6),
- "Multi-Line with Parentheses Around Debug String Compiler String Concat"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG (
- (DEBUG_INFO,
- " %02x: %04x %02x/%02x/%02x %02x/%02x %04x %04x %04x:%04x\n",
- (UINTN)Index,
- (UINTN)LocalBbsTable[Index].BootPriority,
- (UINTN)LocalBbsTable[Index].Bus,
- (UINTN)LocalBbsTable[Index].Device,
- (UINTN)LocalBbsTable[Index].Function,
- (UINTN)LocalBbsTable[Index].Class,
- (UINTN)LocalBbsTable[Index].SubClass,
- (UINTN)LocalBbsTable[Index].DeviceType,
- (UINTN)*(UINT16 *)&LocalBbsTable[Index].StatusFlags,
- (UINTN)LocalBbsTable[Index].BootHandlerSegment,
- (UINTN)LocalBbsTable[Index].BootHandlerOffset,
- (UINTN)((LocalBbsTable[Index].MfgStringSegment << 4) + LocalBbsTable[Index].MfgStringOffset),
- (UINTN)((LocalBbsTable[Index].DescStringSegment << 4) + LocalBbsTable[Index].DescStringOffset))
- );
- ''',
- (1, 11, 13),
- "Multi-line Macro with Many Arguments And Multi-Line Parentheses"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_WARN,
- "0x%-8x:%H%s%N % -64s(%73-.73s){%g}<%H% -70s%N>\n. Size: 0x%-16x (%-,d) bytes.\n\n",
- HexNumber,
- GetUnicodeString (InnerFunctionCall(Arg1, &Arg2)),
- &UnicodeString[4],
- UnicodeString2,
- &Guid,
- AnotherUnicodeString,
- Struct.SomeSize,
- CommaDecimalValue
- ));
- ''',
- (0, 8, 8),
- "Multi-line Macro with Multiple Complex Print Specifiers 1 and 2-Depth Function Calls"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG (
- (DEBUG_NET,
- "TcpFastRecover: enter fast retransmission for TCB %p, recover point is %d\n",
- Tcb,
- Tcb->Recover)
- );
- ''',
- (0, 2, 2),
- "Multi-line Macro with Parentheses Separated"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_VERBOSE,
- "%a: APIC ID " FMT_APIC_ID " was hot-plugged "
- "before; ignoring it\n",
- __func__,
- NewApicId
- ));
- ''',
- (1, 1, 2),
- "Multi-line Imbalanced Macro with Indented String Concatenation"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_VERBOSE,
- "%a: APIC ID was hot-plugged - %a",
- __func__,
- "String with , inside"
- ));
- ''',
- (0, 2, 2),
- "Multi-line with Quoted String Argument Containing Comma"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_VERBOSE,
- "%a: APIC ID was hot-plugged - %a",
- __func__,
- "St,ring, with , ins,ide"
- ));
- ''',
- (0, 2, 2),
- "Multi-line with Quoted String Argument Containing Multiple Commas"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((DEBUG_VERBOSE, "%a: APIC ID was hot-plugged, \"%a\"", __func__, "S\"t,\"ring, with , ins,i\"de"));
- ''',
- (0, 2, 2),
- "Quoted String Argument with Escaped Quotes and Multiple Commas"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_ERROR,
- "%a: AddProcessor(" FMT_APIC_ID "): %r\n",
- __func__,
- Status
- ));
- ''',
- (0, 2, 2),
- "Quoted Parenthesized String Inside Debug Message String"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((
- DEBUG_INFO,
- "%a: hot-added APIC ID " FMT_APIC_ID ", SMBASE 0x%Lx, "
- "EFI_SMM_CPU_SERVICE_PROTOCOL assigned number %Lu\n",
- __func__,
- (UINT64)mCpuHotPlugData->SmBase[NewSlot],
- (UINT64)NewProcessorNumberByProtocol
- ));
- ''',
- (0, 3, 3),
- "Quoted String with Concatenation Inside Debug Message String"
- ),
- SpecialParsingMacroTest(
- r'''
- DEBUG ((DEBUG_INFO, Index == COLUMN_SIZE/2 ? "0" : " %02x", (UINTN)Data[Index]));
- ''',
- (0, 1, 1),
- "Ternary Operating in Debug Message String"
- ),
-
- #####################################################################
- # Section: Code Snippet Tests
- #####################################################################
- CodeSnippetMacroTest(
- r'''
- /**
- Print the BBS Table.
-
- @param LocalBbsTable The BBS table.
- @param BbsCount The count of entry in BBS table.
- **/
- VOID
- LegacyBmPrintBbsTable (
- IN BBS_TABLE *LocalBbsTable,
- IN UINT16 BbsCount
- )
- {
- UINT16 Index;
-
- DEBUG ((DEBUG_INFO, "\n"));
- DEBUG ((DEBUG_INFO, " NO Prio bb/dd/ff cl/sc Type Stat segm:offs\n"));
- DEBUG ((DEBUG_INFO, "=============================================\n"));
- for (Index = 0; Index < BbsCount; Index++) {
- if (!LegacyBmValidBbsEntry (&LocalBbsTable[Index])) {
- continue;
- }
-
- DEBUG (
- (DEBUG_INFO,
- " %02x: %04x %02x/%02x/%02x %02x/%02x %04x %04x %04x:%04x\n",
- (UINTN)Index,
- (UINTN)LocalBbsTable[Index].BootPriority,
- (UINTN)LocalBbsTable[Index].Bus,
- (UINTN)LocalBbsTable[Index].Device,
- (UINTN)LocalBbsTable[Index].Function,
- (UINTN)LocalBbsTable[Index].Class,
- (UINTN)LocalBbsTable[Index].SubClass,
- (UINTN)LocalBbsTable[Index].DeviceType,
- (UINTN)*(UINT16 *)&LocalBbsTable[Index].StatusFlags,
- (UINTN)LocalBbsTable[Index].BootHandlerSegment,
- (UINTN)LocalBbsTable[Index].BootHandlerOffset,
- (UINTN)((LocalBbsTable[Index].MfgStringSegment << 4) + LocalBbsTable[Index].MfgStringOffset),
- (UINTN)((LocalBbsTable[Index].DescStringSegment << 4) + LocalBbsTable[Index].DescStringOffset))
- );
- }
-
- DEBUG ((DEBUG_INFO, "\n"));
- ''',
- (1, 0, 0),
- "Code Section with An Imbalanced Macro"
- ),
- CodeSnippetMacroTest(
- r'''
- if (*Buffer == AML_ROOT_CHAR) {
- //
- // RootChar
- //
- Buffer++;
- DEBUG ((DEBUG_ERROR, "\\"));
- } else if (*Buffer == AML_PARENT_PREFIX_CHAR) {
- //
- // ParentPrefixChar
- //
- do {
- Buffer++;
- DEBUG ((DEBUG_ERROR, "^"));
- } while (*Buffer == AML_PARENT_PREFIX_CHAR);
- }
- DEBUG ((DEBUG_WARN, "Failed to retrieve Variable:\"MebxData\", Status = %r\n", Status));
- ''',
- (0, 1, 1),
- "Code Section with Escaped Backslash and Escaped Quotes"
- ),
- CodeSnippetMacroTest(
- r'''
- if (EFI_ERROR (Status)) {
- UINTN Offset;
- UINTN Start;
-
- DEBUG ((
- DEBUG_INFO,
- "Variable FV header is not valid. It will be reinitialized.\n"
- ));
-
- //
- // Get FvbInfo to provide in FwhInstance.
- //
- Status = GetFvbInfo (Length, &GoodFwVolHeader);
- ASSERT (!EFI_ERROR (Status));
- }
- ''',
- (0, 0, 0),
- "Code Section with Multi-Line Macro with No Arguments"
- )
-]
+# @file DebugMacroDataSet.py +# +# Contains a debug macro test data set for verifying debug macros are +# recognized and parsed properly. +# +# This data is automatically converted into test cases. Just add the new +# data object here and run the tests. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +from .MacroTest import (NoSpecifierNoArgumentMacroTest, + EqualSpecifierEqualArgumentMacroTest, + MoreSpecifiersThanArgumentsMacroTest, + LessSpecifiersThanArgumentsMacroTest, + IgnoredSpecifiersMacroTest, + SpecialParsingMacroTest, + CodeSnippetMacroTest) + + +# Ignore flake8 linter errors for lines that are too long (E501) +# flake8: noqa: E501 + +# Data Set of DEBUG macros and expected results. +# macro: A string representing a DEBUG macro. +# result: A tuple with the following value representations. +# [0]: Count of total formatting errors +# [1]: Count of print specifiers found +# [2]: Count of macro arguments found +DEBUG_MACROS = [ + ##################################################################### + # Section: No Print Specifiers No Arguments + ##################################################################### + NoSpecifierNoArgumentMacroTest( + r'', + (0, 0, 0) + ), + NoSpecifierNoArgumentMacroTest( + r'DEBUG ((DEBUG_ERROR, "\\"));', + (0, 0, 0) + ), + NoSpecifierNoArgumentMacroTest( + r'DEBUG ((DEBUG_EVENT, ""));', + (0, 0, 0) + ), + NoSpecifierNoArgumentMacroTest( + r'DEBUG ((DEBUG_EVENT, "\n"));', + (0, 0, 0) + ), + NoSpecifierNoArgumentMacroTest( + r'DEBUG ((DEBUG_EVENT, "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"));', + (0, 0, 0) + ), + NoSpecifierNoArgumentMacroTest( + r'DEBUG ((DEBUG_EVENT, "GCD:Initial GCD Memory Space Map\n"));', + (0, 0, 0) + ), + NoSpecifierNoArgumentMacroTest( + r'DEBUG ((DEBUG_GCD, "GCD:Initial GCD Memory Space Map\n"));', + (0, 0, 0) + ), + NoSpecifierNoArgumentMacroTest( + r'DEBUG ((DEBUG_INFO, " Retuning TimerCnt Disabled\n"));', + (0, 0, 0) + ), + + ##################################################################### + # Section: Equal Print Specifiers to Arguments + ##################################################################### + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_INFO, "%d", Number));', + (0, 1, 1) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_BLKIO, "NorFlashBlockIoReset(MediaId=0x%x)\n", This->Media->MediaId));', + (0, 1, 1) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_INFO, " Retuning TimerCnt %dseconds\n", 2 * (Capability->TimerCount - 1)));', + (0, 1, 1) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_ERROR, "UsbEnumerateNewDev: failed to reset port %d - %r\n", Port, Status));', + (0, 2, 2) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_ERROR, "UsbEnumerateNewDev: failed to reset port %d - %r\n", Port, Status));', + (0, 2, 2) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_INFO, "Find GPT Partition [0x%lx", PartitionEntryBuffer[Index].StartingLBA));', + (0, 1, 1) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_ERROR, "Failed to locate gEdkiiBootLogo2ProtocolGuid Status = %r. No Progress bar support. \n", Status));', + (0, 1, 1) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_LOAD, " (%s)", Image->ExitData));', + (0, 1, 1) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_DISPATCH, "%a%r%s%lx%p%c%g", Ascii, Status, Unicode, Hex, Pointer, Character, Guid));', + (0, 7, 7) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_INFO, "LoadCapsuleOnDisk - LoadRecoveryCapsule (%d) - %r\n", CapsuleInstance, Status));', + (0, 2, 2) + ), + EqualSpecifierEqualArgumentMacroTest( + r'DEBUG ((DEBUG_DISPATCH, "%a%r%s%lx%p%c%g%a%r%s%lx%p%c%g%a%r%s%lx%p%c%g%a%r%s%lx%p%c%g", Ascii, Status, Unicode, Hex, Pointer, Character, Guid, Ascii, Status, Unicode, Hex, Pointer, Character, Guid, Ascii, Status, Unicode, Hex, Pointer, Character, Guid, Ascii, Status, Unicode, Hex, Pointer, Character, Guid));', + (0, 28, 28) + ), + + ##################################################################### + # Section: More Print Specifiers Than Arguments + ##################################################################### + MoreSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_BLKIO, "NorFlashBlockIoReadBlocks(MediaId=0x%x, Lba=%ld, BufferSize=0x%x bytes (%d kB), BufferPtr @ 0x%08x)\n", MediaId, Lba, BufferSizeInBytes, Buffer));', + (1, 5, 4) + ), + MoreSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_INFO, "%a: Request=%s\n", __func__));', + (1, 2, 1) + ), + MoreSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_ERROR, "%a: Invalid request format %d for %d\n", CertFormat, CertRequest));', + (1, 3, 2) + ), + + ##################################################################### + # Section: Less Print Specifiers Than Arguments + ##################################################################### + LessSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_INFO, "Find GPT Partition [0x%lx", PartitionEntryBuffer[Index].StartingLBA, BlockDevPtr->LastBlock));', + (1, 1, 2) + ), + LessSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_INFO, " Retuning TimerCnt Disabled\n", 2 * (Capability->TimerCount - 1)));', + (1, 0, 1) + ), + LessSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_ERROR, "Failed to locate gEdkiiBootLogo2ProtocolGuid. No Progress bar support. \n", Status));', + (1, 0, 1) + ), + LessSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_ERROR, "UsbEnumeratePort: Critical Over Current\n", Port));', + (1, 0, 1) + ), + LessSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_ERROR, "[TPM2] Submit PP Request failure! Sync PPRQ/PPRM with PP variable.\n", Status));', + (1, 0, 1) + ), + LessSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_ERROR, ": Failed to update debug log index file: %r !\n", __func__, Status));', + (1, 1, 2) + ), + LessSpecifiersThanArgumentsMacroTest( + r'DEBUG ((DEBUG_ERROR, "%a - Failed to extract nonce from policy blob with return status %r\n", __func__, gPolicyBlobFieldName[MFCI_POLICY_TARGET_NONCE], Status));', + (1, 2, 3) + ), + + ##################################################################### + # Section: Macros with Ignored Specifiers + ##################################################################### + IgnoredSpecifiersMacroTest( + r'DEBUG ((DEBUG_INIT, "%HEmuOpenBlock: opened %a%N\n", Private->Filename));', + (0, 1, 1) + ), + IgnoredSpecifiersMacroTest( + r'DEBUG ((DEBUG_LOAD, " (%hs)", Image->ExitData));', + (0, 1, 1) + ), + IgnoredSpecifiersMacroTest( + r'DEBUG ((DEBUG_LOAD, "%H%s%N: Unknown flag - ''%H%s%N''\r\n", String1, String2));', + (0, 2, 2) + ), + + ##################################################################### + # Section: Macros with Special Parsing Scenarios + ##################################################################### + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_INFO, " File Name: %a\n", "Document.txt"))', + (0, 1, 1), + "Malformatted Macro - Missing Semicolon" + ), + SpecialParsingMacroTest( + r'DEBUG (DEBUG_INFO, " File Name: %a\n", "Document.txt");', + (0, 0, 0), + "Malformatted Macro - Missing Two Parentheses" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_INFO, "%a\n", "Removable Slot"));', + (0, 1, 1), + "Single String Argument in Quotes" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_INFO, " SDR50 Tuning %a\n", Capability->TuningSDR50 ? "TRUE" : "FALSE"));', + (0, 1, 1), + "Ternary Operator Present" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_INFO, " SDR50 Tuning %a\n", Capability->TuningSDR50 ? "TRUE" : "FALSE"));', + (0, 1, 1), + "Ternary Operator Present" + ), + SpecialParsingMacroTest( + r''' + DEBUG ((DEBUG_ERROR, "\\")); + DEBUG ((DEBUG_ERROR, "\\")); + DEBUG ((DEBUG_ERROR, "\\")); + DEBUG ((DEBUG_ERROR, "\\")); + ''', + (0, 0, 0), + "Multiple Macros with an Escaped Character" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_INFO, + "UsbEnumerateNewDev: device uses translator (%d, %d)\n", + Child->Translator.TranslatorHubAddress, + Child->Translator.TranslatorPortNumber + )); + ''', + (0, 2, 2), + "Multi-line Macro" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_INFO, + "UsbEnumeratePort: port %d state - %02x, change - %02x on %p\n", + Port, + PortState.PortStatus, + PortState.PortChangeStatus, + HubIf + )); + ''', + (0, 4, 4), + "Multi-line Macro" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_ERROR, + "%a:%a: failed to allocate reserved pages: " + "BufferSize=%Lu LoadFile=\"%s\" FilePath=\"%s\"\n", + gEfiCallerBaseName, + __func__, + (UINT64)BufferSize, + LoadFileText, + FileText + )); + ''', + (0, 5, 5), + "Multi-line Macro with Compiler String Concatenation" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_ERROR, + "ERROR: GTDT: GT Block Frame Info Structures %d and %d have the same " \ + "frame number: 0x%x.\n", + Index1, + Index2, + FrameNumber1 + )); + ''', + (0, 3, 3), + "Multi-line Macro with Backslash String Concatenation" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_ERROR, + "ERROR: PPTT: Too many private resources. Count = %d. " \ + "Maximum supported Processor Node size exceeded. " \ + "Token = %p. Status = %r\n", + ProcInfoNode->NoOfPrivateResources, + ProcInfoNode->ParentToken, + Status + )); + ''', + (0, 3, 3), + "Multi-line Macro with Backslash String Concatenation" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_VERBOSE, + "% 20a % 20a % 20a % 20a\n", + "PhysicalStart(0x)", + "PhysicalSize(0x)", + "CpuStart(0x)", + "RegionState(0x)" + )); + ''', + (0, 4, 4), + "Multi-line Macro with Quoted String Arguments" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_ERROR, + "XenPvBlk: " + "%a error %d on %a at sector %Lx, num bytes %Lx\n", + Response->operation == BLKIF_OP_READ ? "read" : "write", + Status, + IoData->Dev->NodeName, + (UINT64)IoData->Sector, + (UINT64)IoData->Size + )); + ''', + (0, 5, 5), + "Multi-line Macro with Ternary Operator and Quoted String Arguments" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_ERROR, + "%a: Label=\"%s\" OldParentNodeId=%Lu OldName=\"%a\" " + "NewParentNodeId=%Lu NewName=\"%a\" Errno=%d\n", + __func__, + VirtioFs->Label, + OldParentNodeId, + OldName, + NewParentNodeId, + NewName, + CommonResp.Error + )); + ''', + (0, 7, 7), + "Multi-line Macro with Escaped Quotes and String Concatenation" + ), + SpecialParsingMacroTest( + r''' + DEBUG ((DEBUG_WARN, "Failed to retrieve Variable:\"MebxData\", Status = %r\n", Status)); + ''', + (0, 1, 1), + "Escaped Parentheses in Debug Message" + ), + SpecialParsingMacroTest( + r''' + DEBUG((DEBUG_INFO, "%0d %s", XbB_ddr4[1][bankBit][xorBit], xorBit == (XaB_NUM_OF_BITS-1) ? "]": ", ")); + ''', + (0, 2, 2), + "Parentheses in Ternary Operator Expression" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_INFO | DEBUG_EVENT | DEBUG_WARN, " %u\n", &Structure->Block.Value));', + (0, 1, 1), + "Multiple Print Specifier Levels Present" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString()));', + (0, 1, 1), + "Function Call Argument No Params" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1)));', + (0, 1, 1), + "Function Call Argument 1 Param" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, Param2)));', + (0, 1, 1), + "Function Call Argument Multiple Params" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam())));', + (0, 1, 1), + "Function Call Argument 2-Level Depth No 2nd-Level Param" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam(*Param))));', + (0, 1, 1), + "Function Call Argument 2-Level Depth 1 2nd-Level Param" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam(*Param, &ParamNext))));', + (0, 1, 1), + "Function Call Argument 2-Level Depth Multiple 2nd-Level Param" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam(*Param, GetParam(1, 2, 3)))));', + (0, 1, 1), + "Function Call Argument 3-Level Depth Multiple Params" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1, ReturnParam(*Param, GetParam(1, 2, 3), NextParam))));', + (0, 1, 1), + "Function Call Argument 3-Level Depth Multiple Params with Param After Function Call" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s-%a\n", ReturnString(&Param1), ReturnString2(&ParamN)));', + (0, 2, 2), + "Multiple Function Call Arguments" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ReturnString(&Param1), ReturnString2(&ParamN)));', + (1, 1, 2), + "Multiple Function Call Arguments with Imbalance" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s%s\n", (ReturnString(&Param1)), (ReturnString2(&ParamN))));', + (0, 2, 2), + "Multiple Function Call Arguments Surrounded with Parentheses" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, " %s\n", ((((ReturnString(&Param1)))))));', + (0, 1, 1), + "Multiple Function Call Arguments Surrounded with Many Parentheses" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, ""%B%08X%N: %-48a %V*%a*%N"", HexNumber, ReturnString(Array[Index]), &AsciiString[0]));', + (0, 3, 3), + "Complex String Print Specifier 1" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, "0x%-8x:%H%s%N % -64s(%73-.73s){%g}<%H% -70s%N>\n. Size: 0x%-16x (%-,d) bytes.\n\n", HexNumber, GetUnicodeString (), &UnicodeString[4], UnicodeString2, &Guid, AnotherUnicodeString, Struct.SomeSize, CommaDecimalValue));', + (0, 8, 8), + "Multiple Complex Print Specifiers 1" + ), + SpecialParsingMacroTest( + r'DEBUG ((DEBUG_WARN, "0x%-8x:%H%s%N % -64s(%73-.73s){%g}<%H% -70s%N%r>\n. Size: 0x%-16x (%-,d) bytes.\n\n", HexNumber, GetUnicodeString (), &UnicodeString[4], UnicodeString2, &Guid, AnotherUnicodeString, Struct.SomeSize, CommaDecimalValue));', + (1, 9, 8), + "Multiple Complex Print Specifiers Imbalance 1" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_ERROR, + ("%a: Label=\"%s\" CanonicalPathname=\"%a\" FileName=\"%s\" " + "OpenMode=0x%Lx Attributes=0x%Lx: nonsensical request to possibly " + "create a file marked read-only, for read-write access\n"), + __func__, + VirtioFs->Label, + VirtioFsFile->CanonicalPathname, + FileName, + OpenMode, + Attributes + )); + ''', + (0, 6, 6), + "Multi-Line with Parentheses Around Debug String Compiler String Concat" + ), + SpecialParsingMacroTest( + r''' + DEBUG ( + (DEBUG_INFO, + " %02x: %04x %02x/%02x/%02x %02x/%02x %04x %04x %04x:%04x\n", + (UINTN)Index, + (UINTN)LocalBbsTable[Index].BootPriority, + (UINTN)LocalBbsTable[Index].Bus, + (UINTN)LocalBbsTable[Index].Device, + (UINTN)LocalBbsTable[Index].Function, + (UINTN)LocalBbsTable[Index].Class, + (UINTN)LocalBbsTable[Index].SubClass, + (UINTN)LocalBbsTable[Index].DeviceType, + (UINTN)*(UINT16 *)&LocalBbsTable[Index].StatusFlags, + (UINTN)LocalBbsTable[Index].BootHandlerSegment, + (UINTN)LocalBbsTable[Index].BootHandlerOffset, + (UINTN)((LocalBbsTable[Index].MfgStringSegment << 4) + LocalBbsTable[Index].MfgStringOffset), + (UINTN)((LocalBbsTable[Index].DescStringSegment << 4) + LocalBbsTable[Index].DescStringOffset)) + ); + ''', + (1, 11, 13), + "Multi-line Macro with Many Arguments And Multi-Line Parentheses" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_WARN, + "0x%-8x:%H%s%N % -64s(%73-.73s){%g}<%H% -70s%N>\n. Size: 0x%-16x (%-,d) bytes.\n\n", + HexNumber, + GetUnicodeString (InnerFunctionCall(Arg1, &Arg2)), + &UnicodeString[4], + UnicodeString2, + &Guid, + AnotherUnicodeString, + Struct.SomeSize, + CommaDecimalValue + )); + ''', + (0, 8, 8), + "Multi-line Macro with Multiple Complex Print Specifiers 1 and 2-Depth Function Calls" + ), + SpecialParsingMacroTest( + r''' + DEBUG ( + (DEBUG_NET, + "TcpFastRecover: enter fast retransmission for TCB %p, recover point is %d\n", + Tcb, + Tcb->Recover) + ); + ''', + (0, 2, 2), + "Multi-line Macro with Parentheses Separated" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_VERBOSE, + "%a: APIC ID " FMT_APIC_ID " was hot-plugged " + "before; ignoring it\n", + __func__, + NewApicId + )); + ''', + (1, 1, 2), + "Multi-line Imbalanced Macro with Indented String Concatenation" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_VERBOSE, + "%a: APIC ID was hot-plugged - %a", + __func__, + "String with , inside" + )); + ''', + (0, 2, 2), + "Multi-line with Quoted String Argument Containing Comma" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_VERBOSE, + "%a: APIC ID was hot-plugged - %a", + __func__, + "St,ring, with , ins,ide" + )); + ''', + (0, 2, 2), + "Multi-line with Quoted String Argument Containing Multiple Commas" + ), + SpecialParsingMacroTest( + r''' + DEBUG ((DEBUG_VERBOSE, "%a: APIC ID was hot-plugged, \"%a\"", __func__, "S\"t,\"ring, with , ins,i\"de")); + ''', + (0, 2, 2), + "Quoted String Argument with Escaped Quotes and Multiple Commas" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_ERROR, + "%a: AddProcessor(" FMT_APIC_ID "): %r\n", + __func__, + Status + )); + ''', + (0, 2, 2), + "Quoted Parenthesized String Inside Debug Message String" + ), + SpecialParsingMacroTest( + r''' + DEBUG (( + DEBUG_INFO, + "%a: hot-added APIC ID " FMT_APIC_ID ", SMBASE 0x%Lx, " + "EFI_SMM_CPU_SERVICE_PROTOCOL assigned number %Lu\n", + __func__, + (UINT64)mCpuHotPlugData->SmBase[NewSlot], + (UINT64)NewProcessorNumberByProtocol + )); + ''', + (0, 3, 3), + "Quoted String with Concatenation Inside Debug Message String" + ), + SpecialParsingMacroTest( + r''' + DEBUG ((DEBUG_INFO, Index == COLUMN_SIZE/2 ? "0" : " %02x", (UINTN)Data[Index])); + ''', + (0, 1, 1), + "Ternary Operating in Debug Message String" + ), + + ##################################################################### + # Section: Code Snippet Tests + ##################################################################### + CodeSnippetMacroTest( + r''' + /** + Print the BBS Table. + + @param LocalBbsTable The BBS table. + @param BbsCount The count of entry in BBS table. + **/ + VOID + LegacyBmPrintBbsTable ( + IN BBS_TABLE *LocalBbsTable, + IN UINT16 BbsCount + ) + { + UINT16 Index; + + DEBUG ((DEBUG_INFO, "\n")); + DEBUG ((DEBUG_INFO, " NO Prio bb/dd/ff cl/sc Type Stat segm:offs\n")); + DEBUG ((DEBUG_INFO, "=============================================\n")); + for (Index = 0; Index < BbsCount; Index++) { + if (!LegacyBmValidBbsEntry (&LocalBbsTable[Index])) { + continue; + } + + DEBUG ( + (DEBUG_INFO, + " %02x: %04x %02x/%02x/%02x %02x/%02x %04x %04x %04x:%04x\n", + (UINTN)Index, + (UINTN)LocalBbsTable[Index].BootPriority, + (UINTN)LocalBbsTable[Index].Bus, + (UINTN)LocalBbsTable[Index].Device, + (UINTN)LocalBbsTable[Index].Function, + (UINTN)LocalBbsTable[Index].Class, + (UINTN)LocalBbsTable[Index].SubClass, + (UINTN)LocalBbsTable[Index].DeviceType, + (UINTN)*(UINT16 *)&LocalBbsTable[Index].StatusFlags, + (UINTN)LocalBbsTable[Index].BootHandlerSegment, + (UINTN)LocalBbsTable[Index].BootHandlerOffset, + (UINTN)((LocalBbsTable[Index].MfgStringSegment << 4) + LocalBbsTable[Index].MfgStringOffset), + (UINTN)((LocalBbsTable[Index].DescStringSegment << 4) + LocalBbsTable[Index].DescStringOffset)) + ); + } + + DEBUG ((DEBUG_INFO, "\n")); + ''', + (1, 0, 0), + "Code Section with An Imbalanced Macro" + ), + CodeSnippetMacroTest( + r''' + if (*Buffer == AML_ROOT_CHAR) { + // + // RootChar + // + Buffer++; + DEBUG ((DEBUG_ERROR, "\\")); + } else if (*Buffer == AML_PARENT_PREFIX_CHAR) { + // + // ParentPrefixChar + // + do { + Buffer++; + DEBUG ((DEBUG_ERROR, "^")); + } while (*Buffer == AML_PARENT_PREFIX_CHAR); + } + DEBUG ((DEBUG_WARN, "Failed to retrieve Variable:\"MebxData\", Status = %r\n", Status)); + ''', + (0, 1, 1), + "Code Section with Escaped Backslash and Escaped Quotes" + ), + CodeSnippetMacroTest( + r''' + if (EFI_ERROR (Status)) { + UINTN Offset; + UINTN Start; + + DEBUG (( + DEBUG_INFO, + "Variable FV header is not valid. It will be reinitialized.\n" + )); + + // + // Get FvbInfo to provide in FwhInstance. + // + Status = GetFvbInfo (Length, &GoodFwVolHeader); + ASSERT (!EFI_ERROR (Status)); + } + ''', + (0, 0, 0), + "Code Section with Multi-Line Macro with No Arguments" + ) +] diff --git a/BaseTools/Plugin/DebugMacroCheck/tests/MacroTest.py b/BaseTools/Plugin/DebugMacroCheck/tests/MacroTest.py index 3b966d31ff..4b54a4da18 100644 --- a/BaseTools/Plugin/DebugMacroCheck/tests/MacroTest.py +++ b/BaseTools/Plugin/DebugMacroCheck/tests/MacroTest.py @@ -1,131 +1,131 @@ -# @file MacroTest.py
-#
-# Contains the data classes that are used to compose debug macro tests.
-#
-# All data classes inherit from a single abstract base class that expects
-# the subclass to define the category of test it represents.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-from dataclasses import dataclass, field
-from os import linesep
-from typing import Tuple
-
-import abc
-
-
-@dataclass(frozen=True)
-class MacroTest(abc.ABC):
- """Abstract base class for an individual macro test case."""
-
- macro: str
- result: Tuple[int, int, int]
- description: str = field(default='')
-
- @property
- @abc.abstractmethod
- def category(self) -> str:
- """Returns the test class category identifier.
-
- Example: 'equal_specifier_equal_argument_macro_test'
-
- This string is used to bind test objects against this class.
-
- Returns:
- str: Test category identifier string.
- """
- pass
-
- @property
- def category_description(self) -> str:
- """Returns the test class category description.
-
- Example: 'Test case with equal count of print specifiers to arguments.'
-
- This string is a human readable description of the test category.
-
- Returns:
- str: String describing the test category.
- """
- return self.__doc__
-
- def __str__(self):
- """Returns a macro test case description string."""
-
- s = [
- f"{linesep}",
- "=" * 80,
- f"Macro Test Type: {self.category_description}",
- f"{linesep}Macro: {self.macro}",
- f"{linesep}Expected Result: {self.result}"
- ]
-
- if self.description:
- s.insert(3, f"Test Description: {self.description}")
-
- return f'{linesep}'.join(s)
-
-
-@dataclass(frozen=True)
-class NoSpecifierNoArgumentMacroTest(MacroTest):
- """Test case with no print specifier and no arguments."""
-
- @property
- def category(self) -> str:
- return "no_specifier_no_argument_macro_test"
-
-
-@dataclass(frozen=True)
-class EqualSpecifierEqualArgumentMacroTest(MacroTest):
- """Test case with equal count of print specifiers to arguments."""
-
- @property
- def category(self) -> str:
- return "equal_specifier_equal_argument_macro_test"
-
-
-@dataclass(frozen=True)
-class MoreSpecifiersThanArgumentsMacroTest(MacroTest):
- """Test case with more print specifiers than arguments."""
-
- @property
- def category(self) -> str:
- return "more_specifiers_than_arguments_macro_test"
-
-
-@dataclass(frozen=True)
-class LessSpecifiersThanArgumentsMacroTest(MacroTest):
- """Test case with less print specifiers than arguments."""
-
- @property
- def category(self) -> str:
- return "less_specifiers_than_arguments_macro_test"
-
-
-@dataclass(frozen=True)
-class IgnoredSpecifiersMacroTest(MacroTest):
- """Test case to test ignored print specifiers."""
-
- @property
- def category(self) -> str:
- return "ignored_specifiers_macro_test"
-
-
-@dataclass(frozen=True)
-class SpecialParsingMacroTest(MacroTest):
- """Test case with special (complicated) parsing scenarios."""
-
- @property
- def category(self) -> str:
- return "special_parsing_macro_test"
-
-
-@dataclass(frozen=True)
-class CodeSnippetMacroTest(MacroTest):
- """Test case within a larger code snippet."""
-
- @property
- def category(self) -> str:
- return "code_snippet_macro_test"
+# @file MacroTest.py +# +# Contains the data classes that are used to compose debug macro tests. +# +# All data classes inherit from a single abstract base class that expects +# the subclass to define the category of test it represents. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +from dataclasses import dataclass, field +from os import linesep +from typing import Tuple + +import abc + + +@dataclass(frozen=True) +class MacroTest(abc.ABC): + """Abstract base class for an individual macro test case.""" + + macro: str + result: Tuple[int, int, int] + description: str = field(default='') + + @property + @abc.abstractmethod + def category(self) -> str: + """Returns the test class category identifier. + + Example: 'equal_specifier_equal_argument_macro_test' + + This string is used to bind test objects against this class. + + Returns: + str: Test category identifier string. + """ + pass + + @property + def category_description(self) -> str: + """Returns the test class category description. + + Example: 'Test case with equal count of print specifiers to arguments.' + + This string is a human readable description of the test category. + + Returns: + str: String describing the test category. + """ + return self.__doc__ + + def __str__(self): + """Returns a macro test case description string.""" + + s = [ + f"{linesep}", + "=" * 80, + f"Macro Test Type: {self.category_description}", + f"{linesep}Macro: {self.macro}", + f"{linesep}Expected Result: {self.result}" + ] + + if self.description: + s.insert(3, f"Test Description: {self.description}") + + return f'{linesep}'.join(s) + + +@dataclass(frozen=True) +class NoSpecifierNoArgumentMacroTest(MacroTest): + """Test case with no print specifier and no arguments.""" + + @property + def category(self) -> str: + return "no_specifier_no_argument_macro_test" + + +@dataclass(frozen=True) +class EqualSpecifierEqualArgumentMacroTest(MacroTest): + """Test case with equal count of print specifiers to arguments.""" + + @property + def category(self) -> str: + return "equal_specifier_equal_argument_macro_test" + + +@dataclass(frozen=True) +class MoreSpecifiersThanArgumentsMacroTest(MacroTest): + """Test case with more print specifiers than arguments.""" + + @property + def category(self) -> str: + return "more_specifiers_than_arguments_macro_test" + + +@dataclass(frozen=True) +class LessSpecifiersThanArgumentsMacroTest(MacroTest): + """Test case with less print specifiers than arguments.""" + + @property + def category(self) -> str: + return "less_specifiers_than_arguments_macro_test" + + +@dataclass(frozen=True) +class IgnoredSpecifiersMacroTest(MacroTest): + """Test case to test ignored print specifiers.""" + + @property + def category(self) -> str: + return "ignored_specifiers_macro_test" + + +@dataclass(frozen=True) +class SpecialParsingMacroTest(MacroTest): + """Test case with special (complicated) parsing scenarios.""" + + @property + def category(self) -> str: + return "special_parsing_macro_test" + + +@dataclass(frozen=True) +class CodeSnippetMacroTest(MacroTest): + """Test case within a larger code snippet.""" + + @property + def category(self) -> str: + return "code_snippet_macro_test" diff --git a/BaseTools/Plugin/DebugMacroCheck/tests/test_DebugMacroCheck.py b/BaseTools/Plugin/DebugMacroCheck/tests/test_DebugMacroCheck.py index 7abc0d2b87..5e85c1da46 100644 --- a/BaseTools/Plugin/DebugMacroCheck/tests/test_DebugMacroCheck.py +++ b/BaseTools/Plugin/DebugMacroCheck/tests/test_DebugMacroCheck.py @@ -1,201 +1,201 @@ -# @file test_DebugMacroCheck.py
-#
-# Contains unit tests for the DebugMacroCheck build plugin.
-#
-# An example of running these tests from the root of the workspace:
-# python -m unittest discover -s ./BaseTools/Plugin/DebugMacroCheck/tests -v
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-import inspect
-import pathlib
-import sys
-import unittest
-
-# Import the build plugin
-test_file = pathlib.Path(__file__)
-sys.path.append(str(test_file.parent.parent))
-
-# flake8 (E402): Ignore flake8 module level import not at top of file
-import DebugMacroCheck # noqa: E402
-
-from os import linesep # noqa: E402
-from tests import DebugMacroDataSet # noqa: E402
-from tests import MacroTest # noqa: E402
-from typing import Callable, Tuple # noqa: E402
-
-
-#
-# This metaclass is provided to dynamically produce test case container
-# classes. The main purpose of this approach is to:
-# 1. Allow categories of test cases to be defined (test container classes)
-# 2. Allow test cases to automatically (dynamically) be assigned to their
-# corresponding test container class when new test data is defined.
-#
-# The idea being that infrastructure and test data are separated. Adding
-# / removing / modifying test data does not require an infrastructure
-# change (unless new categories are defined).
-# 3. To work with the unittest discovery algorithm and VS Code Test Explorer.
-#
-# Notes:
-# - (1) can roughly be achieved with unittest test suites. In another
-# implementation approach, this solution was tested with relatively minor
-# modifications to use test suites. However, this became a bit overly
-# complicated with the dynamic test case method generation and did not
-# work as well with VS Code Test Explorer.
-# - For (2) and (3), particularly for VS Code Test Explorer to work, the
-# dynamic population of the container class namespace needed to happen prior
-# to class object creation. That is why the metaclass assigns test methods
-# to the new classes based upon the test category specified in the
-# corresponding data class.
-# - This could have been simplified a bit by either using one test case
-# container class and/or testing data in a single, monolithic test function
-# that iterates over the data set. However, the dynamic hierarchy greatly
-# helps organize test results and reporting. The infrastructure though
-# inheriting some complexity to support it, should not need to change (much)
-# as the data set expands.
-# - Test case categories (container classes) are derived from the overall
-# type of macro conditions under test.
-#
-# - This implementation assumes unittest will discover test cases
-# (classes derived from unittest.TestCase) with the name pattern "Test_*"
-# and test functions with the name pattern "test_x". Individual tests are
-# dynamically numbered monotonically within a category.
-# - The final test case description is also able to return fairly clean
-# context information.
-#
-class Meta_TestDebugMacroCheck(type):
- """
- Metaclass for debug macro test case class factory.
- """
- @classmethod
- def __prepare__(mcls, name, bases, **kwargs):
- """Returns the test case namespace for this class."""
- candidate_macros, cls_ns, cnt = [], {}, 0
-
- if "category" in kwargs.keys():
- candidate_macros = [m for m in DebugMacroDataSet.DEBUG_MACROS if
- m.category == kwargs["category"]]
- else:
- candidate_macros = DebugMacroDataSet.DEBUG_MACROS
-
- for cnt, macro_test in enumerate(candidate_macros):
- f_name = f'test_{macro_test.category}_{cnt}'
- t_desc = f'{macro_test!s}'
- cls_ns[f_name] = mcls.build_macro_test(macro_test, t_desc)
- return cls_ns
-
- def __new__(mcls, name, bases, ns, **kwargs):
- """Defined to prevent variable args from bubbling to the base class."""
- return super().__new__(mcls, name, bases, ns)
-
- def __init__(mcls, name, bases, ns, **kwargs):
- """Defined to prevent variable args from bubbling to the base class."""
- return super().__init__(name, bases, ns)
-
- @classmethod
- def build_macro_test(cls, macro_test: MacroTest.MacroTest,
- test_desc: str) -> Callable[[None], None]:
- """Returns a test function for this macro test data."
-
- Args:
- macro_test (MacroTest.MacroTest): The macro test class.
-
- test_desc (str): A test description string.
-
- Returns:
- Callable[[None], None]: A test case function.
- """
- def test_func(self):
- act_result = cls.check_regex(macro_test.macro)
- self.assertCountEqual(
- act_result,
- macro_test.result,
- test_desc + f'{linesep}'.join(
- ["", f"Actual Result: {act_result}", "=" * 80, ""]))
-
- return test_func
-
- @classmethod
- def check_regex(cls, source_str: str) -> Tuple[int, int, int]:
- """Returns the plugin result for the given macro string.
-
- Args:
- source_str (str): A string containing debug macros.
-
- Returns:
- Tuple[int, int, int]: A tuple of the number of formatting errors,
- number of print specifiers, and number of arguments for the macros
- given.
- """
- return DebugMacroCheck.check_debug_macros(
- DebugMacroCheck.get_debug_macros(source_str),
- cls._get_function_name())
-
- @classmethod
- def _get_function_name(cls) -> str:
- """Returns the function name from one level of call depth.
-
- Returns:
- str: The caller function name.
- """
- return "function: " + inspect.currentframe().f_back.f_code.co_name
-
-
-# Test container classes for dynamically generated macro test cases.
-# A class can be removed below to skip / remove it from testing.
-# Test case functions will be added to the appropriate class as they are
-# created.
-class Test_NoSpecifierNoArgument(
- unittest.TestCase,
- metaclass=Meta_TestDebugMacroCheck,
- category="no_specifier_no_argument_macro_test"):
- pass
-
-
-class Test_EqualSpecifierEqualArgument(
- unittest.TestCase,
- metaclass=Meta_TestDebugMacroCheck,
- category="equal_specifier_equal_argument_macro_test"):
- pass
-
-
-class Test_MoreSpecifiersThanArguments(
- unittest.TestCase,
- metaclass=Meta_TestDebugMacroCheck,
- category="more_specifiers_than_arguments_macro_test"):
- pass
-
-
-class Test_LessSpecifiersThanArguments(
- unittest.TestCase,
- metaclass=Meta_TestDebugMacroCheck,
- category="less_specifiers_than_arguments_macro_test"):
- pass
-
-
-class Test_IgnoredSpecifiers(
- unittest.TestCase,
- metaclass=Meta_TestDebugMacroCheck,
- category="ignored_specifiers_macro_test"):
- pass
-
-
-class Test_SpecialParsingMacroTest(
- unittest.TestCase,
- metaclass=Meta_TestDebugMacroCheck,
- category="special_parsing_macro_test"):
- pass
-
-
-class Test_CodeSnippetMacroTest(
- unittest.TestCase,
- metaclass=Meta_TestDebugMacroCheck,
- category="code_snippet_macro_test"):
- pass
-
-
-if __name__ == '__main__':
- unittest.main()
+# @file test_DebugMacroCheck.py +# +# Contains unit tests for the DebugMacroCheck build plugin. +# +# An example of running these tests from the root of the workspace: +# python -m unittest discover -s ./BaseTools/Plugin/DebugMacroCheck/tests -v +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import inspect +import pathlib +import sys +import unittest + +# Import the build plugin +test_file = pathlib.Path(__file__) +sys.path.append(str(test_file.parent.parent)) + +# flake8 (E402): Ignore flake8 module level import not at top of file +import DebugMacroCheck # noqa: E402 + +from os import linesep # noqa: E402 +from tests import DebugMacroDataSet # noqa: E402 +from tests import MacroTest # noqa: E402 +from typing import Callable, Tuple # noqa: E402 + + +# +# This metaclass is provided to dynamically produce test case container +# classes. The main purpose of this approach is to: +# 1. Allow categories of test cases to be defined (test container classes) +# 2. Allow test cases to automatically (dynamically) be assigned to their +# corresponding test container class when new test data is defined. +# +# The idea being that infrastructure and test data are separated. Adding +# / removing / modifying test data does not require an infrastructure +# change (unless new categories are defined). +# 3. To work with the unittest discovery algorithm and VS Code Test Explorer. +# +# Notes: +# - (1) can roughly be achieved with unittest test suites. In another +# implementation approach, this solution was tested with relatively minor +# modifications to use test suites. However, this became a bit overly +# complicated with the dynamic test case method generation and did not +# work as well with VS Code Test Explorer. +# - For (2) and (3), particularly for VS Code Test Explorer to work, the +# dynamic population of the container class namespace needed to happen prior +# to class object creation. That is why the metaclass assigns test methods +# to the new classes based upon the test category specified in the +# corresponding data class. +# - This could have been simplified a bit by either using one test case +# container class and/or testing data in a single, monolithic test function +# that iterates over the data set. However, the dynamic hierarchy greatly +# helps organize test results and reporting. The infrastructure though +# inheriting some complexity to support it, should not need to change (much) +# as the data set expands. +# - Test case categories (container classes) are derived from the overall +# type of macro conditions under test. +# +# - This implementation assumes unittest will discover test cases +# (classes derived from unittest.TestCase) with the name pattern "Test_*" +# and test functions with the name pattern "test_x". Individual tests are +# dynamically numbered monotonically within a category. +# - The final test case description is also able to return fairly clean +# context information. +# +class Meta_TestDebugMacroCheck(type): + """ + Metaclass for debug macro test case class factory. + """ + @classmethod + def __prepare__(mcls, name, bases, **kwargs): + """Returns the test case namespace for this class.""" + candidate_macros, cls_ns, cnt = [], {}, 0 + + if "category" in kwargs.keys(): + candidate_macros = [m for m in DebugMacroDataSet.DEBUG_MACROS if + m.category == kwargs["category"]] + else: + candidate_macros = DebugMacroDataSet.DEBUG_MACROS + + for cnt, macro_test in enumerate(candidate_macros): + f_name = f'test_{macro_test.category}_{cnt}' + t_desc = f'{macro_test!s}' + cls_ns[f_name] = mcls.build_macro_test(macro_test, t_desc) + return cls_ns + + def __new__(mcls, name, bases, ns, **kwargs): + """Defined to prevent variable args from bubbling to the base class.""" + return super().__new__(mcls, name, bases, ns) + + def __init__(mcls, name, bases, ns, **kwargs): + """Defined to prevent variable args from bubbling to the base class.""" + return super().__init__(name, bases, ns) + + @classmethod + def build_macro_test(cls, macro_test: MacroTest.MacroTest, + test_desc: str) -> Callable[[None], None]: + """Returns a test function for this macro test data." + + Args: + macro_test (MacroTest.MacroTest): The macro test class. + + test_desc (str): A test description string. + + Returns: + Callable[[None], None]: A test case function. + """ + def test_func(self): + act_result = cls.check_regex(macro_test.macro) + self.assertCountEqual( + act_result, + macro_test.result, + test_desc + f'{linesep}'.join( + ["", f"Actual Result: {act_result}", "=" * 80, ""])) + + return test_func + + @classmethod + def check_regex(cls, source_str: str) -> Tuple[int, int, int]: + """Returns the plugin result for the given macro string. + + Args: + source_str (str): A string containing debug macros. + + Returns: + Tuple[int, int, int]: A tuple of the number of formatting errors, + number of print specifiers, and number of arguments for the macros + given. + """ + return DebugMacroCheck.check_debug_macros( + DebugMacroCheck.get_debug_macros(source_str), + cls._get_function_name()) + + @classmethod + def _get_function_name(cls) -> str: + """Returns the function name from one level of call depth. + + Returns: + str: The caller function name. + """ + return "function: " + inspect.currentframe().f_back.f_code.co_name + + +# Test container classes for dynamically generated macro test cases. +# A class can be removed below to skip / remove it from testing. +# Test case functions will be added to the appropriate class as they are +# created. +class Test_NoSpecifierNoArgument( + unittest.TestCase, + metaclass=Meta_TestDebugMacroCheck, + category="no_specifier_no_argument_macro_test"): + pass + + +class Test_EqualSpecifierEqualArgument( + unittest.TestCase, + metaclass=Meta_TestDebugMacroCheck, + category="equal_specifier_equal_argument_macro_test"): + pass + + +class Test_MoreSpecifiersThanArguments( + unittest.TestCase, + metaclass=Meta_TestDebugMacroCheck, + category="more_specifiers_than_arguments_macro_test"): + pass + + +class Test_LessSpecifiersThanArguments( + unittest.TestCase, + metaclass=Meta_TestDebugMacroCheck, + category="less_specifiers_than_arguments_macro_test"): + pass + + +class Test_IgnoredSpecifiers( + unittest.TestCase, + metaclass=Meta_TestDebugMacroCheck, + category="ignored_specifiers_macro_test"): + pass + + +class Test_SpecialParsingMacroTest( + unittest.TestCase, + metaclass=Meta_TestDebugMacroCheck, + category="special_parsing_macro_test"): + pass + + +class Test_CodeSnippetMacroTest( + unittest.TestCase, + metaclass=Meta_TestDebugMacroCheck, + category="code_snippet_macro_test"): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py index 2e5c462cd2..d3ba80d006 100644 --- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py +++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py @@ -1,270 +1,270 @@ -# @file HostBasedUnitTestRunner.py
-# Plugin to located any host-based unit tests in the output directory and execute them.
-##
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-#
-##
-import os
-import logging
-import glob
-import stat
-import xml.etree.ElementTree
-from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin
-from edk2toolext import edk2_logging
-import edk2toollib.windows.locate_tools as locate_tools
-from edk2toolext.environment import shell_environment
-from edk2toollib.utility_functions import RunCmd
-from edk2toollib.utility_functions import GetHostInfo
-from textwrap import dedent
-
-
-class HostBasedUnitTestRunner(IUefiBuildPlugin):
-
- def do_pre_build(self, thebuilder):
- '''
- Run Prebuild
- '''
-
- return 0
-
- def do_post_build(self, thebuilder):
- '''
- After a build, will automatically locate and run all host-based unit tests. Logs any
- failures with Warning severity and will return a count of the failures as the return code.
-
- EXPECTS:
- - Build Var 'CI_BUILD_TYPE' - If not set to 'host_unit_test', will not do anything.
-
- UPDATES:
- - Shell Var 'CMOCKA_XML_FILE'
- '''
- ci_type = thebuilder.env.GetValue('CI_BUILD_TYPE')
- if ci_type != 'host_unit_test':
- return 0
-
- shell_env = shell_environment.GetEnvironment()
- logging.log(edk2_logging.get_section_level(),
- "Run Host based Unit Tests")
- path = thebuilder.env.GetValue("BUILD_OUTPUT_BASE")
-
- failure_count = 0
-
- # Set up the reporting type for Cmocka.
- shell_env.set_shell_var('CMOCKA_MESSAGE_OUTPUT', 'xml')
-
- for arch in thebuilder.env.GetValue("TARGET_ARCH").split():
- logging.log(edk2_logging.get_subsection_level(),
- "Testing for architecture: " + arch)
- cp = os.path.join(path, arch)
-
- # If any old results XML files exist, clean them up.
- for old_result in glob.iglob(os.path.join(cp, "*.result.xml")):
- os.remove(old_result)
-
- # Find and Run any Host Tests
- if GetHostInfo().os.upper() == "LINUX":
- testList = glob.glob(os.path.join(cp, "*Test*"))
- for a in testList[:]:
- p = os.path.join(cp, a)
- # It must be a file
- if not os.path.isfile(p):
- testList.remove(a)
- logging.debug(f"Remove directory file: {p}")
- continue
- # It must be executable
- if os.stat(p).st_mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) == 0:
- testList.remove(a)
- logging.debug(f"Remove non-executable file: {p}")
- continue
-
- logging.info(f"Test file found: {p}")
-
- elif GetHostInfo().os.upper() == "WINDOWS":
- testList = glob.glob(os.path.join(cp, "*Test*.exe"))
- else:
- raise NotImplementedError("Unsupported Operating System")
-
- if not testList:
- logging.warning(dedent("""
- UnitTest Coverage:
- No unit tests discovered. Test coverage will not be generated.
-
- Prevent this message by:
- 1. Adding host-based unit tests to this package
- 2. Ensuring tests have the word "Test" in their name
- 3. Disabling HostUnitTestCompilerPlugin in the package CI YAML file
- """).strip())
- return 0
-
- for test in testList:
- # Configure output name if test uses cmocka.
- shell_env.set_shell_var(
- 'CMOCKA_XML_FILE', test + ".CMOCKA.%g." + arch + ".result.xml")
- # Configure output name if test uses gtest.
- shell_env.set_shell_var(
- 'GTEST_OUTPUT', "xml:" + test + ".GTEST." + arch + ".result.xml")
-
- # Run the test.
- ret = RunCmd('"' + test + '"', "", workingdir=cp)
- if ret != 0:
- logging.error("UnitTest Execution Error: " +
- os.path.basename(test))
- else:
- logging.info("UnitTest Completed: " +
- os.path.basename(test))
- file_match_pattern = test + ".*." + arch + ".result.xml"
- xml_results_list = glob.glob(file_match_pattern)
- for xml_result_file in xml_results_list:
- root = xml.etree.ElementTree.parse(
- xml_result_file).getroot()
- for suite in root:
- for case in suite:
- for result in case:
- if result.tag == 'failure':
- logging.warning(
- "%s Test Failed" % os.path.basename(test))
- logging.warning(
- " %s - %s" % (case.attrib['name'], result.text))
- failure_count += 1
-
- if thebuilder.env.GetValue("CODE_COVERAGE") != "FALSE":
- if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5":
- ret = self.gen_code_coverage_gcc(thebuilder)
- if ret != 0:
- failure_count += 1
- elif thebuilder.env.GetValue("TOOL_CHAIN_TAG").startswith ("VS"):
- ret = self.gen_code_coverage_msvc(thebuilder)
- if ret != 0:
- failure_count += 1
- else:
- logging.info("Skipping code coverage. Currently, support GCC and MSVC compiler.")
-
- return failure_count
-
- def gen_code_coverage_gcc(self, thebuilder):
- logging.info("Generating UnitTest code coverage")
-
- buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE")
- workspace = thebuilder.env.GetValue("WORKSPACE")
-
- # Generate base code coverage for all source files
- ret = RunCmd("lcov", f"--no-external --capture --initial --directory {buildOutputBase} --output-file {buildOutputBase}/cov-base.info --rc lcov_branch_coverage=1")
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to build initial coverage data.")
- return 1
-
- # Coverage data for tested files only
- ret = RunCmd("lcov", f"--capture --directory {buildOutputBase}/ --output-file {buildOutputBase}/coverage-test.info --rc lcov_branch_coverage=1")
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to build coverage data for tested files.")
- return 1
-
- # Aggregate all coverage data
- ret = RunCmd("lcov", f"--add-tracefile {buildOutputBase}/cov-base.info --add-tracefile {buildOutputBase}/coverage-test.info --output-file {buildOutputBase}/total-coverage.info --rc lcov_branch_coverage=1")
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to aggregate coverage data.")
- return 1
-
- # Generate coverage XML
- ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info -o {buildOutputBase}/compare.xml")
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to generate coverage XML.")
- return 1
-
- # Filter out auto-generated and test code
- ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {buildOutputBase}/coverage.xml")
- if ret != 0:
- logging.error("UnitTest Coverage: Failed generate filtered coverage XML.")
- return 1
-
- # Generate all coverage file
- testCoverageList = glob.glob (f"{workspace}/Build/**/total-coverage.info", recursive=True)
-
- coverageFile = ""
- for testCoverage in testCoverageList:
- coverageFile += " --add-tracefile " + testCoverage
- ret = RunCmd("lcov", f"{coverageFile} --output-file {workspace}/Build/all-coverage.info --rc lcov_branch_coverage=1")
- if ret != 0:
- logging.error("UnitTest Coverage: Failed generate all coverage file.")
- return 1
-
- # Generate and XML file if requested.for all package
- if os.path.isfile(f"{workspace}/Build/coverage.xml"):
- os.remove(f"{workspace}/Build/coverage.xml")
- ret = RunCmd("lcov_cobertura",f"{workspace}/Build/all-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {workspace}/Build/coverage.xml")
-
- return 0
-
-
- def gen_code_coverage_msvc(self, thebuilder):
- logging.info("Generating UnitTest code coverage")
-
-
- buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE")
- testList = glob.glob(os.path.join(buildOutputBase, "**","*Test*.exe"), recursive=True)
- workspace = thebuilder.env.GetValue("WORKSPACE")
- workspace = (workspace + os.sep) if workspace[-1] != os.sep else workspace
- workspaceBuild = os.path.join(workspace, 'Build')
- # Generate coverage file
- coverageFile = ""
- for testFile in testList:
- ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}")
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to collect coverage data.")
- return 1
-
- coverageFile = f" --input_coverage={testFile}.cov"
- totalCoverageFile = os.path.join(buildOutputBase, 'coverage.cov')
- if os.path.isfile(totalCoverageFile):
- coverageFile += f" --input_coverage={totalCoverageFile}"
- ret = RunCmd(
- "OpenCppCoverage",
- f"--export_type binary:{totalCoverageFile} " +
- f"--working_dir={workspaceBuild} " +
- f"{coverageFile}"
- )
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to collect coverage data.")
- return 1
-
- # Generate and XML file if requested.by each package
- ret = RunCmd(
- "OpenCppCoverage",
- f"--export_type cobertura:{os.path.join(buildOutputBase, 'coverage.xml')} " +
- f"--working_dir={workspaceBuild} " +
- f"--input_coverage={totalCoverageFile} "
- )
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to generate cobertura format xml in single package.")
- return 1
-
- # Generate total report XML file for all package
- testCoverageList = glob.glob(os.path.join(workspace, "Build", "**", "*Test*.exe.cov"), recursive=True)
- coverageFile = ""
- totalCoverageFile = os.path.join(workspaceBuild, 'coverage.cov')
- for testCoverage in testCoverageList:
- coverageFile = f" --input_coverage={testCoverage}"
- if os.path.isfile(totalCoverageFile):
- coverageFile += f" --input_coverage={totalCoverageFile}"
- ret = RunCmd(
- "OpenCppCoverage",
- f"--export_type binary:{totalCoverageFile} " +
- f"--working_dir={workspaceBuild} " +
- f"{coverageFile}"
- )
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to collect coverage data.")
- return 1
-
- ret = RunCmd(
- "OpenCppCoverage",
- f"--export_type cobertura:{os.path.join(workspaceBuild, 'coverage.xml')} " +
- f"--working_dir={workspaceBuild} " +
- f"--input_coverage={totalCoverageFile}"
- )
- if ret != 0:
- logging.error("UnitTest Coverage: Failed to generate cobertura format xml.")
- return 1
-
- return 0
+# @file HostBasedUnitTestRunner.py +# Plugin to located any host-based unit tests in the output directory and execute them. +## +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +## +import os +import logging +import glob +import stat +import xml.etree.ElementTree +from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +from edk2toolext import edk2_logging +import edk2toollib.windows.locate_tools as locate_tools +from edk2toolext.environment import shell_environment +from edk2toollib.utility_functions import RunCmd +from edk2toollib.utility_functions import GetHostInfo +from textwrap import dedent + + +class HostBasedUnitTestRunner(IUefiBuildPlugin): + + def do_pre_build(self, thebuilder): + ''' + Run Prebuild + ''' + + return 0 + + def do_post_build(self, thebuilder): + ''' + After a build, will automatically locate and run all host-based unit tests. Logs any + failures with Warning severity and will return a count of the failures as the return code. + + EXPECTS: + - Build Var 'CI_BUILD_TYPE' - If not set to 'host_unit_test', will not do anything. + + UPDATES: + - Shell Var 'CMOCKA_XML_FILE' + ''' + ci_type = thebuilder.env.GetValue('CI_BUILD_TYPE') + if ci_type != 'host_unit_test': + return 0 + + shell_env = shell_environment.GetEnvironment() + logging.log(edk2_logging.get_section_level(), + "Run Host based Unit Tests") + path = thebuilder.env.GetValue("BUILD_OUTPUT_BASE") + + failure_count = 0 + + # Set up the reporting type for Cmocka. + shell_env.set_shell_var('CMOCKA_MESSAGE_OUTPUT', 'xml') + + for arch in thebuilder.env.GetValue("TARGET_ARCH").split(): + logging.log(edk2_logging.get_subsection_level(), + "Testing for architecture: " + arch) + cp = os.path.join(path, arch) + + # If any old results XML files exist, clean them up. + for old_result in glob.iglob(os.path.join(cp, "*.result.xml")): + os.remove(old_result) + + # Find and Run any Host Tests + if GetHostInfo().os.upper() == "LINUX": + testList = glob.glob(os.path.join(cp, "*Test*")) + for a in testList[:]: + p = os.path.join(cp, a) + # It must be a file + if not os.path.isfile(p): + testList.remove(a) + logging.debug(f"Remove directory file: {p}") + continue + # It must be executable + if os.stat(p).st_mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) == 0: + testList.remove(a) + logging.debug(f"Remove non-executable file: {p}") + continue + + logging.info(f"Test file found: {p}") + + elif GetHostInfo().os.upper() == "WINDOWS": + testList = glob.glob(os.path.join(cp, "*Test*.exe")) + else: + raise NotImplementedError("Unsupported Operating System") + + if not testList: + logging.warning(dedent(""" + UnitTest Coverage: + No unit tests discovered. Test coverage will not be generated. + + Prevent this message by: + 1. Adding host-based unit tests to this package + 2. Ensuring tests have the word "Test" in their name + 3. Disabling HostUnitTestCompilerPlugin in the package CI YAML file + """).strip()) + return 0 + + for test in testList: + # Configure output name if test uses cmocka. + shell_env.set_shell_var( + 'CMOCKA_XML_FILE', test + ".CMOCKA.%g." + arch + ".result.xml") + # Configure output name if test uses gtest. + shell_env.set_shell_var( + 'GTEST_OUTPUT', "xml:" + test + ".GTEST." + arch + ".result.xml") + + # Run the test. + ret = RunCmd('"' + test + '"', "", workingdir=cp) + if ret != 0: + logging.error("UnitTest Execution Error: " + + os.path.basename(test)) + else: + logging.info("UnitTest Completed: " + + os.path.basename(test)) + file_match_pattern = test + ".*." + arch + ".result.xml" + xml_results_list = glob.glob(file_match_pattern) + for xml_result_file in xml_results_list: + root = xml.etree.ElementTree.parse( + xml_result_file).getroot() + for suite in root: + for case in suite: + for result in case: + if result.tag == 'failure': + logging.warning( + "%s Test Failed" % os.path.basename(test)) + logging.warning( + " %s - %s" % (case.attrib['name'], result.text)) + failure_count += 1 + + if thebuilder.env.GetValue("CODE_COVERAGE") != "FALSE": + if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5": + ret = self.gen_code_coverage_gcc(thebuilder) + if ret != 0: + failure_count += 1 + elif thebuilder.env.GetValue("TOOL_CHAIN_TAG").startswith ("VS"): + ret = self.gen_code_coverage_msvc(thebuilder) + if ret != 0: + failure_count += 1 + else: + logging.info("Skipping code coverage. Currently, support GCC and MSVC compiler.") + + return failure_count + + def gen_code_coverage_gcc(self, thebuilder): + logging.info("Generating UnitTest code coverage") + + buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE") + workspace = thebuilder.env.GetValue("WORKSPACE") + + # Generate base code coverage for all source files + ret = RunCmd("lcov", f"--no-external --capture --initial --directory {buildOutputBase} --output-file {buildOutputBase}/cov-base.info --rc lcov_branch_coverage=1") + if ret != 0: + logging.error("UnitTest Coverage: Failed to build initial coverage data.") + return 1 + + # Coverage data for tested files only + ret = RunCmd("lcov", f"--capture --directory {buildOutputBase}/ --output-file {buildOutputBase}/coverage-test.info --rc lcov_branch_coverage=1") + if ret != 0: + logging.error("UnitTest Coverage: Failed to build coverage data for tested files.") + return 1 + + # Aggregate all coverage data + ret = RunCmd("lcov", f"--add-tracefile {buildOutputBase}/cov-base.info --add-tracefile {buildOutputBase}/coverage-test.info --output-file {buildOutputBase}/total-coverage.info --rc lcov_branch_coverage=1") + if ret != 0: + logging.error("UnitTest Coverage: Failed to aggregate coverage data.") + return 1 + + # Generate coverage XML + ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info -o {buildOutputBase}/compare.xml") + if ret != 0: + logging.error("UnitTest Coverage: Failed to generate coverage XML.") + return 1 + + # Filter out auto-generated and test code + ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {buildOutputBase}/coverage.xml") + if ret != 0: + logging.error("UnitTest Coverage: Failed generate filtered coverage XML.") + return 1 + + # Generate all coverage file + testCoverageList = glob.glob (f"{workspace}/Build/**/total-coverage.info", recursive=True) + + coverageFile = "" + for testCoverage in testCoverageList: + coverageFile += " --add-tracefile " + testCoverage + ret = RunCmd("lcov", f"{coverageFile} --output-file {workspace}/Build/all-coverage.info --rc lcov_branch_coverage=1") + if ret != 0: + logging.error("UnitTest Coverage: Failed generate all coverage file.") + return 1 + + # Generate and XML file if requested.for all package + if os.path.isfile(f"{workspace}/Build/coverage.xml"): + os.remove(f"{workspace}/Build/coverage.xml") + ret = RunCmd("lcov_cobertura",f"{workspace}/Build/all-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {workspace}/Build/coverage.xml") + + return 0 + + + def gen_code_coverage_msvc(self, thebuilder): + logging.info("Generating UnitTest code coverage") + + + buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE") + testList = glob.glob(os.path.join(buildOutputBase, "**","*Test*.exe"), recursive=True) + workspace = thebuilder.env.GetValue("WORKSPACE") + workspace = (workspace + os.sep) if workspace[-1] != os.sep else workspace + workspaceBuild = os.path.join(workspace, 'Build') + # Generate coverage file + coverageFile = "" + for testFile in testList: + ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}") + if ret != 0: + logging.error("UnitTest Coverage: Failed to collect coverage data.") + return 1 + + coverageFile = f" --input_coverage={testFile}.cov" + totalCoverageFile = os.path.join(buildOutputBase, 'coverage.cov') + if os.path.isfile(totalCoverageFile): + coverageFile += f" --input_coverage={totalCoverageFile}" + ret = RunCmd( + "OpenCppCoverage", + f"--export_type binary:{totalCoverageFile} " + + f"--working_dir={workspaceBuild} " + + f"{coverageFile}" + ) + if ret != 0: + logging.error("UnitTest Coverage: Failed to collect coverage data.") + return 1 + + # Generate and XML file if requested.by each package + ret = RunCmd( + "OpenCppCoverage", + f"--export_type cobertura:{os.path.join(buildOutputBase, 'coverage.xml')} " + + f"--working_dir={workspaceBuild} " + + f"--input_coverage={totalCoverageFile} " + ) + if ret != 0: + logging.error("UnitTest Coverage: Failed to generate cobertura format xml in single package.") + return 1 + + # Generate total report XML file for all package + testCoverageList = glob.glob(os.path.join(workspace, "Build", "**", "*Test*.exe.cov"), recursive=True) + coverageFile = "" + totalCoverageFile = os.path.join(workspaceBuild, 'coverage.cov') + for testCoverage in testCoverageList: + coverageFile = f" --input_coverage={testCoverage}" + if os.path.isfile(totalCoverageFile): + coverageFile += f" --input_coverage={totalCoverageFile}" + ret = RunCmd( + "OpenCppCoverage", + f"--export_type binary:{totalCoverageFile} " + + f"--working_dir={workspaceBuild} " + + f"{coverageFile}" + ) + if ret != 0: + logging.error("UnitTest Coverage: Failed to collect coverage data.") + return 1 + + ret = RunCmd( + "OpenCppCoverage", + f"--export_type cobertura:{os.path.join(workspaceBuild, 'coverage.xml')} " + + f"--working_dir={workspaceBuild} " + + f"--input_coverage={totalCoverageFile}" + ) + if ret != 0: + logging.error("UnitTest Coverage: Failed to generate cobertura format xml.") + return 1 + + return 0 diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner_plug_in.yaml b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner_plug_in.yaml index a0fbf3d2fe..83ae4b3bb3 100644 --- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner_plug_in.yaml +++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner_plug_in.yaml @@ -1,12 +1,12 @@ -##
-# IUefiBuildPlugin used to run any unittests that
-# were built on this build.
-#
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-{
- "scope": "host-based-test",
- "name": "Host-Based Unit Test Runner",
- "module": "HostBasedUnitTestRunner"
-}
+## +# IUefiBuildPlugin used to run any unittests that +# were built on this build. +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "host-based-test", + "name": "Host-Based Unit Test Runner", + "module": "HostBasedUnitTestRunner" +} diff --git a/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain.py b/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain.py index dab7a87997..01ad29045b 100644 --- a/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain.py +++ b/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain.py @@ -1,154 +1,154 @@ -# @file LinuxGcc5ToolChain.py
-# Plugin to configures paths for GCC5 ARM/AARCH64 Toolchain
-##
-# This plugin works in conjuncture with the tools_def
-#
-# Copyright (c) Microsoft Corporation
-# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>
-# Copyright (c) 2022, Loongson Technology Corporation Limited. All rights reserved.<BR>
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-import os
-import logging
-from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin
-from edk2toolext.environment import shell_environment
-
-
-class LinuxGcc5ToolChain(IUefiBuildPlugin):
-
- def do_post_build(self, thebuilder):
- return 0
-
- def do_pre_build(self, thebuilder):
- self.Logger = logging.getLogger("LinuxGcc5ToolChain")
-
- #
- # GCC5 - The ARM and AARCH64 compilers need their paths set if available
- if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5":
-
- # Start with AARACH64 compiler
- ret = self._check_aarch64()
- if ret != 0:
- self.Logger.critical("Failed in check aarch64")
- return ret
-
- # Check arm compiler
- ret = self._check_arm()
- if ret != 0:
- self.Logger.critical("Failed in check arm")
- return ret
-
- # Check RISCV64 compiler
- ret = self._check_riscv64()
- if ret != 0:
- self.Logger.critical("Failed in check riscv64")
- return ret
-
- # Check LoongArch64 compiler
- ret = self._check_loongarch64()
- if ret != 0:
- self.Logger.critical("Failed in check loongarch64")
- return ret
-
- return 0
-
- def _check_arm(self):
- # check to see if full path already configured
- if shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_PREFIX") is not None:
- self.Logger.info("GCC5_ARM_PREFIX is already set.")
-
- else:
- # now check for install dir. If set then set the Prefix
- install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL")
- if install_path is None:
- return 0
-
- # make GCC5_ARM_PREFIX to align with tools_def.txt
- prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-")
- shell_environment.GetEnvironment().set_shell_var("GCC5_ARM_PREFIX", prefix)
-
- # now confirm it exists
- if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_PREFIX") + "gcc"):
- self.Logger.error("Path for GCC5_ARM_PREFIX toolchain is invalid")
- return -2
-
- return 0
-
- def _check_aarch64(self):
- # check to see if full path already configured
- if shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX") is not None:
- self.Logger.info("GCC5_AARCH64_PREFIX is already set.")
-
- else:
- # now check for install dir. If set then set the Prefix
- install_path = shell_environment.GetEnvironment(
- ).get_shell_var("GCC5_AARCH64_INSTALL")
- if install_path is None:
- return 0
-
- # make GCC5_AARCH64_PREFIX to align with tools_def.txt
- prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-")
- shell_environment.GetEnvironment().set_shell_var("GCC5_AARCH64_PREFIX", prefix)
-
- # now confirm it exists
- if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX") + "gcc"):
- self.Logger.error(
- "Path for GCC5_AARCH64_PREFIX toolchain is invalid")
- return -2
-
- return 0
-
- def _check_riscv64(self):
- # now check for install dir. If set then set the Prefix
- install_path = shell_environment.GetEnvironment(
- ).get_shell_var("GCC5_RISCV64_INSTALL")
- if install_path is None:
- return 0
-
- # check to see if full path already configured
- if shell_environment.GetEnvironment().get_shell_var("GCC5_RISCV64_PREFIX") is not None:
- self.Logger.info("GCC5_RISCV64_PREFIX is already set.")
-
- else:
- # make GCC5_RISCV64_PREFIX to align with tools_def.txt
- prefix = os.path.join(install_path, "bin", "riscv64-unknown-elf-")
- shell_environment.GetEnvironment().set_shell_var("GCC5_RISCV64_PREFIX", prefix)
-
- # now confirm it exists
- if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_RISCV64_PREFIX") + "gcc"):
- self.Logger.error(
- "Path for GCC5_RISCV64_PREFIX toolchain is invalid")
- return -2
-
- # Check if LD_LIBRARY_PATH is set for the libraries of RISC-V GCC toolchain
- if shell_environment.GetEnvironment().get_shell_var("LD_LIBRARY_PATH") is not None:
- self.Logger.info("LD_LIBRARY_PATH is already set.")
-
- prefix = os.path.join(install_path, "lib")
- shell_environment.GetEnvironment().set_shell_var("LD_LIBRARY_PATH", prefix)
-
- return 0
-
- def _check_loongarch64(self):
- # check to see if full path already configured
- if shell_environment.GetEnvironment().get_shell_var("GCC5_LOONGARCH64_PREFIX") is not None:
- self.Logger.info("GCC5_LOONGARCH64_PREFIX is already set.")
-
- else:
- # now check for install dir. If set then set the Prefix
- install_path = shell_environment.GetEnvironment(
- ).get_shell_var("GCC5_LOONGARCH64_INSTALL")
- if install_path is None:
- return 0
-
- # make GCC5_LOONGARCH64_PREFIX to align with tools_def.txt
- prefix = os.path.join(install_path, "bin", "loongarch64-unknown-linux-gnu-")
- shell_environment.GetEnvironment().set_shell_var("GCC5_LOONGARCH64_PREFIX", prefix)
-
- # now confirm it exists
- if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_LOONGARCH64_PREFIX") + "gcc"):
- self.Logger.error(
- "Path for GCC5_LOONGARCH64_PREFIX toolchain is invalid")
- return -2
-
- return 0
+# @file LinuxGcc5ToolChain.py +# Plugin to configures paths for GCC5 ARM/AARCH64 Toolchain +## +# This plugin works in conjuncture with the tools_def +# +# Copyright (c) Microsoft Corporation +# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR> +# Copyright (c) 2022, Loongson Technology Corporation Limited. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +import logging +from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +from edk2toolext.environment import shell_environment + + +class LinuxGcc5ToolChain(IUefiBuildPlugin): + + def do_post_build(self, thebuilder): + return 0 + + def do_pre_build(self, thebuilder): + self.Logger = logging.getLogger("LinuxGcc5ToolChain") + + # + # GCC5 - The ARM and AARCH64 compilers need their paths set if available + if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5": + + # Start with AARACH64 compiler + ret = self._check_aarch64() + if ret != 0: + self.Logger.critical("Failed in check aarch64") + return ret + + # Check arm compiler + ret = self._check_arm() + if ret != 0: + self.Logger.critical("Failed in check arm") + return ret + + # Check RISCV64 compiler + ret = self._check_riscv64() + if ret != 0: + self.Logger.critical("Failed in check riscv64") + return ret + + # Check LoongArch64 compiler + ret = self._check_loongarch64() + if ret != 0: + self.Logger.critical("Failed in check loongarch64") + return ret + + return 0 + + def _check_arm(self): + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_PREFIX") is not None: + self.Logger.info("GCC5_ARM_PREFIX is already set.") + + else: + # now check for install dir. If set then set the Prefix + install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL") + if install_path is None: + return 0 + + # make GCC5_ARM_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-") + shell_environment.GetEnvironment().set_shell_var("GCC5_ARM_PREFIX", prefix) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_PREFIX") + "gcc"): + self.Logger.error("Path for GCC5_ARM_PREFIX toolchain is invalid") + return -2 + + return 0 + + def _check_aarch64(self): + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX") is not None: + self.Logger.info("GCC5_AARCH64_PREFIX is already set.") + + else: + # now check for install dir. If set then set the Prefix + install_path = shell_environment.GetEnvironment( + ).get_shell_var("GCC5_AARCH64_INSTALL") + if install_path is None: + return 0 + + # make GCC5_AARCH64_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-") + shell_environment.GetEnvironment().set_shell_var("GCC5_AARCH64_PREFIX", prefix) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX") + "gcc"): + self.Logger.error( + "Path for GCC5_AARCH64_PREFIX toolchain is invalid") + return -2 + + return 0 + + def _check_riscv64(self): + # now check for install dir. If set then set the Prefix + install_path = shell_environment.GetEnvironment( + ).get_shell_var("GCC5_RISCV64_INSTALL") + if install_path is None: + return 0 + + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("GCC5_RISCV64_PREFIX") is not None: + self.Logger.info("GCC5_RISCV64_PREFIX is already set.") + + else: + # make GCC5_RISCV64_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "bin", "riscv64-unknown-elf-") + shell_environment.GetEnvironment().set_shell_var("GCC5_RISCV64_PREFIX", prefix) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_RISCV64_PREFIX") + "gcc"): + self.Logger.error( + "Path for GCC5_RISCV64_PREFIX toolchain is invalid") + return -2 + + # Check if LD_LIBRARY_PATH is set for the libraries of RISC-V GCC toolchain + if shell_environment.GetEnvironment().get_shell_var("LD_LIBRARY_PATH") is not None: + self.Logger.info("LD_LIBRARY_PATH is already set.") + + prefix = os.path.join(install_path, "lib") + shell_environment.GetEnvironment().set_shell_var("LD_LIBRARY_PATH", prefix) + + return 0 + + def _check_loongarch64(self): + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("GCC5_LOONGARCH64_PREFIX") is not None: + self.Logger.info("GCC5_LOONGARCH64_PREFIX is already set.") + + else: + # now check for install dir. If set then set the Prefix + install_path = shell_environment.GetEnvironment( + ).get_shell_var("GCC5_LOONGARCH64_INSTALL") + if install_path is None: + return 0 + + # make GCC5_LOONGARCH64_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "bin", "loongarch64-unknown-linux-gnu-") + shell_environment.GetEnvironment().set_shell_var("GCC5_LOONGARCH64_PREFIX", prefix) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_LOONGARCH64_PREFIX") + "gcc"): + self.Logger.error( + "Path for GCC5_LOONGARCH64_PREFIX toolchain is invalid") + return -2 + + return 0 diff --git a/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain_plug_in.yaml b/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain_plug_in.yaml index 39c378a926..e07441ae3a 100644 --- a/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain_plug_in.yaml +++ b/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain_plug_in.yaml @@ -1,12 +1,12 @@ -## @file
-# Build Plugin used to set the path
-# for the GCC5 ARM/AARCH64 downloaded compilers
-#
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-{
- "scope": "global-nix",
- "name": "Linux GCC5 Tool Chain Support",
- "module": "LinuxGcc5ToolChain"
-}
+## @file +# Build Plugin used to set the path +# for the GCC5 ARM/AARCH64 downloaded compilers +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global-nix", + "name": "Linux GCC5 Tool Chain Support", + "module": "LinuxGcc5ToolChain" +} diff --git a/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath.py b/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath.py index ec2f2d1298..5f1bb5a5cf 100644 --- a/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath.py +++ b/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath.py @@ -1,29 +1,29 @@ -## @file WinRcPath.py
-# Plugin to find Windows SDK Resource Compiler rc.exe
-##
-# This plugin works in conjuncture with the tools_def to support rc.exe
-#
-# Copyright (c) Microsoft Corporation
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-import os
-from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin
-import edk2toollib.windows.locate_tools as locate_tools
-from edk2toolext.environment import shell_environment
-from edk2toolext.environment import version_aggregator
-
-class WinRcPath(IUefiBuildPlugin):
-
- def do_post_build(self, thebuilder):
- return 0
-
- def do_pre_build(self, thebuilder):
- #get the locate tools module
- path = locate_tools.FindToolInWinSdk("rc.exe")
- if path is None:
- thebuilder.logging.warning("Failed to find rc.exe")
- else:
- p = os.path.abspath(os.path.dirname(path))
- shell_environment.GetEnvironment().set_shell_var("WINSDK_PATH_FOR_RC_EXE", p)
- version_aggregator.GetVersionAggregator().ReportVersion("WINSDK_PATH_FOR_RC_EXE", p, version_aggregator.VersionTypes.INFO)
- return 0
+## @file WinRcPath.py +# Plugin to find Windows SDK Resource Compiler rc.exe +## +# This plugin works in conjuncture with the tools_def to support rc.exe +# +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +import edk2toollib.windows.locate_tools as locate_tools +from edk2toolext.environment import shell_environment +from edk2toolext.environment import version_aggregator + +class WinRcPath(IUefiBuildPlugin): + + def do_post_build(self, thebuilder): + return 0 + + def do_pre_build(self, thebuilder): + #get the locate tools module + path = locate_tools.FindToolInWinSdk("rc.exe") + if path is None: + thebuilder.logging.warning("Failed to find rc.exe") + else: + p = os.path.abspath(os.path.dirname(path)) + shell_environment.GetEnvironment().set_shell_var("WINSDK_PATH_FOR_RC_EXE", p) + version_aggregator.GetVersionAggregator().ReportVersion("WINSDK_PATH_FOR_RC_EXE", p, version_aggregator.VersionTypes.INFO) + return 0 diff --git a/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath_plug_in.yaml b/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath_plug_in.yaml index 3aec35d863..9dc5534729 100644 --- a/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath_plug_in.yaml +++ b/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath_plug_in.yaml @@ -1,13 +1,13 @@ -## @file
-# Build Plugin used to set the path to rc.exe on windows.
-# The plugin is able to use python to locate the tool as to avoid
-# hard-coding the path
-#
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-{
- "scope": "global-win",
- "name": "Windows RC Path Support",
- "module": "WinRcPath"
-}
+## @file +# Build Plugin used to set the path to rc.exe on windows. +# The plugin is able to use python to locate the tool as to avoid +# hard-coding the path +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global-win", + "name": "Windows RC Path Support", + "module": "WinRcPath" +} diff --git a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py index 615b5ed6d1..982929ddf1 100644 --- a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py +++ b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py @@ -1,215 +1,215 @@ -# @file WindowsVsToolChain.py
-# Plugin to configures paths for the VS2017 and VS2019 tool chain
-##
-# This plugin works in conjuncture with the tools_def
-#
-# Copyright (c) Microsoft Corporation
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-import os
-import logging
-from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin
-import edk2toollib.windows.locate_tools as locate_tools
-from edk2toollib.windows.locate_tools import FindWithVsWhere
-from edk2toolext.environment import shell_environment
-from edk2toolext.environment import version_aggregator
-from edk2toollib.utility_functions import GetHostInfo
-
-
-class WindowsVsToolChain(IUefiBuildPlugin):
-
- def do_post_build(self, thebuilder):
- return 0
-
- def do_pre_build(self, thebuilder):
- self.Logger = logging.getLogger("WindowsVsToolChain")
- interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB", "LIBPATH", "UniversalCRTSdkDir",
- "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath", "WindowsSdkDir", "WindowsSdkVerBinPath",
- "WindowsSDKVersion", "VCToolsInstallDir", "Path"]
-
- #
- # VS2017 - Follow VS2017 where there is potential for many versions of the tools.
- # If a specific version is required then the user must set both env variables:
- # VS150INSTALLPATH: base install path on system to VC install dir. Here you will find the VC folder, etc
- # VS150TOOLVER: version number for the VC compiler tools
- # VS2017_PREFIX: path to MSVC compiler folder with trailing slash (can be used instead of two vars above)
- # VS2017_HOST: set the host architecture to use for host tools, and host libs, etc
- if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "VS2017":
-
- # check to see if host is configured
- # HostType for VS2017 should be (defined in tools_def):
- # x86 == 32bit Intel
- # x64 == 64bit Intel
- # arm == 32bit Arm
- # arm64 == 64bit Arm
- #
- HostType = shell_environment.GetEnvironment().get_shell_var("VS2017_HOST")
- if HostType is not None:
- HostType = HostType.lower()
- self.Logger.info(
- f"HOST TYPE defined by environment. Host Type is {HostType}")
- else:
- HostInfo = GetHostInfo()
- if HostInfo.arch == "x86":
- if HostInfo.bit == "32":
- HostType = "x86"
- elif HostInfo.bit == "64":
- HostType = "x64"
- else:
- raise NotImplementedError()
-
- # VS2017_HOST options are not exactly the same as QueryVcVariables. This translates.
- VC_HOST_ARCH_TRANSLATOR = {
- "x86": "x86", "x64": "AMD64", "arm": "not supported", "arm64": "not supported"}
-
- # check to see if full path already configured
- if shell_environment.GetEnvironment().get_shell_var("VS2017_PREFIX") != None:
- self.Logger.info("VS2017_PREFIX is already set.")
-
- else:
- install_path = self._get_vs_install_path(
- "VS2017".lower(), "VS150INSTALLPATH")
- vc_ver = self._get_vc_version(install_path, "VS150TOOLVER")
-
- if install_path is None or vc_ver is None:
- self.Logger.error(
- "Failed to configure environment for VS2017")
- return -1
-
- version_aggregator.GetVersionAggregator().ReportVersion(
- "Visual Studio Install Path", install_path, version_aggregator.VersionTypes.INFO)
- version_aggregator.GetVersionAggregator().ReportVersion(
- "VC Version", vc_ver, version_aggregator.VersionTypes.TOOL)
-
- # make VS2017_PREFIX to align with tools_def.txt
- prefix = os.path.join(install_path, "VC",
- "Tools", "MSVC", vc_ver)
- prefix = prefix + os.path.sep
- shell_environment.GetEnvironment().set_shell_var("VS2017_PREFIX", prefix)
- shell_environment.GetEnvironment().set_shell_var("VS2017_HOST", HostType)
-
- shell_env = shell_environment.GetEnvironment()
- # Use the tools lib to determine the correct values for the vars that interest us.
- vs_vars = locate_tools.QueryVcVariables(
- interesting_keys, VC_HOST_ARCH_TRANSLATOR[HostType], vs_version="vs2017")
- for (k, v) in vs_vars.items():
- shell_env.set_shell_var(k, v)
-
- # now confirm it exists
- if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("VS2017_PREFIX")):
- self.Logger.error("Path for VS2017 toolchain is invalid")
- return -2
-
- #
- # VS2019 - Follow VS2019 where there is potential for many versions of the tools.
- # If a specific version is required then the user must set both env variables:
- # VS160INSTALLPATH: base install path on system to VC install dir. Here you will find the VC folder, etc
- # VS160TOOLVER: version number for the VC compiler tools
- # VS2019_PREFIX: path to MSVC compiler folder with trailing slash (can be used instead of two vars above)
- # VS2017_HOST: set the host architecture to use for host tools, and host libs, etc
- elif thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "VS2019":
-
- # check to see if host is configured
- # HostType for VS2019 should be (defined in tools_def):
- # x86 == 32bit Intel
- # x64 == 64bit Intel
- # arm == 32bit Arm
- # arm64 == 64bit Arm
- #
- HostType = shell_environment.GetEnvironment().get_shell_var("VS2019_HOST")
- if HostType is not None:
- HostType = HostType.lower()
- self.Logger.info(
- f"HOST TYPE defined by environment. Host Type is {HostType}")
- else:
- HostInfo = GetHostInfo()
- if HostInfo.arch == "x86":
- if HostInfo.bit == "32":
- HostType = "x86"
- elif HostInfo.bit == "64":
- HostType = "x64"
- else:
- raise NotImplementedError()
-
- # VS2019_HOST options are not exactly the same as QueryVcVariables. This translates.
- VC_HOST_ARCH_TRANSLATOR = {
- "x86": "x86", "x64": "AMD64", "arm": "not supported", "arm64": "not supported"}
-
- # check to see if full path already configured
- if shell_environment.GetEnvironment().get_shell_var("VS2019_PREFIX") != None:
- self.Logger.info("VS2019_PREFIX is already set.")
-
- else:
- install_path = self._get_vs_install_path(
- "VS2019".lower(), "VS160INSTALLPATH")
- vc_ver = self._get_vc_version(install_path, "VS160TOOLVER")
-
- if install_path is None or vc_ver is None:
- self.Logger.error(
- "Failed to configure environment for VS2019")
- return -1
-
- version_aggregator.GetVersionAggregator().ReportVersion(
- "Visual Studio Install Path", install_path, version_aggregator.VersionTypes.INFO)
- version_aggregator.GetVersionAggregator().ReportVersion(
- "VC Version", vc_ver, version_aggregator.VersionTypes.TOOL)
-
- # make VS2019_PREFIX to align with tools_def.txt
- prefix = os.path.join(install_path, "VC",
- "Tools", "MSVC", vc_ver)
- prefix = prefix + os.path.sep
- shell_environment.GetEnvironment().set_shell_var("VS2019_PREFIX", prefix)
- shell_environment.GetEnvironment().set_shell_var("VS2019_HOST", HostType)
-
- shell_env = shell_environment.GetEnvironment()
- # Use the tools lib to determine the correct values for the vars that interest us.
- vs_vars = locate_tools.QueryVcVariables(
- interesting_keys, VC_HOST_ARCH_TRANSLATOR[HostType], vs_version="vs2019")
- for (k, v) in vs_vars.items():
- shell_env.set_shell_var(k, v)
-
- # now confirm it exists
- if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("VS2019_PREFIX")):
- self.Logger.error("Path for VS2019 toolchain is invalid")
- return -2
-
- return 0
-
- def _get_vs_install_path(self, vs_version, varname):
- # check if already specified
- path = None
- if varname is not None:
- path = shell_environment.GetEnvironment().get_shell_var(varname)
-
- if(path is None):
- # Not specified...find latest
- try:
- path = FindWithVsWhere(vs_version=vs_version)
- except (EnvironmentError, ValueError, RuntimeError) as e:
- self.Logger.error(str(e))
- return None
-
- if path is not None and os.path.exists(path):
- self.Logger.debug("Found VS instance for %s", vs_version)
- else:
- self.Logger.error(
- f"VsWhere successfully executed, but could not find VS instance for {vs_version}.")
- return path
-
- def _get_vc_version(self, path, varname):
- # check if already specified
- vc_ver = shell_environment.GetEnvironment().get_shell_var(varname)
- if (path is None):
- self.Logger.critical(
- "Failed to find Visual Studio tools. Might need to check for VS install")
- return vc_ver
- if(vc_ver is None):
- # Not specified...find latest
- p2 = os.path.join(path, "VC", "Tools", "MSVC")
- if not os.path.isdir(p2):
- self.Logger.critical(
- "Failed to find VC tools. Might need to check for VS install")
- return vc_ver
- vc_ver = os.listdir(p2)[-1].strip() # get last in list
- self.Logger.debug("Found VC Tool version is %s" % vc_ver)
- return vc_ver
+# @file WindowsVsToolChain.py +# Plugin to configures paths for the VS2017 and VS2019 tool chain +## +# This plugin works in conjuncture with the tools_def +# +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +import logging +from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +import edk2toollib.windows.locate_tools as locate_tools +from edk2toollib.windows.locate_tools import FindWithVsWhere +from edk2toolext.environment import shell_environment +from edk2toolext.environment import version_aggregator +from edk2toollib.utility_functions import GetHostInfo + + +class WindowsVsToolChain(IUefiBuildPlugin): + + def do_post_build(self, thebuilder): + return 0 + + def do_pre_build(self, thebuilder): + self.Logger = logging.getLogger("WindowsVsToolChain") + interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB", "LIBPATH", "UniversalCRTSdkDir", + "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath", "WindowsSdkDir", "WindowsSdkVerBinPath", + "WindowsSDKVersion", "VCToolsInstallDir", "Path"] + + # + # VS2017 - Follow VS2017 where there is potential for many versions of the tools. + # If a specific version is required then the user must set both env variables: + # VS150INSTALLPATH: base install path on system to VC install dir. Here you will find the VC folder, etc + # VS150TOOLVER: version number for the VC compiler tools + # VS2017_PREFIX: path to MSVC compiler folder with trailing slash (can be used instead of two vars above) + # VS2017_HOST: set the host architecture to use for host tools, and host libs, etc + if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "VS2017": + + # check to see if host is configured + # HostType for VS2017 should be (defined in tools_def): + # x86 == 32bit Intel + # x64 == 64bit Intel + # arm == 32bit Arm + # arm64 == 64bit Arm + # + HostType = shell_environment.GetEnvironment().get_shell_var("VS2017_HOST") + if HostType is not None: + HostType = HostType.lower() + self.Logger.info( + f"HOST TYPE defined by environment. Host Type is {HostType}") + else: + HostInfo = GetHostInfo() + if HostInfo.arch == "x86": + if HostInfo.bit == "32": + HostType = "x86" + elif HostInfo.bit == "64": + HostType = "x64" + else: + raise NotImplementedError() + + # VS2017_HOST options are not exactly the same as QueryVcVariables. This translates. + VC_HOST_ARCH_TRANSLATOR = { + "x86": "x86", "x64": "AMD64", "arm": "not supported", "arm64": "not supported"} + + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("VS2017_PREFIX") != None: + self.Logger.info("VS2017_PREFIX is already set.") + + else: + install_path = self._get_vs_install_path( + "VS2017".lower(), "VS150INSTALLPATH") + vc_ver = self._get_vc_version(install_path, "VS150TOOLVER") + + if install_path is None or vc_ver is None: + self.Logger.error( + "Failed to configure environment for VS2017") + return -1 + + version_aggregator.GetVersionAggregator().ReportVersion( + "Visual Studio Install Path", install_path, version_aggregator.VersionTypes.INFO) + version_aggregator.GetVersionAggregator().ReportVersion( + "VC Version", vc_ver, version_aggregator.VersionTypes.TOOL) + + # make VS2017_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "VC", + "Tools", "MSVC", vc_ver) + prefix = prefix + os.path.sep + shell_environment.GetEnvironment().set_shell_var("VS2017_PREFIX", prefix) + shell_environment.GetEnvironment().set_shell_var("VS2017_HOST", HostType) + + shell_env = shell_environment.GetEnvironment() + # Use the tools lib to determine the correct values for the vars that interest us. + vs_vars = locate_tools.QueryVcVariables( + interesting_keys, VC_HOST_ARCH_TRANSLATOR[HostType], vs_version="vs2017") + for (k, v) in vs_vars.items(): + shell_env.set_shell_var(k, v) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("VS2017_PREFIX")): + self.Logger.error("Path for VS2017 toolchain is invalid") + return -2 + + # + # VS2019 - Follow VS2019 where there is potential for many versions of the tools. + # If a specific version is required then the user must set both env variables: + # VS160INSTALLPATH: base install path on system to VC install dir. Here you will find the VC folder, etc + # VS160TOOLVER: version number for the VC compiler tools + # VS2019_PREFIX: path to MSVC compiler folder with trailing slash (can be used instead of two vars above) + # VS2017_HOST: set the host architecture to use for host tools, and host libs, etc + elif thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "VS2019": + + # check to see if host is configured + # HostType for VS2019 should be (defined in tools_def): + # x86 == 32bit Intel + # x64 == 64bit Intel + # arm == 32bit Arm + # arm64 == 64bit Arm + # + HostType = shell_environment.GetEnvironment().get_shell_var("VS2019_HOST") + if HostType is not None: + HostType = HostType.lower() + self.Logger.info( + f"HOST TYPE defined by environment. Host Type is {HostType}") + else: + HostInfo = GetHostInfo() + if HostInfo.arch == "x86": + if HostInfo.bit == "32": + HostType = "x86" + elif HostInfo.bit == "64": + HostType = "x64" + else: + raise NotImplementedError() + + # VS2019_HOST options are not exactly the same as QueryVcVariables. This translates. + VC_HOST_ARCH_TRANSLATOR = { + "x86": "x86", "x64": "AMD64", "arm": "not supported", "arm64": "not supported"} + + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("VS2019_PREFIX") != None: + self.Logger.info("VS2019_PREFIX is already set.") + + else: + install_path = self._get_vs_install_path( + "VS2019".lower(), "VS160INSTALLPATH") + vc_ver = self._get_vc_version(install_path, "VS160TOOLVER") + + if install_path is None or vc_ver is None: + self.Logger.error( + "Failed to configure environment for VS2019") + return -1 + + version_aggregator.GetVersionAggregator().ReportVersion( + "Visual Studio Install Path", install_path, version_aggregator.VersionTypes.INFO) + version_aggregator.GetVersionAggregator().ReportVersion( + "VC Version", vc_ver, version_aggregator.VersionTypes.TOOL) + + # make VS2019_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "VC", + "Tools", "MSVC", vc_ver) + prefix = prefix + os.path.sep + shell_environment.GetEnvironment().set_shell_var("VS2019_PREFIX", prefix) + shell_environment.GetEnvironment().set_shell_var("VS2019_HOST", HostType) + + shell_env = shell_environment.GetEnvironment() + # Use the tools lib to determine the correct values for the vars that interest us. + vs_vars = locate_tools.QueryVcVariables( + interesting_keys, VC_HOST_ARCH_TRANSLATOR[HostType], vs_version="vs2019") + for (k, v) in vs_vars.items(): + shell_env.set_shell_var(k, v) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("VS2019_PREFIX")): + self.Logger.error("Path for VS2019 toolchain is invalid") + return -2 + + return 0 + + def _get_vs_install_path(self, vs_version, varname): + # check if already specified + path = None + if varname is not None: + path = shell_environment.GetEnvironment().get_shell_var(varname) + + if(path is None): + # Not specified...find latest + try: + path = FindWithVsWhere(vs_version=vs_version) + except (EnvironmentError, ValueError, RuntimeError) as e: + self.Logger.error(str(e)) + return None + + if path is not None and os.path.exists(path): + self.Logger.debug("Found VS instance for %s", vs_version) + else: + self.Logger.error( + f"VsWhere successfully executed, but could not find VS instance for {vs_version}.") + return path + + def _get_vc_version(self, path, varname): + # check if already specified + vc_ver = shell_environment.GetEnvironment().get_shell_var(varname) + if (path is None): + self.Logger.critical( + "Failed to find Visual Studio tools. Might need to check for VS install") + return vc_ver + if(vc_ver is None): + # Not specified...find latest + p2 = os.path.join(path, "VC", "Tools", "MSVC") + if not os.path.isdir(p2): + self.Logger.critical( + "Failed to find VC tools. Might need to check for VS install") + return vc_ver + vc_ver = os.listdir(p2)[-1].strip() # get last in list + self.Logger.debug("Found VC Tool version is %s" % vc_ver) + return vc_ver diff --git a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain_plug_in.yaml b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain_plug_in.yaml index 72b5c4a092..16a7658a20 100644 --- a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain_plug_in.yaml +++ b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain_plug_in.yaml @@ -1,11 +1,11 @@ -## @file
-# Build Plugin used to set the path to the visual studio tools chain
-#
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-{
- "scope": "global-win",
- "name": "Windows Visual Studio Tool Chain Support",
- "module": "WindowsVsToolChain"
-}
+## @file +# Build Plugin used to set the path to the visual studio tools chain +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global-win", + "name": "Windows Visual Studio Tool Chain Support", + "module": "WindowsVsToolChain" +} |