diff options
Diffstat (limited to 'BaseTools/Plugin/CodeQL')
| -rw-r--r-- | BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py | 444 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml | 26 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py | 338 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml | 26 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/CodeQlQueries.qls | 236 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/Readme.md | 776 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/analyze/analyze_filter.py | 368 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/analyze/globber.py | 254 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml | 52 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml | 48 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml | 48 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/common/codeql_plugin.py | 148 | ||||
| -rw-r--r-- | BaseTools/Plugin/CodeQL/integration/stuart_codeql.py | 158 |
13 files changed, 1461 insertions, 1461 deletions
diff --git a/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py b/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py index 9734478f8b..761b964e58 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py +++ b/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py @@ -1,222 +1,222 @@ -# @file CodeQAnalyzePlugin.py
-#
-# A build plugin that analyzes a CodeQL database.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-import json
-import logging
-import os
-import yaml
-
-from analyze import analyze_filter
-from common import codeql_plugin
-
-from edk2toolext import edk2_logging
-from edk2toolext.environment.plugintypes.uefi_build_plugin import \
- IUefiBuildPlugin
-from edk2toolext.environment.uefi_build import UefiBuilder
-from edk2toollib.uefi.edk2.path_utilities import Edk2Path
-from edk2toollib.utility_functions import RunCmd
-from pathlib import Path
-
-
-class CodeQlAnalyzePlugin(IUefiBuildPlugin):
-
- def do_post_build(self, builder: UefiBuilder) -> int:
- """CodeQL analysis post-build functionality.
-
- Args:
- builder (UefiBuilder): A UEFI builder object for this build.
-
- Returns:
- int: The number of CodeQL errors found. Zero indicates that
- AuditOnly mode is enabled or no failures were found.
- """
- self.builder = builder
- self.package = builder.edk2path.GetContainingPackage(
- builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
- builder.env.GetValue("ACTIVE_PLATFORM")
- )
- )
-
- self.package_path = Path(
- builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
- self.package
- )
- )
- self.target = builder.env.GetValue("TARGET")
-
- self.codeql_db_path = codeql_plugin.get_codeql_db_path(
- builder.ws, self.package, self.target,
- new_path=False)
-
- self.codeql_path = codeql_plugin.get_codeql_cli_path()
- if not self.codeql_path:
- logging.critical("CodeQL build enabled but CodeQL CLI application "
- "not found.")
- return -1
-
- codeql_sarif_dir_path = self.codeql_db_path[
- :self.codeql_db_path.rindex('-')]
- codeql_sarif_dir_path = codeql_sarif_dir_path.replace(
- "-db-", "-analysis-")
- self.codeql_sarif_path = os.path.join(
- codeql_sarif_dir_path,
- (os.path.basename(
- self.codeql_db_path) +
- ".sarif"))
-
- edk2_logging.log_progress(f"Analyzing {self.package} ({self.target}) "
- f"CodeQL database at:\n"
- f" {self.codeql_db_path}")
- edk2_logging.log_progress(f"Results will be written to:\n"
- f" {self.codeql_sarif_path}")
-
- # Packages are allowed to specify package-specific query specifiers
- # in the package CI YAML file that override the global query specifier.
- audit_only = False
- query_specifiers = None
- package_config_file = Path(os.path.join(
- self.package_path, self.package + ".ci.yaml"))
- plugin_data = None
- if package_config_file.is_file():
- with open(package_config_file, 'r') as cf:
- package_config_file_data = yaml.safe_load(cf)
- if "CodeQlAnalyze" in package_config_file_data:
- plugin_data = package_config_file_data["CodeQlAnalyze"]
- if "AuditOnly" in plugin_data:
- audit_only = plugin_data["AuditOnly"]
- if "QuerySpecifiers" in plugin_data:
- logging.debug(f"Loading CodeQL query specifiers in "
- f"{str(package_config_file)}")
- query_specifiers = plugin_data["QuerySpecifiers"]
-
- global_audit_only = builder.env.GetValue("STUART_CODEQL_AUDIT_ONLY")
- if global_audit_only:
- if global_audit_only.strip().lower() == "true":
- audit_only = True
-
- if audit_only:
- logging.info(f"CodeQL Analyze plugin is in audit only mode for "
- f"{self.package} ({self.target}).")
-
- # Builds can override the query specifiers defined in this plugin
- # by setting the value in the STUART_CODEQL_QUERY_SPECIFIERS
- # environment variable.
- if not query_specifiers:
- query_specifiers = builder.env.GetValue(
- "STUART_CODEQL_QUERY_SPECIFIERS")
-
- # Use this plugins query set file as the default fallback if it is
- # not overridden. It is possible the file is not present if modified
- # locally. In that case, skip the plugin.
- plugin_query_set = Path(Path(__file__).parent, "CodeQlQueries.qls")
-
- if not query_specifiers and plugin_query_set.is_file():
- query_specifiers = str(plugin_query_set.resolve())
-
- if not query_specifiers:
- logging.warning("Skipping CodeQL analysis since no CodeQL query "
- "specifiers were provided.")
- return 0
-
- codeql_params = (f'database analyze {self.codeql_db_path} '
- f'{query_specifiers} --format=sarifv2.1.0 '
- f'--output={self.codeql_sarif_path} --download '
- f'--threads=0')
-
- # CodeQL requires the sarif file parent directory to exist already.
- Path(self.codeql_sarif_path).parent.mkdir(exist_ok=True, parents=True)
-
- cmd_ret = RunCmd(self.codeql_path, codeql_params)
- if cmd_ret != 0:
- logging.critical(f"CodeQL CLI analysis failed with return code "
- f"{cmd_ret}.")
-
- if not os.path.isfile(self.codeql_sarif_path):
- logging.critical(f"The sarif file {self.codeql_sarif_path} was "
- f"not created. Analysis cannot continue.")
- return -1
-
- filter_pattern_data = []
- global_filter_file_value = builder.env.GetValue(
- "STUART_CODEQL_FILTER_FILES")
- if global_filter_file_value:
- global_filter_files = global_filter_file_value.strip().split(',')
- global_filter_files = [Path(f) for f in global_filter_files]
-
- for global_filter_file in global_filter_files:
- if global_filter_file.is_file():
- with open(global_filter_file, 'r') as ff:
- global_filter_file_data = yaml.safe_load(ff)
- if "Filters" in global_filter_file_data:
- current_pattern_data = \
- global_filter_file_data["Filters"]
- if type(current_pattern_data) is not list:
- logging.critical(
- f"CodeQL pattern data must be a list of "
- f"strings. Data in "
- f"{str(global_filter_file.resolve())} is "
- f"invalid. CodeQL analysis is incomplete.")
- return -1
- filter_pattern_data += current_pattern_data
- else:
- logging.critical(
- f"CodeQL global filter file "
- f"{str(global_filter_file.resolve())} is "
- f"malformed. Missing Filters section. CodeQL "
- f"analysis is incomplete.")
- return -1
- else:
- logging.critical(
- f"CodeQL global filter file "
- f"{str(global_filter_file.resolve())} was not found. "
- f"CodeQL analysis is incomplete.")
- return -1
-
- if plugin_data and "Filters" in plugin_data:
- if type(plugin_data["Filters"]) is not list:
- logging.critical(
- "CodeQL pattern data must be a list of strings. "
- "CodeQL analysis is incomplete.")
- return -1
- filter_pattern_data.extend(plugin_data["Filters"])
-
- if filter_pattern_data:
- logging.info("Applying CodeQL SARIF result filters.")
- analyze_filter.filter_sarif(
- self.codeql_sarif_path,
- self.codeql_sarif_path,
- filter_pattern_data,
- split_lines=False)
-
- with open(self.codeql_sarif_path, 'r') as sf:
- sarif_file_data = json.load(sf)
-
- try:
- # Perform minimal JSON parsing to find the number of errors.
- total_errors = 0
- for run in sarif_file_data['runs']:
- total_errors += len(run['results'])
- except KeyError:
- logging.critical("Sarif file does not contain expected data. "
- "Analysis cannot continue.")
- return -1
-
- if total_errors > 0:
- if audit_only:
- # Show a warning message so CodeQL analysis is not forgotten.
- # If the repo owners truly do not want to fix CodeQL issues,
- # analysis should be disabled entirely.
- logging.warning(f"{self.package} ({self.target}) CodeQL "
- f"analysis ignored {total_errors} errors due "
- f"to audit mode being enabled.")
- return 0
- else:
- logging.error(f"{self.package} ({self.target}) CodeQL "
- f"analysis failed with {total_errors} errors.")
-
- return total_errors
+# @file CodeQAnalyzePlugin.py +# +# A build plugin that analyzes a CodeQL database. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import json +import logging +import os +import yaml + +from analyze import analyze_filter +from common import codeql_plugin + +from edk2toolext import edk2_logging +from edk2toolext.environment.plugintypes.uefi_build_plugin import \ + IUefiBuildPlugin +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toollib.uefi.edk2.path_utilities import Edk2Path +from edk2toollib.utility_functions import RunCmd +from pathlib import Path + + +class CodeQlAnalyzePlugin(IUefiBuildPlugin): + + def do_post_build(self, builder: UefiBuilder) -> int: + """CodeQL analysis post-build functionality. + + Args: + builder (UefiBuilder): A UEFI builder object for this build. + + Returns: + int: The number of CodeQL errors found. Zero indicates that + AuditOnly mode is enabled or no failures were found. + """ + self.builder = builder + self.package = builder.edk2path.GetContainingPackage( + builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath( + builder.env.GetValue("ACTIVE_PLATFORM") + ) + ) + + self.package_path = Path( + builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath( + self.package + ) + ) + self.target = builder.env.GetValue("TARGET") + + self.codeql_db_path = codeql_plugin.get_codeql_db_path( + builder.ws, self.package, self.target, + new_path=False) + + self.codeql_path = codeql_plugin.get_codeql_cli_path() + if not self.codeql_path: + logging.critical("CodeQL build enabled but CodeQL CLI application " + "not found.") + return -1 + + codeql_sarif_dir_path = self.codeql_db_path[ + :self.codeql_db_path.rindex('-')] + codeql_sarif_dir_path = codeql_sarif_dir_path.replace( + "-db-", "-analysis-") + self.codeql_sarif_path = os.path.join( + codeql_sarif_dir_path, + (os.path.basename( + self.codeql_db_path) + + ".sarif")) + + edk2_logging.log_progress(f"Analyzing {self.package} ({self.target}) " + f"CodeQL database at:\n" + f" {self.codeql_db_path}") + edk2_logging.log_progress(f"Results will be written to:\n" + f" {self.codeql_sarif_path}") + + # Packages are allowed to specify package-specific query specifiers + # in the package CI YAML file that override the global query specifier. + audit_only = False + query_specifiers = None + package_config_file = Path(os.path.join( + self.package_path, self.package + ".ci.yaml")) + plugin_data = None + if package_config_file.is_file(): + with open(package_config_file, 'r') as cf: + package_config_file_data = yaml.safe_load(cf) + if "CodeQlAnalyze" in package_config_file_data: + plugin_data = package_config_file_data["CodeQlAnalyze"] + if "AuditOnly" in plugin_data: + audit_only = plugin_data["AuditOnly"] + if "QuerySpecifiers" in plugin_data: + logging.debug(f"Loading CodeQL query specifiers in " + f"{str(package_config_file)}") + query_specifiers = plugin_data["QuerySpecifiers"] + + global_audit_only = builder.env.GetValue("STUART_CODEQL_AUDIT_ONLY") + if global_audit_only: + if global_audit_only.strip().lower() == "true": + audit_only = True + + if audit_only: + logging.info(f"CodeQL Analyze plugin is in audit only mode for " + f"{self.package} ({self.target}).") + + # Builds can override the query specifiers defined in this plugin + # by setting the value in the STUART_CODEQL_QUERY_SPECIFIERS + # environment variable. + if not query_specifiers: + query_specifiers = builder.env.GetValue( + "STUART_CODEQL_QUERY_SPECIFIERS") + + # Use this plugins query set file as the default fallback if it is + # not overridden. It is possible the file is not present if modified + # locally. In that case, skip the plugin. + plugin_query_set = Path(Path(__file__).parent, "CodeQlQueries.qls") + + if not query_specifiers and plugin_query_set.is_file(): + query_specifiers = str(plugin_query_set.resolve()) + + if not query_specifiers: + logging.warning("Skipping CodeQL analysis since no CodeQL query " + "specifiers were provided.") + return 0 + + codeql_params = (f'database analyze {self.codeql_db_path} ' + f'{query_specifiers} --format=sarifv2.1.0 ' + f'--output={self.codeql_sarif_path} --download ' + f'--threads=0') + + # CodeQL requires the sarif file parent directory to exist already. + Path(self.codeql_sarif_path).parent.mkdir(exist_ok=True, parents=True) + + cmd_ret = RunCmd(self.codeql_path, codeql_params) + if cmd_ret != 0: + logging.critical(f"CodeQL CLI analysis failed with return code " + f"{cmd_ret}.") + + if not os.path.isfile(self.codeql_sarif_path): + logging.critical(f"The sarif file {self.codeql_sarif_path} was " + f"not created. Analysis cannot continue.") + return -1 + + filter_pattern_data = [] + global_filter_file_value = builder.env.GetValue( + "STUART_CODEQL_FILTER_FILES") + if global_filter_file_value: + global_filter_files = global_filter_file_value.strip().split(',') + global_filter_files = [Path(f) for f in global_filter_files] + + for global_filter_file in global_filter_files: + if global_filter_file.is_file(): + with open(global_filter_file, 'r') as ff: + global_filter_file_data = yaml.safe_load(ff) + if "Filters" in global_filter_file_data: + current_pattern_data = \ + global_filter_file_data["Filters"] + if type(current_pattern_data) is not list: + logging.critical( + f"CodeQL pattern data must be a list of " + f"strings. Data in " + f"{str(global_filter_file.resolve())} is " + f"invalid. CodeQL analysis is incomplete.") + return -1 + filter_pattern_data += current_pattern_data + else: + logging.critical( + f"CodeQL global filter file " + f"{str(global_filter_file.resolve())} is " + f"malformed. Missing Filters section. CodeQL " + f"analysis is incomplete.") + return -1 + else: + logging.critical( + f"CodeQL global filter file " + f"{str(global_filter_file.resolve())} was not found. " + f"CodeQL analysis is incomplete.") + return -1 + + if plugin_data and "Filters" in plugin_data: + if type(plugin_data["Filters"]) is not list: + logging.critical( + "CodeQL pattern data must be a list of strings. " + "CodeQL analysis is incomplete.") + return -1 + filter_pattern_data.extend(plugin_data["Filters"]) + + if filter_pattern_data: + logging.info("Applying CodeQL SARIF result filters.") + analyze_filter.filter_sarif( + self.codeql_sarif_path, + self.codeql_sarif_path, + filter_pattern_data, + split_lines=False) + + with open(self.codeql_sarif_path, 'r') as sf: + sarif_file_data = json.load(sf) + + try: + # Perform minimal JSON parsing to find the number of errors. + total_errors = 0 + for run in sarif_file_data['runs']: + total_errors += len(run['results']) + except KeyError: + logging.critical("Sarif file does not contain expected data. " + "Analysis cannot continue.") + return -1 + + if total_errors > 0: + if audit_only: + # Show a warning message so CodeQL analysis is not forgotten. + # If the repo owners truly do not want to fix CodeQL issues, + # analysis should be disabled entirely. + logging.warning(f"{self.package} ({self.target}) CodeQL " + f"analysis ignored {total_errors} errors due " + f"to audit mode being enabled.") + return 0 + else: + logging.error(f"{self.package} ({self.target}) CodeQL " + f"analysis failed with {total_errors} errors.") + + return total_errors diff --git a/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml b/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml index ec01e55c53..e3cac3a2e5 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml +++ b/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml @@ -1,13 +1,13 @@ -## @file CodeQlAnalyze_plug_in.py
-#
-# Build plugin used to analyze CodeQL results.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-analyze",
- "name": "CodeQL Analyze Plugin",
- "module": "CodeQlAnalyzePlugin"
-}
+## @file CodeQlAnalyze_plug_in.py +# +# Build plugin used to analyze CodeQL results. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-analyze", + "name": "CodeQL Analyze Plugin", + "module": "CodeQlAnalyzePlugin" +} diff --git a/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py b/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py index 2f6c928c21..06898dfec6 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py +++ b/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py @@ -1,169 +1,169 @@ -# @file CodeQlBuildPlugin.py
-#
-# A build plugin that produces CodeQL results for the present build.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-import glob
-import logging
-import os
-import stat
-from common import codeql_plugin
-from pathlib import Path
-
-from edk2toolext import edk2_logging
-from edk2toolext.environment.plugintypes.uefi_build_plugin import \
- IUefiBuildPlugin
-from edk2toolext.environment.uefi_build import UefiBuilder
-from edk2toollib.uefi.edk2.path_utilities import Edk2Path
-from edk2toollib.utility_functions import GetHostInfo, RemoveTree
-
-
-class CodeQlBuildPlugin(IUefiBuildPlugin):
-
- def do_pre_build(self, builder: UefiBuilder) -> int:
- """CodeQL pre-build functionality.
-
- Args:
- builder (UefiBuilder): A UEFI builder object for this build.
-
- Returns:
- int: The plugin return code. Zero indicates the plugin ran
- successfully. A non-zero value indicates an unexpected error
- occurred during plugin execution.
- """
-
- if not builder.SkipBuild:
- self.builder = builder
- self.package = builder.edk2path.GetContainingPackage(
- builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
- builder.env.GetValue("ACTIVE_PLATFORM")
- )
- )
-
- self.target = builder.env.GetValue("TARGET")
-
- self.build_output_dir = builder.env.GetValue("BUILD_OUTPUT_BASE")
-
- self.codeql_db_path = codeql_plugin.get_codeql_db_path(
- builder.ws, self.package, self.target)
-
- edk2_logging.log_progress(f"{self.package} will be built for CodeQL")
- edk2_logging.log_progress(f" CodeQL database will be written to "
- f"{self.codeql_db_path}")
-
- self.codeql_path = codeql_plugin.get_codeql_cli_path()
- if not self.codeql_path:
- logging.critical("CodeQL build enabled but CodeQL CLI application "
- "not found.")
- return -1
-
- # CodeQL can only generate a database on clean build
- #
- # Note: builder.CleanTree() cannot be used here as some platforms
- # have build steps that run before this plugin that store
- # files in the build output directory.
- #
- # CodeQL does not care about with those files or many others such
- # as the FV directory, build logs, etc. so instead focus on
- # removing only the directories with compilation/linker output
- # for the architectures being built (that need clean runs for
- # CodeQL to work).
- targets = self.builder.env.GetValue("TARGET_ARCH").split(" ")
- for target in targets:
- directory_to_delete = Path(self.build_output_dir, target)
-
- if directory_to_delete.is_dir():
- logging.debug(f"Removing {str(directory_to_delete)} to have a "
- f"clean build for CodeQL.")
- RemoveTree(str(directory_to_delete))
-
- # CodeQL CLI does not handle spaces passed in CLI commands well
- # (perhaps at all) as discussed here:
- # 1. https://github.com/github/codeql-cli-binaries/issues/73
- # 2. https://github.com/github/codeql/issues/4910
- #
- # Since it's unclear how quotes are handled and may change in the
- # future, this code is going to use the workaround to place the
- # command in an executable file that is instead passed to CodeQL.
- self.codeql_cmd_path = Path(self.build_output_dir, "codeql_build_command")
-
- build_params = self._get_build_params()
-
- codeql_build_cmd = ""
- if GetHostInfo().os == "Windows":
- self.codeql_cmd_path = self.codeql_cmd_path.parent / (
- self.codeql_cmd_path.name + '.bat')
- elif GetHostInfo().os == "Linux":
- self.codeql_cmd_path = self.codeql_cmd_path.parent / (
- self.codeql_cmd_path.name + '.sh')
- codeql_build_cmd += f"#!/bin/bash{os.linesep * 2}"
- codeql_build_cmd += "build " + build_params
-
- self.codeql_cmd_path.parent.mkdir(exist_ok=True, parents=True)
- self.codeql_cmd_path.write_text(encoding='utf8', data=codeql_build_cmd)
-
- if GetHostInfo().os == "Linux":
- os.chmod(self.codeql_cmd_path,
- os.stat(self.codeql_cmd_path).st_mode | stat.S_IEXEC)
- for f in glob.glob(os.path.join(
- os.path.dirname(self.codeql_path), '**/*'), recursive=True):
- os.chmod(f, os.stat(f).st_mode | stat.S_IEXEC)
-
- codeql_params = (f'database create {self.codeql_db_path} '
- f'--language=cpp '
- f'--source-root={builder.ws} '
- f'--command={self.codeql_cmd_path}')
-
- # Set environment variables so the CodeQL build command is picked up
- # as the active build command.
- #
- # Note: Requires recent changes in edk2-pytool-extensions (0.20.0)
- # to support reading these variables.
- builder.env.SetValue(
- "EDK_BUILD_CMD", self.codeql_path, "Set in CodeQL Build Plugin")
- builder.env.SetValue(
- "EDK_BUILD_PARAMS", codeql_params, "Set in CodeQL Build Plugin")
-
- return 0
-
- def _get_build_params(self) -> str:
- """Returns the build command parameters for this build.
-
- Based on the well-defined `build` command-line parameters.
-
- Returns:
- str: A string representing the parameters for the build command.
- """
- build_params = f"-p {self.builder.env.GetValue('ACTIVE_PLATFORM')}"
- build_params += f" -b {self.target}"
- build_params += f" -t {self.builder.env.GetValue('TOOL_CHAIN_TAG')}"
-
- max_threads = self.builder.env.GetValue('MAX_CONCURRENT_THREAD_NUMBER')
- if max_threads is not None:
- build_params += f" -n {max_threads}"
-
- rt = self.builder.env.GetValue("TARGET_ARCH").split(" ")
- for t in rt:
- build_params += " -a " + t
-
- if (self.builder.env.GetValue("BUILDREPORTING") == "TRUE"):
- build_params += (" -y " +
- self.builder.env.GetValue("BUILDREPORT_FILE"))
- rt = self.builder.env.GetValue("BUILDREPORT_TYPES").split(" ")
- for t in rt:
- build_params += " -Y " + t
-
- # add special processing to handle building a single module
- mod = self.builder.env.GetValue("BUILDMODULE")
- if (mod is not None and len(mod.strip()) > 0):
- build_params += " -m " + mod
- edk2_logging.log_progress("Single Module Build: " + mod)
-
- build_vars = self.builder.env.GetAllBuildKeyValues(self.target)
- for key, value in build_vars.items():
- build_params += " -D " + key + "=" + value
-
- return build_params
+# @file CodeQlBuildPlugin.py +# +# A build plugin that produces CodeQL results for the present build. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import glob +import logging +import os +import stat +from common import codeql_plugin +from pathlib import Path + +from edk2toolext import edk2_logging +from edk2toolext.environment.plugintypes.uefi_build_plugin import \ + IUefiBuildPlugin +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toollib.uefi.edk2.path_utilities import Edk2Path +from edk2toollib.utility_functions import GetHostInfo, RemoveTree + + +class CodeQlBuildPlugin(IUefiBuildPlugin): + + def do_pre_build(self, builder: UefiBuilder) -> int: + """CodeQL pre-build functionality. + + Args: + builder (UefiBuilder): A UEFI builder object for this build. + + Returns: + int: The plugin return code. Zero indicates the plugin ran + successfully. A non-zero value indicates an unexpected error + occurred during plugin execution. + """ + + if not builder.SkipBuild: + self.builder = builder + self.package = builder.edk2path.GetContainingPackage( + builder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath( + builder.env.GetValue("ACTIVE_PLATFORM") + ) + ) + + self.target = builder.env.GetValue("TARGET") + + self.build_output_dir = builder.env.GetValue("BUILD_OUTPUT_BASE") + + self.codeql_db_path = codeql_plugin.get_codeql_db_path( + builder.ws, self.package, self.target) + + edk2_logging.log_progress(f"{self.package} will be built for CodeQL") + edk2_logging.log_progress(f" CodeQL database will be written to " + f"{self.codeql_db_path}") + + self.codeql_path = codeql_plugin.get_codeql_cli_path() + if not self.codeql_path: + logging.critical("CodeQL build enabled but CodeQL CLI application " + "not found.") + return -1 + + # CodeQL can only generate a database on clean build + # + # Note: builder.CleanTree() cannot be used here as some platforms + # have build steps that run before this plugin that store + # files in the build output directory. + # + # CodeQL does not care about with those files or many others such + # as the FV directory, build logs, etc. so instead focus on + # removing only the directories with compilation/linker output + # for the architectures being built (that need clean runs for + # CodeQL to work). + targets = self.builder.env.GetValue("TARGET_ARCH").split(" ") + for target in targets: + directory_to_delete = Path(self.build_output_dir, target) + + if directory_to_delete.is_dir(): + logging.debug(f"Removing {str(directory_to_delete)} to have a " + f"clean build for CodeQL.") + RemoveTree(str(directory_to_delete)) + + # CodeQL CLI does not handle spaces passed in CLI commands well + # (perhaps at all) as discussed here: + # 1. https://github.com/github/codeql-cli-binaries/issues/73 + # 2. https://github.com/github/codeql/issues/4910 + # + # Since it's unclear how quotes are handled and may change in the + # future, this code is going to use the workaround to place the + # command in an executable file that is instead passed to CodeQL. + self.codeql_cmd_path = Path(self.build_output_dir, "codeql_build_command") + + build_params = self._get_build_params() + + codeql_build_cmd = "" + if GetHostInfo().os == "Windows": + self.codeql_cmd_path = self.codeql_cmd_path.parent / ( + self.codeql_cmd_path.name + '.bat') + elif GetHostInfo().os == "Linux": + self.codeql_cmd_path = self.codeql_cmd_path.parent / ( + self.codeql_cmd_path.name + '.sh') + codeql_build_cmd += f"#!/bin/bash{os.linesep * 2}" + codeql_build_cmd += "build " + build_params + + self.codeql_cmd_path.parent.mkdir(exist_ok=True, parents=True) + self.codeql_cmd_path.write_text(encoding='utf8', data=codeql_build_cmd) + + if GetHostInfo().os == "Linux": + os.chmod(self.codeql_cmd_path, + os.stat(self.codeql_cmd_path).st_mode | stat.S_IEXEC) + for f in glob.glob(os.path.join( + os.path.dirname(self.codeql_path), '**/*'), recursive=True): + os.chmod(f, os.stat(f).st_mode | stat.S_IEXEC) + + codeql_params = (f'database create {self.codeql_db_path} ' + f'--language=cpp ' + f'--source-root={builder.ws} ' + f'--command={self.codeql_cmd_path}') + + # Set environment variables so the CodeQL build command is picked up + # as the active build command. + # + # Note: Requires recent changes in edk2-pytool-extensions (0.20.0) + # to support reading these variables. + builder.env.SetValue( + "EDK_BUILD_CMD", self.codeql_path, "Set in CodeQL Build Plugin") + builder.env.SetValue( + "EDK_BUILD_PARAMS", codeql_params, "Set in CodeQL Build Plugin") + + return 0 + + def _get_build_params(self) -> str: + """Returns the build command parameters for this build. + + Based on the well-defined `build` command-line parameters. + + Returns: + str: A string representing the parameters for the build command. + """ + build_params = f"-p {self.builder.env.GetValue('ACTIVE_PLATFORM')}" + build_params += f" -b {self.target}" + build_params += f" -t {self.builder.env.GetValue('TOOL_CHAIN_TAG')}" + + max_threads = self.builder.env.GetValue('MAX_CONCURRENT_THREAD_NUMBER') + if max_threads is not None: + build_params += f" -n {max_threads}" + + rt = self.builder.env.GetValue("TARGET_ARCH").split(" ") + for t in rt: + build_params += " -a " + t + + if (self.builder.env.GetValue("BUILDREPORTING") == "TRUE"): + build_params += (" -y " + + self.builder.env.GetValue("BUILDREPORT_FILE")) + rt = self.builder.env.GetValue("BUILDREPORT_TYPES").split(" ") + for t in rt: + build_params += " -Y " + t + + # add special processing to handle building a single module + mod = self.builder.env.GetValue("BUILDMODULE") + if (mod is not None and len(mod.strip()) > 0): + build_params += " -m " + mod + edk2_logging.log_progress("Single Module Build: " + mod) + + build_vars = self.builder.env.GetAllBuildKeyValues(self.target) + for key, value in build_vars.items(): + build_params += " -D " + key + "=" + value + + return build_params diff --git a/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml b/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml index 13baa58d0c..25a35ad27a 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml +++ b/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml @@ -1,13 +1,13 @@ -## @file CodeQlBuild_plug_in.py
-#
-# Build plugin used to produce a CodeQL database from a build.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-build",
- "name": "CodeQL Build Plugin",
- "module": "CodeQlBuildPlugin"
-}
+## @file CodeQlBuild_plug_in.py +# +# Build plugin used to produce a CodeQL database from a build. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-build", + "name": "CodeQL Build Plugin", + "module": "CodeQlBuildPlugin" +} diff --git a/BaseTools/Plugin/CodeQL/CodeQlQueries.qls b/BaseTools/Plugin/CodeQL/CodeQlQueries.qls index 1a50983221..84db69f86a 100644 --- a/BaseTools/Plugin/CodeQL/CodeQlQueries.qls +++ b/BaseTools/Plugin/CodeQL/CodeQlQueries.qls @@ -1,118 +1,118 @@ ----
-- description: C++ queries
-
-- queries: '.'
- from: codeql/cpp-queries
-
-##########################################################################################
-# Queries
-##########################################################################################
-
-## Errors
-- include:
- id: cpp/badoverflowguard
-- include:
- id: cpp/infiniteloop
-- include:
- id: cpp/likely-bugs/memory-management/v2/conditionally-uninitialized-variable
-- include:
- id: cpp/missing-null-test
-- include:
- id: cpp/missing-return
-- include:
- id: cpp/no-space-for-terminator
-- include:
- id: cpp/pointer-overflow-check
-- include:
- id: cpp/redundant-null-check-simple
-- include:
- id: cpp/sizeof/const-int-argument
-- include:
- id: cpp/sizeof/sizeof-or-operation-as-argument
-- include:
- id: cpp/unguardednullreturndereferenc
-- include:
- id: cpp/very-likely-overrunning-write
-
-## Warnings
-- include:
- id: cpp/comparison-with-wider-type
-- include:
- id: cpp/conditionallyuninitializedvariable
-- include:
- id: cpp/comparison-precedence
-- include:
- id: cpp/implicit-bitfield-downcast
-- include:
- id: cpp/infinite-loop-with-unsatisfiable-exit-condition
-- include:
- id: cpp/offset-use-before-range-check
-- include:
- id: cpp/overflow-buffer
-- include:
- id: cpp/overflow-calculated
-- include:
- id: cpp/overflow-destination
-- include:
- id: cpp/paddingbyteinformationdisclosure
-- include:
- id: cpp/return-stack-allocated-memory
-- include:
- id: cpp/static-buffer-overflow
-- include:
- id: cpp/unsigned-comparison-zero
-- include:
- id: cpp/uselesstest
-
-## Recommendations
-- include:
- id: cpp/missing-header-guard
-- include:
- id: cpp/unused-local-variable
-- include:
- id: cpp/unused-static-function
-- include:
- id: cpp/unused-static-variable
-
-# Note: Some queries above are not active by default with the below filter.
-# Update the filter and run the queries again to get all results.
-- include:
- tags:
- - "security"
- - "correctness"
- severity:
- - "error"
- - "warning"
- - "recommendation"
-
-# Specifically hide the results of these.
-#
-# The following rules have been evaluated and explicitly not included for the following reasons:
-# - `cpp/allocation-too-small` - Appears to be hardcoded for C standard library functions `malloc`, `calloc`,
-# `realloc`, so it consumes time without much value with custom allocation functions in the codebase.
-# - `cpp/commented-out-code` - Triggers often. Needs further review.
-# - `cpp/duplicate-include-guard` - The <Phase>EntryPoint.h files includes a common include guard value
-# `__MODULE_ENTRY_POINT_H__`. This was the only occurrence found. So not very useful.
-# - `cpp/invalid-pointer-deref` - Very limited results with what appear to be false positives.
-# - `cpp/use-of-goto` - Goto is valid and allowed in the codebase.
-# - `cpp/useless-expression` - Triggers too often on cases where a NULL lib implementation is provided for a function.
-# Because the implementation simply returns, the check considers it useless.
-# - `cpp/weak-crypto/*` - Crypto algorithms are tracked outside CodeQL.
-- exclude:
- id: cpp/allocation-too-small
-- exclude:
- id: cpp/commented-out-code
-- exclude:
- id: cpp/duplicate-include-guard
-- exclude:
- id: cpp/invalid-pointer-deref
-- exclude:
- id: cpp/use-of-goto
-- exclude:
- id: cpp/useless-expression
-- exclude:
- id: cpp/weak-crypto/banned-hash-algorithms
-- exclude:
- id: cpp/weak-crypto/capi/banned-modes
-- exclude:
- id: cpp/weak-crypto/openssl/banned-hash-algorithms
+--- +- description: C++ queries + +- queries: '.' + from: codeql/cpp-queries + +########################################################################################## +# Queries +########################################################################################## + +## Errors +- include: + id: cpp/badoverflowguard +- include: + id: cpp/infiniteloop +- include: + id: cpp/likely-bugs/memory-management/v2/conditionally-uninitialized-variable +- include: + id: cpp/missing-null-test +- include: + id: cpp/missing-return +- include: + id: cpp/no-space-for-terminator +- include: + id: cpp/pointer-overflow-check +- include: + id: cpp/redundant-null-check-simple +- include: + id: cpp/sizeof/const-int-argument +- include: + id: cpp/sizeof/sizeof-or-operation-as-argument +- include: + id: cpp/unguardednullreturndereferenc +- include: + id: cpp/very-likely-overrunning-write + +## Warnings +- include: + id: cpp/comparison-with-wider-type +- include: + id: cpp/conditionallyuninitializedvariable +- include: + id: cpp/comparison-precedence +- include: + id: cpp/implicit-bitfield-downcast +- include: + id: cpp/infinite-loop-with-unsatisfiable-exit-condition +- include: + id: cpp/offset-use-before-range-check +- include: + id: cpp/overflow-buffer +- include: + id: cpp/overflow-calculated +- include: + id: cpp/overflow-destination +- include: + id: cpp/paddingbyteinformationdisclosure +- include: + id: cpp/return-stack-allocated-memory +- include: + id: cpp/static-buffer-overflow +- include: + id: cpp/unsigned-comparison-zero +- include: + id: cpp/uselesstest + +## Recommendations +- include: + id: cpp/missing-header-guard +- include: + id: cpp/unused-local-variable +- include: + id: cpp/unused-static-function +- include: + id: cpp/unused-static-variable + +# Note: Some queries above are not active by default with the below filter. +# Update the filter and run the queries again to get all results. +- include: + tags: + - "security" + - "correctness" + severity: + - "error" + - "warning" + - "recommendation" + +# Specifically hide the results of these. +# +# The following rules have been evaluated and explicitly not included for the following reasons: +# - `cpp/allocation-too-small` - Appears to be hardcoded for C standard library functions `malloc`, `calloc`, +# `realloc`, so it consumes time without much value with custom allocation functions in the codebase. +# - `cpp/commented-out-code` - Triggers often. Needs further review. +# - `cpp/duplicate-include-guard` - The <Phase>EntryPoint.h files includes a common include guard value +# `__MODULE_ENTRY_POINT_H__`. This was the only occurrence found. So not very useful. +# - `cpp/invalid-pointer-deref` - Very limited results with what appear to be false positives. +# - `cpp/use-of-goto` - Goto is valid and allowed in the codebase. +# - `cpp/useless-expression` - Triggers too often on cases where a NULL lib implementation is provided for a function. +# Because the implementation simply returns, the check considers it useless. +# - `cpp/weak-crypto/*` - Crypto algorithms are tracked outside CodeQL. +- exclude: + id: cpp/allocation-too-small +- exclude: + id: cpp/commented-out-code +- exclude: + id: cpp/duplicate-include-guard +- exclude: + id: cpp/invalid-pointer-deref +- exclude: + id: cpp/use-of-goto +- exclude: + id: cpp/useless-expression +- exclude: + id: cpp/weak-crypto/banned-hash-algorithms +- exclude: + id: cpp/weak-crypto/capi/banned-modes +- exclude: + id: cpp/weak-crypto/openssl/banned-hash-algorithms diff --git a/BaseTools/Plugin/CodeQL/Readme.md b/BaseTools/Plugin/CodeQL/Readme.md index 18587e2b25..046c28e057 100644 --- a/BaseTools/Plugin/CodeQL/Readme.md +++ b/BaseTools/Plugin/CodeQL/Readme.md @@ -1,388 +1,388 @@ -# CodeQL Plugin
-
-The set of CodeQL plugins provided include two main plugins that seamlessly integrate into a Stuart build environment:
-
-1. `CodeQlBuildPlugin` - Used to produce a CodeQL database from a build.
-2. `CodeQlAnalyzePlugin` - Used to analyze a CodeQL database.
-
-While CodeQL can be run in a CI environment with other approaches. This plugin offers the following advantages:
-
-1. Provides exactly the same results locally as on a CI server.
-2. Integrates very well into VS Code.
-3. Very simple to use - just use normal Stuart update and build commands.
-4. Very simple to understand - minimally wraps the official CodeQL CLI.
-5. Very simple to integrate - works like any other Stuart build plugin.
- - Integration is usually just a few lines of code.
-6. Portable - not tied to Azure DevOps specific, GitHub specific, or other host infrastructure.
-7. Versioned - the query and filters are versioned in source control so easy to find and track.
-
-It is very important to read the Integration Instructions in this file and determine how to best integrate the
-CodeQL plugin into your environment.
-
-Due to the total size of dependencies required to run CodeQL and the flexibility needed by a platform to determine what
-CodeQL queries to run and how to interpret results, a number of configuration options are provided to allow a high
-degree of flexibility during platform integration.
-
-This document is focused on those setting up the CodeQL plugin in their environment. Once setup, end users simply need
-to use their normal build commands and process and CodeQL will be integrated with it. The most relevant section for
-such users is [Local Development Tips](#local-development-tips).
-
-## Table of Contents
-
-1. [Database and Analysis Result Locations](#database-and-analysis-result-locations)
-2. [Global Configuration](#global-configuration)
-3. [Package-Specific Configuration](#package-specific-configuration)
-4. [Filter Patterns](#filter-patterns)
-5. [Integration Instructions](#integration-instructions)
- - [Integration Step 1 - Choose Scopes](#integration-step-1---choose-scopes)
- - [Scopes Available](#scopes-available)
- - [Integration Step 2 - Choose CodeQL Queries](#integration-step-2---choose-codeql-queries)
- - [Integration Step 3 - Determine Global Configuration Values](#integration-step-3---determine-global-configuration-values)
- - [Integration Step 4 - Determine Package-Specific Configuration Values](#integration-step-4---determine-package-specific-configuration-values)
- - [Integration Step 5 - Testing](#integration-step-5---testing)
- - [Integration Step 6 - Define Inclusion and Exclusion Filter Patterns](#integration-step-6---define-inclusion-and-exclusion-filter-patterns)
-6. [High-Level Operation](#high-level-operation)
- - [CodeQlBuildPlugin](#codeqlbuildplugin)
- - [CodeQlAnalyzePlugin](#codeqlanalyzeplugin)
-7. [Local Development Tips](#local-development-tips)
-8. [Resolution Guidelines](#resolution-guidelines)
-
-## Database and Analysis Result Locations
-
-The CodeQL database is written to a directory unique to the package and target being built:
-
- `Build/codeql-db-<package>-<target>-<instance>`
-
-For example: `Build/codeql-db-mdemodulepkg-debug-0`
-
-The plugin does not delete or overwrite existing databases, the instance value is simply increased. This is
-because databases are large, take a long time to generate, and are important for reproducing analysis results. The user
-is responsible for deleting database directories when they are no longer needed.
-
-Similarly, analysis results are written to a directory unique to the package and target. For analysis, results are
-stored in individual files so those files are stored in a single directory.
-
-For example, all analysis results for the above package and target will be stored in:
- `codeql-analysis-mdemodulepkg-debug`
-
-CodeQL results are stored in [SARIF](https://sarifweb.azurewebsites.net/) (Static Analysis Results Interchange Format)
-([CodeQL SARIF documentation](https://codeql.github.com/docs/codeql-cli/sarif-output/)) files. Each SARIF file
-corresponding to a database will be stored in a file with an instance matching the database instance.
-
-For example, the analysis result file for the above database would be stored in this file:
- `codeql-analysis-mdemodulepkg-debug/codeql-db-mdemodulepkg-debug-0.sarif`
-
-Result files are overwritten. This is because result files are quick to generate and need to represent the latest
-results for the last analysis operation performed. The user is responsible for backing up SARIF result files if they
-need to saved.
-
-## Global Configuration
-
-Global configuration values are specified with build environment variables.
-
-These values are all optional. They provide a convenient mechanism for a build script to set the value for all packages
-built by the script.
-
-- `STUART_CODEQL_AUDIT_ONLY` - If `true` (case insensitive), `CodeQlAnalyzePlugin` will be in audit-only mode. In this
- mode all CodeQL failures are ignored.
-- `STUART_CODEQL_PATH` - The path to the CodeQL CLI application to use.
-- `STUART_CODEQL_QUERY_SPECIFIERS` - The CodeQL CLI query specifiers to use. See [Running codeql database analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-with-the-codeql-cli/#running-codeql-database-analyze)
- for possible options.
-- `STUART_CODEQL_FILTER_FILES` - The path to "filter" files that contains filter patterns as described in
- [Filter Patterns](#filter-patterns).
- - More than one file may be specified by separating each absolute file path with a comma.
- - This might be useful to reference a global filter file from an upstream repo and also include a global filter
- file for the local repo.
- - Filters are concatenated in the order of files in the variable. Patterns in later files can override patterns
- in earlier files.
- - The file only needs to contain a list of filter pattern strings under a `"Filters"` key. For example:
-
- ```yaml
- {
- "Filters": [
- "<pattern-line-1>",
- "<pattern-line-2>"
- ]
- }
- ...
- ```
-
- Comments are allowed in the filter files and begin with `#` (like a normal YAML file).
-
-## Package-Specific Configuration
-
-Package-specific configuration values reuse existing package-level configuration approaches to simplify adjusting
-CodeQL plugin behavior per package.
-
-These values are all optional. They provide a convenient mechanism for a package owner to adjust settings specific to
-the package.
-
-``` yaml
- "CodeQlAnalyze": {
- "AuditOnly": False, # Don't fail the build if there are errors. Just log them.
- "QuerySpecifiers": "" # Query specifiers to pass to CodeQL CLI.
- "Filters": "" # Inclusion/exclusion filters
- }
-```
-
-> _NOTE:_ If a global filter set is provided via `STUART_CODEQL_FILTER_FILES` and a package has a package-specific
-> list, then the package-specific filter list (in a package CI YAML file) is appended onto the global filter list and
-> may be used to override settings in the global list.
-
-The format used to specify items in `"Filters"` is specified in [Filter Patterns](#filter-patterns).
-
-## Filter Patterns
-
-As you inspect results, you may want to include or exclude certain sets of results. For example, exclude some files by
-file path entirely or adjust the CodeQL rule applied to a certain file. This plugin reuses logic from a popular
-GitHub Action called [`filter-sarif`](https://github.com/advanced-security/filter-sarif) to allow filtering as part of
-the plugin analysis process.
-
-If any results are excluded using filters, the results are removed from the SARIF file. This allows the exclude results
-seen locally to exactly match the results on the CI server.
-
-Read the ["Patterns"](https://github.com/advanced-security/filter-sarif#patterns) section there for more details. The
-patterns section is also copied below with some updates to make the information more relevant for an edk2 codebase
-for convenience.
-
-Each pattern line is of the form:
-
-```plaintext
-[+/-]<file pattern>[:<rule pattern>]
-```
-
-For example:
-
-```yaml
--**/*Test*.c:** # exclusion pattern: remove all alerts from all test files
--**/*Test*.c # ditto, short form of the line above
-+**/*.c:cpp/infiniteloop # inclusion pattern: This line has precedence over the first two
- # and thus "allow lists" alerts of type "cpp/infiniteloop"
-**/*.c:cpp/infiniteloop # ditto, the "+" in inclusion patterns is optional
-** # allow all alerts in all files (reverses all previous lines)
-```
-
-- The path separator character in patterns is always `/`, independent of the platform the code is running on and
- independent of the paths in the SARIF file.
-- `*` matches any character, except a path separator
-- `**` matches any character and is only allowed between path separators, e.g. `/**/file.txt`, `**/file.txt` or `**`.
- NOT allowed: `**.txt`, `/etc**`
-- The rule pattern is optional. If omitted, it will apply to alerts of all types.
-- Subsequent lines override earlier ones. By default all alerts are included.
-- If you need to use the literals `+`, `-`, `\` or `:` in your pattern, you can escape them with `\`, e.g.
- `\-this/is/an/inclusion/file/pattern\:with-a-semicolon:and/a/rule/pattern/with/a/\\/backslash`. For `+` and `-`, this
- is only necessary if they appear at the beginning of the pattern line.
-
-## Integration Instructions
-
-First, note that most CodeQL CLI operations will take a long time the first time they are run. This is due to:
-
-1. Downloads - Downloading the CodeQL CLI binary (during `stuart_update`) and downloading CodeQL queries during
- CodeQL plugin execution
-2. Cache not established - CodeQL CLI caches data as it performs analysis. The first time analysis is performed will
- take more time than in the future.
-
-Second, these are build plugins. This means a build needs to take place for the plugins to run. This typically happens
-in the following two scenarios:
-
-1. `stuart_build` - A single package is built and the build process is started by the stuart tools.
-2. `stuart_ci_build` - A number of packages may be built and the build process is started by the `CompilerPlugin`.
-
-In any case, each time a package is built, the CodeQL plugins will be run if their scopes are active.
-
-### Integration Step 1 - Choose Scopes
-
-Decide which scopes need to be enabled in your platform, see [Scopes Available](#scopes-available).
-
-Consider using a build profile to enable CodeQL so developers and pipelines can use the profile when they are
-interested in CodeQL results but in other cases they can easily work without CodeQL in the way.
-
-Furthermore, build-script specific command-line parameters might be useful to control CodeQL scopes and other
-behavior.
-
-#### Scopes Available
-
-This CodeQL plugin leverages scopes to control major pieces of functionality. Any combination of scopes can be
-returned from the `GetActiveScopes()` function in the platform settings manager to add and remove functionality.
-
-Plugin scopes:
-
-- `codeql-analyze` - Activate `CodeQlAnalyzePlugin` to perform post-build analysis of the last generated database for
- the package and target specified.
-- `codeql-build` - Activate `CodeQlBuildPlugin` to hook the firmware build in pre-build such that the build will
- generate a CodeQL database during build.
-
-In most cases, to perform a full CodeQL run, `codeql-build` should be enabled so a new CodeQL database is generated
-during build and `codeql-analyze` should be be enabled so analysis of that database is performed after the build is
-completed.
-
-External dependency scopes:
-
-- `codeql-ext-dep` - Downloads the cross-platform CodeQL CLI as an external dependency.
-- `codeql-linux-ext-dep` - Downloads the Linux CodeQL CLI as an external dependency.
-- `codeql-windows-ext-dep` - Downloads the Windows CodeQL CLI as an external dependency.
-
-Note, that the CodeQL CLI is large in size. Sizes as of the [v2.11.2 release](https://github.com/github/codeql-cli-binaries/releases/tag/v2.11.2).
-
-| Cross-platform | Linux | Windows |
-|:--------------:|:------:|:-------:|
-| 934 MB | 415 MB | 290 MB |
-
-Therefore, the following is recommended:
-
-1. **Ideal** - Create container images for build agents and install the CodeQL CLI for the container OS into the
- container.
-2. Leverage host-OS detection (e.g. [`GetHostInfo()`](https://github.com/tianocore/edk2-pytool-library/blob/42ad6561af73ba34564f1577f64f7dbaf1d0a5a2/edk2toollib/utility_functions.py#L112))
-to set the scope for the appropriate operating system. This will download the much smaller OS-specific application.
-
-> _NOTE:_ You should never have more than one CodeQL external dependency scope enabled at a time.
-
-### Integration Step 2 - Choose CodeQL Queries
-
-Determine which queries need to be run against packages in your repo. In most cases, the same set of queries will be
-run against all packages. It is also possible to customize the queries run at the package level.
-
-The default set of Project Mu CodeQL queries is specified in the `MuCodeQlQueries.qls` file in this plugin.
-
-> _NOTE:_ The queries in `MuCodeQlQueries.qls` may change at any time. If you do not want these changes to impact
-> your platform, do not relay on option (3).
-
-The plugin decides what queries to run based on the following, in order of preference:
-
-1. Package CI YAML file query specifier
-2. Build environment variable query specifier
-3. Plugin default query set file
-
-For details on how to set (1) and (2), see the Package CI Configuration and Environment Variable sections respectively.
-
-> _NOTE:_ The value specified is directly passed as a `query specifier` to CodeQL CLI. Therefore, the arguments
-> allowed by the `<query-specifiers>` argument of CodeQL CLI are allowed here. See
-> [Running codeql database analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-with-the-codeql-cli/#running-codeql-database-analyze).
-
-A likely scenario is that a platform needs to run local/closed source queries in addition to the open-source queries.
-There's various ways to handle that:
-
-1. Create a query specifier that includes all the queries needed, both public and private and use that query specifier,
- either globally or at package-level.
-
- For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` = _"Absolute_path_to_AllMyQueries.qls"_
-
-2. Specify a query specifier that includes the closed sources queries and reuse the public query list provided by
- this plugin.
-
- For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` = _"Absolute_path_to_MuCodeQlQueries.qls
- Absolute_path_to_ClosedSourceQueries.qls"_
-
-Refer to the CodeQL documentation noted above on query specifiers to devise other options.
-
-### Integration Step 3 - Determine Global Configuration Values
-
-Review the Environment Variable section to determine which, if any, global values need to be set in your build script.
-
-### Integration Step 4 - Determine Package-Specific Configuration Values
-
-Review the Package CI Configuration section to determine which, if any, global values need to be set in your
-package's CI YAML file.
-
-### Integration Step 5 - Testing
-
-Verify a `stuart_update` and `stuart_build` (or `stuart_ci_build`) command work.
-
-### Integration Step 6 - Define Inclusion and Exclusion Filter Patterns
-
-After reviewing the test results from Step 5, determine if you need to apply any filters as described in
-[Filter Patterns](#filter-patterns).
-
-## High-Level Operation
-
-This section summarizes the complete CodeQL plugin flow. This is to help developers understand basic theory of
-operation behind the plugin and can be skipped by anyone not interested in those details.
-
-### CodeQlBuildPlugin
-
-1. Register a pre-build hook
-2. Determine the package and target being built
-3. Determine the best CodeQL CLI path to use
- - First choice, the `STUART_CODEQL_PATH` environment variable
- - Note: This is set by the CodeQL CLI external dependency if that is used
- - Second choice, `codeql` as found on the system path
-4. Determine the directory name for the CodeQL database
- - Format: `Build/codeql-db-<package>-<target>-<instance>`
-5. Clean the build directory of the active platform and target
- - CodeQL database generation only works on clean builds
-6. Ensure the "build" step is not skipped as a build is needed to generate a CodeQL database
-7. Build a CodeQL file that wraps around the edk2 build
- - Written to the package build directory
- - Example: `Build/MdeModulePkg/VS2022/codeql_build_command.bat`
-8. Set the variables necessary for stuart to call CodeQL CLI during the build phase
- - Sets `EDK_BUILD_CMD` and `EDK_BUILD_PARAMS`
-
-### CodeQlAnalyzePlugin
-
-1. Register a post-build hook
-2. Determine the package and target being built
-3. Determine the best CodeQL CLI path to use
- - First choice, the `STUART_CODEQL_PATH` environment variable
- - Note: This is set by the CodeQL CLI external dependency if that is used
- - Second choice, `codeql` as found on the system path
-4. Determine the directory name for the most recent CodeQL database
- - Format: `Build/codeql-db-<package>-<target>-<instance>`
-5. Determine plugin audit status for the given package and target
- - Check if `AuditOnly` is enabled either globally or for the package
-6. Determine the CodeQL query specifiers to use for the given package and target
- - First choice, the package CI YAML file value
- - Second choice, the `STUART_CODEQL_QUERY_SPECIFIERS`
- - Third choice, use `CodeQlQueries.qls` (in the plugin directory)
-7. Run CodeQL CLI to perform database analysis
-8. Parse the analysis SARIF file to determine the number of CodeQL failures
-9. Return the number of failures (or zero if `AuditOnly` is enabled)
-
-## Local Development Tips
-
-This section contains helpful tips to expedite common scenarios when working with CodeQL locally.
-
-1. Pre-build, Build, and Post-Build
-
- Generating a database requires the pre-build and build steps. Analyzing a database requires the post-build step.
-
- Therefore, if you are making tweaks that don't affect the build, such as modifying the CodeQL queries used or level
- of severity reported, you can save time by skipping pre-build and post-build (e.g. `--skipprebuild` and
- `--skipbuild`).
-
-2. Scopes
-
- Similar to (1), add/remove `codeql-build` and `codeql-analyze` from the active scopes to save time depending on what
- you are trying to do.
-
- If you are focusing on coding, remove the code CodeQL scopes if they are active. If you are ready to check your
- changes against CodeQL, simply add the scopes back. It is recommended to use build profiles to do this more
- conveniently.
-
- If you already have CodeQL CLI enabled, you can remove the `codeql-ext-dep` scope locally. The build will use the
- `codeql` command on your path.
-
-3. CodeQL Output is in the CI Build Log
-
- To see exactly which queries CodeQL ran or why it might be taking longer than expected, look in the CI build log
- (i.e. `Build/CI_BUILDLOG.txt`) where the CodeQL CLI application output is written.
-
- Search for the text you see in the progress output (e.g. "Analyzing _MdeModulePkg_ (_DEBUG_) CodeQL database at")
- to jump to the section of the log just before the CodeQL CLI is invoked.
-
-4. Use a SARIF Viewer to Read Results
-
-The [SARIF Viewer extension for VS Code](https://marketplace.visualstudio.com/items?itemName=MS-SarifVSCode.sarif-viewer)
-can open the .sarif file generated by this plugin and allow you to click links directly to the problem area in source
-files.
-
-## Resolution Guidelines
-
-This section captures brief guidelines to keep in mind while resolving CodeQL issues.
-
-1. Look at surrounding code. Changes should always take into account the context of nearby code. The new logic may
- need to account conditions not immediately obvious based on the issue alone. It is easy to focus only on the line
- of code highlighted by CodeQL and miss the code's role in the big picture.
-2. A CodeQL alert may be benign but the code can be refactored to prevent the alert. Often refactoring the code makes
- the code intention clearer and avoids an unnecessary exception.
-3. Consider adding unit tests while making CodeQL fixes especially for commonly used code and code with a high volume
- of CodeQL alerts.
+# CodeQL Plugin + +The set of CodeQL plugins provided include two main plugins that seamlessly integrate into a Stuart build environment: + +1. `CodeQlBuildPlugin` - Used to produce a CodeQL database from a build. +2. `CodeQlAnalyzePlugin` - Used to analyze a CodeQL database. + +While CodeQL can be run in a CI environment with other approaches. This plugin offers the following advantages: + +1. Provides exactly the same results locally as on a CI server. +2. Integrates very well into VS Code. +3. Very simple to use - just use normal Stuart update and build commands. +4. Very simple to understand - minimally wraps the official CodeQL CLI. +5. Very simple to integrate - works like any other Stuart build plugin. + - Integration is usually just a few lines of code. +6. Portable - not tied to Azure DevOps specific, GitHub specific, or other host infrastructure. +7. Versioned - the query and filters are versioned in source control so easy to find and track. + +It is very important to read the Integration Instructions in this file and determine how to best integrate the +CodeQL plugin into your environment. + +Due to the total size of dependencies required to run CodeQL and the flexibility needed by a platform to determine what +CodeQL queries to run and how to interpret results, a number of configuration options are provided to allow a high +degree of flexibility during platform integration. + +This document is focused on those setting up the CodeQL plugin in their environment. Once setup, end users simply need +to use their normal build commands and process and CodeQL will be integrated with it. The most relevant section for +such users is [Local Development Tips](#local-development-tips). + +## Table of Contents + +1. [Database and Analysis Result Locations](#database-and-analysis-result-locations) +2. [Global Configuration](#global-configuration) +3. [Package-Specific Configuration](#package-specific-configuration) +4. [Filter Patterns](#filter-patterns) +5. [Integration Instructions](#integration-instructions) + - [Integration Step 1 - Choose Scopes](#integration-step-1---choose-scopes) + - [Scopes Available](#scopes-available) + - [Integration Step 2 - Choose CodeQL Queries](#integration-step-2---choose-codeql-queries) + - [Integration Step 3 - Determine Global Configuration Values](#integration-step-3---determine-global-configuration-values) + - [Integration Step 4 - Determine Package-Specific Configuration Values](#integration-step-4---determine-package-specific-configuration-values) + - [Integration Step 5 - Testing](#integration-step-5---testing) + - [Integration Step 6 - Define Inclusion and Exclusion Filter Patterns](#integration-step-6---define-inclusion-and-exclusion-filter-patterns) +6. [High-Level Operation](#high-level-operation) + - [CodeQlBuildPlugin](#codeqlbuildplugin) + - [CodeQlAnalyzePlugin](#codeqlanalyzeplugin) +7. [Local Development Tips](#local-development-tips) +8. [Resolution Guidelines](#resolution-guidelines) + +## Database and Analysis Result Locations + +The CodeQL database is written to a directory unique to the package and target being built: + + `Build/codeql-db-<package>-<target>-<instance>` + +For example: `Build/codeql-db-mdemodulepkg-debug-0` + +The plugin does not delete or overwrite existing databases, the instance value is simply increased. This is +because databases are large, take a long time to generate, and are important for reproducing analysis results. The user +is responsible for deleting database directories when they are no longer needed. + +Similarly, analysis results are written to a directory unique to the package and target. For analysis, results are +stored in individual files so those files are stored in a single directory. + +For example, all analysis results for the above package and target will be stored in: + `codeql-analysis-mdemodulepkg-debug` + +CodeQL results are stored in [SARIF](https://sarifweb.azurewebsites.net/) (Static Analysis Results Interchange Format) +([CodeQL SARIF documentation](https://codeql.github.com/docs/codeql-cli/sarif-output/)) files. Each SARIF file +corresponding to a database will be stored in a file with an instance matching the database instance. + +For example, the analysis result file for the above database would be stored in this file: + `codeql-analysis-mdemodulepkg-debug/codeql-db-mdemodulepkg-debug-0.sarif` + +Result files are overwritten. This is because result files are quick to generate and need to represent the latest +results for the last analysis operation performed. The user is responsible for backing up SARIF result files if they +need to saved. + +## Global Configuration + +Global configuration values are specified with build environment variables. + +These values are all optional. They provide a convenient mechanism for a build script to set the value for all packages +built by the script. + +- `STUART_CODEQL_AUDIT_ONLY` - If `true` (case insensitive), `CodeQlAnalyzePlugin` will be in audit-only mode. In this + mode all CodeQL failures are ignored. +- `STUART_CODEQL_PATH` - The path to the CodeQL CLI application to use. +- `STUART_CODEQL_QUERY_SPECIFIERS` - The CodeQL CLI query specifiers to use. See [Running codeql database analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-with-the-codeql-cli/#running-codeql-database-analyze) + for possible options. +- `STUART_CODEQL_FILTER_FILES` - The path to "filter" files that contains filter patterns as described in + [Filter Patterns](#filter-patterns). + - More than one file may be specified by separating each absolute file path with a comma. + - This might be useful to reference a global filter file from an upstream repo and also include a global filter + file for the local repo. + - Filters are concatenated in the order of files in the variable. Patterns in later files can override patterns + in earlier files. + - The file only needs to contain a list of filter pattern strings under a `"Filters"` key. For example: + + ```yaml + { + "Filters": [ + "<pattern-line-1>", + "<pattern-line-2>" + ] + } + ... + ``` + + Comments are allowed in the filter files and begin with `#` (like a normal YAML file). + +## Package-Specific Configuration + +Package-specific configuration values reuse existing package-level configuration approaches to simplify adjusting +CodeQL plugin behavior per package. + +These values are all optional. They provide a convenient mechanism for a package owner to adjust settings specific to +the package. + +``` yaml + "CodeQlAnalyze": { + "AuditOnly": False, # Don't fail the build if there are errors. Just log them. + "QuerySpecifiers": "" # Query specifiers to pass to CodeQL CLI. + "Filters": "" # Inclusion/exclusion filters + } +``` + +> _NOTE:_ If a global filter set is provided via `STUART_CODEQL_FILTER_FILES` and a package has a package-specific +> list, then the package-specific filter list (in a package CI YAML file) is appended onto the global filter list and +> may be used to override settings in the global list. + +The format used to specify items in `"Filters"` is specified in [Filter Patterns](#filter-patterns). + +## Filter Patterns + +As you inspect results, you may want to include or exclude certain sets of results. For example, exclude some files by +file path entirely or adjust the CodeQL rule applied to a certain file. This plugin reuses logic from a popular +GitHub Action called [`filter-sarif`](https://github.com/advanced-security/filter-sarif) to allow filtering as part of +the plugin analysis process. + +If any results are excluded using filters, the results are removed from the SARIF file. This allows the exclude results +seen locally to exactly match the results on the CI server. + +Read the ["Patterns"](https://github.com/advanced-security/filter-sarif#patterns) section there for more details. The +patterns section is also copied below with some updates to make the information more relevant for an edk2 codebase +for convenience. + +Each pattern line is of the form: + +```plaintext +[+/-]<file pattern>[:<rule pattern>] +``` + +For example: + +```yaml +-**/*Test*.c:** # exclusion pattern: remove all alerts from all test files +-**/*Test*.c # ditto, short form of the line above ++**/*.c:cpp/infiniteloop # inclusion pattern: This line has precedence over the first two + # and thus "allow lists" alerts of type "cpp/infiniteloop" +**/*.c:cpp/infiniteloop # ditto, the "+" in inclusion patterns is optional +** # allow all alerts in all files (reverses all previous lines) +``` + +- The path separator character in patterns is always `/`, independent of the platform the code is running on and + independent of the paths in the SARIF file. +- `*` matches any character, except a path separator +- `**` matches any character and is only allowed between path separators, e.g. `/**/file.txt`, `**/file.txt` or `**`. + NOT allowed: `**.txt`, `/etc**` +- The rule pattern is optional. If omitted, it will apply to alerts of all types. +- Subsequent lines override earlier ones. By default all alerts are included. +- If you need to use the literals `+`, `-`, `\` or `:` in your pattern, you can escape them with `\`, e.g. + `\-this/is/an/inclusion/file/pattern\:with-a-semicolon:and/a/rule/pattern/with/a/\\/backslash`. For `+` and `-`, this + is only necessary if they appear at the beginning of the pattern line. + +## Integration Instructions + +First, note that most CodeQL CLI operations will take a long time the first time they are run. This is due to: + +1. Downloads - Downloading the CodeQL CLI binary (during `stuart_update`) and downloading CodeQL queries during + CodeQL plugin execution +2. Cache not established - CodeQL CLI caches data as it performs analysis. The first time analysis is performed will + take more time than in the future. + +Second, these are build plugins. This means a build needs to take place for the plugins to run. This typically happens +in the following two scenarios: + +1. `stuart_build` - A single package is built and the build process is started by the stuart tools. +2. `stuart_ci_build` - A number of packages may be built and the build process is started by the `CompilerPlugin`. + +In any case, each time a package is built, the CodeQL plugins will be run if their scopes are active. + +### Integration Step 1 - Choose Scopes + +Decide which scopes need to be enabled in your platform, see [Scopes Available](#scopes-available). + +Consider using a build profile to enable CodeQL so developers and pipelines can use the profile when they are +interested in CodeQL results but in other cases they can easily work without CodeQL in the way. + +Furthermore, build-script specific command-line parameters might be useful to control CodeQL scopes and other +behavior. + +#### Scopes Available + +This CodeQL plugin leverages scopes to control major pieces of functionality. Any combination of scopes can be +returned from the `GetActiveScopes()` function in the platform settings manager to add and remove functionality. + +Plugin scopes: + +- `codeql-analyze` - Activate `CodeQlAnalyzePlugin` to perform post-build analysis of the last generated database for + the package and target specified. +- `codeql-build` - Activate `CodeQlBuildPlugin` to hook the firmware build in pre-build such that the build will + generate a CodeQL database during build. + +In most cases, to perform a full CodeQL run, `codeql-build` should be enabled so a new CodeQL database is generated +during build and `codeql-analyze` should be be enabled so analysis of that database is performed after the build is +completed. + +External dependency scopes: + +- `codeql-ext-dep` - Downloads the cross-platform CodeQL CLI as an external dependency. +- `codeql-linux-ext-dep` - Downloads the Linux CodeQL CLI as an external dependency. +- `codeql-windows-ext-dep` - Downloads the Windows CodeQL CLI as an external dependency. + +Note, that the CodeQL CLI is large in size. Sizes as of the [v2.11.2 release](https://github.com/github/codeql-cli-binaries/releases/tag/v2.11.2). + +| Cross-platform | Linux | Windows | +|:--------------:|:------:|:-------:| +| 934 MB | 415 MB | 290 MB | + +Therefore, the following is recommended: + +1. **Ideal** - Create container images for build agents and install the CodeQL CLI for the container OS into the + container. +2. Leverage host-OS detection (e.g. [`GetHostInfo()`](https://github.com/tianocore/edk2-pytool-library/blob/42ad6561af73ba34564f1577f64f7dbaf1d0a5a2/edk2toollib/utility_functions.py#L112)) +to set the scope for the appropriate operating system. This will download the much smaller OS-specific application. + +> _NOTE:_ You should never have more than one CodeQL external dependency scope enabled at a time. + +### Integration Step 2 - Choose CodeQL Queries + +Determine which queries need to be run against packages in your repo. In most cases, the same set of queries will be +run against all packages. It is also possible to customize the queries run at the package level. + +The default set of Project Mu CodeQL queries is specified in the `MuCodeQlQueries.qls` file in this plugin. + +> _NOTE:_ The queries in `MuCodeQlQueries.qls` may change at any time. If you do not want these changes to impact +> your platform, do not relay on option (3). + +The plugin decides what queries to run based on the following, in order of preference: + +1. Package CI YAML file query specifier +2. Build environment variable query specifier +3. Plugin default query set file + +For details on how to set (1) and (2), see the Package CI Configuration and Environment Variable sections respectively. + +> _NOTE:_ The value specified is directly passed as a `query specifier` to CodeQL CLI. Therefore, the arguments +> allowed by the `<query-specifiers>` argument of CodeQL CLI are allowed here. See +> [Running codeql database analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-with-the-codeql-cli/#running-codeql-database-analyze). + +A likely scenario is that a platform needs to run local/closed source queries in addition to the open-source queries. +There's various ways to handle that: + +1. Create a query specifier that includes all the queries needed, both public and private and use that query specifier, + either globally or at package-level. + + For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` = _"Absolute_path_to_AllMyQueries.qls"_ + +2. Specify a query specifier that includes the closed sources queries and reuse the public query list provided by + this plugin. + + For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` = _"Absolute_path_to_MuCodeQlQueries.qls + Absolute_path_to_ClosedSourceQueries.qls"_ + +Refer to the CodeQL documentation noted above on query specifiers to devise other options. + +### Integration Step 3 - Determine Global Configuration Values + +Review the Environment Variable section to determine which, if any, global values need to be set in your build script. + +### Integration Step 4 - Determine Package-Specific Configuration Values + +Review the Package CI Configuration section to determine which, if any, global values need to be set in your +package's CI YAML file. + +### Integration Step 5 - Testing + +Verify a `stuart_update` and `stuart_build` (or `stuart_ci_build`) command work. + +### Integration Step 6 - Define Inclusion and Exclusion Filter Patterns + +After reviewing the test results from Step 5, determine if you need to apply any filters as described in +[Filter Patterns](#filter-patterns). + +## High-Level Operation + +This section summarizes the complete CodeQL plugin flow. This is to help developers understand basic theory of +operation behind the plugin and can be skipped by anyone not interested in those details. + +### CodeQlBuildPlugin + +1. Register a pre-build hook +2. Determine the package and target being built +3. Determine the best CodeQL CLI path to use + - First choice, the `STUART_CODEQL_PATH` environment variable + - Note: This is set by the CodeQL CLI external dependency if that is used + - Second choice, `codeql` as found on the system path +4. Determine the directory name for the CodeQL database + - Format: `Build/codeql-db-<package>-<target>-<instance>` +5. Clean the build directory of the active platform and target + - CodeQL database generation only works on clean builds +6. Ensure the "build" step is not skipped as a build is needed to generate a CodeQL database +7. Build a CodeQL file that wraps around the edk2 build + - Written to the package build directory + - Example: `Build/MdeModulePkg/VS2022/codeql_build_command.bat` +8. Set the variables necessary for stuart to call CodeQL CLI during the build phase + - Sets `EDK_BUILD_CMD` and `EDK_BUILD_PARAMS` + +### CodeQlAnalyzePlugin + +1. Register a post-build hook +2. Determine the package and target being built +3. Determine the best CodeQL CLI path to use + - First choice, the `STUART_CODEQL_PATH` environment variable + - Note: This is set by the CodeQL CLI external dependency if that is used + - Second choice, `codeql` as found on the system path +4. Determine the directory name for the most recent CodeQL database + - Format: `Build/codeql-db-<package>-<target>-<instance>` +5. Determine plugin audit status for the given package and target + - Check if `AuditOnly` is enabled either globally or for the package +6. Determine the CodeQL query specifiers to use for the given package and target + - First choice, the package CI YAML file value + - Second choice, the `STUART_CODEQL_QUERY_SPECIFIERS` + - Third choice, use `CodeQlQueries.qls` (in the plugin directory) +7. Run CodeQL CLI to perform database analysis +8. Parse the analysis SARIF file to determine the number of CodeQL failures +9. Return the number of failures (or zero if `AuditOnly` is enabled) + +## Local Development Tips + +This section contains helpful tips to expedite common scenarios when working with CodeQL locally. + +1. Pre-build, Build, and Post-Build + + Generating a database requires the pre-build and build steps. Analyzing a database requires the post-build step. + + Therefore, if you are making tweaks that don't affect the build, such as modifying the CodeQL queries used or level + of severity reported, you can save time by skipping pre-build and post-build (e.g. `--skipprebuild` and + `--skipbuild`). + +2. Scopes + + Similar to (1), add/remove `codeql-build` and `codeql-analyze` from the active scopes to save time depending on what + you are trying to do. + + If you are focusing on coding, remove the code CodeQL scopes if they are active. If you are ready to check your + changes against CodeQL, simply add the scopes back. It is recommended to use build profiles to do this more + conveniently. + + If you already have CodeQL CLI enabled, you can remove the `codeql-ext-dep` scope locally. The build will use the + `codeql` command on your path. + +3. CodeQL Output is in the CI Build Log + + To see exactly which queries CodeQL ran or why it might be taking longer than expected, look in the CI build log + (i.e. `Build/CI_BUILDLOG.txt`) where the CodeQL CLI application output is written. + + Search for the text you see in the progress output (e.g. "Analyzing _MdeModulePkg_ (_DEBUG_) CodeQL database at") + to jump to the section of the log just before the CodeQL CLI is invoked. + +4. Use a SARIF Viewer to Read Results + +The [SARIF Viewer extension for VS Code](https://marketplace.visualstudio.com/items?itemName=MS-SarifVSCode.sarif-viewer) +can open the .sarif file generated by this plugin and allow you to click links directly to the problem area in source +files. + +## Resolution Guidelines + +This section captures brief guidelines to keep in mind while resolving CodeQL issues. + +1. Look at surrounding code. Changes should always take into account the context of nearby code. The new logic may + need to account conditions not immediately obvious based on the issue alone. It is easy to focus only on the line + of code highlighted by CodeQL and miss the code's role in the big picture. +2. A CodeQL alert may be benign but the code can be refactored to prevent the alert. Often refactoring the code makes + the code intention clearer and avoids an unnecessary exception. +3. Consider adding unit tests while making CodeQL fixes especially for commonly used code and code with a high volume + of CodeQL alerts. diff --git a/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py b/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py index f363dd378f..e0e9dfbc55 100644 --- a/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py +++ b/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py @@ -1,184 +1,184 @@ -# @file analyze_filter.py
-#
-# Filters results in a SARIF file.
-#
-# Apache License
-# Version 2.0, January 2004
-# http://www.apache.org/licenses/
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This file has been altered from its original form. Based on code in:
-# https://github.com/advanced-security/filter-sarif
-#
-# It primarily contains modifications made to integrate with the CodeQL plugin.
-#
-# Specifically:
-# https://github.com/advanced-security/filter-sarif/blob/main/filter_sarif.py
-#
-# View the full and complete license as provided by that repository here:
-# https://github.com/advanced-security/filter-sarif/blob/main/LICENSE
-#
-# SPDX-License-Identifier: Apache-2.0
-##
-
-import json
-import logging
-import re
-from os import PathLike
-from typing import Iterable, List, Tuple
-
-from analyze.globber import match
-
-
-def _match_path_and_rule(
- path: str, rule: str, patterns: Iterable[str]) -> bool:
- """Returns whether a given path matches a given rule.
-
- Args:
- path (str): A file path string.
- rule (str): A rule file path string.
- patterns (Iterable[str]): An iterable of pattern strings.
-
- Returns:
- bool: True if the path matches a rule. Otherwise, False.
- """
- result = True
- for s, fp, rp in patterns:
- if match(rp, rule) and match(fp, path):
- result = s
- return result
-
-
-def _parse_pattern(line: str) -> Tuple[str]:
- """Parses a given pattern line.
-
- Args:
- line (str): The line string that contains the rule.
-
- Returns:
- Tuple[str]: The parsed sign, file pattern, and rule pattern from the
- line.
- """
- sep_char = ':'
- esc_char = '\\'
- file_pattern = ''
- rule_pattern = ''
- seen_separator = False
- sign = True
-
- # inclusion or exclusion pattern?
- u_line = line
- if line:
- if line[0] == '-':
- sign = False
- u_line = line[1:]
- elif line[0] == '+':
- u_line = line[1:]
-
- i = 0
- while i < len(u_line):
- c = u_line[i]
- i = i + 1
- if c == sep_char:
- if seen_separator:
- raise Exception(
- 'Invalid pattern: "' + line + '" Contains more than one '
- 'separator!')
- seen_separator = True
- continue
- elif c == esc_char:
- next_c = u_line[i] if (i < len(u_line)) else None
- if next_c in ['+' , '-', esc_char, sep_char]:
- i = i + 1
- c = next_c
- if seen_separator:
- rule_pattern = rule_pattern + c
- else:
- file_pattern = file_pattern + c
-
- if not rule_pattern:
- rule_pattern = '**'
-
- return sign, file_pattern, rule_pattern
-
-
-def filter_sarif(input_sarif: PathLike,
- output_sarif: PathLike,
- patterns: List[str],
- split_lines: bool) -> None:
- """Filters a SARIF file with a given set of filter patterns.
-
- Args:
- input_sarif (PathLike): Input SARIF file path.
- output_sarif (PathLike): Output SARIF file path.
- patterns (PathLike): List of filter pattern strings.
- split_lines (PathLike): Whether to split lines in individual patterns.
- """
- if split_lines:
- tmp = []
- for p in patterns:
- tmp = tmp + re.split('\r?\n', p)
- patterns = tmp
-
- patterns = [_parse_pattern(p) for p in patterns if p]
-
- logging.debug('Given patterns:')
- for s, fp, rp in patterns:
- logging.debug(
- 'files: {file_pattern} rules: {rule_pattern} ({sign})'.format(
- file_pattern=fp,
- rule_pattern=rp,
- sign='positive' if s else 'negative'))
-
- with open(input_sarif, 'r') as f:
- s = json.load(f)
-
- for run in s.get('runs', []):
- if run.get('results', []):
- new_results = []
- for r in run['results']:
- if r.get('locations', []):
- new_locations = []
- for l in r['locations']:
- # TODO: The uri field is optional. We might have to
- # fetch the actual uri from "artifacts" via
- # "index"
- # (see https://github.com/microsoft/sarif-tutorials/blob/main/docs/2-Basics.md#-linking-results-to-artifacts)
- uri = l.get(
- 'physicalLocation', {}).get(
- 'artifactLocation', {}).get(
- 'uri', None)
-
- # TODO: The ruleId field is optional and potentially
- # ambiguous. We might have to fetch the actual
- # ruleId from the rule metadata via the ruleIndex
- # field.
- # (see https://github.com/microsoft/sarif-tutorials/blob/main/docs/2-Basics.md#rule-metadata)
- ruleId = r['ruleId']
-
- if (uri is None or
- _match_path_and_rule(uri, ruleId, patterns)):
- new_locations.append(l)
- r['locations'] = new_locations
- if new_locations:
- new_results.append(r)
- else:
- # locations array doesn't exist or is empty, so we can't
- # match on anything. Therefore, we include the result in
- # the output.
- new_results.append(r)
- run['results'] = new_results
-
- with open(output_sarif, 'w') as f:
- json.dump(s, f, indent=2)
+# @file analyze_filter.py +# +# Filters results in a SARIF file. +# +# Apache License +# Version 2.0, January 2004 +# http://www.apache.org/licenses/ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been altered from its original form. Based on code in: +# https://github.com/advanced-security/filter-sarif +# +# It primarily contains modifications made to integrate with the CodeQL plugin. +# +# Specifically: +# https://github.com/advanced-security/filter-sarif/blob/main/filter_sarif.py +# +# View the full and complete license as provided by that repository here: +# https://github.com/advanced-security/filter-sarif/blob/main/LICENSE +# +# SPDX-License-Identifier: Apache-2.0 +## + +import json +import logging +import re +from os import PathLike +from typing import Iterable, List, Tuple + +from analyze.globber import match + + +def _match_path_and_rule( + path: str, rule: str, patterns: Iterable[str]) -> bool: + """Returns whether a given path matches a given rule. + + Args: + path (str): A file path string. + rule (str): A rule file path string. + patterns (Iterable[str]): An iterable of pattern strings. + + Returns: + bool: True if the path matches a rule. Otherwise, False. + """ + result = True + for s, fp, rp in patterns: + if match(rp, rule) and match(fp, path): + result = s + return result + + +def _parse_pattern(line: str) -> Tuple[str]: + """Parses a given pattern line. + + Args: + line (str): The line string that contains the rule. + + Returns: + Tuple[str]: The parsed sign, file pattern, and rule pattern from the + line. + """ + sep_char = ':' + esc_char = '\\' + file_pattern = '' + rule_pattern = '' + seen_separator = False + sign = True + + # inclusion or exclusion pattern? + u_line = line + if line: + if line[0] == '-': + sign = False + u_line = line[1:] + elif line[0] == '+': + u_line = line[1:] + + i = 0 + while i < len(u_line): + c = u_line[i] + i = i + 1 + if c == sep_char: + if seen_separator: + raise Exception( + 'Invalid pattern: "' + line + '" Contains more than one ' + 'separator!') + seen_separator = True + continue + elif c == esc_char: + next_c = u_line[i] if (i < len(u_line)) else None + if next_c in ['+' , '-', esc_char, sep_char]: + i = i + 1 + c = next_c + if seen_separator: + rule_pattern = rule_pattern + c + else: + file_pattern = file_pattern + c + + if not rule_pattern: + rule_pattern = '**' + + return sign, file_pattern, rule_pattern + + +def filter_sarif(input_sarif: PathLike, + output_sarif: PathLike, + patterns: List[str], + split_lines: bool) -> None: + """Filters a SARIF file with a given set of filter patterns. + + Args: + input_sarif (PathLike): Input SARIF file path. + output_sarif (PathLike): Output SARIF file path. + patterns (PathLike): List of filter pattern strings. + split_lines (PathLike): Whether to split lines in individual patterns. + """ + if split_lines: + tmp = [] + for p in patterns: + tmp = tmp + re.split('\r?\n', p) + patterns = tmp + + patterns = [_parse_pattern(p) for p in patterns if p] + + logging.debug('Given patterns:') + for s, fp, rp in patterns: + logging.debug( + 'files: {file_pattern} rules: {rule_pattern} ({sign})'.format( + file_pattern=fp, + rule_pattern=rp, + sign='positive' if s else 'negative')) + + with open(input_sarif, 'r') as f: + s = json.load(f) + + for run in s.get('runs', []): + if run.get('results', []): + new_results = [] + for r in run['results']: + if r.get('locations', []): + new_locations = [] + for l in r['locations']: + # TODO: The uri field is optional. We might have to + # fetch the actual uri from "artifacts" via + # "index" + # (see https://github.com/microsoft/sarif-tutorials/blob/main/docs/2-Basics.md#-linking-results-to-artifacts) + uri = l.get( + 'physicalLocation', {}).get( + 'artifactLocation', {}).get( + 'uri', None) + + # TODO: The ruleId field is optional and potentially + # ambiguous. We might have to fetch the actual + # ruleId from the rule metadata via the ruleIndex + # field. + # (see https://github.com/microsoft/sarif-tutorials/blob/main/docs/2-Basics.md#rule-metadata) + ruleId = r['ruleId'] + + if (uri is None or + _match_path_and_rule(uri, ruleId, patterns)): + new_locations.append(l) + r['locations'] = new_locations + if new_locations: + new_results.append(r) + else: + # locations array doesn't exist or is empty, so we can't + # match on anything. Therefore, we include the result in + # the output. + new_results.append(r) + run['results'] = new_results + + with open(output_sarif, 'w') as f: + json.dump(s, f, indent=2) diff --git a/BaseTools/Plugin/CodeQL/analyze/globber.py b/BaseTools/Plugin/CodeQL/analyze/globber.py index 5d45abaa1f..11ef9d6cef 100644 --- a/BaseTools/Plugin/CodeQL/analyze/globber.py +++ b/BaseTools/Plugin/CodeQL/analyze/globber.py @@ -1,127 +1,127 @@ -# @file globber.py
-#
-# Provides global functionality for use by the CodeQL plugin.
-#
-# Copyright 2019 Jaakko Kangasharju
-#
-# Apache License
-# Version 2.0, January 2004
-# http://www.apache.org/licenses/
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This file has been altered from its original form. Based on code in:
-# https://github.com/advanced-security/filter-sarif
-#
-# Specifically:
-# https://github.com/advanced-security/filter-sarif/blob/main/filter_sarif.py
-#
-# It primarily contains modifications made to integrate with the CodeQL plugin.
-#
-# SPDX-License-Identifier: Apache-2.0
-##
-
-import re
-
-_double_star_after_invalid_regex = re.compile(r'[^/\\]\*\*')
-_double_star_first_before_invalid_regex = re.compile('^\\*\\*[^/]')
-_double_star_middle_before_invalid_regex = re.compile(r'[^\\]\*\*[^/]')
-
-
-def _match_component(pattern_component, file_name_component):
- if len(pattern_component) == 0 and len(file_name_component) == 0:
- return True
- elif len(pattern_component) == 0:
- return False
- elif len(file_name_component) == 0:
- return pattern_component == '*'
- elif pattern_component[0] == '*':
- return (_match_component(pattern_component, file_name_component[1:]) or
- _match_component(pattern_component[1:], file_name_component))
- elif pattern_component[0] == '?':
- return _match_component(pattern_component[1:], file_name_component[1:])
- elif pattern_component[0] == '\\':
- return (len(pattern_component) >= 2 and
- pattern_component[1] == file_name_component[0] and
- _match_component(
- pattern_component[2:], file_name_component[1:]))
- elif pattern_component[0] != file_name_component[0]:
- return False
- else:
- return _match_component(pattern_component[1:], file_name_component[1:])
-
-
-def _match_components(pattern_components, file_name_components):
- if len(pattern_components) == 0 and len(file_name_components) == 0:
- return True
- if len(pattern_components) == 0:
- return False
- if len(file_name_components) == 0:
- return len(pattern_components) == 1 and pattern_components[0] == '**'
- if pattern_components[0] == '**':
- return (_match_components(pattern_components, file_name_components[1:])
- or _match_components(
- pattern_components[1:], file_name_components))
- else:
- return (
- _match_component(
- pattern_components[0], file_name_components[0]) and
- _match_components(
- pattern_components[1:], file_name_components[1:]))
-
-
-def match(pattern: str, file_name: str):
- """Match a glob pattern against a file name.
-
- Glob pattern matching is for file names, which do not need to exist as
- files on the file system.
-
- A file name is a sequence of directory names, possibly followed by the name
- of a file, with the components separated by a path separator. A glob
- pattern is similar, except it may contain special characters: A '?' matches
- any character in a name. A '*' matches any sequence of characters (possibly
- empty) in a name. Both of these match only within a single component, i.e.,
- they will not match a path separator. A component in a pattern may also be
- a literal '**', which matches zero or more components in the complete file
- name. A backslash '\\' in a pattern acts as an escape character, and
- indicates that the following character is to be matched literally, even if
- it is a special character.
-
- Args:
- pattern (str): The pattern to match. The path separator in patterns is
- always '/'.
- file_name (str): The file name to match against. The path separator in
- file names is the platform separator
-
- Returns:
- bool: True if the pattern matches, False otherwise.
- """
- if (_double_star_after_invalid_regex.search(pattern) is not None or
- _double_star_first_before_invalid_regex.search(
- pattern) is not None or
- _double_star_middle_before_invalid_regex.search(pattern) is not None):
- raise ValueError(
- '** in {} not alone between path separators'.format(pattern))
-
- pattern = pattern.rstrip('/')
- file_name = file_name.rstrip('/')
-
- while '**/**' in pattern:
- pattern = pattern.replace('**/**', '**')
-
- pattern_components = pattern.split('/')
-
- # We split on '\' as well as '/' to support unix and windows-style paths
- file_name_components = re.split(r'[\\/]', file_name)
-
- return _match_components(pattern_components, file_name_components)
+# @file globber.py +# +# Provides global functionality for use by the CodeQL plugin. +# +# Copyright 2019 Jaakko Kangasharju +# +# Apache License +# Version 2.0, January 2004 +# http://www.apache.org/licenses/ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been altered from its original form. Based on code in: +# https://github.com/advanced-security/filter-sarif +# +# Specifically: +# https://github.com/advanced-security/filter-sarif/blob/main/filter_sarif.py +# +# It primarily contains modifications made to integrate with the CodeQL plugin. +# +# SPDX-License-Identifier: Apache-2.0 +## + +import re + +_double_star_after_invalid_regex = re.compile(r'[^/\\]\*\*') +_double_star_first_before_invalid_regex = re.compile('^\\*\\*[^/]') +_double_star_middle_before_invalid_regex = re.compile(r'[^\\]\*\*[^/]') + + +def _match_component(pattern_component, file_name_component): + if len(pattern_component) == 0 and len(file_name_component) == 0: + return True + elif len(pattern_component) == 0: + return False + elif len(file_name_component) == 0: + return pattern_component == '*' + elif pattern_component[0] == '*': + return (_match_component(pattern_component, file_name_component[1:]) or + _match_component(pattern_component[1:], file_name_component)) + elif pattern_component[0] == '?': + return _match_component(pattern_component[1:], file_name_component[1:]) + elif pattern_component[0] == '\\': + return (len(pattern_component) >= 2 and + pattern_component[1] == file_name_component[0] and + _match_component( + pattern_component[2:], file_name_component[1:])) + elif pattern_component[0] != file_name_component[0]: + return False + else: + return _match_component(pattern_component[1:], file_name_component[1:]) + + +def _match_components(pattern_components, file_name_components): + if len(pattern_components) == 0 and len(file_name_components) == 0: + return True + if len(pattern_components) == 0: + return False + if len(file_name_components) == 0: + return len(pattern_components) == 1 and pattern_components[0] == '**' + if pattern_components[0] == '**': + return (_match_components(pattern_components, file_name_components[1:]) + or _match_components( + pattern_components[1:], file_name_components)) + else: + return ( + _match_component( + pattern_components[0], file_name_components[0]) and + _match_components( + pattern_components[1:], file_name_components[1:])) + + +def match(pattern: str, file_name: str): + """Match a glob pattern against a file name. + + Glob pattern matching is for file names, which do not need to exist as + files on the file system. + + A file name is a sequence of directory names, possibly followed by the name + of a file, with the components separated by a path separator. A glob + pattern is similar, except it may contain special characters: A '?' matches + any character in a name. A '*' matches any sequence of characters (possibly + empty) in a name. Both of these match only within a single component, i.e., + they will not match a path separator. A component in a pattern may also be + a literal '**', which matches zero or more components in the complete file + name. A backslash '\\' in a pattern acts as an escape character, and + indicates that the following character is to be matched literally, even if + it is a special character. + + Args: + pattern (str): The pattern to match. The path separator in patterns is + always '/'. + file_name (str): The file name to match against. The path separator in + file names is the platform separator + + Returns: + bool: True if the pattern matches, False otherwise. + """ + if (_double_star_after_invalid_regex.search(pattern) is not None or + _double_star_first_before_invalid_regex.search( + pattern) is not None or + _double_star_middle_before_invalid_regex.search(pattern) is not None): + raise ValueError( + '** in {} not alone between path separators'.format(pattern)) + + pattern = pattern.rstrip('/') + file_name = file_name.rstrip('/') + + while '**/**' in pattern: + pattern = pattern.replace('**/**', '**') + + pattern_components = pattern.split('/') + + # We split on '\' as well as '/' to support unix and windows-style paths + file_name_components = re.split(r'[\\/]', file_name) + + return _match_components(pattern_components, file_name_components) diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml b/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml index dbc9c2ba02..a6e711b78b 100644 --- a/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml +++ b/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml @@ -1,26 +1,26 @@ -## @file codeqlcli_ext_dep.yaml
-#
-# Downloads the CodeQL Command-Line Interface (CLI) application that support Linux, Windows, and Mac OS X.
-#
-# This download is very large but conveniently provides support for all operating systems. Use it if you
-# need CodeQL CLI support without concern for the host operating system.
-#
-# In an environment where a platform might build in different operating systems, it is recommended to set
-# the scope for the appropriate CodeQL external dependency based on the host operating system being used.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-ext-dep",
- "type": "web",
- "name": "codeql_cli",
- "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql.zip",
- "version": "2.17.3",
- "sha256": "e5ac1d87ab38e405c9af5db234a338b10dffabc98a648903f1664dd2a566dfd5",
- "compression_type": "zip",
- "internal_path": "/codeql/",
- "flags": ["set_shell_var", ],
- "var_name": "STUART_CODEQL_PATH"
-}
+## @file codeqlcli_ext_dep.yaml +# +# Downloads the CodeQL Command-Line Interface (CLI) application that support Linux, Windows, and Mac OS X. +# +# This download is very large but conveniently provides support for all operating systems. Use it if you +# need CodeQL CLI support without concern for the host operating system. +# +# In an environment where a platform might build in different operating systems, it is recommended to set +# the scope for the appropriate CodeQL external dependency based on the host operating system being used. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-ext-dep", + "type": "web", + "name": "codeql_cli", + "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql.zip", + "version": "2.17.3", + "sha256": "e5ac1d87ab38e405c9af5db234a338b10dffabc98a648903f1664dd2a566dfd5", + "compression_type": "zip", + "internal_path": "/codeql/", + "flags": ["set_shell_var", ], + "var_name": "STUART_CODEQL_PATH" +} diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml b/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml index 536322f2b3..16f1f5643d 100644 --- a/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml +++ b/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml @@ -1,24 +1,24 @@ -## @file codeqlcli_linux_ext_dep.yaml
-#
-# Downloads the Linux CodeQL Command-Line Interface (CLI) application.
-#
-# This download only supports Linux. In an environment where a platform might build in different operating
-# systems, it is recommended to set the scope for the appropriate CodeQL external dependency based on the
-# host operating system being used.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-linux-ext-dep",
- "type": "web",
- "name": "codeql_linux_cli",
- "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql-linux64.zip",
- "version": "2.17.3",
- "sha256": "9fba000c4b821534d354bc16821aa066fdb1304446226ea449870e64a8ad3c7a",
- "compression_type": "zip",
- "internal_path": "/codeql/",
- "flags": ["set_shell_var", ],
- "var_name": "STUART_CODEQL_PATH"
-}
+## @file codeqlcli_linux_ext_dep.yaml +# +# Downloads the Linux CodeQL Command-Line Interface (CLI) application. +# +# This download only supports Linux. In an environment where a platform might build in different operating +# systems, it is recommended to set the scope for the appropriate CodeQL external dependency based on the +# host operating system being used. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-linux-ext-dep", + "type": "web", + "name": "codeql_linux_cli", + "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql-linux64.zip", + "version": "2.17.3", + "sha256": "9fba000c4b821534d354bc16821aa066fdb1304446226ea449870e64a8ad3c7a", + "compression_type": "zip", + "internal_path": "/codeql/", + "flags": ["set_shell_var", ], + "var_name": "STUART_CODEQL_PATH" +} diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml b/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml index 93a81ffd50..e997a9c7ef 100644 --- a/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml +++ b/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml @@ -1,24 +1,24 @@ -## @file codeqlcli_windows_ext_dep.yaml
-#
-# Downloads the Windows CodeQL Command-Line Interface (CLI) application.
-#
-# This download only supports Windows. In an environment where a platform might build in different operating
-# systems, it is recommended to set the scope for the appropriate CodeQL external dependency based on the
-# host operating system being used.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-{
- "scope": "codeql-windows-ext-dep",
- "type": "web",
- "name": "codeql_windows_cli",
- "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql-win64.zip",
- "version": "2.17.3",
- "sha256": "4c6fbf2ea2eaf0f47bf0347eacf54c6b9d6bdf7acb6b63e17f9e6f2dd83b34e7",
- "compression_type": "zip",
- "internal_path": "/codeql/",
- "flags": ["set_shell_var", ],
- "var_name": "STUART_CODEQL_PATH"
-}
+## @file codeqlcli_windows_ext_dep.yaml +# +# Downloads the Windows CodeQL Command-Line Interface (CLI) application. +# +# This download only supports Windows. In an environment where a platform might build in different operating +# systems, it is recommended to set the scope for the appropriate CodeQL external dependency based on the +# host operating system being used. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{ + "scope": "codeql-windows-ext-dep", + "type": "web", + "name": "codeql_windows_cli", + "source": "https://github.com/github/codeql-cli-binaries/releases/download/v2.17.3/codeql-win64.zip", + "version": "2.17.3", + "sha256": "4c6fbf2ea2eaf0f47bf0347eacf54c6b9d6bdf7acb6b63e17f9e6f2dd83b34e7", + "compression_type": "zip", + "internal_path": "/codeql/", + "flags": ["set_shell_var", ], + "var_name": "STUART_CODEQL_PATH" +} diff --git a/BaseTools/Plugin/CodeQL/common/codeql_plugin.py b/BaseTools/Plugin/CodeQL/common/codeql_plugin.py index c827cc30ae..72347ec13e 100644 --- a/BaseTools/Plugin/CodeQL/common/codeql_plugin.py +++ b/BaseTools/Plugin/CodeQL/common/codeql_plugin.py @@ -1,74 +1,74 @@ -# @file codeql_plugin.py
-#
-# Common logic shared across the CodeQL plugin.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-import os
-import shutil
-from os import PathLike
-
-from edk2toollib.utility_functions import GetHostInfo
-
-
-def get_codeql_db_path(workspace: PathLike, package: str, target: str,
- new_path: bool = True) -> str:
- """Return the CodeQL database path for this build.
-
- Args:
- workspace (PathLike): The workspace path.
- package (str): The package name (e.g. "MdeModulePkg")
- target (str): The target (e.g. "DEBUG")
- new_path (bool, optional): Whether to create a new database path or
- return an existing path. Defaults to True.
-
- Returns:
- str: The absolute path to the CodeQL database directory.
- """
- codeql_db_dir_name = "codeql-db-" + package + "-" + target
- codeql_db_dir_name = codeql_db_dir_name.lower()
- codeql_db_path = os.path.join("Build", codeql_db_dir_name)
- codeql_db_path = os.path.join(workspace, codeql_db_path)
-
- i = 0
- while os.path.isdir(f"{codeql_db_path + '-%s' % i}"):
- i += 1
-
- if not new_path:
- if i == 0:
- return None
- else:
- i -= 1
-
- return codeql_db_path + f"-{i}"
-
-
-def get_codeql_cli_path() -> str:
- """Return the current CodeQL CLI path.
-
- Returns:
- str: The absolute path to the CodeQL CLI application to use for
- this build.
- """
- # The CodeQL executable path can be passed via the
- # STUART_CODEQL_PATH environment variable (to override with a
- # custom value for this run) or read from the system path.
- codeql_path = None
-
- if "STUART_CODEQL_PATH" in os.environ:
- codeql_path = os.environ["STUART_CODEQL_PATH"]
-
- if GetHostInfo().os == "Windows":
- codeql_path = os.path.join(codeql_path, "codeql.exe")
- else:
- codeql_path = os.path.join(codeql_path, "codeql")
-
- if not os.path.isfile(codeql_path):
- codeql_path = None
-
- if not codeql_path:
- codeql_path = shutil.which("codeql")
-
- return codeql_path
+# @file codeql_plugin.py +# +# Common logic shared across the CodeQL plugin. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import os +import shutil +from os import PathLike + +from edk2toollib.utility_functions import GetHostInfo + + +def get_codeql_db_path(workspace: PathLike, package: str, target: str, + new_path: bool = True) -> str: + """Return the CodeQL database path for this build. + + Args: + workspace (PathLike): The workspace path. + package (str): The package name (e.g. "MdeModulePkg") + target (str): The target (e.g. "DEBUG") + new_path (bool, optional): Whether to create a new database path or + return an existing path. Defaults to True. + + Returns: + str: The absolute path to the CodeQL database directory. + """ + codeql_db_dir_name = "codeql-db-" + package + "-" + target + codeql_db_dir_name = codeql_db_dir_name.lower() + codeql_db_path = os.path.join("Build", codeql_db_dir_name) + codeql_db_path = os.path.join(workspace, codeql_db_path) + + i = 0 + while os.path.isdir(f"{codeql_db_path + '-%s' % i}"): + i += 1 + + if not new_path: + if i == 0: + return None + else: + i -= 1 + + return codeql_db_path + f"-{i}" + + +def get_codeql_cli_path() -> str: + """Return the current CodeQL CLI path. + + Returns: + str: The absolute path to the CodeQL CLI application to use for + this build. + """ + # The CodeQL executable path can be passed via the + # STUART_CODEQL_PATH environment variable (to override with a + # custom value for this run) or read from the system path. + codeql_path = None + + if "STUART_CODEQL_PATH" in os.environ: + codeql_path = os.environ["STUART_CODEQL_PATH"] + + if GetHostInfo().os == "Windows": + codeql_path = os.path.join(codeql_path, "codeql.exe") + else: + codeql_path = os.path.join(codeql_path, "codeql") + + if not os.path.isfile(codeql_path): + codeql_path = None + + if not codeql_path: + codeql_path = shutil.which("codeql") + + return codeql_path diff --git a/BaseTools/Plugin/CodeQL/integration/stuart_codeql.py b/BaseTools/Plugin/CodeQL/integration/stuart_codeql.py index a3941d1315..d778d7d949 100644 --- a/BaseTools/Plugin/CodeQL/integration/stuart_codeql.py +++ b/BaseTools/Plugin/CodeQL/integration/stuart_codeql.py @@ -1,79 +1,79 @@ -# @file stuart_codeql.py
-#
-# Exports functions commonly needed for Stuart-based platforms to easily
-# enable CodeQL in their platform build.
-#
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-##
-
-from edk2toolext.environment.uefi_build import UefiBuilder
-from edk2toollib.utility_functions import GetHostInfo
-from argparse import ArgumentParser, Namespace
-from typing import Tuple
-
-
-def add_command_line_option(parser: ArgumentParser) -> None:
- """Adds the CodeQL command to the platform command line options.
-
- Args:
- parser (ArgumentParser): The argument parser used in this build.
-
- """
- parser.add_argument(
- '--codeql',
- dest='codeql',
- action='store_true',
- default=False,
- help="Optional - Produces CodeQL results from the build. See "
- "BaseTools/Plugin/CodeQL/Readme.md for more info.")
-
-
-def get_scopes(codeql_enabled: bool) -> Tuple[str]:
- """Returns the active CodeQL scopes for this build.
-
- Args:
- codeql_enabled (bool): Whether CodeQL is enabled.
-
- Returns:
- Tuple[str]: A tuple of strings containing scopes that enable the
- CodeQL plugin.
- """
- active_scopes = ()
-
- if codeql_enabled:
- if GetHostInfo().os == "Linux":
- active_scopes += ("codeql-linux-ext-dep",)
- else:
- active_scopes += ("codeql-windows-ext-dep",)
- active_scopes += ("codeql-build", "codeql-analyze")
-
- return active_scopes
-
-
-def is_codeql_enabled_on_command_line(args: Namespace) -> bool:
- """Returns whether CodeQL was enabled on the command line.
-
- Args:
- args (Namespace): Object holding a string representation of command
- line arguments.
-
- Returns:
- bool: True if CodeQL is enabled on the command line. Otherwise, false.
- """
- return args.codeql
-
-
-def set_audit_only_mode(uefi_builder: UefiBuilder) -> None:
- """Configures the CodeQL plugin to run in audit only mode.
-
- Args:
- uefi_builder (UefiBuilder): The UefiBuilder object for this platform
- build.
-
- """
-
- uefi_builder.env.SetValue(
- "STUART_CODEQL_AUDIT_ONLY",
- "true",
- "Platform Defined")
+# @file stuart_codeql.py +# +# Exports functions commonly needed for Stuart-based platforms to easily +# enable CodeQL in their platform build. +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toollib.utility_functions import GetHostInfo +from argparse import ArgumentParser, Namespace +from typing import Tuple + + +def add_command_line_option(parser: ArgumentParser) -> None: + """Adds the CodeQL command to the platform command line options. + + Args: + parser (ArgumentParser): The argument parser used in this build. + + """ + parser.add_argument( + '--codeql', + dest='codeql', + action='store_true', + default=False, + help="Optional - Produces CodeQL results from the build. See " + "BaseTools/Plugin/CodeQL/Readme.md for more info.") + + +def get_scopes(codeql_enabled: bool) -> Tuple[str]: + """Returns the active CodeQL scopes for this build. + + Args: + codeql_enabled (bool): Whether CodeQL is enabled. + + Returns: + Tuple[str]: A tuple of strings containing scopes that enable the + CodeQL plugin. + """ + active_scopes = () + + if codeql_enabled: + if GetHostInfo().os == "Linux": + active_scopes += ("codeql-linux-ext-dep",) + else: + active_scopes += ("codeql-windows-ext-dep",) + active_scopes += ("codeql-build", "codeql-analyze") + + return active_scopes + + +def is_codeql_enabled_on_command_line(args: Namespace) -> bool: + """Returns whether CodeQL was enabled on the command line. + + Args: + args (Namespace): Object holding a string representation of command + line arguments. + + Returns: + bool: True if CodeQL is enabled on the command line. Otherwise, false. + """ + return args.codeql + + +def set_audit_only_mode(uefi_builder: UefiBuilder) -> None: + """Configures the CodeQL plugin to run in audit only mode. + + Args: + uefi_builder (UefiBuilder): The UefiBuilder object for this platform + build. + + """ + + uefi_builder.env.SetValue( + "STUART_CODEQL_AUDIT_ONLY", + "true", + "Platform Defined") |
