diff options
| author | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2022-08-03 16:55:16 +0300 |
|---|---|---|
| committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2022-08-03 17:56:03 +0300 |
| commit | bec4ebc22c43c1ff5c3fddb820d44a88bd3aebf0 (patch) | |
| tree | cd378e3e0eaff8fe11880bd397f41671e2347a39 /meta-arm/ci | |
| parent | 79161d7a7126cad324ff0c11a93d8e57d80203ed (diff) | |
| download | openbmc-bec4ebc22c43c1ff5c3fddb820d44a88bd3aebf0.tar.xz | |
Import 80d60e7 from yoctoproject.org meta-arm
To support ARMv8 SoCs.
meta-arm has several patch files. Since they are maintained by the
upstream meta-arm community, add meta-arm to the ignore list in
run-repotest.
Change-Id: Ia87a2e947bbabd347d256eccc47a343e1c885479
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Diffstat (limited to 'meta-arm/ci')
51 files changed, 1019 insertions, 0 deletions
diff --git a/meta-arm/ci/armgcc.yml b/meta-arm/ci/armgcc.yml new file mode 100644 index 0000000000..d46f0b36da --- /dev/null +++ b/meta-arm/ci/armgcc.yml @@ -0,0 +1,6 @@ +header: + version: 11 + +local_conf_header: + cc: | + GCCVERSION = "arm-11.2" diff --git a/meta-arm/ci/base.yml b/meta-arm/ci/base.yml new file mode 100644 index 0000000000..eff466ed8b --- /dev/null +++ b/meta-arm/ci/base.yml @@ -0,0 +1,53 @@ +header: + version: 11 + +distro: poky + +defaults: + repos: + refspec: master + +repos: + meta-arm: + layers: + meta-arm: + meta-arm-bsp: + meta-arm-toolchain: + + poky: + url: https://git.yoctoproject.org/git/poky + layers: + meta: + meta-poky: + +env: + BB_LOGCONFIG: "" + TOOLCHAIN_DIR: "" + +local_conf_header: + base: | + BB_SERVER_TIMEOUT = "60" + CONF_VERSION = "2" + BB_NUMBER_THREADS = "16" + PARALLEL_MAKE = "-j16" + LICENSE_FLAGS_ACCEPTED += "Arm-FVP-EULA" + setup: | + PACKAGE_CLASSES = "package_ipk" + PACKAGECONFIG:remove:pn-qemu-system-native = "gtk+ sdl" + EXTRA_IMAGE_FEATURES:append = " debug-tweaks" + PACKAGECONFIG:append:pn-perf = " coresight" + INHERIT += "rm_work" + DISTRO_FEATURES:remove = "ptest" + kvm: | + QEMU_USE_KVM = "" + perf: | + CORE_IMAGE_EXTRA_INSTALL += "perf" + sshkeys: | + CORE_IMAGE_EXTRA_INSTALL += "ssh-pregen-hostkeys" + universally_failing_tests: | + TEST_SUITES:remove = "opkg" + +machine: unset + +target: + - core-image-sato diff --git a/meta-arm/ci/check-machine-coverage b/meta-arm/ci/check-machine-coverage new file mode 100755 index 0000000000..f329fcebac --- /dev/null +++ b/meta-arm/ci/check-machine-coverage @@ -0,0 +1,28 @@ +#! /usr/bin/env python3 + +from pathlib import Path +import sys +from listmachines import list_machines + +metaarm = Path.cwd() + +if metaarm.name != "meta-arm": + print("Not running inside meta-arm") + sys.exit(1) + +# Find all layers +layers = (p.name for p in metaarm.glob("meta-*") if p.is_dir()) +# All machine configurations +machines = list_machines(layers) + +# All kas files +kas = metaarm.glob("ci/*.yml") +kas = set(p.stem for p in kas) + +missing = machines - kas +print(f"The following machines are missing: {', '.join(sorted(missing))}.") + +covered = len(machines) - len(missing) +total = len(machines) +percent = int(covered / total * 100) +print(f"Coverage: {percent}%") diff --git a/meta-arm/ci/check-warnings b/meta-arm/ci/check-warnings new file mode 100755 index 0000000000..89ae955dfe --- /dev/null +++ b/meta-arm/ci/check-warnings @@ -0,0 +1,19 @@ +#! /bin/bash + +# Expects the path to a log file as $1, and if this file has any content +# then display the contents and exit with an error code. + +set -e -u + +LOGFILE=$1 + +LINES=$(grep --invert-match "attempting MIRRORS if available" $LOGFILE | wc -l) +if test "$LINES" -ne 0; then + echo ============================== + echo The build had warnings/errors: + echo ============================== + cat $LOGFILE + exit 1 +fi + +exit 0 diff --git a/meta-arm/ci/clang.yml b/meta-arm/ci/clang.yml new file mode 100644 index 0000000000..80b9a4e347 --- /dev/null +++ b/meta-arm/ci/clang.yml @@ -0,0 +1,16 @@ +header: + version: 11 + +repos: + meta-clang: + url: https://github.com/kraj/meta-clang + +local_conf_header: + clang: | + TOOLCHAIN = "clang" + # Clang causes more binaries to have buildpaths in the debug symbols + # https://github.com/llvm/llvm-project/issues/56609 + WARN_QA:remove = "buildpaths" + +target: + - core-image-base diff --git a/meta-arm/ci/corstone1000-common.yml b/meta-arm/ci/corstone1000-common.yml new file mode 100644 index 0000000000..6599d97ac3 --- /dev/null +++ b/meta-arm/ci/corstone1000-common.yml @@ -0,0 +1,13 @@ +header: + version: 11 + includes: + - ci/base.yml + - ci/meta-openembedded.yml +local_conf_header: + perf: | + +distro: poky-tiny + +target: + - perf + - corstone1000-image diff --git a/meta-arm/ci/corstone1000-fvp.yml b/meta-arm/ci/corstone1000-fvp.yml new file mode 100644 index 0000000000..1586504302 --- /dev/null +++ b/meta-arm/ci/corstone1000-fvp.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - ci/corstone1000-common.yml + +local_conf_header: + fvp-config: | + # Remove Dropbear SSH as it will not fit into the corstone1000 image. + IMAGE_FEATURES:remove = " ssh-server-dropbear" + INHERIT += "fvpboot" + +machine: corstone1000-fvp diff --git a/meta-arm/ci/corstone1000-mps3.yml b/meta-arm/ci/corstone1000-mps3.yml new file mode 100644 index 0000000000..2df7d97671 --- /dev/null +++ b/meta-arm/ci/corstone1000-mps3.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - ci/corstone1000-common.yml + +machine: corstone1000-mps3 diff --git a/meta-arm/ci/corstone500.yml b/meta-arm/ci/corstone500.yml new file mode 100644 index 0000000000..ef78cacdf7 --- /dev/null +++ b/meta-arm/ci/corstone500.yml @@ -0,0 +1,18 @@ +header: + version: 11 + includes: + - ci/base.yml + +local_conf_header: + testimagefvp: | + INHERIT += "fvpboot" + IMAGE_FEATURES:remove = " ssh-server-dropbear" + perf: | + +machine: corstone500 + +distro: poky-tiny + +target: + - core-image-minimal + - perf diff --git a/meta-arm/ci/edk2.yml b/meta-arm/ci/edk2.yml new file mode 100644 index 0000000000..1261bf10fc --- /dev/null +++ b/meta-arm/ci/edk2.yml @@ -0,0 +1,17 @@ +header: + version: 11 + +local_conf_header: + bootfirmware: | + PREFERRED_PROVIDER_virtual/bootloader = "edk2-firmware" + MACHINE_FEATURES += "efi" + TFA_UBOOT = "0" + TFA_UEFI = "1" + + EXTRA_IMAGEDEPENDS += "edk2-firmware" + EFI_PROVIDER ?= "grub-efi" + + QB_DEFAULT_BIOS = "QEMU_EFI.fd" + WKS_FILE ?= "efi-disk.wks.in" + failing_tests: | + TEST_SUITES:remove = "xorg" diff --git a/meta-arm/ci/external-gccarm.yml b/meta-arm/ci/external-gccarm.yml new file mode 100644 index 0000000000..2af8b5e8e9 --- /dev/null +++ b/meta-arm/ci/external-gccarm.yml @@ -0,0 +1,8 @@ +header: + version: 11 + +local_conf_header: + cc: | + SKIP_RECIPE[gcc-cross-arm] = "Using external toolchain" + TCMODE = "external-arm" + EXTERNAL_TOOLCHAIN = "${TOPDIR}/toolchains/${TARGET_ARCH}" diff --git a/meta-arm/ci/fvp-base-arm32.yml b/meta-arm/ci/fvp-base-arm32.yml new file mode 100644 index 0000000000..be4b0088bb --- /dev/null +++ b/meta-arm/ci/fvp-base-arm32.yml @@ -0,0 +1,18 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: fvp-base-arm32 + +local_conf_header: + testimagefvp: | + INHERIT = "fvpboot" + # This fails but we can't add to the ignorelist from meta-arm yet + # https://bugzilla.yoctoproject.org/show_bug.cgi?id=14604 + TEST_SUITES:remove = "parselogs" + # Tell testimage to connect to localhost:8122, and forward that to SSH in the FVP. + TEST_TARGET_IP = "127.0.0.1:8122" + FVP_CONFIG[bp.virtio_net.hostbridge.userNetPorts] = "8122=22" + failing_tests: | + TEST_SUITES:remove = "xorg" diff --git a/meta-arm/ci/fvp-base.yml b/meta-arm/ci/fvp-base.yml new file mode 100644 index 0000000000..fa2ddb3e31 --- /dev/null +++ b/meta-arm/ci/fvp-base.yml @@ -0,0 +1,18 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: fvp-base + +local_conf_header: + testimagefvp: | + INHERIT += "fvpboot" + # This fails but we can't add to the ignorelist from meta-arm yet + # https://bugzilla.yoctoproject.org/show_bug.cgi?id=14604 + TEST_SUITES:remove = "parselogs" + # Tell testimage to connect to localhost:8022, and forward that to SSH in the FVP. + TEST_TARGET_IP = "localhost:8022" + FVP_CONFIG[bp.virtio_net.hostbridge.userNetPorts] ?= "8022=22" + failing_tests: | + TEST_SUITES:remove = "xorg" diff --git a/meta-arm/ci/fvp-baser-aemv8r64.yml b/meta-arm/ci/fvp-baser-aemv8r64.yml new file mode 100644 index 0000000000..40818bc6a3 --- /dev/null +++ b/meta-arm/ci/fvp-baser-aemv8r64.yml @@ -0,0 +1,7 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: fvp-baser-aemv8r64 + diff --git a/meta-arm/ci/fvps.yml b/meta-arm/ci/fvps.yml new file mode 100644 index 0000000000..576faa3119 --- /dev/null +++ b/meta-arm/ci/fvps.yml @@ -0,0 +1,20 @@ +# Simple target to build the FVPs that are publically available + +header: + version: 11 + includes: + - ci/base.yml + +machine: qemuarm64 + +local_conf_header: + sdk: | + SDKMACHINE = "x86_64" + +target: + - nativesdk-fvp-base-a-aem + - nativesdk-fvp-n1-edge + - nativesdk-fvp-sgi575 + - nativesdk-fvp-corstone500 + - nativesdk-fvp-corstone1000 + - nativesdk-fvp-tc0 diff --git a/meta-arm/ci/gem5-arm64.yml b/meta-arm/ci/gem5-arm64.yml new file mode 100644 index 0000000000..90913f37c2 --- /dev/null +++ b/meta-arm/ci/gem5-arm64.yml @@ -0,0 +1,16 @@ +header: + version: 11 + includes: + - ci/base.yml + - ci/meta-openembedded.yml + +repos: + meta-arm: + layers: + meta-gem5: + +machine: gem5-arm64 + +target: + - core-image-minimal + - gem5-aarch64-native diff --git a/meta-arm/ci/gem5-atp-arm64.yml b/meta-arm/ci/gem5-atp-arm64.yml new file mode 100644 index 0000000000..626947b5bb --- /dev/null +++ b/meta-arm/ci/gem5-atp-arm64.yml @@ -0,0 +1,15 @@ +header: + version: 11 + includes: + - ci/gem5-arm64.yml + +repos: + meta-arm: + layers: + meta-atp: + +machine: gem5-atp-arm64 + +target: + - atp-native + - core-image-minimal diff --git a/meta-arm/ci/generic-arm64.yml b/meta-arm/ci/generic-arm64.yml new file mode 100644 index 0000000000..873c9fd193 --- /dev/null +++ b/meta-arm/ci/generic-arm64.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: generic-arm64 diff --git a/meta-arm/ci/get-binary-toolchains b/meta-arm/ci/get-binary-toolchains new file mode 100755 index 0000000000..bfdd8c573b --- /dev/null +++ b/meta-arm/ci/get-binary-toolchains @@ -0,0 +1,46 @@ +#!/bin/bash +set -u + +HOST_ARCH=$(uname -m) +VER="11.2-2022.02" + +DOWNLOAD_DIR=$1 +TOOLCHAIN_DIR=$2 +TOOLCHAIN_LINK_DIR=$3 + +# These should be already created by .gitlab-ci.yml, but do here if run outside of that env +mkdir -p $DOWNLOAD_DIR $TOOLCHAIN_DIR $TOOLCHAIN_LINK_DIR + +if [ $HOST_ARCH = "aarch64" ]; then + #AArch64 Linux hosted cross compilers + + #AArch32 target with hard float (arm-none-linux-gnueabihf) + wget -P $DOWNLOAD_DIR -nc https://developer.arm.com/-/media/Files/downloads/gnu/$VER/binrel/gcc-arm-$VER-$HOST_ARCH-arm-none-linux-gnueabihf.tar.xz +elif [ $HOST_ARCH = "x86_64" ]; then + #x86_64 Linux hosted cross compilers + + #AArch32 target with hard float (arm-linux-none-gnueabihf) + wget -P $DOWNLOAD_DIR -nc https://developer.arm.com/-/media/Files/downloads/gnu/$VER/binrel/gcc-arm-$VER-$HOST_ARCH-arm-none-linux-gnueabihf.tar.xz + + #AArch64 GNU/Linux target (aarch64-none-linux-gnu) + wget -P $DOWNLOAD_DIR -nc https://developer.arm.com/-/media/Files/downloads/gnu/$VER/binrel/gcc-arm-$VER-$HOST_ARCH-aarch64-none-linux-gnu.tar.xz + + #AArch64 GNU/Linux target (aarch64_be-none-linux-gnu) + wget -P $DOWNLOAD_DIR -nc https://developer.arm.com/-/media/Files/downloads/gnu/$VER/binrel/gcc-arm-$VER-$HOST_ARCH-aarch64_be-none-linux-gnu.tar.xz +else + echo "ERROR - Unknown build arch of $HOST_ARCH" + exit 1 +fi + +for i in arm aarch64 aarch64_be; do + if [ ! -d $TOOLCHAIN_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*/ ]; then + if [ ! -f $DOWNLOAD_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*.tar.xz ]; then + continue + fi + + tar -C $TOOLCHAIN_DIR -axvf $DOWNLOAD_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*.tar.xz + fi + + # Setup a link for the toolchain to use local to the building machine (e.g., not in a shared location) + ln -s $TOOLCHAIN_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu* $TOOLCHAIN_LINK_DIR/$i +done diff --git a/meta-arm/ci/jobs-to-kas b/meta-arm/ci/jobs-to-kas new file mode 100755 index 0000000000..d6896b7728 --- /dev/null +++ b/meta-arm/ci/jobs-to-kas @@ -0,0 +1,27 @@ +#! /bin/bash + +# This script is expecting an input of machine name, optionally followed by a +# colon and a list of one or more parameters separated by commas between +# brackets. For example, the following are acceptable: +# corstone500 +# fvp-base: [testimage] +# qemuarm64-secureboot: [clang, glibc, testimage] +# +# Turn this list into a series of yml files separated by colons to pass to kas + +set -e -u + +FILES="ci/$(echo $1 | cut -d ':' -f 1).yml" + +for i in $(echo $1 | cut -s -d ':' -f 2 | sed 's/[][,]//g'); do + # Given that there are no yml files for gcc or glibc, as those are the + # defaults, we can simply ignore those parameters. They are necessary + # to pass in so that matrix can correctly setup all of the permutations + # of each individual run. + if [[ $i == 'none' || $i == 'gcc' || $i == 'glibc' || $i == 'uboot' ]]; then + continue + fi + FILES+=":ci/$i.yml" +done + +echo $FILES diff --git a/meta-arm/ci/juno.yml b/meta-arm/ci/juno.yml new file mode 100644 index 0000000000..b2ee60a7e9 --- /dev/null +++ b/meta-arm/ci/juno.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: juno diff --git a/meta-arm/ci/listmachines.py b/meta-arm/ci/listmachines.py new file mode 100755 index 0000000000..04d207e48b --- /dev/null +++ b/meta-arm/ci/listmachines.py @@ -0,0 +1,29 @@ +#! /usr/bin/env python3 + +import pathlib +import typing +import sys + +""" +List all of the machines available under the listed sub-layers of meta-arm. +""" +def list_machines(layers: typing.Sequence[str]) -> typing.Set[str]: + machines = set() + + # We know we're in meta-arm/scripts, so find the top-level directory + metaarm = pathlib.Path(__file__).resolve().parent.parent + if metaarm.name != "meta-arm": + raise Exception("Not running inside meta-arm") + + for layer in layers: + machines |= set(p.stem for p in (metaarm / layer / "conf" / "machine").glob("*.conf")) + return machines + +if __name__ == "__main__": + if len(sys.argv) > 1: + machines = list_machines(sys.argv[1:]) + print(" ".join(sorted(machines))) + sys.exit(0) + else: + print("Usage:\n$ %s [layer name ...] " % sys.argv[0]) + sys.exit(1) diff --git a/meta-arm/ci/logging.yml b/meta-arm/ci/logging.yml new file mode 100644 index 0000000000..3af10295f8 --- /dev/null +++ b/meta-arm/ci/logging.yml @@ -0,0 +1,13 @@ +# Python logging configuration to write all warnings to a separate file +version: 1 + +handlers: + warnings: + class: logging.FileHandler + level: WARNING + filename: warnings.log + formatter: BitBake.logfileFormatter + +loggers: + BitBake: + handlers: [warnings] diff --git a/meta-arm/ci/meta-openembedded.yml b/meta-arm/ci/meta-openembedded.yml new file mode 100644 index 0000000000..bed338dae0 --- /dev/null +++ b/meta-arm/ci/meta-openembedded.yml @@ -0,0 +1,11 @@ +header: + version: 11 + +repos: + meta-openembedded: + url: https://git.openembedded.org/meta-openembedded + layers: + meta-filesystems: + meta-networking: + meta-oe: + meta-python: diff --git a/meta-arm/ci/meta-virtualization.yml b/meta-arm/ci/meta-virtualization.yml new file mode 100644 index 0000000000..1cd0e21a89 --- /dev/null +++ b/meta-arm/ci/meta-virtualization.yml @@ -0,0 +1,8 @@ +header: + version: 11 + includes: + - ci/meta-openembedded.yml + +repos: + meta-virtualization: + url: git://git.yoctoproject.org/meta-virtualization diff --git a/meta-arm/ci/meta-zephyr.yml b/meta-arm/ci/meta-zephyr.yml new file mode 100644 index 0000000000..bdd0e2bbb9 --- /dev/null +++ b/meta-arm/ci/meta-zephyr.yml @@ -0,0 +1,13 @@ +header: + version: 11 + includes: + - ci/meta-openembedded.yml + +repos: + meta-zephyr: + url: https://git.yoctoproject.org/git/meta-zephyr + layers: + meta-zephyr-core: + +target: + - zephyr-kernel-test-all diff --git a/meta-arm/ci/microbit-v1.yml b/meta-arm/ci/microbit-v1.yml new file mode 100644 index 0000000000..b850148650 --- /dev/null +++ b/meta-arm/ci/microbit-v1.yml @@ -0,0 +1,7 @@ +header: + version: 11 + includes: + - ci/base.yml + - ci/meta-zephyr.yml + +machine: microbit-v1 diff --git a/meta-arm/ci/musca-b1.yml b/meta-arm/ci/musca-b1.yml new file mode 100644 index 0000000000..dc9814d2db --- /dev/null +++ b/meta-arm/ci/musca-b1.yml @@ -0,0 +1,15 @@ +header: + version: 11 + includes: + - ci/base.yml + - ci/meta-zephyr.yml + +local_conf_header: + nonbuilding_tests: | + ZEPHYRTESTS:remove = "common sleep poll device queue" + +machine: musca-b1 + +target: + - trusted-firmware-m + - zephyr-kernel-test-all diff --git a/meta-arm/ci/musca-s1.yml b/meta-arm/ci/musca-s1.yml new file mode 100644 index 0000000000..80a59c487e --- /dev/null +++ b/meta-arm/ci/musca-s1.yml @@ -0,0 +1,15 @@ +header: + version: 11 + includes: + - ci/base.yml + - ci/meta-zephyr.yml + +local_conf_header: + nonbuilding_tests: | + ZEPHYRTESTS:remove = "common sleep poll device queue" + +machine: musca-s1 + +target: + - trusted-firmware-m + - zephyr-kernel-test-all diff --git a/meta-arm/ci/musl.yml b/meta-arm/ci/musl.yml new file mode 100644 index 0000000000..ee7905ec8d --- /dev/null +++ b/meta-arm/ci/musl.yml @@ -0,0 +1,6 @@ +header: + version: 11 + +local_conf_header: + libc: | + TCLIBC = "musl" diff --git a/meta-arm/ci/n1sdp.yml b/meta-arm/ci/n1sdp.yml new file mode 100644 index 0000000000..797a52255c --- /dev/null +++ b/meta-arm/ci/n1sdp.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: n1sdp diff --git a/meta-arm/ci/patchreview b/meta-arm/ci/patchreview new file mode 100755 index 0000000000..b23eda1f14 --- /dev/null +++ b/meta-arm/ci/patchreview @@ -0,0 +1,286 @@ +#! /usr/bin/env python3 +# +# SPDX-License-Identifier: GPL-2.0-only +# + +# TODO +# - option to just list all broken files +# - test suite +# - validate signed-off-by + +import argparse +import collections +import json +import os +import re +import subprocess + +status_values = ( + "accepted", + "pending", + "inappropriate", + "backport", + "submitted", + "denied", +) + + +class PatchResult: + # Whether the patch has an Upstream-Status or not + missing_upstream_status = False + # If the Upstream-Status tag is malformed in some way (string for bad bit) + malformed_upstream_status = None + # If the Upstream-Status value is unknown (boolean) + unknown_upstream_status = False + # The upstream status value (Pending, etc) + upstream_status = None + # Whether the patch has a Signed-off-by or not + missing_sob = False + # Whether the Signed-off-by tag is malformed in some way + malformed_sob = False + # The Signed-off-by tag value + sob = None + # Whether a patch looks like a CVE but doesn't have a CVE tag + missing_cve = False + + +class Summary: + total = 0 + cve_missing = 0 + sob_missing = 0 + sob_malformed = 0 + status_missing = 0 + status_malformed = 0 + status_pending = 0 + +def blame_patch(patch): + """ + From a patch filename, return a list of "commit summary (author name <author + email>)" strings representing the history. + """ + return subprocess.check_output(("git", "log", + "--follow", "--find-renames", "--diff-filter=A", + "--format=%s (%aN <%aE>)", + "--", patch)).decode("utf-8").splitlines() + +def patchreview(patches): + # General pattern: start of line, optional whitespace, tag with optional + # hyphen or spaces, maybe a colon, some whitespace, then the value, all case + # insensitive. + sob_re = re.compile(r"^[\t ]*(Signed[-_ ]off[-_ ]by:?)[\t ]*(.+)", re.IGNORECASE | re.MULTILINE) + status_re = re.compile(r"^[\t ]*(Upstream[-_ ]Status:?)[\t ]*(\w*)", re.IGNORECASE | re.MULTILINE) + cve_tag_re = re.compile(r"^[\t ]*(CVE:)[\t ]*(.*)", re.IGNORECASE | re.MULTILINE) + cve_re = re.compile(r"cve-[0-9]{4}-[0-9]{4,6}", re.IGNORECASE) + + results = {} + + for patch in patches: + + result = PatchResult() + results[patch] = result + + content = open(patch, encoding="ascii", errors="ignore").read() + + # Find the Signed-off-by tag + match = sob_re.search(content) + if match: + value = match.group(1) + if value != "Signed-off-by:": + result.malformed_sob = value + result.sob = match.group(2) + else: + result.missing_sob = True + + # Find the Upstream-Status tag + match = status_re.search(content) + if match: + value = match.group(1) + if value != "Upstream-Status:": + result.malformed_upstream_status = value + + value = match.group(2).lower() + # TODO: check case + if value not in status_values: + result.unknown_upstream_status = True + result.upstream_status = value + else: + result.missing_upstream_status = True + + # Check that patches which looks like CVEs have CVE tags + if cve_re.search(patch) or cve_re.search(content): + if not cve_tag_re.search(content): + result.missing_cve = True + # TODO: extract CVE list + + return results + + +def analyse(results, want_blame=False, verbose=True): + """ + want_blame: display blame data for each malformed patch + verbose: display per-file results instead of just summary + """ + + # want_blame requires verbose, so disable blame if we're not verbose + if want_blame and not verbose: + want_blame = False + + summary = Summary() + + for patch in sorted(results): + r = results[patch] + summary.total += 1 + need_blame = False + + # Build statistics + if r.missing_sob: + summary.sob_missing += 1 + if r.malformed_sob: + summary.sob_malformed += 1 + if r.missing_upstream_status: + summary.status_missing += 1 + if r.malformed_upstream_status or r.unknown_upstream_status: + summary.status_malformed += 1 + # Count patches with no status as pending + summary.status_pending += 1 + if r.missing_cve: + summary.cve_missing += 1 + if r.upstream_status == "pending": + summary.status_pending += 1 + + # Output warnings + if r.missing_sob: + need_blame = True + if verbose: + print("Missing Signed-off-by tag (%s)" % patch) + if r.malformed_sob: + need_blame = True + if verbose: + print("Malformed Signed-off-by '%s' (%s)" % (r.malformed_sob, patch)) + if r.missing_cve: + need_blame = True + if verbose: + print("Missing CVE tag (%s)" % patch) + if r.missing_upstream_status: + need_blame = True + if verbose: + print("Missing Upstream-Status tag (%s)" % patch) + if r.malformed_upstream_status: + need_blame = True + if verbose: + print("Malformed Upstream-Status '%s' (%s)" % (r.malformed_upstream_status, patch)) + if r.unknown_upstream_status: + need_blame = True + if verbose: + print("Unknown Upstream-Status value '%s' (%s)" % (r.upstream_status, patch)) + + if want_blame and need_blame: + print("\n".join(blame_patch(patch)) + "\n") + + return summary + + +def display_summary(summary, verbose): + def percent(num): + try: + return "%d (%d%%)" % (num, round(num * 100.0 / summary.total)) + except ZeroDivisionError: + return "N/A" + + if verbose: + print() + + print("""Total patches found: %d +Patches missing Signed-off-by: %s +Patches with malformed Signed-off-by: %s +Patches missing CVE: %s +Patches missing Upstream-Status: %s +Patches with malformed Upstream-Status: %s +Patches in Pending state: %s""" % (summary.total, + percent(summary.sob_missing), + percent(summary.sob_malformed), + percent(summary.cve_missing), + percent(summary.status_missing), + percent(summary.status_malformed), + percent(summary.status_pending))) + + +def generate_metrics(summary, output): + # https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md + # Summary attribute name, MetricPoint help + mapping = ( + ("total", "Total patches"), + ("cve_missing", "Patches missing CVE tag"), + ("sob_malformed", "Patches with malformed Signed-off-by"), + ("sob_missing", "Patches with missing Signed-off-by"), + ("status_malformed", "Patches with malformed Upstream-Status"), + ("status_missing", "Patches with missing Upstream-Status"), + ("status_pending", "Patches with Pending Upstream-Status") + ) + for attr, help in mapping: + metric = f"patch_check_{attr}" + value = getattr(summary, attr) + output.write(f""" +# TYPE {metric} gauge +# HELP {help} +{metric} {value} +""") + output.write("\n# EOF\n") + +def histogram(results): + import math + + from toolz import dicttoolz, recipes + counts = recipes.countby(lambda r: r.upstream_status, results.values()) + bars = dicttoolz.valmap(lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts) + for k in bars: + print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k])) + +def gather_patches(directories): + patches = [] + for directory in directories: + filenames = subprocess.check_output(("git", "-C", directory, "ls-files", "recipes-*/**/*.patch", "recipes-*/**/*.diff")).decode("utf-8").split() + patches += [os.path.join(directory, f) for f in filenames] + return patches + +if __name__ == "__main__": + args = argparse.ArgumentParser(description="Patch Review Tool") + args.add_argument("-b", "--blame", action="store_true", help="show blame for malformed patches") + args.add_argument("-v", "--verbose", action="store_true", help="show per-patch results") + args.add_argument("-g", "--histogram", action="store_true", help="show patch histogram") + args.add_argument("-j", "--json", help="update JSON") + args.add_argument("-m", "--metrics", type=argparse.FileType('w'), help="write OpenMetrics") + args.add_argument("dirs", metavar="DIRECTORY", nargs="+", help="directory to scan") + args = args.parse_args() + + patches = gather_patches(args.dirs) + results = patchreview(patches) + summary = analyse(results, want_blame=args.blame, verbose=args.verbose) + display_summary(summary, verbose=args.verbose) + + if args.json: + if os.path.isfile(args.json): + data = json.load(open(args.json)) + else: + data = [] + + row = collections.Counter() + row["total"] = len(results) + row["date"] = subprocess.check_output(["git", "-C", args.dirs[0], "show", "-s", "--pretty=format:%cd", "--date=format:%s"]).decode("utf-8").strip() + for r in results.values(): + if r.upstream_status in status_values: + row[r.upstream_status] += 1 + if r.malformed_upstream_status or r.missing_upstream_status: + row["malformed-upstream-status"] += 1 + if r.malformed_sob or r.missing_sob: + row["malformed-sob"] += 1 + + data.append(row) + json.dump(data, open(args.json, "w")) + + if args.metrics: + generate_metrics(summary, args.metrics) + + if args.histogram: + print() + histogram(results) diff --git a/meta-arm/ci/qemu-cortex-a53.yml b/meta-arm/ci/qemu-cortex-a53.yml new file mode 100644 index 0000000000..3eec5801e4 --- /dev/null +++ b/meta-arm/ci/qemu-cortex-a53.yml @@ -0,0 +1,9 @@ +header: + version: 11 + includes: + - ci/base.yml + - ci/meta-zephyr.yml + +# FIXME - testimage fails all tests currently, but all the tests build + +machine: qemu-cortex-a53 diff --git a/meta-arm/ci/qemu-cortex-m3.yml b/meta-arm/ci/qemu-cortex-m3.yml new file mode 100644 index 0000000000..f45cb9e15c --- /dev/null +++ b/meta-arm/ci/qemu-cortex-m3.yml @@ -0,0 +1,20 @@ +header: + version: 11 + includes: + - ci/base.yml + - ci/meta-zephyr.yml + +repos: + meta-zephyr: + layers: + meta-zephyr-bsp: + +local_conf_header: + tclibc: | + TCLIBC = "newlib" + nonbuilding_tests: | + ZEPHYRTESTS:remove = "common context pending poll sleep" + qemu_opts: | + QB_OPT_APPEND = "-icount shift=3,align=off,sleep=on -rtc clock=vm" + +machine: qemu-cortex-m3 diff --git a/meta-arm/ci/qemu-cortex-r5.yml b/meta-arm/ci/qemu-cortex-r5.yml new file mode 100644 index 0000000000..cfcb7b9396 --- /dev/null +++ b/meta-arm/ci/qemu-cortex-r5.yml @@ -0,0 +1,11 @@ +header: + version: 11 + includes: + - ci/base.yml + - ci/meta-zephyr.yml + +local_conf_header: + nonbuilding_tests: | + ZEPHYRTESTS:remove = "common poll sleep queue device" + +machine: qemu-cortex-r5 diff --git a/meta-arm/ci/qemu-generic-arm64.yml b/meta-arm/ci/qemu-generic-arm64.yml new file mode 100644 index 0000000000..87cbd2f892 --- /dev/null +++ b/meta-arm/ci/qemu-generic-arm64.yml @@ -0,0 +1,14 @@ +header: + version: 11 + includes: + - ci/generic-arm64.yml + +local_conf_header: + failing_tests: | + DEFAULT_TEST_SUITES:remove = "parselogs" + +machine: qemu-generic-arm64 + +target: + - core-image-base + - sbsa-acs diff --git a/meta-arm/ci/qemuarm-secureboot.yml b/meta-arm/ci/qemuarm-secureboot.yml new file mode 100644 index 0000000000..044661cdc7 --- /dev/null +++ b/meta-arm/ci/qemuarm-secureboot.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: qemuarm-secureboot + +target: + - core-image-base + - optee-examples + - optee-test + - optee-os-tadevkit diff --git a/meta-arm/ci/qemuarm.yml b/meta-arm/ci/qemuarm.yml new file mode 100644 index 0000000000..4155847c19 --- /dev/null +++ b/meta-arm/ci/qemuarm.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: qemuarm diff --git a/meta-arm/ci/qemuarm64-secureboot.yml b/meta-arm/ci/qemuarm64-secureboot.yml new file mode 100644 index 0000000000..7e15a76baf --- /dev/null +++ b/meta-arm/ci/qemuarm64-secureboot.yml @@ -0,0 +1,18 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: qemuarm64-secureboot + +local_conf_header: + failing_tests: | + # software IO TLB: Cannot allocate buffer + DEFAULT_TEST_SUITES:remove = "parselogs" + +target: + - core-image-base + - optee-examples + - optee-test + - optee-spdevkit + - optee-os-tadevkit diff --git a/meta-arm/ci/qemuarm64.yml b/meta-arm/ci/qemuarm64.yml new file mode 100644 index 0000000000..6639034432 --- /dev/null +++ b/meta-arm/ci/qemuarm64.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: qemuarm64 diff --git a/meta-arm/ci/qemuarmv5.yml b/meta-arm/ci/qemuarmv5.yml new file mode 100644 index 0000000000..18c7a15a2d --- /dev/null +++ b/meta-arm/ci/qemuarmv5.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: qemuarmv5 diff --git a/meta-arm/ci/selftest.yml b/meta-arm/ci/selftest.yml new file mode 100644 index 0000000000..9a587354bd --- /dev/null +++ b/meta-arm/ci/selftest.yml @@ -0,0 +1,7 @@ +header: + version: 11 + +local_conf_header: + setup: | + BB_LOGCONFIG = "" + SANITY_TESTED_DISTROS = "" diff --git a/meta-arm/ci/sgi575.yml b/meta-arm/ci/sgi575.yml new file mode 100644 index 0000000000..1895fc523e --- /dev/null +++ b/meta-arm/ci/sgi575.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: sgi575 diff --git a/meta-arm/ci/tc0.yml b/meta-arm/ci/tc0.yml new file mode 100644 index 0000000000..b2f75d554a --- /dev/null +++ b/meta-arm/ci/tc0.yml @@ -0,0 +1,9 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: tc0 + +target: + - tc-artifacts-image diff --git a/meta-arm/ci/tc1.yml b/meta-arm/ci/tc1.yml new file mode 100644 index 0000000000..fd9acbd27e --- /dev/null +++ b/meta-arm/ci/tc1.yml @@ -0,0 +1,9 @@ +header: + version: 11 + includes: + - ci/base.yml + +machine: tc1 + +target: + - tc-artifacts-image diff --git a/meta-arm/ci/testimage-zephyr.yml b/meta-arm/ci/testimage-zephyr.yml new file mode 100644 index 0000000000..83e17a70d5 --- /dev/null +++ b/meta-arm/ci/testimage-zephyr.yml @@ -0,0 +1,9 @@ +header: + version: 11 + +local_conf_header: + testimage: | + IMAGE_CLASSES += "testimage" + TEST_TARGET = "QemuTargetZephyr" + TEST_SUITES = "zephyr" + TESTIMAGE_AUTO = "1" diff --git a/meta-arm/ci/testimage.yml b/meta-arm/ci/testimage.yml new file mode 100644 index 0000000000..a26dcaf9b2 --- /dev/null +++ b/meta-arm/ci/testimage.yml @@ -0,0 +1,13 @@ +header: + version: 11 + +local_conf_header: + testimage: | + IMAGE_CLASSES += "testimage" + TESTIMAGE_AUTO = "1" + slirp: | + TEST_RUNQEMUPARAMS = "slirp" + TEST_SERVER_IP = "127.0.0.1" + QEMU_USE_SLIRP = "1" + sshd: | + IMAGE_FEATURES:append = " ssh-server-dropbear" diff --git a/meta-arm/ci/tftf.yml b/meta-arm/ci/tftf.yml new file mode 100644 index 0000000000..6e42d9c688 --- /dev/null +++ b/meta-arm/ci/tftf.yml @@ -0,0 +1,7 @@ +header: + version: 9 + +local_conf_header: + tftf: | + TFA_UBOOT = "0" + TFTF_TESTS = "1" diff --git a/meta-arm/ci/toolchains.yml b/meta-arm/ci/toolchains.yml new file mode 100644 index 0000000000..080d8d978e --- /dev/null +++ b/meta-arm/ci/toolchains.yml @@ -0,0 +1,19 @@ +header: + version: 11 + includes: + - ci/base.yml + +# Target is arm64 and SDK is x86-64 to ensure that we exercise both +# architectures + +machine: qemuarm64 +local_conf_header: + toolchains: | + SDKMACHINE = "x86_64" + +target: + - gcc-aarch64-none-elf + - nativesdk-gcc-aarch64-none-elf + - gcc-arm-none-eabi + - nativesdk-gcc-arm-none-eabi + - nativesdk-androidclang diff --git a/meta-arm/ci/update-repos b/meta-arm/ci/update-repos new file mode 100755 index 0000000000..a68257bbab --- /dev/null +++ b/meta-arm/ci/update-repos @@ -0,0 +1,43 @@ +#! /usr/bin/env python3 + +# Update clones of the repositories we need in KAS_REPO_REF_DIR to speed up fetches + +import sys +import os +import subprocess +import pathlib + +def repo_shortname(url): + # Taken from Kas (Repo.__getattr__) to ensure the logic is right + from urllib.parse import urlparse + url = urlparse(url) + return ('{url.netloc}{url.path}' + .format(url=url) + .replace('@', '.') + .replace(':', '.') + .replace('/', '.') + .replace('*', '.')) + +repositories = ( + "https://git.yoctoproject.org/git/poky", + "https://git.openembedded.org/meta-openembedded", + "https://git.yoctoproject.org/git/meta-virtualization", + "https://git.yoctoproject.org/git/meta-zephyr", + "https://github.com/kraj/meta-clang", +) + +if __name__ == "__main__": + if "KAS_REPO_REF_DIR" not in os.environ: + print("KAS_REPO_REF_DIR needs to be set") + sys.exit(1) + + base_repodir = pathlib.Path(os.environ["KAS_REPO_REF_DIR"]) + + for repo in repositories: + repodir = base_repodir / repo_shortname(repo) + if repodir.exists(): + print("Updating %s..." % repo) + subprocess.run(["git", "-C", repodir, "fetch"], check=True) + else: + print("Cloning %s..." % repo) + subprocess.run(["git", "clone", "--bare", repo, repodir], check=True) diff --git a/meta-arm/ci/xen.yml b/meta-arm/ci/xen.yml new file mode 100644 index 0000000000..d8b75d44dc --- /dev/null +++ b/meta-arm/ci/xen.yml @@ -0,0 +1,11 @@ +header: + version: 11 + includes: + - ci/meta-virtualization.yml + +local_conf_header: + meta-virt: | + DISTRO_FEATURES:append = " virtualization xen" + +target: + - xen-image-minimal |