diff --git a/.github/actions/stuart-ci/action.yml b/.github/actions/stuart-ci/action.yml new file mode 100644 index 00000000..6b64ceaa --- /dev/null +++ b/.github/actions/stuart-ci/action.yml @@ -0,0 +1,195 @@ +# A callable Github Action to run CI tests on EDK II package(s) using stuart. +# +# Produces an output, `log-path`, which is the path to the folder containing all logs produced. +# +## +# Copyright (c) Microsoft Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +name: EDK II Package CI +description: Prepare and run CI tests on EDK II package(s) using stuart. + +inputs: + ci-config: + description: Path to the CI configuration file (relative to the repository root). No value means '.pytool/CISettings.py'. + required: false + default: '.pytool/CISettings.py' + packages: + description: Comma-separated list of EDK II packages to run CI tests on. No value means all packages. Do not include spaces between package names. + required: false + default: '' + targets: + description: Comma-separated list of build targets to run CI tests on. No value means all targets. Do not include spaces between target names. + required: false + default: '' + architectures: + description: Comma-separated list of architectures to run CI tests on. No value means all architectures. Do not include spaces between architecture names. + required: false + default: '' + toolchain: + description: The toolchain to use for the CI build. + required: false + default: 'CLANGPDB' + stuart-args: + description: Additional arguments to pass to `stuart_ci_build`. Should be space-separated. For example, `--test-filter MyTest*`. + required: false + default: '' + stuart-setup: + description: Whether to run `stuart_setup` before running CI tests ('true' or 'false'). + required: false + default: 'false' + stuart-ci-setup: + description: Whether to run `stuart_ci_setup` before running CI tests ('true' or 'false'). + required: false + default: 'true' + +outputs: + log-path: + description: 'Path to the folder containing the build logs' + value: ${{ steps.tempdir.outputs.path }} + +runs: + using: composite + + steps: + - name: Create temporary directory for log files + id: tempdir + shell: bash + env: + PACKAGES: ${{ inputs.packages }} + run: | + TEMP_DIR="ci-logs" + if [ -n "${PACKAGES}" ]; then + TEMP_DIR="${TEMP_DIR}-$(echo "${PACKAGES}" | tr ',' '-')" + fi + mkdir -p "$RUNNER_TEMP/$TEMP_DIR" + echo "path=$RUNNER_TEMP/$TEMP_DIR" >> $GITHUB_OUTPUT + + - name: Gather submodule hashes + if: ${{ fromJson(inputs.stuart-setup) }} + id: submodules-hash + shell: bash + run: | + HASH=$(git submodule foreach --quiet 'echo $sha1' | sha256sum | awk '{print $1}') + echo "hash=$HASH" >> $GITHUB_OUTPUT + + - name: Cache submodules + if: ${{ fromJson(inputs.stuart-setup) }} + uses: actions/cache@v5 + with: + path: .git/modules + key: ${{ runner.os }}-submodules-${{ steps.submodules-hash.outputs.hash }} + + - name: Download required submodules (stuart_setup) + if: ${{ fromJson(inputs.stuart-setup) }} + env: + CI_CONFIG: ${{ inputs.ci-config }} + shell: bash + run: | + stuart_setup -c "${CI_CONFIG}" + + - name: Move setup log + if: always() && fromJson(inputs.stuart-setup) + shell: bash + env: + TEMP_DIR: ${{ steps.tempdir.outputs.path }} + run: | + shopt -s globstar nullglob + files=(Build/SETUP*.txt) + + if ((${#files[@]})); then + echo "Moving ${#files[@]} log files" + mv "${files[@]}" "$TEMP_DIR" + else + echo "No matching log files found" + fi + + - name: Download CI dependencies (stuart_ci_setup) + if: ${{ fromJson(inputs.stuart-ci-setup) }} + shell: bash + env: + CI_CONFIG: ${{ inputs.ci-config }} + run: | + stuart_ci_setup -c "${CI_CONFIG}" + + - name: Move CI setup log + if: always() && fromJson(inputs.stuart-ci-setup) + shell: bash + env: + TEMP_DIR: ${{ steps.tempdir.outputs.path }} + run: | + shopt -s globstar nullglob + files=(Build/CI_SETUP*.txt) + + if ((${#files[@]})); then + echo "Moving ${#files[@]} log files" + mv "${files[@]}" "$TEMP_DIR" + else + echo "No matching log files found" + fi + + - name: Download external dependencies (stuart_update) + shell: bash + env: + CI_CONFIG: ${{ inputs.ci-config }} + run: | + stuart_update -c "${CI_CONFIG}" + + - name: Move update log + if: always() + shell: bash + env: + TEMP_DIR: ${{ steps.tempdir.outputs.path }} + run: | + shopt -s globstar nullglob + files=(Build/UPDATE*.txt) + + if ((${#files[@]})); then + echo "Moving ${#files[@]} log files" + mv "${files[@]}" "$TEMP_DIR" + else + echo "No matching log files found" + fi + + - name: Run CI tests + shell: bash + env: + CI_CONFIG: ${{ inputs.ci-config }} + PACKAGES: ${{ inputs.packages }} + TARGETS: ${{ inputs.targets }} + ARCHITECTURES: ${{ inputs.architectures }} + TOOLCHAIN: ${{ inputs.toolchain }} + STUART_ARGS: ${{ inputs.stuart-args }} + run: | + ARGS=(-c "${CI_CONFIG}") + if [ -n "${PACKAGES}" ]; then + ARGS+=(-p "${PACKAGES}") + fi + if [ -n "${TARGETS}" ]; then + ARGS+=(-t "${TARGETS}") + fi + if [ -n "${ARCHITECTURES}" ]; then + ARGS+=(-a "${ARCHITECTURES}") + fi + if [ -n "${TOOLCHAIN}" ]; then + ARGS+=(TOOL_CHAIN_TAG="${TOOLCHAIN}") + fi + echo "stuart_ci_build ${ARGS[*]} ${STUART_ARGS}" + stuart_ci_build "${ARGS[@]}" ${STUART_ARGS} + + - name: Move build log + if: always() + shell: bash + env: + TEMP_DIR: ${{ steps.tempdir.outputs.path }} + run: | + shopt -s globstar nullglob + files=(Build/**/CI_BUILDLOG*.txt Build/**/BUILDLOG_*.txt) + + if ((${#files[@]})); then + echo "Moving ${#files[@]} log files" + mv "${files[@]}" "$TEMP_DIR" + else + echo "No matching log files found" + fi diff --git a/.github/workflows/PackageCi.yml b/.github/workflows/PackageCi.yml new file mode 100644 index 00000000..b0de99be --- /dev/null +++ b/.github/workflows/PackageCi.yml @@ -0,0 +1,122 @@ +# A workflow to build EDK II packages in this repository using certain configurations. +# +# NOTE: This file is automatically synchronized from Mu DevOps to keep the version of the +# workflow up to date. Update the original file there instead of the file in this repo. +# +# - Mu DevOps Repo: https://github.com/microsoft/mu_devops +# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml +# +# Copyright (c) Microsoft Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +name: Package CI + +on: + workflow_call: + inputs: + ci-config: + description: > + Path to the CI configuration file used to run the CI build. Default is + '.pytool/CISettings.py'. + type: string + required: false + default: '.pytool/CISettings.py' + container: + description: A container image to run the build in when using a linux runner. + type: string + required: false + default: '' + package-config: + description: > + A YAML array of objects. Each object must have a "packages" key. + All other keys are passed through to the output. + Example: + - packages: 'MdePkg,UefiCpuPkg' + targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' + architectures: 'X64,AARCH64' + - packages: 'ShellPkg' + targets: 'DEBUG' + type: string + required: true + python-version: + description: Python version to use when not running in a container + type: string + required: false + default: '3.12' + runner: + description: > + The type of runner to use for the CI build. + type: string + required: false + default: 'ubuntu-latest' + setup-cmd: + description: > + The stuart setup command to run, e.g. `setup`, `ci-setup`, or `both`. + Any other value will result in no setup command being run. Default is `both`. + type: string + required: false + default: 'both' + stuart-args: + description: Additional arguments to pass to `stuart_ci_build`. Should be space-separated. For example, `--test-filter MyTest*`. + type: string + required: false + default: '' + +jobs: + package-ci: + + name: "${{ matrix.packages }} ${{ matrix.architectures }} ${{ matrix.targets }}" + + runs-on: ${{ inputs.runner }} + + container: + image: ${{ inputs.container || null }} + options: --security-opt seccomp=unconfined + + strategy: + fail-fast: false + matrix: ${{ fromJson(inputs.package-config) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Python ${{ inputs.python-version }} + if: ${{ inputs.container == null }} + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + cache: 'pip' + cache-dependency-path: pip-requirements.txt + + - name: Install Python dependencies + run: pip install -r pip-requirements.txt + + - name: Container Configuration + if: ${{ inputs.container != null }} + shell: bash + run: | + git config --global --add safe.directory '*' + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + + - name: EDK II Package CI + id: package-ci + uses: microsoft/mu_devops/.github/actions/stuart-ci@v18.0.4 + with: + ci-config: ${{ inputs.ci-config }} + packages: ${{ matrix.packages }} + targets: ${{ matrix.targets }} + architectures: ${{ matrix.architectures }} + toolchain: ${{ matrix.toolchain }} + stuart-setup: ${{ inputs.setup-cmd == 'setup' || inputs.setup-cmd == 'both' }} + stuart-ci-setup: ${{ inputs.setup-cmd == 'ci-setup' || inputs.setup-cmd == 'both' }} + stuart-args: ${{ inputs.stuart-args }} + + - name: Upload CI build logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-logs-${{ matrix.packages }}-${{ matrix.architectures }}-${{ matrix.targets }}-${{ inputs.runner }} + path: ${{ steps.package-ci.outputs.log-path }} diff --git a/.github/workflows/PackageMatrix.yml b/.github/workflows/PackageMatrix.yml new file mode 100644 index 00000000..97fb5b22 --- /dev/null +++ b/.github/workflows/PackageMatrix.yml @@ -0,0 +1,173 @@ +# A reusable workflow to discover and validate EDK II packages in a repository. +# +# Produces a matrix object where: +# - default-config arrays become top-level matrix dimensions (cross-product) +# for any undeclared *Pkg directories +# - package-config entries go into the matrix "include" for exact combinations. +# Any package names here are excluded from the default-config cross-product. +# +# If `ci-config` is provided and the event is a pull request, `stuart_pr_eval` is +# used to filter the produced job matrix to only include packages affected by the +# PR changes. +# +## +# Copyright (c) Microsoft Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +name: Build Matrix Configuration + +on: + workflow_call: + inputs: + ci-config: + description: > + Path to the CI configuration file. If provided, it will be used + to determine which packages need testing based on the changed + files in a pull request. No value means '.pytool/CISettings.py'. + type: string + required: false + default: '' + default-config: + description: > + A YAML object whose values are arrays used as top-level matrix + dimensions. Undeclared *Pkg directories are added as the + "packages" array. These dimensions form a cross-product. + Example: + targets: [DEBUG, RELEASE] + toolchain: [CLANGPDB] + architectures: [IA32, X64, AARCH64] + required: true + type: string + package-config: + description: > + A YAML array of objects placed into the matrix "include". Each + object must have a "packages" key. All other keys are passed + through as-is. + Example: + - packages: MdeModulePkg + targets: RELEASE + toolchain: CLANGPDB + - packages: MdeModulePkg + targets: DEBUG + toolchain: CLANGPDB + required: false + default: '[]' + type: string + python-version: + description: Python version to use when not running in a container + type: string + required: false + default: '3.12' + + outputs: + matrix: + description: > + A JSON matrix object with top-level dimensions from default-config, + a "packages" array of undeclared packages, and an "include" array + from package-config. Suitable for use with fromJson() in a matrix + strategy. + value: ${{ jobs.gather-packages.outputs.matrix }} + +jobs: + gather-packages: + name: Build Matrix Configuration + + runs-on: ubuntu-latest + + outputs: + matrix: ${{ steps.filter-matrix.outputs.matrix || steps.set-packages.outputs.matrix }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Fetch PR target branch + if: ${{ inputs.ci-config != '' && github.event_name == 'pull_request' }} + run: git fetch --depth=1 origin ${{ github.base_ref }}:${{ github.base_ref }} + + - name: Setup Python ${{ inputs.python-version }} + if: ${{ inputs.ci-config != '' && github.event_name == 'pull_request' }} + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + cache: 'pip' + cache-dependency-path: pip-requirements.txt + + - name: Install Python dependencies + if: ${{ inputs.ci-config != '' && github.event_name == 'pull_request' }} + run: pip install -r pip-requirements.txt + + - name: Filter to packages that need testing + if: ${{ inputs.ci-config != '' && github.event_name == 'pull_request' }} + id: filter-packages + shell: bash + run: | + stuart_pr_eval \ + -c ${{ inputs.ci-config }} \ + --pr-target ${{ github.base_ref }} \ + --output-csv-format-string "{pkgcsv}" \ + 2>&1 | tee stuart_pr_eval.log + + CSV=$(tail -1 stuart_pr_eval.log) + echo "csv=$CSV" >> "$GITHUB_OUTPUT" + + - name: Build matrix from all packages + id: set-packages + shell: bash + env: + PACKAGE_CONFIG: ${{ inputs.package-config }} + DEFAULT_CONFIG: ${{ inputs.default-config }} + run: | + # Convert YAML inputs to JSON + PACKAGE_CONFIG_JSON=$(echo "$PACKAGE_CONFIG" | yq -o=json -I=0 '.') + DEFAULT_JSON=$(echo "$DEFAULT_CONFIG" | yq -o=json -I=0 '.') + + # Collect all package names mentioned across include entries + DECLARED=$(echo "$PACKAGE_CONFIG_JSON" | jq -r '.[].packages // empty' | tr ',' '\n' | sort -u) + + # Find *Pkg directories not covered by any include entry + UNDECLARED="[]" + for dir in *Pkg/; do + pkg="${dir%/}" + if ! echo "$DECLARED" | grep -qxF "$pkg"; then + UNDECLARED=$(echo "$UNDECLARED" | jq -c --arg p "$pkg" '. + [$p]') + fi + done + + # Build matrix: default-config dimensions + undeclared packages + include + OUTPUT=$(echo "$DEFAULT_JSON" | jq -c \ + --argjson pkgs "$UNDECLARED" \ + --argjson inc "$PACKAGE_CONFIG_JSON" \ + '. + {"packages": $pkgs, "include": $inc}') + + echo "::group::Generated matrix JSON" + echo "$OUTPUT" | jq . + echo "::endgroup::" + echo "" + echo "matrix=$OUTPUT" >> "$GITHUB_OUTPUT" + + - name: Filter matrix to PR-affected packages + if: ${{ inputs.ci-config != '' && github.event_name == 'pull_request' }} + id: filter-matrix + shell: bash + env: + MATRIX: ${{ steps.set-packages.outputs.matrix }} + PACKAGE_CSV: ${{ steps.filter-packages.outputs.csv }} + run: | + echo "Filtering matrix to packages that need testing: $PACKAGE_CSV" + + # Convert CSV to a JSON array of package names + CSV_JSON=$(echo "$PACKAGE_CSV" | tr ',' '\n' | jq -R . | jq -sc '.') + + # Filter "packages" array to only those in CSV + # Filter "include" array to only entries with at least one package in CSV + OUTPUT=$(echo "$MATRIX" | jq -c --argjson csv "$CSV_JSON" \ + '.packages = [.packages[] | select(. as $p | $csv | index($p))] + | .include = [.include[] | select(.packages | split(",") | any(. as $p | $csv | index($p)))]') + + echo "::group::Filtered matrix JSON" + echo "$OUTPUT" | jq . + echo "::endgroup::" + echo "" + echo "matrix=$OUTPUT" >> "$GITHUB_OUTPUT" diff --git a/.sync/Files.yml b/.sync/Files.yml index 9e58e325..1c075471 100644 --- a/.sync/Files.yml +++ b/.sync/Files.yml @@ -580,6 +580,15 @@ group: repos: | microsoft/mu_tiano_platforms +# Leaf Worfklow - Package CI +# NOTE: This is sync'd only to mu_devops purely to keep the inner workflow versions up to date. + - files: + - source: .sync/workflows/leaf/PackageCi.yml + dest: .github/workflows/PackageCi.yml + template: true + repos: | + microsoft/mu_devops + # Pull Request Template - Common Template - files: - source: .sync/github_templates/pull_requests/pull_request_template.md diff --git a/.sync/Version.njk b/.sync/Version.njk index 962e347c..e26492ef 100644 --- a/.sync/Version.njk +++ b/.sync/Version.njk @@ -30,7 +30,7 @@ #} {# The git ref value that files dependent on this repo will use. #} -{% set mu_devops = "v18.0.3" %} +{% set mu_devops = "v18.0.4" %} {# The latest Project Mu release branch value. #} {% set latest_mu_release_branch = "release/202511" %} diff --git a/.sync/workflows/leaf/PackageCi.yml b/.sync/workflows/leaf/PackageCi.yml new file mode 100644 index 00000000..be5d9444 --- /dev/null +++ b/.sync/workflows/leaf/PackageCi.yml @@ -0,0 +1,132 @@ +# A workflow to build EDK II packages in this repository using certain configurations. +# +# NOTE: This file is automatically synchronized from Mu DevOps to keep the version of the +# workflow up to date. Update the original file there instead of the file in this repo. +# +# - Mu DevOps Repo: https://github.com/microsoft/mu_devops +# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml +# +# Copyright (c) Microsoft Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +{% import '../Version.njk' as sync_version -%} +{% raw %} +# NOTE: Because this pipeline YAML file is a Nunjucks template, the pipeline syntax of `{{}}` will conflict with +# Nunjucks style. Surround pipeline YAML code that uses `{{}}` within `raw` and `endraw` tags +# to allow it to pass through Nunjucks processing. +{% endraw %} +name: Package CI + +on: + workflow_call: + inputs: + ci-config: + description: > + Path to the CI configuration file used to run the CI build. Default is + '.pytool/CISettings.py'. + type: string + required: false + default: '.pytool/CISettings.py' + container: + description: A container image to run the build in when using a linux runner. + type: string + required: false + default: '' + package-config: + description: > + A YAML array of objects. Each object must have a "packages" key. + All other keys are passed through to the output. + Example: + - packages: 'MdePkg,UefiCpuPkg' + targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' + architectures: 'X64,AARCH64' + - packages: 'ShellPkg' + targets: 'DEBUG' + type: string + required: true + python-version: + description: Python version to use when not running in a container + type: string + required: false + default: '3.12' + runner: + description: > + The type of runner to use for the CI build. + type: string + required: false + default: 'ubuntu-latest' + setup-cmd: + description: > + The stuart setup command to run, e.g. `setup`, `ci-setup`, or `both`. + Any other value will result in no setup command being run. Default is `both`. + type: string + required: false + default: 'both' + stuart-args: + description: Additional arguments to pass to `stuart_ci_build`. Should be space-separated. For example, `--test-filter MyTest*`. + type: string + required: false + default: '' + +{% raw %} +jobs: + package-ci: + + name: "${{ matrix.packages }} ${{ matrix.architectures }} ${{ matrix.targets }}" + + runs-on: ${{ inputs.runner }} + + container: + image: ${{ inputs.container || null }} + options: --security-opt seccomp=unconfined + + strategy: + fail-fast: false + matrix: ${{ fromJson(inputs.package-config) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Python ${{ inputs.python-version }} + if: ${{ inputs.container == null }} + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + cache: 'pip' + cache-dependency-path: pip-requirements.txt + + - name: Install Python dependencies + run: pip install -r pip-requirements.txt + + - name: Container Configuration + if: ${{ inputs.container != null }} + shell: bash + run: | + git config --global --add safe.directory '*' + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + + - name: EDK II Package CI + id: package-ci +{% endraw %} + uses: microsoft/mu_devops/.github/actions/stuart-ci@{{ sync_version.mu_devops }} +{% raw %} + with: + ci-config: ${{ inputs.ci-config }} + packages: ${{ matrix.packages }} + targets: ${{ matrix.targets }} + architectures: ${{ matrix.architectures }} + toolchain: ${{ matrix.toolchain }} + stuart-setup: ${{ inputs.setup-cmd == 'setup' || inputs.setup-cmd == 'both' }} + stuart-ci-setup: ${{ inputs.setup-cmd == 'ci-setup' || inputs.setup-cmd == 'both' }} + stuart-args: ${{ inputs.stuart-args }} + + - name: Upload CI build logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-logs-${{ matrix.packages }}-${{ matrix.architectures }}-${{ matrix.targets }}-${{ inputs.runner }} + path: ${{ steps.package-ci.outputs.log-path }} +{% endraw %}