run.yml

name: Run Benchpark and Simple Benchmark Suite
on:
  workflow_call:
    secrets:
      BENCHPARK_CODECOV_TOKEN:
        required: true
jobs:
  saxpy:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Build Saxpy Workspace
        run: |
          ./bin/benchpark system init --dest=generic-x86 generic-x86
          ./bin/benchpark experiment init --dest=saxpy/openmp generic-x86 saxpy+openmp
          ./bin/benchpark setup generic-x86/saxpy/openmp workspace/
      - name: Setup Ramble & Spack
        run: |
          . workspace/setup.sh
          spack mirror add ci-buildcache oci://ghcr.io/llnl/benchpark-binary-cache
          spack config add "packages:all:target:[x86_64_v3]"
          env | grep SPACK >> "$GITHUB_ENV"
          env | grep -P 'RAMBLE(?!_WORKSPACE)' >> "$GITHUB_ENV"
          echo "PATH=$PATH" >> "$GITHUB_ENV"
      - name: Setup Saxpy Workspace
        working-directory: ./workspace/generic-x86/saxpy/openmp/workspace/
        run: |
          ramble \
            --workspace-dir . \
            --disable-logger \
            -c config:spack_flags:install:'--no-check-signature' \
            workspace setup
      - name: Run Saxpy Experiments
        working-directory: ./workspace/generic-x86/saxpy/openmp/workspace/
        run: |
          ramble \
            --workspace-dir . \
            --disable-logger \
            on \
            --executor '{execute_experiment}' \
            --where '{n_nodes} == 1'
      - name: Analyze Saxpy Results
        working-directory: ./workspace/generic-x86/saxpy/openmp/workspace/
        run: |
          ramble \
            --workspace-dir . \
            --disable-logger \
            workspace analyze
      - name: Archive Experiment Workspace
        working-directory: ./workspace/generic-x86/saxpy/openmp/workspace/
        if: always()
        run: |
          ramble \
            --workspace-dir . \
            --disable-logger \
            workspace archive
      - name: Upload Workspace Archive as CI Artifact
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
        with:
          name: workspace-archive
          path: ./workspace/generic-x86/saxpy/openmp/workspace/archive/**
      - name: Upload Binaries to CI Cache
        if: github.ref == 'refs/heads/develop'
        env:
          SPACK_OCI_USER: ${{ github.actor }}
          SPACK_OCI_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          spack mirror set --push \
            --oci-username-variable SPACK_OCI_USER \
            --oci-password-variable SPACK_OCI_TOKEN \
            ci-buildcache
          spack buildcache push \
            -j $(($(nproc) + 1)) \
            --base-image ubuntu:24.04 \
            --unsigned \
            --update-index ci-buildcache \
            $(spack find --format '/{hash}')
      - name: Test Execution Using Binary
        run: |
          OUTPUT=$(spack find --paths saxpy)
          SAXPY_PATH=$(echo "$OUTPUT" | grep 'saxpy@' | awk '{print $2}')/bin/saxpy
          echo "Extracted Path: $SAXPY_PATH"
          BENCHPARK_RUN_COVERAGE=binary ./bin/benchpark experiment init --dest=saxpy/binary generic-x86 saxpy+openmp package_manager="user-managed" append_path="$SAXPY_PATH"
          ./bin/benchpark setup generic-x86/saxpy/binary workspace-binary/
          ./workspace-binary/ramble/bin/ramble \
            --workspace-dir \
            /home/runner/work/benchpark/benchpark/workspace-binary/generic-x86/saxpy/binary/workspace \
            workspace setup
          ./workspace-binary/ramble/bin/ramble \
            --workspace-dir \
            /home/runner/work/benchpark/benchpark/workspace-binary/generic-x86/saxpy/binary/workspace \
            on \
            --executor '{execute_experiment}' \
            --where '{n_nodes} == 1'
      - name: Upload binary coverage to Codecov
        if: ${{ !contains(github.head_ref, 'dependabot/') }}
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
          directory: ./coverage-data-binary
          flags: dryrunexperiments-binary
          verbose: true
          fail_ci_if_error: true
      - name: Clean up binary coverage artifacts
        run: |
          echo "Cleaning up binary coverage artifacts"
          rm -rf ./coverage-data-binary
      - name: Test aggregate Command
        run: |
          ./bin/benchpark system init --dest=dane llnl-cluster cluster=dane
          ./bin/benchpark experiment init --dest=saxpy-agg dane saxpy n_repeats=5
          ./bin/benchpark setup dane/saxpy-agg workspace2/
          ./workspace2/ramble/bin/ramble --workspace-dir /home/runner/work/benchpark/benchpark/workspace2/dane/saxpy-agg/workspace workspace setup --dry-run
          BENCHPARK_RUN_COVERAGE=aggregate ./bin/benchpark aggregate --dest agg workspace2  # Aggregate multi-trial workspace
      - name: Upload aggregate coverage to Codecov
        if: ${{ !contains(github.head_ref, 'dependabot/') }}
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
          directory: ./coverage-data-aggregate
          flags: dryrunexperiments-aggregate
          verbose: true
          fail_ci_if_error: true
      - name: Clean up aggregate coverage artifacts
        run: |
          echo "Cleaning up aggregate coverage artifacts"
          rm -rf ./coverage-data-aggregate
  dryrunexperiments:
    runs-on: ubuntu-24.04
    strategy:
      # Don't fail other running testtype tests
      fail-fast: false
      matrix:
        testtype:
          - cuda
          - rocm
          - openmp
          - mpi
          - strong
          - weak
          - throughput
          - modifiers
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Run Dryruns
        run: |
          ./bin/benchpark bootstrap
          BENCHPARK_RUN_COVERAGE=${{ matrix.testtype }} ./bin/benchpark-python .github/utils/dryruns.py --test ${{ matrix.testtype }}
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.head_ref, 'dependabot/') }}
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
          directory: ./coverage-data-${{ matrix.testtype }}
          flags: dryrunexperiments-${{ matrix.testtype }}
          verbose: true
          fail_ci_if_error: true
  pytest:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Run
        run: |-
          ./bin/benchpark unit-test
  test_bootstrap:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Run
        run: |-
          output=$(./bin/benchpark configure --bootstrap-location ./bootstrap && ./bin/benchpark bootstrap 2>&1)

          # 1. Check the path is present
          echo "$output" | grep -q "/home/runner/work/benchpark/benchpark/bootstrap/ramble" \
            || { echo "Expected path not found"; exit 1; }

          # 2. Check "Cloning Ramble" appears exactly once
          count=$(echo "$output" | grep -c "Cloning Ramble")
          if [[ "$count" -ne 1 ]]; then
            echo "'Cloning Ramble' appeared $count times (expected 1)"
            exit 1
          fi
  validate_docs:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      - name: Setup Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
        with:
          python-version: '3.12'
          cache: pip
          cache-dependency-path: .github/workflows/requirements/docs.txt
      - name: Install deps
        run: |
          pip install -r .github/workflows/requirements/docs.txt
      - name: Test Docs Scripts
        run: |-
          cd docs/
          python generate-sys-defs-list.py
          python generate-benchmark-list.py