From f3163f3d28fa2949163979298d3f88757f7a309d Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:20:45 +0000 Subject: [PATCH 01/10] Resurrection with new tools --- .github/workflows/_sbom-benchmark.yml | 327 ++++++++++++++++++++++++++ .github/workflows/docker.yml | 199 +--------------- .github/workflows/go.yml | 23 ++ .github/workflows/java.yml | 23 ++ .github/workflows/javascript.yml | 23 ++ .github/workflows/python.yml | 253 +------------------- .github/workflows/rust.yml | 23 ++ README.md | 66 +++++- go/README.md | 39 +++ java/README.md | 39 +++ javascript/README.md | 40 ++++ python/README.md | 15 +- python/requirements.txt | 4 + rust/README.md | 40 ++++ 14 files changed, 669 insertions(+), 445 deletions(-) create mode 100644 .github/workflows/_sbom-benchmark.yml create mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/java.yml create mode 100644 .github/workflows/javascript.yml create mode 100644 .github/workflows/rust.yml create mode 100644 go/README.md create mode 100644 java/README.md create mode 100644 javascript/README.md create mode 100644 rust/README.md diff --git a/.github/workflows/_sbom-benchmark.yml b/.github/workflows/_sbom-benchmark.yml new file mode 100644 index 0000000..f25af1b --- /dev/null +++ b/.github/workflows/_sbom-benchmark.yml @@ -0,0 +1,327 @@ +--- +name: SBOM Benchmark (Reusable) + +on: + workflow_call: + inputs: + name: + description: 'Benchmark name (used for artifact prefix and display)' + required: true + type: string + target_type: + description: 'Type of target: filesystem or docker' + required: true + type: string + target_path: + description: 'Path to scan (filesystem) or image name (docker)' + required: true + type: string + setup_commands: + description: 'Commands to run before scanning (e.g., clone repo, build image)' + required: false + type: string + default: '' + sbomify_input: + description: 'Input for sbomify: LOCK_FILE path or DOCKER_IMAGE name' + required: true + type: string + sbomify_input_type: + description: 'Type of sbomify input: lock_file or docker_image' + required: true + type: string + baseline_command: + description: 'Command to generate baseline (optional)' + required: false + type: string + default: '' + extra_tools: + description: 'Additional tools to run (comma-separated: cyclonedx-python,sbom4python)' + required: false + type: string + default: '' + title: + description: 'Title for the summary table' + required: false + type: string + default: '' + +env: + TRIVY_VERSION: 0.67.2 + SYFT_VERSION: 1.39.0 + SBOMQS_VERSION: 2.0.2 + CYCLONEDX_PYTHON_VERSION: 7.2.1 + SBOM4PYTHON_VERSION: 0.12.4 + +jobs: + trivy: + name: Trivy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup + if: inputs.setup_commands != '' + run: ${{ inputs.setup_commands }} + + - name: Install Trivy + run: | + curl -L -o /tmp/trivy.tgz \ + "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" + tar xvf /tmp/trivy.tgz -C /tmp + chmod +x /tmp/trivy + + - name: "CycloneDX: Generate SBOM" + run: | + /tmp/trivy ${{ inputs.target_type == 'docker' && 'image' || 'fs' }} \ + --format cyclonedx \ + --output /tmp/trivy.cdx.json \ + ${{ inputs.target_path }} + + - name: "SPDX: Generate SBOM" + run: | + /tmp/trivy ${{ inputs.target_type == 'docker' && 'image' || 'fs' }} \ + --format spdx-json \ + --output /tmp/trivy.spdx.json \ + ${{ inputs.target_path }} + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-trivy + path: "/tmp/trivy.*.json" + + syft: + name: Syft + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup + if: inputs.setup_commands != '' + run: ${{ inputs.setup_commands }} + + - name: Install Syft + run: | + curl -L -o /tmp/syft.tgz \ + "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_amd64.tar.gz" + tar xvf /tmp/syft.tgz -C /tmp + chmod +x /tmp/syft + + - name: "CycloneDX: Generate SBOM" + run: | + /tmp/syft ${{ inputs.target_path }} \ + -o cyclonedx-json > /tmp/syft.cdx.json + + - name: "SPDX: Generate SBOM" + run: | + /tmp/syft ${{ inputs.target_path }} \ + -o spdx-json > /tmp/syft.spdx.json + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-syft + path: "/tmp/syft.*.json" + + sbomify: + name: sbomify + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup + if: inputs.setup_commands != '' + run: ${{ inputs.setup_commands }} + + - name: "CycloneDX: Generate SBOM" + uses: sbomify/github-action@master + env: + LOCK_FILE: ${{ inputs.sbomify_input_type == 'lock_file' && inputs.sbomify_input || '' }} + DOCKER_IMAGE: ${{ inputs.sbomify_input_type == 'docker_image' && inputs.sbomify_input || '' }} + OUTPUT_FILE: /tmp/sbomify.cdx.json + UPLOAD: "false" + AUGMENT: "false" + ENRICH: "true" + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-sbomify + path: "/tmp/sbomify.cdx.json" + + cyclonedx-python: + name: CycloneDX Python + if: contains(inputs.extra_tools, 'cyclonedx-python') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install CycloneDX Python + run: | + python -m pip install cyclonedx-bom==${CYCLONEDX_PYTHON_VERSION} + + - name: "CycloneDX: Generate SBOM" + run: | + cyclonedx-py requirements ${{ inputs.sbomify_input }} \ + --schema-version 1.6 > /tmp/cyclonedx.cdx.json + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-cyclonedx + path: "/tmp/cyclonedx.*.json" + + sbom4python: + name: sbom4python + if: contains(inputs.extra_tools, 'sbom4python') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install sbom4python + run: | + python -m pip install sbom4python==${SBOM4PYTHON_VERSION} + + - name: "CycloneDX: Generate SBOM" + run: | + sbom4python -r ${{ inputs.sbomify_input }} \ + --sbom cyclonedx --format json \ + -o /tmp/sbom4python.cdx.json + + - name: "SPDX: Generate SBOM" + run: | + sbom4python -r ${{ inputs.sbomify_input }} \ + --sbom spdx --format json \ + -o /tmp/sbom4python.spdx.json + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-sbom4python + path: "/tmp/sbom4python.*.json" + + baseline: + name: Baseline + if: inputs.baseline_command != '' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup + if: inputs.setup_commands != '' + run: ${{ inputs.setup_commands }} + + - name: Generate baseline + run: | + ${{ inputs.baseline_command }} > /tmp/baseline.txt + + - name: Upload baseline + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-baseline + path: "/tmp/baseline.txt" + + score: + name: Score + needs: + - trivy + - syft + - sbomify + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download SBOMs + uses: actions/download-artifact@v4 + + - name: Install sbomqs + run: | + curl -L -o /tmp/sbomqs \ + "https://github.com/interlynk-io/sbomqs/releases/download/v${SBOMQS_VERSION}/sbomqs-linux-amd64" + chmod +x /tmp/sbomqs + + - name: Generate Summary Table + env: + BENCHMARK_NAME: ${{ inputs.name }} + BENCHMARK_TITLE: ${{ inputs.title }} + EXTRA_TOOLS: ${{ inputs.extra_tools }} + run: | + set +e # Don't exit on error - some files may not exist + + calc_dup() { + local total=$1 unique=$2 + if [ "$total" -eq 0 ]; then echo "N/A"; return; fi + local dups=$((total - unique)) + [ "$dups" -lt 0 ] && dups=0 + echo "scale=2; ($dups / $total) * 100" | bc | xargs printf "%.2f%%" + } + + score_cdx() { + local path=$1 + if [ ! -f "$path" ]; then echo ""; return; fi + local result=$(/tmp/sbomqs score "$path" -j 2>/dev/null) + local total=$(jq '.components[] | .name' "$path" 2>/dev/null | wc -l) + local unique=$(jq '.components[] | .name' "$path" 2>/dev/null | sort -u | wc -l) + local version=$(echo "$result" | jq -r '.files[0].spec_version // "N/A"') + local score=$(echo "$result" | jq -r '.files[0].avg_score // "N/A"') + echo "$total|$unique|$version|$score" + } + + score_spdx() { + local path=$1 + if [ ! -f "$path" ]; then echo ""; return; fi + local result=$(/tmp/sbomqs score "$path" -j 2>/dev/null) + local total=$(jq '.packages[] | .name' "$path" 2>/dev/null | wc -l) + local unique=$(jq '.packages[] | .name' "$path" 2>/dev/null | sort -u | wc -l) + local version=$(echo "$result" | jq -r '.files[0].spec_version // "N/A"') + local score=$(echo "$result" | jq -r '.files[0].avg_score // "N/A"') + echo "$total|$unique|$version|$score" + } + + add_row() { + local tool=$1 format=$2 data=$3 + if [ -z "$data" ]; then return; fi + IFS='|' read -r total unique version score <<< "$data" + local dup=$(calc_dup "$total" "$unique") + echo "| $tool | $format ($version) | $total | $unique | $dup | $score |" >> ${GITHUB_STEP_SUMMARY} + } + + # Header + if [ -n "$BENCHMARK_TITLE" ]; then + echo "## $BENCHMARK_TITLE" >> ${GITHUB_STEP_SUMMARY} + echo "" >> ${GITHUB_STEP_SUMMARY} + fi + + # Baseline (if exists) + BASELINE_FILE="${BENCHMARK_NAME}-baseline/baseline.txt" + if [ -f "$BASELINE_FILE" ]; then + BASELINE_COUNT=$(wc -l < "$BASELINE_FILE") + echo "**Baseline:** $BASELINE_COUNT packages" >> ${GITHUB_STEP_SUMMARY} + echo "" >> ${GITHUB_STEP_SUMMARY} + fi + + echo "| Tool | Format | Packages | Unique | Duplication % | Quality Score |" >> ${GITHUB_STEP_SUMMARY} + echo "| -- | -- | -- | -- | -- | -- |" >> ${GITHUB_STEP_SUMMARY} + + # Core tools - CycloneDX + add_row "Trivy (${TRIVY_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-trivy/trivy.cdx.json")" + add_row "Syft (${SYFT_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-syft/syft.cdx.json")" + add_row "sbomify" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-sbomify/sbomify.cdx.json")" + + # Extra tools - CycloneDX + if [[ "$EXTRA_TOOLS" == *"cyclonedx-python"* ]]; then + add_row "cyclonedx-python (${CYCLONEDX_PYTHON_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-cyclonedx/cyclonedx.cdx.json")" + fi + if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then + add_row "sbom4python (${SBOM4PYTHON_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-sbom4python/sbom4python.cdx.json")" + fi + + # Core tools - SPDX + add_row "Trivy (${TRIVY_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-trivy/trivy.spdx.json")" + add_row "Syft (${SYFT_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-syft/syft.spdx.json")" + + # Extra tools - SPDX + if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then + add_row "sbom4python (${SBOM4PYTHON_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-sbom4python/sbom4python.spdx.json")" + fi diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 42298b9..cd7470d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,192 +7,19 @@ on: paths: - 'docker/**' - '.github/workflows/docker.yml' - -env: - TRIVY_VERSION: 0.54.1 - SYFT_VERSION: 1.11.1 - SBOMQS_VERSION: 0.1.9 + - '.github/workflows/_sbom-benchmark.yml' jobs: - trivy: - name: Trivy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Build Docker image - working-directory: "docker" - run: - docker build . -t nginx-test - - - name: Install Trivy - run: | - curl -L -o /tmp/trivy.tgz \ - "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" - tar xvf /tmp/trivy.tgz -C /tmp - chmod +x /tmp/trivy - - - name: "CycloneDX: Generate SBOM" - run: | - /tmp/trivy image \ - --format cyclonedx \ - --output /tmp/trivy.cdx.json \ - nginx-test - - - name: "SPDX: Generate SBOM" - run: | - /tmp/trivy image \ - --format spdx-json \ - --output /tmp/trivy.spdx.json \ - nginx-test - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: container-trivy - path: "/tmp/trivy.*.json" - - syft: - name: Syft - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Build Docker image - working-directory: "docker" - run: - docker build . -t nginx-test - - - name: Install Syft - run: | - curl -L -o /tmp/syft.tgz \ - "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_amd64.tar.gz" - tar xvf /tmp/syft.tgz -C /tmp - chmod +x /tmp/syft - - - name: "CycloneDX: Generate SBOM" - run: | - /tmp/syft \ - nginx-test \ - -o cyclonedx-json \ - > /tmp/syft.cdx.json - - - name: "SPDX: Generate SBOM" - run: | - /tmp/syft \ - nginx-test \ - -o spdx-json \ - > /tmp/syft.spdx.json - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: container-syft - path: "/tmp/syft.*.json" - benchmark: - name: Benchmark - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Build Docker image - working-directory: "docker" - run: | - docker build . -t nginx-test - - # Provide a baseline of what system packages Debian think is installed - - name: - run: | - docker run \ - --rm nginx-test \ - dpkg-query -W -f='${binary:Package} ${Version}\n' | sort > /tmp/baseline.txt - - - name: Upload baseline - uses: actions/upload-artifact@v4 - with: - name: baseline - path: "/tmp/baseline.txt" - - Score: - needs: - - trivy - - syft - - benchmark - runs-on: ubuntu-latest - steps: - - name: Download SBOMs - uses: actions/download-artifact@v4 - - - name: Install sbomqs - run: | - curl -L -o /tmp/sbomqs \ - "https://github.com/interlynk-io/sbomqs/releases/download/v${SBOMQS_VERSION}/sbomqs-linux-amd64" - chmod +x /tmp/sbomqs - - - name: "Generate Summary Table" - run: | - calculate_duplication_percentage() { - local total=$1 - local unique=$2 - - if [ "$total" -eq 0 ]; then - echo "Duplication Percentage: N/A (Total count is zero)" - return 1 - fi - - local duplicates=$((total - unique)) - - if [ "$duplicates" -lt 0 ]; then - duplicates=0 - fi - - local duplication_percentage=$(echo "scale=2; ($duplicates / $total) * 100" | bc) - - echo "$duplication_percentage%" - } - - # Baseline - SYSTEM_BASELINE=$(cat baseline/baseline.txt | wc -l) - - # Syft - SYFT_CDX_PATH="container-syft/syft.cdx.json" - SYFT_CDX_TOTAL=$(jq '.components[] | .name' $SYFT_CDX_PATH | wc -l) - SYFT_CDX_UNIQUE=$(jq '.components[] | .name' $SYFT_CDX_PATH | uniq | wc -l) - SYFT_CDX_SBOMQS=$(/tmp/sbomqs score $SYFT_CDX_PATH -j) - SYFT_CDX_VERSION=$(echo $SYFT_CDX_SBOMQS | jq -r '.files[0].spec_version') - SYFT_CDX_QUALITY_SCORE=$(echo $SYFT_CDX_SBOMQS | jq -r '.files[0].avg_score') - - SYFT_SPDX_PATH="container-syft/syft.spdx.json" - SYFT_SPDX_TOTAL=$(jq '.packages[] | .name' $SYFT_SPDX_PATH | wc -l) - SYFT_SPDX_UNIQUE=$(jq '.packages[] | .name' $SYFT_SPDX_PATH | uniq | wc -l) - SYFT_SPDX_SBOMQS=$(/tmp/sbomqs score $SYFT_SPDX_PATH -j) - SYFT_SPDX_VERSION=$(echo $SYFT_SPDX_SBOMQS | jq -r '.files[0].spec_version') - SYFT_SPDX_QUALITY_SCORE=$(echo $SYFT_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - # Trivy - TRIVY_CDX_PATH="container-trivy/trivy.cdx.json" - TRIVY_CDX_TOTAL=$(jq '.components[] | .name' $TRIVY_CDX_PATH | wc -l) - TRIVY_CDX_UNIQUE=$(jq '.components[] | .name' $TRIVY_CDX_PATH | uniq | wc -l) - TRIVY_CDX_SBOMQS=$(/tmp/sbomqs score $TRIVY_CDX_PATH -j) - TRIVY_CDX_VERSION=$(echo $TRIVY_CDX_SBOMQS | jq -r '.files[0].spec_version') - TRIVY_CDX_QUALITY_SCORE=$(echo $TRIVY_CDX_SBOMQS | jq -r '.files[0].avg_score') - - TRIVY_SPDX_PATH="container-trivy/trivy.spdx.json" - TRIVY_SPDX_TOTAL=$(jq '.packages[] | .name' $TRIVY_SPDX_PATH | wc -l) - TRIVY_SPDX_UNIQUE=$(jq '.packages[] | .name' $TRIVY_SPDX_PATH | uniq | wc -l) - TRIVY_SPDX_SBOMQS=$(/tmp/sbomqs score $TRIVY_SPDX_PATH -j) - TRIVY_SPDX_VERSION=$(echo $TRIVY_SPDX_SBOMQS | jq -r '.files[0].spec_version') - TRIVY_SPDX_QUALITY_SCORE=$(echo $TRIVY_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - - # Header - echo "| Tool | Format | Packages | Unique Packages | Duplication % | Avg Quality Score |" >> ${GITHUB_STEP_SUMMARY} - echo "| -- | -- | -- | -- | -- |-- |" >> ${GITHUB_STEP_SUMMARY} - - # Construct table - echo "| System Baseline | N/A | $SYSTEM_BASELINE | $SYSTEM_BASELINE | 0% | N/A |" >> ${GITHUB_STEP_SUMMARY} - echo "| Syft ($SYFT_VERSION) | CycloneDX ($SYFT_CDX_VERSION) | $SYFT_CDX_TOTAL | $SYFT_CDX_UNIQUE | $(calculate_duplication_percentage $SYFT_CDX_TOTAL $SYFT_CDX_UNIQUE) | $SYFT_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Trivy ($TRIVY_VERSION) | CycloneDX ($TRIVY_CDX_VERSION) | $TRIVY_CDX_TOTAL | $TRIVY_CDX_UNIQUE | $(calculate_duplication_percentage $TRIVY_CDX_TOTAL $TRIVY_CDX_UNIQUE) | $TRIVY_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Syft ($SYFT_VERSION) | SPDX ($SYFT_SPDX_VERSION) | $SYFT_SPDX_TOTAL | $SYFT_SPDX_UNIQUE | $(calculate_duplication_percentage $SYFT_SPDX_TOTAL $SYFT_SPDX_UNIQUE) | $SYFT_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Trivy ($TRIVY_VERSION) | SPDX ($TRIVY_SPDX_VERSION) | $TRIVY_SPDX_TOTAL | $TRIVY_SPDX_UNIQUE | $(calculate_duplication_percentage $TRIVY_SPDX_TOTAL $TRIVY_SPDX_UNIQUE)| $TRIVY_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: container + target_type: docker + target_path: nginx-test + setup_commands: | + cd docker && docker build . -t nginx-test + sbomify_input: nginx-test + sbomify_input_type: docker_image + baseline_command: | + docker run --rm nginx-test dpkg-query -W -f='${binary:Package} ${Version}\n' | sort + title: Docker (nginx + vim) Benchmark diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..b0a216d --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,23 @@ +--- +name: Go +on: + push: + branches: + - master + paths: + - 'go/**' + - '.github/workflows/go.yml' + - '.github/workflows/_sbom-benchmark.yml' + +jobs: + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: go + target_type: filesystem + target_path: /tmp/osv-scanner + setup_commands: | + git clone --depth 1 --branch v2.3.1 https://github.com/google/osv-scanner.git /tmp/osv-scanner + sbomify_input: /tmp/osv-scanner/go.mod + sbomify_input_type: lock_file + title: Go (OSV Scanner 2.3.1) Benchmark diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml new file mode 100644 index 0000000..6d73068 --- /dev/null +++ b/.github/workflows/java.yml @@ -0,0 +1,23 @@ +--- +name: Java +on: + push: + branches: + - master + paths: + - 'java/**' + - '.github/workflows/java.yml' + - '.github/workflows/_sbom-benchmark.yml' + +jobs: + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: java + target_type: filesystem + target_path: /tmp/keycloak + setup_commands: | + git clone --depth 1 --branch 26.4.7 https://github.com/keycloak/keycloak.git /tmp/keycloak + sbomify_input: /tmp/keycloak/pom.xml + sbomify_input_type: lock_file + title: Java (Keycloak 26.4.7) Benchmark diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml new file mode 100644 index 0000000..1478aed --- /dev/null +++ b/.github/workflows/javascript.yml @@ -0,0 +1,23 @@ +--- +name: JavaScript +on: + push: + branches: + - master + paths: + - 'javascript/**' + - '.github/workflows/javascript.yml' + - '.github/workflows/_sbom-benchmark.yml' + +jobs: + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: javascript + target_type: filesystem + target_path: /tmp/workers-sdk + setup_commands: | + git clone --depth 1 --branch wrangler@3.99.0 https://github.com/cloudflare/workers-sdk.git /tmp/workers-sdk + sbomify_input: /tmp/workers-sdk/pnpm-lock.yaml + sbomify_input_type: lock_file + title: JavaScript (Cloudflare workers-sdk) Benchmark diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9c8f713..60128c7 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -7,249 +7,16 @@ on: paths: - 'python/**' - '.github/workflows/python.yml' - -env: - TRIVY_VERSION: 0.54.1 - SYFT_VERSION: 1.11.1 - CYCLONEDX_PYTHON_VERSION: 4.5.0 - SBOM4PYTHON_VERSION: 0.11.1 - SBOMQS_VERSION: 0.1.9 + - '.github/workflows/_sbom-benchmark.yml' jobs: - trivy: - name: Trivy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Trivy - run: | - curl -L -o /tmp/trivy.tgz \ - "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" - tar xvf /tmp/trivy.tgz -C /tmp - chmod +x /tmp/trivy - - - name: "CycloneDX: Generate SBOM" - run: | - /tmp/trivy fs \ - --format cyclonedx \ - --output /tmp/trivy.cdx.json \ - python/requirements.txt - - - name: "SPDX: Generate SBOM" - run: | - /tmp/trivy fs \ - --format spdx-json \ - --output /tmp/trivy.spdx.json \ - python/requirements.txt - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: python-trivy - path: "/tmp/trivy.*.json" - - syft: - name: Syft - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Syft - run: | - curl -L -o /tmp/syft.tgz \ - "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_amd64.tar.gz" - tar xvf /tmp/syft.tgz -C /tmp - chmod +x /tmp/syft - - - name: "CycloneDX: Generate SBOM" - run: | - /tmp/syft python/requirements.txt \ - -o cyclonedx-json > \ - /tmp/syft.cdx.json - - - name: "SPDX: Generate SBOM" - run: | - /tmp/syft python/requirements.txt \ - -o spdx-json > \ - /tmp/syft.spdx.json - - - name: Upload SBOM - uses: actions/upload-artifact@v4 + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml with: - name: python-syft - path: "/tmp/syft.*.json" - - cyclonedx-python: - name: CycloneDX Python - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install CycloneDX Python - run: | - python -m pip install \ - cyclonedx-bom==${CYCLONEDX_PYTHON_VERSION} - - - name: "CycloneDX: Generate SBOM" - run: | - cyclonedx-py requirements \ - python/requirements.txt \ - --schema-version 1.6 \ - > /tmp/cyclonedx.cdx.json - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: python-cyclonedx - path: "/tmp/cyclonedx.*.json" - - sbom4python: - name: sbom4python - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install sbom4python - run: | - python -m pip install \ - sbom4python==${SBOM4PYTHON_VERSION} - - - name: "CycloneDX: Generate SBOM" - run: | - sbom4python \ - -r python/requirements.txt \ - --sbom cyclonedx \ - --format json \ - -o /tmp/sbom4python.cdx.json - - - name: "SPDX: Generate SBOM" - run: | - sbom4python \ - -r python/requirements.txt \ - --sbom spdx \ - --format json \ - -o /tmp/sbom4python.spdx.json - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: sbom4python - path: "/tmp/sbom4python.*.json" - - Score: - needs: - - trivy - - syft - - sbom4python - - cyclonedx-python - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Download SBOMs - uses: actions/download-artifact@v4 - - - - name: Install sbomqs - run: | - curl -L -o /tmp/sbomqs \ - "https://github.com/interlynk-io/sbomqs/releases/download/v${SBOMQS_VERSION}/sbomqs-linux-amd64" - chmod +x /tmp/sbomqs - - - name: "Generate Summary Table" - run: | - # DEBUG - find . - - calculate_duplication_percentage() { - local total=$1 - local unique=$2 - - if [ "$total" -eq 0 ]; then - echo "Duplication Percentage: N/A (Total count is zero)" - return 1 - fi - - local duplicates=$((total - unique)) - - if [ "$duplicates" -lt 0 ]; then - duplicates=0 - fi - - local duplication_percentage=$(echo "scale=2; ($duplicates / $total) * 100" | bc) - - echo "$duplication_percentage%" - } - - - - # Baseline - SYSTEM_BASELINE=$(cat python/requirements.txt | wc -l) - - # Syft - SYFT_CDX_PATH="python-syft/syft.cdx.json" - SYFT_CDX_TOTAL=$(jq '.components[] | .name' $SYFT_CDX_PATH | wc -l) - SYFT_CDX_UNIQUE=$(jq '.components[] | .name' $SYFT_CDX_PATH | uniq | wc -l) - SYFT_CDX_SBOMQS=$(/tmp/sbomqs score $SYFT_CDX_PATH -j) - SYFT_CDX_VERSION=$(echo $SYFT_CDX_SBOMQS | jq -r '.files[0].spec_version') - SYFT_CDX_QUALITY_SCORE=$(echo $SYFT_CDX_SBOMQS | jq -r '.files[0].avg_score') - - SYFT_SPDX_PATH="python-syft/syft.spdx.json" - SYFT_SPDX_TOTAL=$(jq '.packages[] | .name' $SYFT_SPDX_PATH | wc -l) - SYFT_SPDX_UNIQUE=$(jq '.packages[] | .name' $SYFT_SPDX_PATH | uniq | wc -l) - SYFT_SPDX_SBOMQS=$(/tmp/sbomqs score $SYFT_SPDX_PATH -j) - SYFT_SPDX_VERSION=$(echo $SYFT_SPDX_SBOMQS | jq -r '.files[0].spec_version') - SYFT_SPDX_QUALITY_SCORE=$(echo $SYFT_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - # Trivy - TRIVY_CDX_PATH="python-trivy/trivy.cdx.json" - TRIVY_CDX_TOTAL=$(jq '.components[] | .name' $TRIVY_CDX_PATH | wc -l) - TRIVY_CDX_UNIQUE=$(jq '.components[] | .name' $TRIVY_CDX_PATH | uniq | wc -l) - TRIVY_CDX_SBOMQS=$(/tmp/sbomqs score $TRIVY_CDX_PATH -j) - TRIVY_CDX_VERSION=$(echo $TRIVY_CDX_SBOMQS | jq -r '.files[0].spec_version') - TRIVY_CDX_QUALITY_SCORE=$(echo $TRIVY_CDX_SBOMQS | jq -r '.files[0].avg_score') - - TRIVY_SPDX_PATH="python-trivy/trivy.spdx.json" - TRIVY_SPDX_TOTAL=$(jq '.packages[] | .name' $TRIVY_SPDX_PATH | wc -l) - TRIVY_SPDX_UNIQUE=$(jq '.packages[] | .name' $TRIVY_SPDX_PATH | uniq | wc -l) - TRIVY_SPDX_SBOMQS=$(/tmp/sbomqs score $TRIVY_SPDX_PATH -j) - TRIVY_SPDX_VERSION=$(echo $TRIVY_SPDX_SBOMQS | jq -r '.files[0].spec_version') - TRIVY_SPDX_QUALITY_SCORE=$(echo $TRIVY_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - # sbom4python - SBOM4PYTHON_CDX_PATH="python-trivy/trivy.cdx.json" - SBOM4PYTHON_CDX_TOTAL=$(jq '.components[] | .name' $SBOM4PYTHON_CDX_PATH | wc -l) - SBOM4PYTHON_CDX_UNIQUE=$(jq '.components[] | .name' $SBOM4PYTHON_CDX_PATH | uniq | wc -l) - SBOM4PYTHON_CDX_SBOMQS=$(/tmp/sbomqs score $SBOM4PYTHON_CDX_PATH -j) - SBOM4PYTHON_CDX_VERSION=$(echo $SBOM4PYTHON_CDX_SBOMQS | jq -r '.files[0].spec_version') - SBOM4PYTHON_CDX_QUALITY_SCORE=$(echo $SBOM4PYTHON_CDX_SBOMQS | jq -r '.files[0].avg_score') - - SBOM4PYTHON_SPDX_PATH="python-trivy/trivy.spdx.json" - SBOM4PYTHON_SPDX_TOTAL=$(jq '.packages[] | .name' $SBOM4PYTHON_SPDX_PATH | wc -l) - SBOM4PYTHON_SPDX_UNIQUE=$(jq '.packages[] | .name' $SBOM4PYTHON_SPDX_PATH | uniq | wc -l) - SBOM4PYTHON_SPDX_SBOMQS=$(/tmp/sbomqs score $SBOM4PYTHON_SPDX_PATH -j) - SBOM4PYTHON_SPDX_VERSION=$(echo $SBOM4PYTHON_SPDX_SBOMQS | jq -r '.files[0].spec_version') - SBOM4PYTHON_SPDX_QUALITY_SCORE=$(echo $SBOM4PYTHON_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - # CycloneDX Python - PYTHON_CYCLONEDX_CDX_PATH="python-cyclonedx/cyclonedx.cdx.json" - PYTHON_CYCLONEDX_CDX_TOTAL=$(jq '.components[] | .name' $PYTHON_CYCLONEDX_CDX_PATH | wc -l) - PYTHON_CYCLONEDX_CDX_UNIQUE=$(jq '.components[] | .name' $PYTHON_CYCLONEDX_CDX_PATH | uniq | wc -l) - PYTHON_CYCLONEDX_CDX_SBOMQS=$(/tmp/sbomqs score $PYTHON_CYCLONEDX_CDX_PATH -j) - PYTHON_CYCLONEDX_CDX_VERSION=$(echo $PYTHON_CYCLONEDX_CDX_SBOMQS | jq -r '.files[0].spec_version') - PYTHON_CYCLONEDX_CDX_QUALITY_SCORE=$(echo $PYTHON_CYCLONEDX_CDX_SBOMQS | jq -r '.files[0].avg_score') - - # Header - echo "| Tool | Format | Packages | Unique Packages | Duplication % | Avg Quality Score |" >> ${GITHUB_STEP_SUMMARY} - echo "| -- | -- | -- | -- | -- |-- |" >> ${GITHUB_STEP_SUMMARY} - - # Construct table - echo "| System Baseline | N/A | $SYSTEM_BASELINE | $SYSTEM_BASELINE | 0% | N/A |" >> ${GITHUB_STEP_SUMMARY} - echo "| Syft ($SYFT_VERSION) | CycloneDX ($SYFT_CDX_VERSION) | $SYFT_CDX_TOTAL | $SYFT_CDX_UNIQUE | $(calculate_duplication_percentage $SYFT_CDX_TOTAL $SYFT_CDX_UNIQUE) | $SYFT_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Trivy ($TRIVY_VERSION) | CycloneDX ($TRIVY_CDX_VERSION) | $TRIVY_CDX_TOTAL | $TRIVY_CDX_UNIQUE | $(calculate_duplication_percentage $TRIVY_CDX_TOTAL $TRIVY_CDX_UNIQUE) | $TRIVY_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| sbom4python ($SBOM4PYTHON_VERSION) | CycloneDX ($SBOM4PYTHON_CDX_VERSION) | $SBOM4PYTHON_CDX_TOTAL | $SBOM4PYTHON_CDX_UNIQUE | $(calculate_duplication_percentage $SBOM4PYTHON_CDX_TOTAL $SBOM4PYTHON_CDX_UNIQUE) | $SBOM4PYTHON_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| python-cyclonedx ($CYCLONEDX_PYTHON_VERSION) | CycloneDX ($PYTHON_CYCLONEDX_CDX_VERSION) | $PYTHON_CYCLONEDX_CDX_TOTAL | $PYTHON_CYCLONEDX_CDX_UNIQUE | $(calculate_duplication_percentage $PYTHON_CYCLONEDX_CDX_TOTAL $PYTHON_CYCLONEDX_CDX_UNIQUE) | $PYTHON_CYCLONEDX_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Syft ($SYFT_VERSION) | SPDX ($SYFT_SPDX_VERSION) | $SYFT_SPDX_TOTAL | $SYFT_SPDX_UNIQUE | $(calculate_duplication_percentage $SYFT_SPDX_TOTAL $SYFT_SPDX_UNIQUE) | $SYFT_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Trivy ($TRIVY_VERSION) | SPDX ($TRIVY_SPDX_VERSION) | $TRIVY_SPDX_TOTAL | $TRIVY_SPDX_UNIQUE | $(calculate_duplication_percentage $TRIVY_SPDX_TOTAL $TRIVY_SPDX_UNIQUE)| $TRIVY_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| sbom4python ($SBOM4PYTHON_VERSION) | SPDX ($SBOM4PYTHON_SPDX_VERSION) | $SBOM4PYTHON_SPDX_TOTAL | $SBOM4PYTHON_SPDX_UNIQUE | $(calculate_duplication_percentage $SBOM4PYTHON_SPDX_TOTAL $SBOM4PYTHON_SPDX_UNIQUE)| $SBOM4PYTHON_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} + name: python + target_type: filesystem + target_path: python/requirements.txt + sbomify_input: python/requirements.txt + sbomify_input_type: lock_file + extra_tools: cyclonedx-python,sbom4python + title: Python (Django) Benchmark diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..b7ef1c8 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,23 @@ +--- +name: Rust +on: + push: + branches: + - master + paths: + - 'rust/**' + - '.github/workflows/rust.yml' + - '.github/workflows/_sbom-benchmark.yml' + +jobs: + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: rust + target_type: filesystem + target_path: /tmp/quiche + setup_commands: | + git clone --depth 1 --branch 0.24.5 https://github.com/cloudflare/quiche.git /tmp/quiche + sbomify_input: /tmp/quiche/Cargo.lock + sbomify_input_type: lock_file + title: Rust (Cloudflare quiche 0.24.5) Benchmark diff --git a/README.md b/README.md index 92d1d8c..24a8f01 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,72 @@ # SBOM Benchmarking [![Python](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml) +[![JavaScript](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml) +[![Java](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml) +[![Go](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml) +[![Rust](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml) [![Docker](https://github.com/sbomify/sbom-benchmarks/actions/workflows/docker.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/docker.yml) -This repository is designed to generate Software Bill of Materials (SBOMs) using a comprehensive benchmark across a wide variety of tools on a defined target across multiple programming languages. The goal is to provide a consistent and standardized method for evaluating and comparing the effectiveness and accuracy of various SBOM generation tools, helping users to identify the best tool for their specific needs. +This repository is designed to generate Software Bill of Materials (SBOMs) using a comprehensive benchmark across a wide variety of tools on defined targets across multiple programming languages. The goal is to provide a consistent and standardized method for evaluating and comparing the effectiveness and accuracy of various SBOM generation tools, helping users to identify the best tool for their specific needs. The list of tools used is pulled from our [SBOM resources](https://sbomify.com/resources/) page that includes a comprehensive list of SBOM tools. ## Features -* Multi-Tool Support: Run benchmarks across a diverse set of SBOM generation tools. -* Cross-Language Compatibility: Supports multiple programming languages, allowing for comprehensive analysis regardless of the technology stack. -* Automated Workflow: Easily set up and execute benchmarks with minimal manual intervention. -* Detailed Reports: Generate detailed comparisons and summaries of the SBOMs produced by different tools, highlighting strengths and weaknesses. -* Extensibility: Add support for new tools or languages with minimal configuration changes. +* **Multi-Tool Support**: Run benchmarks across a diverse set of SBOM generation tools including Trivy, Syft, and sbomify. +* **Cross-Language Compatibility**: Supports multiple programming languages (Python, JavaScript, Java, Go, Rust) and container images. +* **Automated Workflow**: Easily set up and execute benchmarks with minimal manual intervention. +* **Detailed Reports**: Generate detailed comparisons and summaries of the SBOMs produced by different tools, highlighting strengths and weaknesses. +* **Quality Scoring**: Each SBOM is scored using [sbomqs](https://github.com/interlynk-io/sbomqs) to measure SBOM quality. +* **Extensibility**: Add support for new tools or languages with minimal configuration changes. +## Benchmarked Tools + +| Tool | Description | +|------|-------------| +| [Trivy](https://github.com/aquasecurity/trivy) | Comprehensive security scanner with SBOM generation | +| [Syft](https://github.com/anchore/syft) | CLI tool and library for generating SBOMs | +| [sbomify](https://github.com/sbomify/github-action) | SBOM generation with enrichment from package registries | +| [cyclonedx-python](https://github.com/CycloneDX/cyclonedx-python) | Native Python SBOM generator (Python benchmarks only) | +| [sbom4python](https://github.com/anthonyharrison/sbom4python) | Python-specific SBOM generator (Python benchmarks only) | + +## Benchmark Targets + +| Target | Language/Type | Project | Description | +|--------|--------------|---------|-------------| +| [Python](python/) | Python | Django | Python web framework dependencies | +| [JavaScript](javascript/) | JavaScript/TypeScript | workers-sdk | Cloudflare's Wrangler CLI monorepo | +| [Java](java/) | Java/Maven | Keycloak | Enterprise IAM with complex Maven dependencies | +| [Go](go/) | Go | OSV Scanner | Go modules-based security tool | +| [Rust](rust/) | Rust | quiche | Cloudflare's QUIC/HTTP3 implementation | +| [Docker](docker/) | Container | nginx + vim | Container image with added packages | ## Run Details -* [Python](https://github.com/sbomify/sbom-benchmarks/tree/master/python) -* [Docker](https://github.com/sbomify/sbom-benchmarks/tree/master/docker) +Each benchmark runs automatically on push to master and produces: +- SBOMs in both CycloneDX and SPDX formats (where supported) +- Quality scores from sbomqs +- Comparison tables in the GitHub Actions summary + +Click on any badge above to see the latest benchmark results. + +### Detailed Documentation + +* [Python Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/python) - Django requirements.txt +* [JavaScript Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/javascript) - Cloudflare workers-sdk (pnpm-lock.yaml) +* [Java Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/java) - Keycloak (Maven/pom.xml) +* [Go Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/go) - OSV Scanner (go.mod) +* [Rust Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/rust) - Cloudflare quiche (Cargo.lock) +* [Docker Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/docker) - nginx container with vim installed + +## Tool Versions + +Current tool versions used in benchmarks: + +| Tool | Version | +|------|---------| +| Trivy | 0.67.2 | +| Syft | 1.39.0 | +| sbomqs | 2.0.2 | +| cyclonedx-bom | 7.2.1 | +| sbom4python | 0.12.4 | diff --git a/go/README.md b/go/README.md new file mode 100644 index 0000000..1675fe4 --- /dev/null +++ b/go/README.md @@ -0,0 +1,39 @@ +# SBOM Generation for Go + +[![Go](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml) + +## Target Project + +This benchmark uses [OSV Scanner](https://github.com/google/osv-scanner), Google's vulnerability scanner that uses the OSV database to find known vulnerabilities in project dependencies. It's a production-grade Go project with a well-structured dependency tree. + +**Version:** 2.3.1 + +## Tools + +Tools from the sbomify [resource list](https://sbomify.com/resources/), specifically: + +* Trivy +* Syft +* sbomify github-action + +## Process + +The benchmark workflow: + +1. Clones the OSV Scanner repository at the specified tag +2. Runs each SBOM generator against the project's `go.mod` and source tree +3. Scores each generated SBOM using sbomqs +4. Produces a comparison table in the workflow summary + +The full process is automated and you can see the exact commands in [go.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/go.yml). + +If you look at the [Go CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. + +## Why OSV Scanner? + +OSV Scanner was chosen as a benchmark target because: + +- **Go modules**: Clean Go module dependency management via `go.mod` +- **Security tooling**: Ironically, it's a security tool itself - interesting to SBOM a vulnerability scanner +- **Google-maintained**: High-quality codebase with good dependency hygiene +- **Moderate complexity**: Not too simple, not too complex - good middle-ground benchmark diff --git a/java/README.md b/java/README.md new file mode 100644 index 0000000..af647f1 --- /dev/null +++ b/java/README.md @@ -0,0 +1,39 @@ +# SBOM Generation for Java + +[![Java](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml) + +## Target Project + +This benchmark uses [Keycloak](https://github.com/keycloak/keycloak), an Open Source Identity and Access Management solution for modern applications and services. Keycloak is a large-scale Java/Maven project with complex dependency management, making it an excellent benchmark target. + +**Version:** 26.4.7 + +## Tools + +Tools from the sbomify [resource list](https://sbomify.com/resources/), specifically: + +* Trivy +* Syft +* sbomify github-action + +## Process + +The benchmark workflow: + +1. Clones the Keycloak repository at the specified tag +2. Runs each SBOM generator against the project's `pom.xml` and source tree +3. Scores each generated SBOM using sbomqs +4. Produces a comparison table in the workflow summary + +The full process is automated and you can see the exact commands in [java.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/java.yml). + +If you look at the [Java CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. + +## Why Keycloak? + +Keycloak was chosen as a benchmark target because: + +- **Complex dependency tree**: Hundreds of Maven dependencies across multiple modules +- **Real-world project**: Actively maintained, widely deployed enterprise software +- **Mixed ecosystem**: Combines Java libraries, JavaScript frontend components, and more +- **Multi-module Maven**: Tests SBOM generators' ability to handle complex build structures diff --git a/javascript/README.md b/javascript/README.md new file mode 100644 index 0000000..3cb895c --- /dev/null +++ b/javascript/README.md @@ -0,0 +1,40 @@ +# SBOM Generation for JavaScript + +[![JavaScript](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml) + +## Target Project + +This benchmark uses [workers-sdk](https://github.com/cloudflare/workers-sdk), Cloudflare's monorepo containing Wrangler (the CLI for Cloudflare Workers), Miniflare, and related tooling. It's a large-scale TypeScript/JavaScript project with complex dependency management. + +**Version:** wrangler@3.99.0 + +## Tools + +Tools from the sbomify [resource list](https://sbomify.com/resources/), specifically: + +* Trivy +* Syft +* sbomify github-action + +## Process + +The benchmark workflow: + +1. Clones the workers-sdk repository at the specified tag +2. Runs each SBOM generator against the project's `pnpm-lock.yaml` and source tree +3. Scores each generated SBOM using sbomqs +4. Produces a comparison table in the workflow summary + +The full process is automated and you can see the exact commands in [javascript.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/javascript.yml). + +If you look at the [JavaScript CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. + +## Why workers-sdk? + +workers-sdk was chosen as a benchmark target because: + +- **Monorepo structure**: Multiple packages (Wrangler, Miniflare, C3) in a pnpm workspace +- **Production TypeScript**: Powers Cloudflare's developer tooling used by millions +- **Complex dependencies**: Hundreds of npm packages across multiple workspaces +- **pnpm lockfile**: Tests SBOM generators' support for modern package managers +- **Widely used**: 106k+ dependent repositories on GitHub diff --git a/python/README.md b/python/README.md index 993defd..acec90c 100644 --- a/python/README.md +++ b/python/README.md @@ -1,24 +1,23 @@ # SBOM Generation for Python + [![Python](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml) -## Setup +## Target -Generate `requirements.txt`: +The `requirements.txt` is **intentionally minimal** - it only lists direct dependencies (Django and its immediate deps). This tests each SBOM tool's ability to resolve **transitive dependencies**. -```bash -$ pip install Django -$ pip freeze > requirements.txt -``` +A quality SBOM generator should discover all indirect dependencies, not just what's explicitly listed in the lockfile. ## Tools -Tools from the sbomify [resource list](https://sbomify.com/resources/#Python), but in short they are: +Tools from the sbomify [resource list](https://sbomify.com/resources/#Python): * Trivy * Syft +* sbomify github-action * sbom4python * cyclonedx-python The full process is automated and you can see the exact commands we run in [python.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/python.yml). -If you look at the [Python CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml), you can also see the quality score of the SBOMs (from `sbomqsq`) as well as downloading the actual SBOMs as artifact. +If you look at the [Python CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. diff --git a/python/requirements.txt b/python/requirements.txt index c3c8b20..744b9a6 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,3 +1,7 @@ +# Intentionally minimal - only direct dependencies listed. +# This tests SBOM tools' ability to resolve transitive dependencies. +# A complete SBOM should include all indirect dependencies (e.g., pytz, typing-extensions, etc.) + asgiref==3.8.1 Django==5.1 sqlparse==0.5.1 diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..78fa6be --- /dev/null +++ b/rust/README.md @@ -0,0 +1,40 @@ +# SBOM Generation for Rust + +[![Rust](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml) + +## Target Project + +This benchmark uses [quiche](https://github.com/cloudflare/quiche), Cloudflare's implementation of the QUIC transport protocol and HTTP/3. It's a production-grade Rust project used in Cloudflare's infrastructure. + +**Version:** 0.24.5 + +## Tools + +Tools from the sbomify [resource list](https://sbomify.com/resources/), specifically: + +* Trivy +* Syft +* sbomify github-action + +## Process + +The benchmark workflow: + +1. Clones the quiche repository at the specified tag +2. Runs each SBOM generator against the project's `Cargo.lock` and source tree +3. Scores each generated SBOM using sbomqs +4. Produces a comparison table in the workflow summary + +The full process is automated and you can see the exact commands in [rust.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/rust.yml). + +If you look at the [Rust CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. + +## Why quiche? + +quiche was chosen as a benchmark target because: + +- **Production Rust**: Used in Cloudflare's production infrastructure serving millions of requests +- **Complex dependencies**: Multiple crates with native dependencies (BoringSSL) +- **Workspace structure**: Multi-crate Cargo workspace tests SBOM generators' Rust support +- **Well-maintained**: Active development with regular releases +- **Popular**: 11k+ GitHub stars, widely used in the Rust ecosystem From df2cf4bf835caf1a0d31d1140a317cb9acf7f4bf Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:22:22 +0000 Subject: [PATCH 02/10] Fixes formatting --- .github/workflows/docker.yml | 5 +++++ .github/workflows/go.yml | 5 +++++ .github/workflows/java.yml | 5 +++++ .github/workflows/javascript.yml | 5 +++++ .github/workflows/python.yml | 7 ++++++- .github/workflows/rust.yml | 5 +++++ 6 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cd7470d..c2b1aed 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,6 +8,11 @@ on: - 'docker/**' - '.github/workflows/docker.yml' - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'docker/**' + - '.github/workflows/docker.yml' + - '.github/workflows/_sbom-benchmark.yml' jobs: benchmark: diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b0a216d..0905128 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,6 +8,11 @@ on: - 'go/**' - '.github/workflows/go.yml' - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'go/**' + - '.github/workflows/go.yml' + - '.github/workflows/_sbom-benchmark.yml' jobs: benchmark: diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 6d73068..183c891 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -8,6 +8,11 @@ on: - 'java/**' - '.github/workflows/java.yml' - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'java/**' + - '.github/workflows/java.yml' + - '.github/workflows/_sbom-benchmark.yml' jobs: benchmark: diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index 1478aed..fcfea2c 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -8,6 +8,11 @@ on: - 'javascript/**' - '.github/workflows/javascript.yml' - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'javascript/**' + - '.github/workflows/javascript.yml' + - '.github/workflows/_sbom-benchmark.yml' jobs: benchmark: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 60128c7..b937454 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -8,11 +8,16 @@ on: - 'python/**' - '.github/workflows/python.yml' - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'python/**' + - '.github/workflows/python.yml' + - '.github/workflows/_sbom-benchmark.yml' jobs: benchmark: uses: ./.github/workflows/_sbom-benchmark.yml - with: + with: name: python target_type: filesystem target_path: python/requirements.txt diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b7ef1c8..7e376ac 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -8,6 +8,11 @@ on: - 'rust/**' - '.github/workflows/rust.yml' - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'rust/**' + - '.github/workflows/rust.yml' + - '.github/workflows/_sbom-benchmark.yml' jobs: benchmark: From 77160e7bb61f428944cb18d11cc595ea16384ec0 Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:31:42 +0000 Subject: [PATCH 03/10] Fix paths, bumps trivy and improves formatting --- .github/workflows/_sbom-benchmark.yml | 300 ++++++++++++++++++++++---- .github/workflows/go.yml | 6 +- .github/workflows/java.yml | 6 +- .github/workflows/javascript.yml | 6 +- .github/workflows/rust.yml | 6 +- README.md | 2 +- 6 files changed, 277 insertions(+), 49 deletions(-) diff --git a/.github/workflows/_sbom-benchmark.yml b/.github/workflows/_sbom-benchmark.yml index f25af1b..cf32120 100644 --- a/.github/workflows/_sbom-benchmark.yml +++ b/.github/workflows/_sbom-benchmark.yml @@ -46,7 +46,7 @@ on: default: '' env: - TRIVY_VERSION: 0.67.2 + TRIVY_VERSION: 0.68.2 SYFT_VERSION: 1.39.0 SBOMQS_VERSION: 2.0.2 CYCLONEDX_PYTHON_VERSION: 7.2.1 @@ -133,7 +133,7 @@ jobs: if: inputs.setup_commands != '' run: ${{ inputs.setup_commands }} - - name: "CycloneDX: Generate SBOM" + - name: Generate SBOM uses: sbomify/github-action@master env: LOCK_FILE: ${{ inputs.sbomify_input_type == 'lock_file' && inputs.sbomify_input || '' }} @@ -163,7 +163,7 @@ jobs: - name: "CycloneDX: Generate SBOM" run: | cyclonedx-py requirements ${{ inputs.sbomify_input }} \ - --schema-version 1.6 > /tmp/cyclonedx.cdx.json + -o /tmp/cyclonedx.cdx.json - name: Upload SBOM uses: actions/upload-artifact@v4 @@ -249,60 +249,244 @@ jobs: run: | set +e # Don't exit on error - some files may not exist + # Helper: safely convert value to integer, defaulting to 0 + to_int() { + local val="${1:-0}" + # Remove any non-numeric characters and handle null/empty + val="${val//[^0-9]/}" + echo "${val:-0}" + } + + # Helper: calculate duplication percentage calc_dup() { - local total=$1 unique=$2 - if [ "$total" -eq 0 ]; then echo "N/A"; return; fi + local total=$(to_int "$1") + local unique=$(to_int "$2") + if [[ "$total" -eq 0 ]]; then + echo "N/A" + return + fi local dups=$((total - unique)) - [ "$dups" -lt 0 ] && dups=0 - echo "scale=2; ($dups / $total) * 100" | bc | xargs printf "%.2f%%" + [[ "$dups" -lt 0 ]] && dups=0 + printf "%.2f%%" "$(echo "scale=2; $dups * 100 / $total" | bc)" + } + + # Helper: calculate percentage safely + calc_pct() { + local count=$(to_int "$1") + local total=$(to_int "$2") + if [[ "$total" -eq 0 ]]; then + echo "0" + return + fi + echo "scale=0; $count * 100 / $total" | bc } + # Score CycloneDX SBOM score_cdx() { - local path=$1 - if [ ! -f "$path" ]; then echo ""; return; fi - local result=$(/tmp/sbomqs score "$path" -j 2>/dev/null) - local total=$(jq '.components[] | .name' "$path" 2>/dev/null | wc -l) - local unique=$(jq '.components[] | .name' "$path" 2>/dev/null | sort -u | wc -l) - local version=$(echo "$result" | jq -r '.files[0].spec_version // "N/A"') - local score=$(echo "$result" | jq -r '.files[0].avg_score // "N/A"') - echo "$total|$unique|$version|$score" + local path="$1" + [[ ! -f "$path" ]] && return + + # Get sbomqs results + local result + result=$(/tmp/sbomqs score "$path" -j 2>/dev/null) || result="{}" + + # Extract component names once, then count + local names + names=$(jq -r '.components[]?.name // empty' "$path" 2>/dev/null) + local total=$(echo "$names" | grep -c . || echo 0) + local unique=$(echo "$names" | sort -u | grep -c . || echo 0) + + local version + version=$(echo "$result" | jq -r '.files[0].spec_version // "N/A"' 2>/dev/null) || version="N/A" + local score + score=$(echo "$result" | jq -r '.files[0].avg_score // "N/A"' 2>/dev/null) || score="N/A" + + echo "${total}|${unique}|${version}|${score}" } + # Score SPDX SBOM score_spdx() { - local path=$1 - if [ ! -f "$path" ]; then echo ""; return; fi - local result=$(/tmp/sbomqs score "$path" -j 2>/dev/null) - local total=$(jq '.packages[] | .name' "$path" 2>/dev/null | wc -l) - local unique=$(jq '.packages[] | .name' "$path" 2>/dev/null | sort -u | wc -l) - local version=$(echo "$result" | jq -r '.files[0].spec_version // "N/A"') - local score=$(echo "$result" | jq -r '.files[0].avg_score // "N/A"') - echo "$total|$unique|$version|$score" + local path="$1" + [[ ! -f "$path" ]] && return + + # Get sbomqs results + local result + result=$(/tmp/sbomqs score "$path" -j 2>/dev/null) || result="{}" + + # Extract package names once, then count + local names + names=$(jq -r '.packages[]?.name // empty' "$path" 2>/dev/null) + local total=$(echo "$names" | grep -c . || echo 0) + local unique=$(echo "$names" | sort -u | grep -c . || echo 0) + + local version + version=$(echo "$result" | jq -r '.files[0].spec_version // "N/A"' 2>/dev/null) || version="N/A" + local score + score=$(echo "$result" | jq -r '.files[0].avg_score // "N/A"' 2>/dev/null) || score="N/A" + + echo "${total}|${unique}|${version}|${score}" + } + + # NTIA Minimum Elements check for CycloneDX + ntia_cdx() { + local path="$1" + [[ ! -f "$path" ]] && return + + # Get total component count + local total + total=$(jq '.components | length // 0' "$path" 2>/dev/null) || total=0 + total=$(to_int "$total") + + # SBOM-level checks + local has_timestamp + has_timestamp=$(jq -r '.metadata.timestamp // empty' "$path" 2>/dev/null) + local has_author + has_author=$(jq -r ' + .metadata.authors[0].name // + .metadata.manufacturer.name // + .metadata.supplier.name // + empty + ' "$path" 2>/dev/null) + + # Component-level NTIA fields - single jq call for efficiency + local counts + counts=$(jq '{ + with_supplier: [.components[]? | select(.supplier.name != null or .publisher != null)] | length, + with_version: [.components[]? | select(.version != null and .version != "")] | length, + with_purl: [.components[]? | select(.purl != null)] | length, + has_deps: (.dependencies | length // 0) + }' "$path" 2>/dev/null) || counts='{"with_supplier":0,"with_version":0,"with_purl":0,"has_deps":0}' + + local with_supplier=$(echo "$counts" | jq -r '.with_supplier // 0') + local with_version=$(echo "$counts" | jq -r '.with_version // 0') + local with_purl=$(echo "$counts" | jq -r '.with_purl // 0') + local has_deps=$(echo "$counts" | jq -r '.has_deps // 0') + + # Calculate percentages + local supplier_pct=$(calc_pct "$with_supplier" "$total") + local version_pct=$(calc_pct "$with_version" "$total") + local purl_pct=$(calc_pct "$with_purl" "$total") + + # Format output with Y/N indicators + local ts_icon="N" + [[ -n "$has_timestamp" ]] && ts_icon="Y" + local author_icon="N" + [[ -n "$has_author" ]] && author_icon="Y" + local deps_icon="N" + [[ $(to_int "$has_deps") -gt 0 ]] && deps_icon="Y" + + echo "${ts_icon}|${author_icon}|${supplier_pct}%|${version_pct}%|${purl_pct}%|${deps_icon}" } + # NTIA Minimum Elements check for SPDX + ntia_spdx() { + local path="$1" + [[ ! -f "$path" ]] && return + + # Get total package count + local total + total=$(jq '.packages | length // 0' "$path" 2>/dev/null) || total=0 + total=$(to_int "$total") + + # SBOM-level checks + local has_timestamp + has_timestamp=$(jq -r '.creationInfo.created // empty' "$path" 2>/dev/null) + local has_author + has_author=$(jq -r '.creationInfo.creators[0] // empty' "$path" 2>/dev/null) + + # Component-level NTIA fields - single jq call for efficiency + local counts + counts=$(jq '{ + with_supplier: [.packages[]? | select(.supplier != null and .supplier != "" and .supplier != "NOASSERTION")] | length, + with_version: [.packages[]? | select(.versionInfo != null and .versionInfo != "" and .versionInfo != "NOASSERTION")] | length, + with_purl: [.packages[]? | select(.externalRefs[]? | .referenceType == "purl")] | length, + has_deps: (.relationships | length // 0) + }' "$path" 2>/dev/null) || counts='{"with_supplier":0,"with_version":0,"with_purl":0,"has_deps":0}' + + local with_supplier=$(echo "$counts" | jq -r '.with_supplier // 0') + local with_version=$(echo "$counts" | jq -r '.with_version // 0') + local with_purl=$(echo "$counts" | jq -r '.with_purl // 0') + local has_deps=$(echo "$counts" | jq -r '.has_deps // 0') + + # Calculate percentages + local supplier_pct=$(calc_pct "$with_supplier" "$total") + local version_pct=$(calc_pct "$with_version" "$total") + local purl_pct=$(calc_pct "$with_purl" "$total") + + # Format output with Y/N indicators + local ts_icon="N" + [[ -n "$has_timestamp" ]] && ts_icon="Y" + local author_icon="N" + [[ -n "$has_author" ]] && author_icon="Y" + local deps_icon="N" + [[ $(to_int "$has_deps") -gt 0 ]] && deps_icon="Y" + + echo "${ts_icon}|${author_icon}|${supplier_pct}%|${version_pct}%|${purl_pct}%|${deps_icon}" + } + + # Add a row to the package detection table add_row() { - local tool=$1 format=$2 data=$3 - if [ -z "$data" ]; then return; fi + local tool="$1" + local format="$2" + local data="$3" + [[ -z "$data" ]] && return + + local total unique version score IFS='|' read -r total unique version score <<< "$data" - local dup=$(calc_dup "$total" "$unique") - echo "| $tool | $format ($version) | $total | $unique | $dup | $score |" >> ${GITHUB_STEP_SUMMARY} + + # Validate we got all fields + [[ -z "$total" ]] && total=0 + [[ -z "$unique" ]] && unique=0 + [[ -z "$version" ]] && version="N/A" + [[ -z "$score" ]] && score="N/A" + + local dup + dup=$(calc_dup "$total" "$unique") + echo "| $tool | $format ($version) | $total | $unique | $dup | $score |" >> "${GITHUB_STEP_SUMMARY}" } + # Add a row to the NTIA table + add_ntia_row() { + local tool="$1" + local format="$2" + local data="$3" + [[ -z "$data" ]] && return + + local ts author supplier version purl deps + IFS='|' read -r ts author supplier version purl deps <<< "$data" + + # Set defaults for empty fields + [[ -z "$ts" ]] && ts="N" + [[ -z "$author" ]] && author="N" + [[ -z "$supplier" ]] && supplier="0%" + [[ -z "$version" ]] && version="0%" + [[ -z "$purl" ]] && purl="0%" + [[ -z "$deps" ]] && deps="N" + + echo "| $tool | $format | $ts | $author | $supplier | $version | $purl | $deps |" >> "${GITHUB_STEP_SUMMARY}" + } + + # === Generate Report === + # Header - if [ -n "$BENCHMARK_TITLE" ]; then - echo "## $BENCHMARK_TITLE" >> ${GITHUB_STEP_SUMMARY} - echo "" >> ${GITHUB_STEP_SUMMARY} + if [[ -n "$BENCHMARK_TITLE" ]]; then + echo "## $BENCHMARK_TITLE" >> "${GITHUB_STEP_SUMMARY}" + echo "" >> "${GITHUB_STEP_SUMMARY}" fi # Baseline (if exists) BASELINE_FILE="${BENCHMARK_NAME}-baseline/baseline.txt" - if [ -f "$BASELINE_FILE" ]; then - BASELINE_COUNT=$(wc -l < "$BASELINE_FILE") - echo "**Baseline:** $BASELINE_COUNT packages" >> ${GITHUB_STEP_SUMMARY} - echo "" >> ${GITHUB_STEP_SUMMARY} + if [[ -f "$BASELINE_FILE" ]]; then + BASELINE_COUNT=$(wc -l < "$BASELINE_FILE" | tr -d ' ') + echo "**Baseline:** ${BASELINE_COUNT} packages" >> "${GITHUB_STEP_SUMMARY}" + echo "" >> "${GITHUB_STEP_SUMMARY}" fi - echo "| Tool | Format | Packages | Unique | Duplication % | Quality Score |" >> ${GITHUB_STEP_SUMMARY} - echo "| -- | -- | -- | -- | -- | -- |" >> ${GITHUB_STEP_SUMMARY} + # === Package Detection Table === + echo "### Package Detection" >> "${GITHUB_STEP_SUMMARY}" + echo "" >> "${GITHUB_STEP_SUMMARY}" + echo "| Tool | Format | Packages | Unique | Duplication % | Quality Score |" >> "${GITHUB_STEP_SUMMARY}" + echo "| -- | -- | -- | -- | -- | -- |" >> "${GITHUB_STEP_SUMMARY}" # Core tools - CycloneDX add_row "Trivy (${TRIVY_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-trivy/trivy.cdx.json")" @@ -325,3 +509,47 @@ jobs: if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then add_row "sbom4python (${SBOM4PYTHON_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-sbom4python/sbom4python.spdx.json")" fi + + # === NTIA Minimum Elements Table === + echo "" >> "${GITHUB_STEP_SUMMARY}" + echo "### NTIA Minimum Elements" >> "${GITHUB_STEP_SUMMARY}" + echo "" >> "${GITHUB_STEP_SUMMARY}" + echo "| Tool | Format | Timestamp | Author | Supplier % | Version % | PURL % | Dependencies |" >> "${GITHUB_STEP_SUMMARY}" + echo "| -- | -- | -- | -- | -- | -- | -- | -- |" >> "${GITHUB_STEP_SUMMARY}" + + # Core tools - CycloneDX + add_ntia_row "Trivy" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-trivy/trivy.cdx.json")" + add_ntia_row "Syft" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-syft/syft.cdx.json")" + add_ntia_row "sbomify" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-sbomify/sbomify.cdx.json")" + + # Extra tools - CycloneDX + if [[ "$EXTRA_TOOLS" == *"cyclonedx-python"* ]]; then + add_ntia_row "cyclonedx-python" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-cyclonedx/cyclonedx.cdx.json")" + fi + if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then + add_ntia_row "sbom4python" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-sbom4python/sbom4python.cdx.json")" + fi + + # Core tools - SPDX + add_ntia_row "Trivy" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-trivy/trivy.spdx.json")" + add_ntia_row "Syft" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-syft/syft.spdx.json")" + + # Extra tools - SPDX + if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then + add_ntia_row "sbom4python" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-sbom4python/sbom4python.spdx.json")" + fi + + # Legend + { + echo "" + echo "
NTIA Field Legend" + echo "" + echo "- **Timestamp**: SBOM creation timestamp present (Y/N)" + echo "- **Author**: SBOM author/creator identified (Y/N)" + echo "- **Supplier %**: Percentage of components with supplier/publisher info" + echo "- **Version %**: Percentage of components with version info" + echo "- **PURL %**: Percentage of components with Package URL identifier" + echo "- **Dependencies**: Dependency relationships present (Y/N)" + echo "" + echo "
" + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0905128..4423f09 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,9 +20,9 @@ jobs: with: name: go target_type: filesystem - target_path: /tmp/osv-scanner + target_path: ./target-repo setup_commands: | - git clone --depth 1 --branch v2.3.1 https://github.com/google/osv-scanner.git /tmp/osv-scanner - sbomify_input: /tmp/osv-scanner/go.mod + git clone --depth 1 --branch v2.3.1 https://github.com/google/osv-scanner.git ./target-repo + sbomify_input: ./target-repo/go.mod sbomify_input_type: lock_file title: Go (OSV Scanner 2.3.1) Benchmark diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 183c891..66c6c74 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -20,9 +20,9 @@ jobs: with: name: java target_type: filesystem - target_path: /tmp/keycloak + target_path: ./target-repo setup_commands: | - git clone --depth 1 --branch 26.4.7 https://github.com/keycloak/keycloak.git /tmp/keycloak - sbomify_input: /tmp/keycloak/pom.xml + git clone --depth 1 --branch 26.4.7 https://github.com/keycloak/keycloak.git ./target-repo + sbomify_input: ./target-repo/pom.xml sbomify_input_type: lock_file title: Java (Keycloak 26.4.7) Benchmark diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index fcfea2c..774a661 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -20,9 +20,9 @@ jobs: with: name: javascript target_type: filesystem - target_path: /tmp/workers-sdk + target_path: ./target-repo setup_commands: | - git clone --depth 1 --branch wrangler@3.99.0 https://github.com/cloudflare/workers-sdk.git /tmp/workers-sdk - sbomify_input: /tmp/workers-sdk/pnpm-lock.yaml + git clone --depth 1 --branch wrangler@3.99.0 https://github.com/cloudflare/workers-sdk.git ./target-repo + sbomify_input: ./target-repo/pnpm-lock.yaml sbomify_input_type: lock_file title: JavaScript (Cloudflare workers-sdk) Benchmark diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7e376ac..7173f2d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,9 +20,9 @@ jobs: with: name: rust target_type: filesystem - target_path: /tmp/quiche + target_path: ./target-repo setup_commands: | - git clone --depth 1 --branch 0.24.5 https://github.com/cloudflare/quiche.git /tmp/quiche - sbomify_input: /tmp/quiche/Cargo.lock + git clone --depth 1 --branch 0.24.5 https://github.com/cloudflare/quiche.git ./target-repo + sbomify_input: ./target-repo/Cargo.lock sbomify_input_type: lock_file title: Rust (Cloudflare quiche 0.24.5) Benchmark diff --git a/README.md b/README.md index 24a8f01..e2cf527 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Current tool versions used in benchmarks: | Tool | Version | |------|---------| -| Trivy | 0.67.2 | +| Trivy | 0.68.2 | | Syft | 1.39.0 | | sbomqs | 2.0.2 | | cyclonedx-bom | 7.2.1 | From 9f472673597310faf3dbd90c32a91b2b8a17be3e Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:33:57 +0000 Subject: [PATCH 04/10] Fixes sbomify path --- .github/workflows/_sbom-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_sbom-benchmark.yml b/.github/workflows/_sbom-benchmark.yml index cf32120..fca9205 100644 --- a/.github/workflows/_sbom-benchmark.yml +++ b/.github/workflows/_sbom-benchmark.yml @@ -138,7 +138,7 @@ jobs: env: LOCK_FILE: ${{ inputs.sbomify_input_type == 'lock_file' && inputs.sbomify_input || '' }} DOCKER_IMAGE: ${{ inputs.sbomify_input_type == 'docker_image' && inputs.sbomify_input || '' }} - OUTPUT_FILE: /tmp/sbomify.cdx.json + OUTPUT_FILE: sbomify.cdx.json UPLOAD: "false" AUGMENT: "false" ENRICH: "true" @@ -147,7 +147,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ inputs.name }}-sbomify - path: "/tmp/sbomify.cdx.json" + path: "sbomify.cdx.json" cyclonedx-python: name: CycloneDX Python From 53566795566c3de52eef598ab6e64db47ca0e24e Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:39:01 +0000 Subject: [PATCH 05/10] Improves parsing --- .github/workflows/_sbom-benchmark.yml | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/_sbom-benchmark.yml b/.github/workflows/_sbom-benchmark.yml index fca9205..ddd1c61 100644 --- a/.github/workflows/_sbom-benchmark.yml +++ b/.github/workflows/_sbom-benchmark.yml @@ -286,9 +286,9 @@ jobs: local path="$1" [[ ! -f "$path" ]] && return - # Get sbomqs results + # Get sbomqs results (use --json flag for v2.0+) local result - result=$(/tmp/sbomqs score "$path" -j 2>/dev/null) || result="{}" + result=$(/tmp/sbomqs score --json "$path" 2>/dev/null) || result="{}" # Extract component names once, then count local names @@ -296,10 +296,17 @@ jobs: local total=$(echo "$names" | grep -c . || echo 0) local unique=$(echo "$names" | sort -u | grep -c . || echo 0) + # Get spec version directly from SBOM local version - version=$(echo "$result" | jq -r '.files[0].spec_version // "N/A"' 2>/dev/null) || version="N/A" + version=$(jq -r '.specVersion // "N/A"' "$path" 2>/dev/null) || version="N/A" + + # Get quality score from sbomqs (v2.0 uses sbom_quality_score, v1.x used avg_score) local score - score=$(echo "$result" | jq -r '.files[0].avg_score // "N/A"' 2>/dev/null) || score="N/A" + score=$(echo "$result" | jq -r '.files[0].sbom_quality_score // .files[0].avg_score // "N/A"' 2>/dev/null) || score="N/A" + # Round to 1 decimal place if numeric + if [[ "$score" =~ ^[0-9.]+$ ]]; then + score=$(printf "%.1f" "$score") + fi echo "${total}|${unique}|${version}|${score}" } @@ -309,9 +316,9 @@ jobs: local path="$1" [[ ! -f "$path" ]] && return - # Get sbomqs results + # Get sbomqs results (use --json flag for v2.0+) local result - result=$(/tmp/sbomqs score "$path" -j 2>/dev/null) || result="{}" + result=$(/tmp/sbomqs score --json "$path" 2>/dev/null) || result="{}" # Extract package names once, then count local names @@ -319,10 +326,17 @@ jobs: local total=$(echo "$names" | grep -c . || echo 0) local unique=$(echo "$names" | sort -u | grep -c . || echo 0) + # Get spec version directly from SBOM (strip "SPDX-" prefix) local version - version=$(echo "$result" | jq -r '.files[0].spec_version // "N/A"' 2>/dev/null) || version="N/A" + version=$(jq -r '.spdxVersion // "N/A"' "$path" 2>/dev/null | sed 's/^SPDX-//') || version="N/A" + + # Get quality score from sbomqs (v2.0 uses sbom_quality_score, v1.x used avg_score) local score - score=$(echo "$result" | jq -r '.files[0].avg_score // "N/A"' 2>/dev/null) || score="N/A" + score=$(echo "$result" | jq -r '.files[0].sbom_quality_score // .files[0].avg_score // "N/A"' 2>/dev/null) || score="N/A" + # Round to 1 decimal place if numeric + if [[ "$score" =~ ^[0-9.]+$ ]]; then + score=$(printf "%.1f" "$score") + fi echo "${total}|${unique}|${version}|${score}" } From 6cc3dbabc20aee896f6dbe103764acb778080fb1 Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:41:46 +0000 Subject: [PATCH 06/10] SPDX with sbomify --- .github/workflows/_sbom-benchmark.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_sbom-benchmark.yml b/.github/workflows/_sbom-benchmark.yml index ddd1c61..1fe1842 100644 --- a/.github/workflows/_sbom-benchmark.yml +++ b/.github/workflows/_sbom-benchmark.yml @@ -133,21 +133,33 @@ jobs: if: inputs.setup_commands != '' run: ${{ inputs.setup_commands }} - - name: Generate SBOM + - name: "CycloneDX: Generate SBOM" uses: sbomify/github-action@master env: LOCK_FILE: ${{ inputs.sbomify_input_type == 'lock_file' && inputs.sbomify_input || '' }} DOCKER_IMAGE: ${{ inputs.sbomify_input_type == 'docker_image' && inputs.sbomify_input || '' }} OUTPUT_FILE: sbomify.cdx.json + OUTPUT_FORMAT: cyclonedx UPLOAD: "false" AUGMENT: "false" ENRICH: "true" - - name: Upload SBOM + - name: "SPDX: Generate SBOM" + uses: sbomify/github-action@master + env: + LOCK_FILE: ${{ inputs.sbomify_input_type == 'lock_file' && inputs.sbomify_input || '' }} + DOCKER_IMAGE: ${{ inputs.sbomify_input_type == 'docker_image' && inputs.sbomify_input || '' }} + OUTPUT_FILE: sbomify.spdx.json + OUTPUT_FORMAT: spdx + UPLOAD: "false" + AUGMENT: "false" + ENRICH: "true" + + - name: Upload SBOMs uses: actions/upload-artifact@v4 with: name: ${{ inputs.name }}-sbomify - path: "sbomify.cdx.json" + path: "sbomify.*.json" cyclonedx-python: name: CycloneDX Python @@ -518,6 +530,7 @@ jobs: # Core tools - SPDX add_row "Trivy (${TRIVY_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-trivy/trivy.spdx.json")" add_row "Syft (${SYFT_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-syft/syft.spdx.json")" + add_row "sbomify" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-sbomify/sbomify.spdx.json")" # Extra tools - SPDX if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then @@ -547,6 +560,7 @@ jobs: # Core tools - SPDX add_ntia_row "Trivy" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-trivy/trivy.spdx.json")" add_ntia_row "Syft" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-syft/syft.spdx.json")" + add_ntia_row "sbomify" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-sbomify/sbomify.spdx.json")" # Extra tools - SPDX if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then From 481b1845cc5db74d7bc00e8f06606b8118295ee4 Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:47:43 +0000 Subject: [PATCH 07/10] Fixes sbomqs link --- .github/workflows/_sbom-benchmark.yml | 35 +++++++++------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/.github/workflows/_sbom-benchmark.yml b/.github/workflows/_sbom-benchmark.yml index 1fe1842..5707cc6 100644 --- a/.github/workflows/_sbom-benchmark.yml +++ b/.github/workflows/_sbom-benchmark.yml @@ -124,7 +124,7 @@ jobs: path: "/tmp/syft.*.json" sbomify: - name: sbomify + name: sbomify action runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -133,33 +133,21 @@ jobs: if: inputs.setup_commands != '' run: ${{ inputs.setup_commands }} - - name: "CycloneDX: Generate SBOM" + - name: Generate SBOM uses: sbomify/github-action@master env: LOCK_FILE: ${{ inputs.sbomify_input_type == 'lock_file' && inputs.sbomify_input || '' }} DOCKER_IMAGE: ${{ inputs.sbomify_input_type == 'docker_image' && inputs.sbomify_input || '' }} OUTPUT_FILE: sbomify.cdx.json - OUTPUT_FORMAT: cyclonedx UPLOAD: "false" AUGMENT: "false" ENRICH: "true" - - name: "SPDX: Generate SBOM" - uses: sbomify/github-action@master - env: - LOCK_FILE: ${{ inputs.sbomify_input_type == 'lock_file' && inputs.sbomify_input || '' }} - DOCKER_IMAGE: ${{ inputs.sbomify_input_type == 'docker_image' && inputs.sbomify_input || '' }} - OUTPUT_FILE: sbomify.spdx.json - OUTPUT_FORMAT: spdx - UPLOAD: "false" - AUGMENT: "false" - ENRICH: "true" - - - name: Upload SBOMs + - name: Upload SBOM uses: actions/upload-artifact@v4 with: name: ${{ inputs.name }}-sbomify - path: "sbomify.*.json" + path: "sbomify.cdx.json" cyclonedx-python: name: CycloneDX Python @@ -249,8 +237,9 @@ jobs: - name: Install sbomqs run: | - curl -L -o /tmp/sbomqs \ - "https://github.com/interlynk-io/sbomqs/releases/download/v${SBOMQS_VERSION}/sbomqs-linux-amd64" + curl -L -o /tmp/sbomqs.tar.gz \ + "https://github.com/interlynk-io/sbomqs/releases/download/v${SBOMQS_VERSION}/sbomqs_${SBOMQS_VERSION}_Linux_x86_64.tar.gz" + tar xzf /tmp/sbomqs.tar.gz -C /tmp chmod +x /tmp/sbomqs - name: Generate Summary Table @@ -517,7 +506,7 @@ jobs: # Core tools - CycloneDX add_row "Trivy (${TRIVY_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-trivy/trivy.cdx.json")" add_row "Syft (${SYFT_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-syft/syft.cdx.json")" - add_row "sbomify" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-sbomify/sbomify.cdx.json")" + add_row "sbomify action" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-sbomify/sbomify.cdx.json")" # Extra tools - CycloneDX if [[ "$EXTRA_TOOLS" == *"cyclonedx-python"* ]]; then @@ -527,10 +516,9 @@ jobs: add_row "sbom4python (${SBOM4PYTHON_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-sbom4python/sbom4python.cdx.json")" fi - # Core tools - SPDX + # Core tools - SPDX (sbomify only supports CycloneDX output) add_row "Trivy (${TRIVY_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-trivy/trivy.spdx.json")" add_row "Syft (${SYFT_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-syft/syft.spdx.json")" - add_row "sbomify" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-sbomify/sbomify.spdx.json")" # Extra tools - SPDX if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then @@ -547,7 +535,7 @@ jobs: # Core tools - CycloneDX add_ntia_row "Trivy" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-trivy/trivy.cdx.json")" add_ntia_row "Syft" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-syft/syft.cdx.json")" - add_ntia_row "sbomify" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-sbomify/sbomify.cdx.json")" + add_ntia_row "sbomify action" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-sbomify/sbomify.cdx.json")" # Extra tools - CycloneDX if [[ "$EXTRA_TOOLS" == *"cyclonedx-python"* ]]; then @@ -557,10 +545,9 @@ jobs: add_ntia_row "sbom4python" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-sbom4python/sbom4python.cdx.json")" fi - # Core tools - SPDX + # Core tools - SPDX (sbomify only supports CycloneDX output) add_ntia_row "Trivy" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-trivy/trivy.spdx.json")" add_ntia_row "Syft" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-syft/syft.spdx.json")" - add_ntia_row "sbomify" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-sbomify/sbomify.spdx.json")" # Extra tools - SPDX if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then From 6923485ea32fd612816afb322895c22be95dd80c Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:51:51 +0000 Subject: [PATCH 08/10] Fixes author field and path --- .github/workflows/_sbom-benchmark.yml | 3 ++- .github/workflows/go.yml | 6 +++--- .github/workflows/java.yml | 6 +++--- .github/workflows/javascript.yml | 6 +++--- .github/workflows/rust.yml | 6 +++--- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_sbom-benchmark.yml b/.github/workflows/_sbom-benchmark.yml index 5707cc6..14063bf 100644 --- a/.github/workflows/_sbom-benchmark.yml +++ b/.github/workflows/_sbom-benchmark.yml @@ -406,8 +406,9 @@ jobs: # SBOM-level checks local has_timestamp has_timestamp=$(jq -r '.creationInfo.created // empty' "$path" 2>/dev/null) + # Author should be Organization or Person, not just Tool local has_author - has_author=$(jq -r '.creationInfo.creators[0] // empty' "$path" 2>/dev/null) + has_author=$(jq -r '.creationInfo.creators[]? | select(startswith("Organization:") or startswith("Person:")) | . // empty' "$path" 2>/dev/null | head -1) # Component-level NTIA fields - single jq call for efficiency local counts diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4423f09..4c13a77 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,9 +20,9 @@ jobs: with: name: go target_type: filesystem - target_path: ./target-repo + target_path: target-repo setup_commands: | - git clone --depth 1 --branch v2.3.1 https://github.com/google/osv-scanner.git ./target-repo - sbomify_input: ./target-repo/go.mod + git clone --depth 1 --branch v2.3.1 https://github.com/google/osv-scanner.git target-repo + sbomify_input: target-repo/go.mod sbomify_input_type: lock_file title: Go (OSV Scanner 2.3.1) Benchmark diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 66c6c74..da05da5 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -20,9 +20,9 @@ jobs: with: name: java target_type: filesystem - target_path: ./target-repo + target_path: target-repo setup_commands: | - git clone --depth 1 --branch 26.4.7 https://github.com/keycloak/keycloak.git ./target-repo - sbomify_input: ./target-repo/pom.xml + git clone --depth 1 --branch 26.4.7 https://github.com/keycloak/keycloak.git target-repo + sbomify_input: target-repo/pom.xml sbomify_input_type: lock_file title: Java (Keycloak 26.4.7) Benchmark diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index 774a661..a320782 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -20,9 +20,9 @@ jobs: with: name: javascript target_type: filesystem - target_path: ./target-repo + target_path: target-repo setup_commands: | - git clone --depth 1 --branch wrangler@3.99.0 https://github.com/cloudflare/workers-sdk.git ./target-repo - sbomify_input: ./target-repo/pnpm-lock.yaml + git clone --depth 1 --branch wrangler@3.99.0 https://github.com/cloudflare/workers-sdk.git target-repo + sbomify_input: target-repo/pnpm-lock.yaml sbomify_input_type: lock_file title: JavaScript (Cloudflare workers-sdk) Benchmark diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7173f2d..caf7f3a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,9 +20,9 @@ jobs: with: name: rust target_type: filesystem - target_path: ./target-repo + target_path: target-repo setup_commands: | - git clone --depth 1 --branch 0.24.5 https://github.com/cloudflare/quiche.git ./target-repo - sbomify_input: ./target-repo/Cargo.lock + git clone --depth 1 --branch 0.24.5 https://github.com/cloudflare/quiche.git target-repo + sbomify_input: target-repo/Cargo.lock sbomify_input_type: lock_file title: Rust (Cloudflare quiche 0.24.5) Benchmark From f8df3fafd7574bd0e18972ae453bd5b233c544fa Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 15:57:20 +0000 Subject: [PATCH 09/10] Generate lockfile --- .github/workflows/rust.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index caf7f3a..92b21ce 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,6 +23,7 @@ jobs: target_path: target-repo setup_commands: | git clone --depth 1 --branch 0.24.5 https://github.com/cloudflare/quiche.git target-repo + cd target-repo && cargo generate-lockfile sbomify_input: target-repo/Cargo.lock sbomify_input_type: lock_file title: Rust (Cloudflare quiche 0.24.5) Benchmark From 7c6ab72a8e0a0e6700489ab6f82b2f003ad43784 Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Fri, 9 Jan 2026 16:07:47 +0000 Subject: [PATCH 10/10] Point to lockfile --- .github/workflows/go.yml | 2 +- .github/workflows/java.yml | 2 +- .github/workflows/javascript.yml | 2 +- .github/workflows/rust.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4c13a77..8f9c754 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,7 +20,7 @@ jobs: with: name: go target_type: filesystem - target_path: target-repo + target_path: target-repo/go.mod setup_commands: | git clone --depth 1 --branch v2.3.1 https://github.com/google/osv-scanner.git target-repo sbomify_input: target-repo/go.mod diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index da05da5..3bbe70b 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -20,7 +20,7 @@ jobs: with: name: java target_type: filesystem - target_path: target-repo + target_path: target-repo/pom.xml setup_commands: | git clone --depth 1 --branch 26.4.7 https://github.com/keycloak/keycloak.git target-repo sbomify_input: target-repo/pom.xml diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index a320782..16d7d2f 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -20,7 +20,7 @@ jobs: with: name: javascript target_type: filesystem - target_path: target-repo + target_path: target-repo/pnpm-lock.yaml setup_commands: | git clone --depth 1 --branch wrangler@3.99.0 https://github.com/cloudflare/workers-sdk.git target-repo sbomify_input: target-repo/pnpm-lock.yaml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 92b21ce..0555b49 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,7 +20,7 @@ jobs: with: name: rust target_type: filesystem - target_path: target-repo + target_path: target-repo/Cargo.lock setup_commands: | git clone --depth 1 --branch 0.24.5 https://github.com/cloudflare/quiche.git target-repo cd target-repo && cargo generate-lockfile