diff --git a/.github/workflows/_sbom-benchmark.yml b/.github/workflows/_sbom-benchmark.yml new file mode 100644 index 0000000..14063bf --- /dev/null +++ b/.github/workflows/_sbom-benchmark.yml @@ -0,0 +1,571 @@ +--- +name: SBOM Benchmark (Reusable) + +on: + workflow_call: + inputs: + name: + description: 'Benchmark name (used for artifact prefix and display)' + required: true + type: string + target_type: + description: 'Type of target: filesystem or docker' + required: true + type: string + target_path: + description: 'Path to scan (filesystem) or image name (docker)' + required: true + type: string + setup_commands: + description: 'Commands to run before scanning (e.g., clone repo, build image)' + required: false + type: string + default: '' + sbomify_input: + description: 'Input for sbomify: LOCK_FILE path or DOCKER_IMAGE name' + required: true + type: string + sbomify_input_type: + description: 'Type of sbomify input: lock_file or docker_image' + required: true + type: string + baseline_command: + description: 'Command to generate baseline (optional)' + required: false + type: string + default: '' + extra_tools: + description: 'Additional tools to run (comma-separated: cyclonedx-python,sbom4python)' + required: false + type: string + default: '' + title: + description: 'Title for the summary table' + required: false + type: string + default: '' + +env: + TRIVY_VERSION: 0.68.2 + SYFT_VERSION: 1.39.0 + SBOMQS_VERSION: 2.0.2 + CYCLONEDX_PYTHON_VERSION: 7.2.1 + SBOM4PYTHON_VERSION: 0.12.4 + +jobs: + trivy: + name: Trivy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup + if: inputs.setup_commands != '' + run: ${{ inputs.setup_commands }} + + - name: Install Trivy + run: | + curl -L -o /tmp/trivy.tgz \ + "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" + tar xvf /tmp/trivy.tgz -C /tmp + chmod +x /tmp/trivy + + - name: "CycloneDX: Generate SBOM" + run: | + /tmp/trivy ${{ inputs.target_type == 'docker' && 'image' || 'fs' }} \ + --format cyclonedx \ + --output /tmp/trivy.cdx.json \ + ${{ inputs.target_path }} + + - name: "SPDX: Generate SBOM" + run: | + /tmp/trivy ${{ inputs.target_type == 'docker' && 'image' || 'fs' }} \ + --format spdx-json \ + --output /tmp/trivy.spdx.json \ + ${{ inputs.target_path }} + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-trivy + path: "/tmp/trivy.*.json" + + syft: + name: Syft + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup + if: inputs.setup_commands != '' + run: ${{ inputs.setup_commands }} + + - name: Install Syft + run: | + curl -L -o /tmp/syft.tgz \ + "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_amd64.tar.gz" + tar xvf /tmp/syft.tgz -C /tmp + chmod +x /tmp/syft + + - name: "CycloneDX: Generate SBOM" + run: | + /tmp/syft ${{ inputs.target_path }} \ + -o cyclonedx-json > /tmp/syft.cdx.json + + - name: "SPDX: Generate SBOM" + run: | + /tmp/syft ${{ inputs.target_path }} \ + -o spdx-json > /tmp/syft.spdx.json + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-syft + path: "/tmp/syft.*.json" + + sbomify: + name: sbomify action + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup + if: inputs.setup_commands != '' + run: ${{ inputs.setup_commands }} + + - name: Generate SBOM + uses: sbomify/github-action@master + env: + LOCK_FILE: ${{ inputs.sbomify_input_type == 'lock_file' && inputs.sbomify_input || '' }} + DOCKER_IMAGE: ${{ inputs.sbomify_input_type == 'docker_image' && inputs.sbomify_input || '' }} + OUTPUT_FILE: sbomify.cdx.json + UPLOAD: "false" + AUGMENT: "false" + ENRICH: "true" + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-sbomify + path: "sbomify.cdx.json" + + cyclonedx-python: + name: CycloneDX Python + if: contains(inputs.extra_tools, 'cyclonedx-python') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install CycloneDX Python + run: | + python -m pip install cyclonedx-bom==${CYCLONEDX_PYTHON_VERSION} + + - name: "CycloneDX: Generate SBOM" + run: | + cyclonedx-py requirements ${{ inputs.sbomify_input }} \ + -o /tmp/cyclonedx.cdx.json + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-cyclonedx + path: "/tmp/cyclonedx.*.json" + + sbom4python: + name: sbom4python + if: contains(inputs.extra_tools, 'sbom4python') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install sbom4python + run: | + python -m pip install sbom4python==${SBOM4PYTHON_VERSION} + + - name: "CycloneDX: Generate SBOM" + run: | + sbom4python -r ${{ inputs.sbomify_input }} \ + --sbom cyclonedx --format json \ + -o /tmp/sbom4python.cdx.json + + - name: "SPDX: Generate SBOM" + run: | + sbom4python -r ${{ inputs.sbomify_input }} \ + --sbom spdx --format json \ + -o /tmp/sbom4python.spdx.json + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-sbom4python + path: "/tmp/sbom4python.*.json" + + baseline: + name: Baseline + if: inputs.baseline_command != '' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup + if: inputs.setup_commands != '' + run: ${{ inputs.setup_commands }} + + - name: Generate baseline + run: | + ${{ inputs.baseline_command }} > /tmp/baseline.txt + + - name: Upload baseline + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }}-baseline + path: "/tmp/baseline.txt" + + score: + name: Score + needs: + - trivy + - syft + - sbomify + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download SBOMs + uses: actions/download-artifact@v4 + + - name: Install sbomqs + run: | + curl -L -o /tmp/sbomqs.tar.gz \ + "https://github.com/interlynk-io/sbomqs/releases/download/v${SBOMQS_VERSION}/sbomqs_${SBOMQS_VERSION}_Linux_x86_64.tar.gz" + tar xzf /tmp/sbomqs.tar.gz -C /tmp + chmod +x /tmp/sbomqs + + - name: Generate Summary Table + env: + BENCHMARK_NAME: ${{ inputs.name }} + BENCHMARK_TITLE: ${{ inputs.title }} + EXTRA_TOOLS: ${{ inputs.extra_tools }} + run: | + set +e # Don't exit on error - some files may not exist + + # Helper: safely convert value to integer, defaulting to 0 + to_int() { + local val="${1:-0}" + # Remove any non-numeric characters and handle null/empty + val="${val//[^0-9]/}" + echo "${val:-0}" + } + + # Helper: calculate duplication percentage + calc_dup() { + local total=$(to_int "$1") + local unique=$(to_int "$2") + if [[ "$total" -eq 0 ]]; then + echo "N/A" + return + fi + local dups=$((total - unique)) + [[ "$dups" -lt 0 ]] && dups=0 + printf "%.2f%%" "$(echo "scale=2; $dups * 100 / $total" | bc)" + } + + # Helper: calculate percentage safely + calc_pct() { + local count=$(to_int "$1") + local total=$(to_int "$2") + if [[ "$total" -eq 0 ]]; then + echo "0" + return + fi + echo "scale=0; $count * 100 / $total" | bc + } + + # Score CycloneDX SBOM + score_cdx() { + local path="$1" + [[ ! -f "$path" ]] && return + + # Get sbomqs results (use --json flag for v2.0+) + local result + result=$(/tmp/sbomqs score --json "$path" 2>/dev/null) || result="{}" + + # Extract component names once, then count + local names + names=$(jq -r '.components[]?.name // empty' "$path" 2>/dev/null) + local total=$(echo "$names" | grep -c . || echo 0) + local unique=$(echo "$names" | sort -u | grep -c . || echo 0) + + # Get spec version directly from SBOM + local version + version=$(jq -r '.specVersion // "N/A"' "$path" 2>/dev/null) || version="N/A" + + # Get quality score from sbomqs (v2.0 uses sbom_quality_score, v1.x used avg_score) + local score + score=$(echo "$result" | jq -r '.files[0].sbom_quality_score // .files[0].avg_score // "N/A"' 2>/dev/null) || score="N/A" + # Round to 1 decimal place if numeric + if [[ "$score" =~ ^[0-9.]+$ ]]; then + score=$(printf "%.1f" "$score") + fi + + echo "${total}|${unique}|${version}|${score}" + } + + # Score SPDX SBOM + score_spdx() { + local path="$1" + [[ ! -f "$path" ]] && return + + # Get sbomqs results (use --json flag for v2.0+) + local result + result=$(/tmp/sbomqs score --json "$path" 2>/dev/null) || result="{}" + + # Extract package names once, then count + local names + names=$(jq -r '.packages[]?.name // empty' "$path" 2>/dev/null) + local total=$(echo "$names" | grep -c . || echo 0) + local unique=$(echo "$names" | sort -u | grep -c . || echo 0) + + # Get spec version directly from SBOM (strip "SPDX-" prefix) + local version + version=$(jq -r '.spdxVersion // "N/A"' "$path" 2>/dev/null | sed 's/^SPDX-//') || version="N/A" + + # Get quality score from sbomqs (v2.0 uses sbom_quality_score, v1.x used avg_score) + local score + score=$(echo "$result" | jq -r '.files[0].sbom_quality_score // .files[0].avg_score // "N/A"' 2>/dev/null) || score="N/A" + # Round to 1 decimal place if numeric + if [[ "$score" =~ ^[0-9.]+$ ]]; then + score=$(printf "%.1f" "$score") + fi + + echo "${total}|${unique}|${version}|${score}" + } + + # NTIA Minimum Elements check for CycloneDX + ntia_cdx() { + local path="$1" + [[ ! -f "$path" ]] && return + + # Get total component count + local total + total=$(jq '.components | length // 0' "$path" 2>/dev/null) || total=0 + total=$(to_int "$total") + + # SBOM-level checks + local has_timestamp + has_timestamp=$(jq -r '.metadata.timestamp // empty' "$path" 2>/dev/null) + local has_author + has_author=$(jq -r ' + .metadata.authors[0].name // + .metadata.manufacturer.name // + .metadata.supplier.name // + empty + ' "$path" 2>/dev/null) + + # Component-level NTIA fields - single jq call for efficiency + local counts + counts=$(jq '{ + with_supplier: [.components[]? | select(.supplier.name != null or .publisher != null)] | length, + with_version: [.components[]? | select(.version != null and .version != "")] | length, + with_purl: [.components[]? | select(.purl != null)] | length, + has_deps: (.dependencies | length // 0) + }' "$path" 2>/dev/null) || counts='{"with_supplier":0,"with_version":0,"with_purl":0,"has_deps":0}' + + local with_supplier=$(echo "$counts" | jq -r '.with_supplier // 0') + local with_version=$(echo "$counts" | jq -r '.with_version // 0') + local with_purl=$(echo "$counts" | jq -r '.with_purl // 0') + local has_deps=$(echo "$counts" | jq -r '.has_deps // 0') + + # Calculate percentages + local supplier_pct=$(calc_pct "$with_supplier" "$total") + local version_pct=$(calc_pct "$with_version" "$total") + local purl_pct=$(calc_pct "$with_purl" "$total") + + # Format output with Y/N indicators + local ts_icon="N" + [[ -n "$has_timestamp" ]] && ts_icon="Y" + local author_icon="N" + [[ -n "$has_author" ]] && author_icon="Y" + local deps_icon="N" + [[ $(to_int "$has_deps") -gt 0 ]] && deps_icon="Y" + + echo "${ts_icon}|${author_icon}|${supplier_pct}%|${version_pct}%|${purl_pct}%|${deps_icon}" + } + + # NTIA Minimum Elements check for SPDX + ntia_spdx() { + local path="$1" + [[ ! -f "$path" ]] && return + + # Get total package count + local total + total=$(jq '.packages | length // 0' "$path" 2>/dev/null) || total=0 + total=$(to_int "$total") + + # SBOM-level checks + local has_timestamp + has_timestamp=$(jq -r '.creationInfo.created // empty' "$path" 2>/dev/null) + # Author should be Organization or Person, not just Tool + local has_author + has_author=$(jq -r '.creationInfo.creators[]? | select(startswith("Organization:") or startswith("Person:")) | . // empty' "$path" 2>/dev/null | head -1) + + # Component-level NTIA fields - single jq call for efficiency + local counts + counts=$(jq '{ + with_supplier: [.packages[]? | select(.supplier != null and .supplier != "" and .supplier != "NOASSERTION")] | length, + with_version: [.packages[]? | select(.versionInfo != null and .versionInfo != "" and .versionInfo != "NOASSERTION")] | length, + with_purl: [.packages[]? | select(.externalRefs[]? | .referenceType == "purl")] | length, + has_deps: (.relationships | length // 0) + }' "$path" 2>/dev/null) || counts='{"with_supplier":0,"with_version":0,"with_purl":0,"has_deps":0}' + + local with_supplier=$(echo "$counts" | jq -r '.with_supplier // 0') + local with_version=$(echo "$counts" | jq -r '.with_version // 0') + local with_purl=$(echo "$counts" | jq -r '.with_purl // 0') + local has_deps=$(echo "$counts" | jq -r '.has_deps // 0') + + # Calculate percentages + local supplier_pct=$(calc_pct "$with_supplier" "$total") + local version_pct=$(calc_pct "$with_version" "$total") + local purl_pct=$(calc_pct "$with_purl" "$total") + + # Format output with Y/N indicators + local ts_icon="N" + [[ -n "$has_timestamp" ]] && ts_icon="Y" + local author_icon="N" + [[ -n "$has_author" ]] && author_icon="Y" + local deps_icon="N" + [[ $(to_int "$has_deps") -gt 0 ]] && deps_icon="Y" + + echo "${ts_icon}|${author_icon}|${supplier_pct}%|${version_pct}%|${purl_pct}%|${deps_icon}" + } + + # Add a row to the package detection table + add_row() { + local tool="$1" + local format="$2" + local data="$3" + [[ -z "$data" ]] && return + + local total unique version score + IFS='|' read -r total unique version score <<< "$data" + + # Validate we got all fields + [[ -z "$total" ]] && total=0 + [[ -z "$unique" ]] && unique=0 + [[ -z "$version" ]] && version="N/A" + [[ -z "$score" ]] && score="N/A" + + local dup + dup=$(calc_dup "$total" "$unique") + echo "| $tool | $format ($version) | $total | $unique | $dup | $score |" >> "${GITHUB_STEP_SUMMARY}" + } + + # Add a row to the NTIA table + add_ntia_row() { + local tool="$1" + local format="$2" + local data="$3" + [[ -z "$data" ]] && return + + local ts author supplier version purl deps + IFS='|' read -r ts author supplier version purl deps <<< "$data" + + # Set defaults for empty fields + [[ -z "$ts" ]] && ts="N" + [[ -z "$author" ]] && author="N" + [[ -z "$supplier" ]] && supplier="0%" + [[ -z "$version" ]] && version="0%" + [[ -z "$purl" ]] && purl="0%" + [[ -z "$deps" ]] && deps="N" + + echo "| $tool | $format | $ts | $author | $supplier | $version | $purl | $deps |" >> "${GITHUB_STEP_SUMMARY}" + } + + # === Generate Report === + + # Header + if [[ -n "$BENCHMARK_TITLE" ]]; then + echo "## $BENCHMARK_TITLE" >> "${GITHUB_STEP_SUMMARY}" + echo "" >> "${GITHUB_STEP_SUMMARY}" + fi + + # Baseline (if exists) + BASELINE_FILE="${BENCHMARK_NAME}-baseline/baseline.txt" + if [[ -f "$BASELINE_FILE" ]]; then + BASELINE_COUNT=$(wc -l < "$BASELINE_FILE" | tr -d ' ') + echo "**Baseline:** ${BASELINE_COUNT} packages" >> "${GITHUB_STEP_SUMMARY}" + echo "" >> "${GITHUB_STEP_SUMMARY}" + fi + + # === Package Detection Table === + echo "### Package Detection" >> "${GITHUB_STEP_SUMMARY}" + echo "" >> "${GITHUB_STEP_SUMMARY}" + echo "| Tool | Format | Packages | Unique | Duplication % | Quality Score |" >> "${GITHUB_STEP_SUMMARY}" + echo "| -- | -- | -- | -- | -- | -- |" >> "${GITHUB_STEP_SUMMARY}" + + # Core tools - CycloneDX + add_row "Trivy (${TRIVY_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-trivy/trivy.cdx.json")" + add_row "Syft (${SYFT_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-syft/syft.cdx.json")" + add_row "sbomify action" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-sbomify/sbomify.cdx.json")" + + # Extra tools - CycloneDX + if [[ "$EXTRA_TOOLS" == *"cyclonedx-python"* ]]; then + add_row "cyclonedx-python (${CYCLONEDX_PYTHON_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-cyclonedx/cyclonedx.cdx.json")" + fi + if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then + add_row "sbom4python (${SBOM4PYTHON_VERSION})" "CycloneDX" "$(score_cdx "${BENCHMARK_NAME}-sbom4python/sbom4python.cdx.json")" + fi + + # Core tools - SPDX (sbomify only supports CycloneDX output) + add_row "Trivy (${TRIVY_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-trivy/trivy.spdx.json")" + add_row "Syft (${SYFT_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-syft/syft.spdx.json")" + + # Extra tools - SPDX + if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then + add_row "sbom4python (${SBOM4PYTHON_VERSION})" "SPDX" "$(score_spdx "${BENCHMARK_NAME}-sbom4python/sbom4python.spdx.json")" + fi + + # === NTIA Minimum Elements Table === + echo "" >> "${GITHUB_STEP_SUMMARY}" + echo "### NTIA Minimum Elements" >> "${GITHUB_STEP_SUMMARY}" + echo "" >> "${GITHUB_STEP_SUMMARY}" + echo "| Tool | Format | Timestamp | Author | Supplier % | Version % | PURL % | Dependencies |" >> "${GITHUB_STEP_SUMMARY}" + echo "| -- | -- | -- | -- | -- | -- | -- | -- |" >> "${GITHUB_STEP_SUMMARY}" + + # Core tools - CycloneDX + add_ntia_row "Trivy" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-trivy/trivy.cdx.json")" + add_ntia_row "Syft" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-syft/syft.cdx.json")" + add_ntia_row "sbomify action" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-sbomify/sbomify.cdx.json")" + + # Extra tools - CycloneDX + if [[ "$EXTRA_TOOLS" == *"cyclonedx-python"* ]]; then + add_ntia_row "cyclonedx-python" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-cyclonedx/cyclonedx.cdx.json")" + fi + if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then + add_ntia_row "sbom4python" "CycloneDX" "$(ntia_cdx "${BENCHMARK_NAME}-sbom4python/sbom4python.cdx.json")" + fi + + # Core tools - SPDX (sbomify only supports CycloneDX output) + add_ntia_row "Trivy" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-trivy/trivy.spdx.json")" + add_ntia_row "Syft" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-syft/syft.spdx.json")" + + # Extra tools - SPDX + if [[ "$EXTRA_TOOLS" == *"sbom4python"* ]]; then + add_ntia_row "sbom4python" "SPDX" "$(ntia_spdx "${BENCHMARK_NAME}-sbom4python/sbom4python.spdx.json")" + fi + + # Legend + { + echo "" + echo "
NTIA Field Legend" + echo "" + echo "- **Timestamp**: SBOM creation timestamp present (Y/N)" + echo "- **Author**: SBOM author/creator identified (Y/N)" + echo "- **Supplier %**: Percentage of components with supplier/publisher info" + echo "- **Version %**: Percentage of components with version info" + echo "- **PURL %**: Percentage of components with Package URL identifier" + echo "- **Dependencies**: Dependency relationships present (Y/N)" + echo "" + echo "
" + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 42298b9..c2b1aed 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,192 +7,24 @@ on: paths: - 'docker/**' - '.github/workflows/docker.yml' - -env: - TRIVY_VERSION: 0.54.1 - SYFT_VERSION: 1.11.1 - SBOMQS_VERSION: 0.1.9 + - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'docker/**' + - '.github/workflows/docker.yml' + - '.github/workflows/_sbom-benchmark.yml' jobs: - trivy: - name: Trivy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Build Docker image - working-directory: "docker" - run: - docker build . -t nginx-test - - - name: Install Trivy - run: | - curl -L -o /tmp/trivy.tgz \ - "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" - tar xvf /tmp/trivy.tgz -C /tmp - chmod +x /tmp/trivy - - - name: "CycloneDX: Generate SBOM" - run: | - /tmp/trivy image \ - --format cyclonedx \ - --output /tmp/trivy.cdx.json \ - nginx-test - - - name: "SPDX: Generate SBOM" - run: | - /tmp/trivy image \ - --format spdx-json \ - --output /tmp/trivy.spdx.json \ - nginx-test - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: container-trivy - path: "/tmp/trivy.*.json" - - syft: - name: Syft - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Build Docker image - working-directory: "docker" - run: - docker build . -t nginx-test - - - name: Install Syft - run: | - curl -L -o /tmp/syft.tgz \ - "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_amd64.tar.gz" - tar xvf /tmp/syft.tgz -C /tmp - chmod +x /tmp/syft - - - name: "CycloneDX: Generate SBOM" - run: | - /tmp/syft \ - nginx-test \ - -o cyclonedx-json \ - > /tmp/syft.cdx.json - - - name: "SPDX: Generate SBOM" - run: | - /tmp/syft \ - nginx-test \ - -o spdx-json \ - > /tmp/syft.spdx.json - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: container-syft - path: "/tmp/syft.*.json" - benchmark: - name: Benchmark - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Build Docker image - working-directory: "docker" - run: | - docker build . -t nginx-test - - # Provide a baseline of what system packages Debian think is installed - - name: - run: | - docker run \ - --rm nginx-test \ - dpkg-query -W -f='${binary:Package} ${Version}\n' | sort > /tmp/baseline.txt - - - name: Upload baseline - uses: actions/upload-artifact@v4 - with: - name: baseline - path: "/tmp/baseline.txt" - - Score: - needs: - - trivy - - syft - - benchmark - runs-on: ubuntu-latest - steps: - - name: Download SBOMs - uses: actions/download-artifact@v4 - - - name: Install sbomqs - run: | - curl -L -o /tmp/sbomqs \ - "https://github.com/interlynk-io/sbomqs/releases/download/v${SBOMQS_VERSION}/sbomqs-linux-amd64" - chmod +x /tmp/sbomqs - - - name: "Generate Summary Table" - run: | - calculate_duplication_percentage() { - local total=$1 - local unique=$2 - - if [ "$total" -eq 0 ]; then - echo "Duplication Percentage: N/A (Total count is zero)" - return 1 - fi - - local duplicates=$((total - unique)) - - if [ "$duplicates" -lt 0 ]; then - duplicates=0 - fi - - local duplication_percentage=$(echo "scale=2; ($duplicates / $total) * 100" | bc) - - echo "$duplication_percentage%" - } - - # Baseline - SYSTEM_BASELINE=$(cat baseline/baseline.txt | wc -l) - - # Syft - SYFT_CDX_PATH="container-syft/syft.cdx.json" - SYFT_CDX_TOTAL=$(jq '.components[] | .name' $SYFT_CDX_PATH | wc -l) - SYFT_CDX_UNIQUE=$(jq '.components[] | .name' $SYFT_CDX_PATH | uniq | wc -l) - SYFT_CDX_SBOMQS=$(/tmp/sbomqs score $SYFT_CDX_PATH -j) - SYFT_CDX_VERSION=$(echo $SYFT_CDX_SBOMQS | jq -r '.files[0].spec_version') - SYFT_CDX_QUALITY_SCORE=$(echo $SYFT_CDX_SBOMQS | jq -r '.files[0].avg_score') - - SYFT_SPDX_PATH="container-syft/syft.spdx.json" - SYFT_SPDX_TOTAL=$(jq '.packages[] | .name' $SYFT_SPDX_PATH | wc -l) - SYFT_SPDX_UNIQUE=$(jq '.packages[] | .name' $SYFT_SPDX_PATH | uniq | wc -l) - SYFT_SPDX_SBOMQS=$(/tmp/sbomqs score $SYFT_SPDX_PATH -j) - SYFT_SPDX_VERSION=$(echo $SYFT_SPDX_SBOMQS | jq -r '.files[0].spec_version') - SYFT_SPDX_QUALITY_SCORE=$(echo $SYFT_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - # Trivy - TRIVY_CDX_PATH="container-trivy/trivy.cdx.json" - TRIVY_CDX_TOTAL=$(jq '.components[] | .name' $TRIVY_CDX_PATH | wc -l) - TRIVY_CDX_UNIQUE=$(jq '.components[] | .name' $TRIVY_CDX_PATH | uniq | wc -l) - TRIVY_CDX_SBOMQS=$(/tmp/sbomqs score $TRIVY_CDX_PATH -j) - TRIVY_CDX_VERSION=$(echo $TRIVY_CDX_SBOMQS | jq -r '.files[0].spec_version') - TRIVY_CDX_QUALITY_SCORE=$(echo $TRIVY_CDX_SBOMQS | jq -r '.files[0].avg_score') - - TRIVY_SPDX_PATH="container-trivy/trivy.spdx.json" - TRIVY_SPDX_TOTAL=$(jq '.packages[] | .name' $TRIVY_SPDX_PATH | wc -l) - TRIVY_SPDX_UNIQUE=$(jq '.packages[] | .name' $TRIVY_SPDX_PATH | uniq | wc -l) - TRIVY_SPDX_SBOMQS=$(/tmp/sbomqs score $TRIVY_SPDX_PATH -j) - TRIVY_SPDX_VERSION=$(echo $TRIVY_SPDX_SBOMQS | jq -r '.files[0].spec_version') - TRIVY_SPDX_QUALITY_SCORE=$(echo $TRIVY_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - - # Header - echo "| Tool | Format | Packages | Unique Packages | Duplication % | Avg Quality Score |" >> ${GITHUB_STEP_SUMMARY} - echo "| -- | -- | -- | -- | -- |-- |" >> ${GITHUB_STEP_SUMMARY} - - # Construct table - echo "| System Baseline | N/A | $SYSTEM_BASELINE | $SYSTEM_BASELINE | 0% | N/A |" >> ${GITHUB_STEP_SUMMARY} - echo "| Syft ($SYFT_VERSION) | CycloneDX ($SYFT_CDX_VERSION) | $SYFT_CDX_TOTAL | $SYFT_CDX_UNIQUE | $(calculate_duplication_percentage $SYFT_CDX_TOTAL $SYFT_CDX_UNIQUE) | $SYFT_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Trivy ($TRIVY_VERSION) | CycloneDX ($TRIVY_CDX_VERSION) | $TRIVY_CDX_TOTAL | $TRIVY_CDX_UNIQUE | $(calculate_duplication_percentage $TRIVY_CDX_TOTAL $TRIVY_CDX_UNIQUE) | $TRIVY_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Syft ($SYFT_VERSION) | SPDX ($SYFT_SPDX_VERSION) | $SYFT_SPDX_TOTAL | $SYFT_SPDX_UNIQUE | $(calculate_duplication_percentage $SYFT_SPDX_TOTAL $SYFT_SPDX_UNIQUE) | $SYFT_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Trivy ($TRIVY_VERSION) | SPDX ($TRIVY_SPDX_VERSION) | $TRIVY_SPDX_TOTAL | $TRIVY_SPDX_UNIQUE | $(calculate_duplication_percentage $TRIVY_SPDX_TOTAL $TRIVY_SPDX_UNIQUE)| $TRIVY_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: container + target_type: docker + target_path: nginx-test + setup_commands: | + cd docker && docker build . -t nginx-test + sbomify_input: nginx-test + sbomify_input_type: docker_image + baseline_command: | + docker run --rm nginx-test dpkg-query -W -f='${binary:Package} ${Version}\n' | sort + title: Docker (nginx + vim) Benchmark diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..8f9c754 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,28 @@ +--- +name: Go +on: + push: + branches: + - master + paths: + - 'go/**' + - '.github/workflows/go.yml' + - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'go/**' + - '.github/workflows/go.yml' + - '.github/workflows/_sbom-benchmark.yml' + +jobs: + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: go + target_type: filesystem + target_path: target-repo/go.mod + setup_commands: | + git clone --depth 1 --branch v2.3.1 https://github.com/google/osv-scanner.git target-repo + sbomify_input: target-repo/go.mod + sbomify_input_type: lock_file + title: Go (OSV Scanner 2.3.1) Benchmark diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml new file mode 100644 index 0000000..3bbe70b --- /dev/null +++ b/.github/workflows/java.yml @@ -0,0 +1,28 @@ +--- +name: Java +on: + push: + branches: + - master + paths: + - 'java/**' + - '.github/workflows/java.yml' + - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'java/**' + - '.github/workflows/java.yml' + - '.github/workflows/_sbom-benchmark.yml' + +jobs: + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: java + target_type: filesystem + target_path: target-repo/pom.xml + setup_commands: | + git clone --depth 1 --branch 26.4.7 https://github.com/keycloak/keycloak.git target-repo + sbomify_input: target-repo/pom.xml + sbomify_input_type: lock_file + title: Java (Keycloak 26.4.7) Benchmark diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml new file mode 100644 index 0000000..16d7d2f --- /dev/null +++ b/.github/workflows/javascript.yml @@ -0,0 +1,28 @@ +--- +name: JavaScript +on: + push: + branches: + - master + paths: + - 'javascript/**' + - '.github/workflows/javascript.yml' + - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'javascript/**' + - '.github/workflows/javascript.yml' + - '.github/workflows/_sbom-benchmark.yml' + +jobs: + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: javascript + target_type: filesystem + target_path: target-repo/pnpm-lock.yaml + setup_commands: | + git clone --depth 1 --branch wrangler@3.99.0 https://github.com/cloudflare/workers-sdk.git target-repo + sbomify_input: target-repo/pnpm-lock.yaml + sbomify_input_type: lock_file + title: JavaScript (Cloudflare workers-sdk) Benchmark diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9c8f713..b937454 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -7,249 +7,21 @@ on: paths: - 'python/**' - '.github/workflows/python.yml' - -env: - TRIVY_VERSION: 0.54.1 - SYFT_VERSION: 1.11.1 - CYCLONEDX_PYTHON_VERSION: 4.5.0 - SBOM4PYTHON_VERSION: 0.11.1 - SBOMQS_VERSION: 0.1.9 + - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'python/**' + - '.github/workflows/python.yml' + - '.github/workflows/_sbom-benchmark.yml' jobs: - trivy: - name: Trivy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Trivy - run: | - curl -L -o /tmp/trivy.tgz \ - "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" - tar xvf /tmp/trivy.tgz -C /tmp - chmod +x /tmp/trivy - - - name: "CycloneDX: Generate SBOM" - run: | - /tmp/trivy fs \ - --format cyclonedx \ - --output /tmp/trivy.cdx.json \ - python/requirements.txt - - - name: "SPDX: Generate SBOM" - run: | - /tmp/trivy fs \ - --format spdx-json \ - --output /tmp/trivy.spdx.json \ - python/requirements.txt - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: python-trivy - path: "/tmp/trivy.*.json" - - syft: - name: Syft - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Syft - run: | - curl -L -o /tmp/syft.tgz \ - "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_amd64.tar.gz" - tar xvf /tmp/syft.tgz -C /tmp - chmod +x /tmp/syft - - - name: "CycloneDX: Generate SBOM" - run: | - /tmp/syft python/requirements.txt \ - -o cyclonedx-json > \ - /tmp/syft.cdx.json - - - name: "SPDX: Generate SBOM" - run: | - /tmp/syft python/requirements.txt \ - -o spdx-json > \ - /tmp/syft.spdx.json - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: python-syft - path: "/tmp/syft.*.json" - - cyclonedx-python: - name: CycloneDX Python - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install CycloneDX Python - run: | - python -m pip install \ - cyclonedx-bom==${CYCLONEDX_PYTHON_VERSION} - - - name: "CycloneDX: Generate SBOM" - run: | - cyclonedx-py requirements \ - python/requirements.txt \ - --schema-version 1.6 \ - > /tmp/cyclonedx.cdx.json - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: python-cyclonedx - path: "/tmp/cyclonedx.*.json" - - sbom4python: - name: sbom4python - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install sbom4python - run: | - python -m pip install \ - sbom4python==${SBOM4PYTHON_VERSION} - - - name: "CycloneDX: Generate SBOM" - run: | - sbom4python \ - -r python/requirements.txt \ - --sbom cyclonedx \ - --format json \ - -o /tmp/sbom4python.cdx.json - - - name: "SPDX: Generate SBOM" - run: | - sbom4python \ - -r python/requirements.txt \ - --sbom spdx \ - --format json \ - -o /tmp/sbom4python.spdx.json - - - name: Upload SBOM - uses: actions/upload-artifact@v4 - with: - name: sbom4python - path: "/tmp/sbom4python.*.json" - - Score: - needs: - - trivy - - syft - - sbom4python - - cyclonedx-python - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Download SBOMs - uses: actions/download-artifact@v4 - - - - name: Install sbomqs - run: | - curl -L -o /tmp/sbomqs \ - "https://github.com/interlynk-io/sbomqs/releases/download/v${SBOMQS_VERSION}/sbomqs-linux-amd64" - chmod +x /tmp/sbomqs - - - name: "Generate Summary Table" - run: | - # DEBUG - find . - - calculate_duplication_percentage() { - local total=$1 - local unique=$2 - - if [ "$total" -eq 0 ]; then - echo "Duplication Percentage: N/A (Total count is zero)" - return 1 - fi - - local duplicates=$((total - unique)) - - if [ "$duplicates" -lt 0 ]; then - duplicates=0 - fi - - local duplication_percentage=$(echo "scale=2; ($duplicates / $total) * 100" | bc) - - echo "$duplication_percentage%" - } - - - - # Baseline - SYSTEM_BASELINE=$(cat python/requirements.txt | wc -l) - - # Syft - SYFT_CDX_PATH="python-syft/syft.cdx.json" - SYFT_CDX_TOTAL=$(jq '.components[] | .name' $SYFT_CDX_PATH | wc -l) - SYFT_CDX_UNIQUE=$(jq '.components[] | .name' $SYFT_CDX_PATH | uniq | wc -l) - SYFT_CDX_SBOMQS=$(/tmp/sbomqs score $SYFT_CDX_PATH -j) - SYFT_CDX_VERSION=$(echo $SYFT_CDX_SBOMQS | jq -r '.files[0].spec_version') - SYFT_CDX_QUALITY_SCORE=$(echo $SYFT_CDX_SBOMQS | jq -r '.files[0].avg_score') - - SYFT_SPDX_PATH="python-syft/syft.spdx.json" - SYFT_SPDX_TOTAL=$(jq '.packages[] | .name' $SYFT_SPDX_PATH | wc -l) - SYFT_SPDX_UNIQUE=$(jq '.packages[] | .name' $SYFT_SPDX_PATH | uniq | wc -l) - SYFT_SPDX_SBOMQS=$(/tmp/sbomqs score $SYFT_SPDX_PATH -j) - SYFT_SPDX_VERSION=$(echo $SYFT_SPDX_SBOMQS | jq -r '.files[0].spec_version') - SYFT_SPDX_QUALITY_SCORE=$(echo $SYFT_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - # Trivy - TRIVY_CDX_PATH="python-trivy/trivy.cdx.json" - TRIVY_CDX_TOTAL=$(jq '.components[] | .name' $TRIVY_CDX_PATH | wc -l) - TRIVY_CDX_UNIQUE=$(jq '.components[] | .name' $TRIVY_CDX_PATH | uniq | wc -l) - TRIVY_CDX_SBOMQS=$(/tmp/sbomqs score $TRIVY_CDX_PATH -j) - TRIVY_CDX_VERSION=$(echo $TRIVY_CDX_SBOMQS | jq -r '.files[0].spec_version') - TRIVY_CDX_QUALITY_SCORE=$(echo $TRIVY_CDX_SBOMQS | jq -r '.files[0].avg_score') - - TRIVY_SPDX_PATH="python-trivy/trivy.spdx.json" - TRIVY_SPDX_TOTAL=$(jq '.packages[] | .name' $TRIVY_SPDX_PATH | wc -l) - TRIVY_SPDX_UNIQUE=$(jq '.packages[] | .name' $TRIVY_SPDX_PATH | uniq | wc -l) - TRIVY_SPDX_SBOMQS=$(/tmp/sbomqs score $TRIVY_SPDX_PATH -j) - TRIVY_SPDX_VERSION=$(echo $TRIVY_SPDX_SBOMQS | jq -r '.files[0].spec_version') - TRIVY_SPDX_QUALITY_SCORE=$(echo $TRIVY_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - # sbom4python - SBOM4PYTHON_CDX_PATH="python-trivy/trivy.cdx.json" - SBOM4PYTHON_CDX_TOTAL=$(jq '.components[] | .name' $SBOM4PYTHON_CDX_PATH | wc -l) - SBOM4PYTHON_CDX_UNIQUE=$(jq '.components[] | .name' $SBOM4PYTHON_CDX_PATH | uniq | wc -l) - SBOM4PYTHON_CDX_SBOMQS=$(/tmp/sbomqs score $SBOM4PYTHON_CDX_PATH -j) - SBOM4PYTHON_CDX_VERSION=$(echo $SBOM4PYTHON_CDX_SBOMQS | jq -r '.files[0].spec_version') - SBOM4PYTHON_CDX_QUALITY_SCORE=$(echo $SBOM4PYTHON_CDX_SBOMQS | jq -r '.files[0].avg_score') - - SBOM4PYTHON_SPDX_PATH="python-trivy/trivy.spdx.json" - SBOM4PYTHON_SPDX_TOTAL=$(jq '.packages[] | .name' $SBOM4PYTHON_SPDX_PATH | wc -l) - SBOM4PYTHON_SPDX_UNIQUE=$(jq '.packages[] | .name' $SBOM4PYTHON_SPDX_PATH | uniq | wc -l) - SBOM4PYTHON_SPDX_SBOMQS=$(/tmp/sbomqs score $SBOM4PYTHON_SPDX_PATH -j) - SBOM4PYTHON_SPDX_VERSION=$(echo $SBOM4PYTHON_SPDX_SBOMQS | jq -r '.files[0].spec_version') - SBOM4PYTHON_SPDX_QUALITY_SCORE=$(echo $SBOM4PYTHON_SPDX_SBOMQS | jq -r '.files[0].avg_score') - - # CycloneDX Python - PYTHON_CYCLONEDX_CDX_PATH="python-cyclonedx/cyclonedx.cdx.json" - PYTHON_CYCLONEDX_CDX_TOTAL=$(jq '.components[] | .name' $PYTHON_CYCLONEDX_CDX_PATH | wc -l) - PYTHON_CYCLONEDX_CDX_UNIQUE=$(jq '.components[] | .name' $PYTHON_CYCLONEDX_CDX_PATH | uniq | wc -l) - PYTHON_CYCLONEDX_CDX_SBOMQS=$(/tmp/sbomqs score $PYTHON_CYCLONEDX_CDX_PATH -j) - PYTHON_CYCLONEDX_CDX_VERSION=$(echo $PYTHON_CYCLONEDX_CDX_SBOMQS | jq -r '.files[0].spec_version') - PYTHON_CYCLONEDX_CDX_QUALITY_SCORE=$(echo $PYTHON_CYCLONEDX_CDX_SBOMQS | jq -r '.files[0].avg_score') - - # Header - echo "| Tool | Format | Packages | Unique Packages | Duplication % | Avg Quality Score |" >> ${GITHUB_STEP_SUMMARY} - echo "| -- | -- | -- | -- | -- |-- |" >> ${GITHUB_STEP_SUMMARY} - - # Construct table - echo "| System Baseline | N/A | $SYSTEM_BASELINE | $SYSTEM_BASELINE | 0% | N/A |" >> ${GITHUB_STEP_SUMMARY} - echo "| Syft ($SYFT_VERSION) | CycloneDX ($SYFT_CDX_VERSION) | $SYFT_CDX_TOTAL | $SYFT_CDX_UNIQUE | $(calculate_duplication_percentage $SYFT_CDX_TOTAL $SYFT_CDX_UNIQUE) | $SYFT_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Trivy ($TRIVY_VERSION) | CycloneDX ($TRIVY_CDX_VERSION) | $TRIVY_CDX_TOTAL | $TRIVY_CDX_UNIQUE | $(calculate_duplication_percentage $TRIVY_CDX_TOTAL $TRIVY_CDX_UNIQUE) | $TRIVY_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| sbom4python ($SBOM4PYTHON_VERSION) | CycloneDX ($SBOM4PYTHON_CDX_VERSION) | $SBOM4PYTHON_CDX_TOTAL | $SBOM4PYTHON_CDX_UNIQUE | $(calculate_duplication_percentage $SBOM4PYTHON_CDX_TOTAL $SBOM4PYTHON_CDX_UNIQUE) | $SBOM4PYTHON_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| python-cyclonedx ($CYCLONEDX_PYTHON_VERSION) | CycloneDX ($PYTHON_CYCLONEDX_CDX_VERSION) | $PYTHON_CYCLONEDX_CDX_TOTAL | $PYTHON_CYCLONEDX_CDX_UNIQUE | $(calculate_duplication_percentage $PYTHON_CYCLONEDX_CDX_TOTAL $PYTHON_CYCLONEDX_CDX_UNIQUE) | $PYTHON_CYCLONEDX_CDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Syft ($SYFT_VERSION) | SPDX ($SYFT_SPDX_VERSION) | $SYFT_SPDX_TOTAL | $SYFT_SPDX_UNIQUE | $(calculate_duplication_percentage $SYFT_SPDX_TOTAL $SYFT_SPDX_UNIQUE) | $SYFT_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| Trivy ($TRIVY_VERSION) | SPDX ($TRIVY_SPDX_VERSION) | $TRIVY_SPDX_TOTAL | $TRIVY_SPDX_UNIQUE | $(calculate_duplication_percentage $TRIVY_SPDX_TOTAL $TRIVY_SPDX_UNIQUE)| $TRIVY_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} - echo "| sbom4python ($SBOM4PYTHON_VERSION) | SPDX ($SBOM4PYTHON_SPDX_VERSION) | $SBOM4PYTHON_SPDX_TOTAL | $SBOM4PYTHON_SPDX_UNIQUE | $(calculate_duplication_percentage $SBOM4PYTHON_SPDX_TOTAL $SBOM4PYTHON_SPDX_UNIQUE)| $SBOM4PYTHON_SPDX_QUALITY_SCORE |" >> ${GITHUB_STEP_SUMMARY} + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: python + target_type: filesystem + target_path: python/requirements.txt + sbomify_input: python/requirements.txt + sbomify_input_type: lock_file + extra_tools: cyclonedx-python,sbom4python + title: Python (Django) Benchmark diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..0555b49 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,29 @@ +--- +name: Rust +on: + push: + branches: + - master + paths: + - 'rust/**' + - '.github/workflows/rust.yml' + - '.github/workflows/_sbom-benchmark.yml' + pull_request: + paths: + - 'rust/**' + - '.github/workflows/rust.yml' + - '.github/workflows/_sbom-benchmark.yml' + +jobs: + benchmark: + uses: ./.github/workflows/_sbom-benchmark.yml + with: + name: rust + target_type: filesystem + target_path: target-repo/Cargo.lock + setup_commands: | + git clone --depth 1 --branch 0.24.5 https://github.com/cloudflare/quiche.git target-repo + cd target-repo && cargo generate-lockfile + sbomify_input: target-repo/Cargo.lock + sbomify_input_type: lock_file + title: Rust (Cloudflare quiche 0.24.5) Benchmark diff --git a/README.md b/README.md index 92d1d8c..e2cf527 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,72 @@ # SBOM Benchmarking [![Python](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml) +[![JavaScript](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml) +[![Java](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml) +[![Go](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml) +[![Rust](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml) [![Docker](https://github.com/sbomify/sbom-benchmarks/actions/workflows/docker.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/docker.yml) -This repository is designed to generate Software Bill of Materials (SBOMs) using a comprehensive benchmark across a wide variety of tools on a defined target across multiple programming languages. The goal is to provide a consistent and standardized method for evaluating and comparing the effectiveness and accuracy of various SBOM generation tools, helping users to identify the best tool for their specific needs. +This repository is designed to generate Software Bill of Materials (SBOMs) using a comprehensive benchmark across a wide variety of tools on defined targets across multiple programming languages. The goal is to provide a consistent and standardized method for evaluating and comparing the effectiveness and accuracy of various SBOM generation tools, helping users to identify the best tool for their specific needs. The list of tools used is pulled from our [SBOM resources](https://sbomify.com/resources/) page that includes a comprehensive list of SBOM tools. ## Features -* Multi-Tool Support: Run benchmarks across a diverse set of SBOM generation tools. -* Cross-Language Compatibility: Supports multiple programming languages, allowing for comprehensive analysis regardless of the technology stack. -* Automated Workflow: Easily set up and execute benchmarks with minimal manual intervention. -* Detailed Reports: Generate detailed comparisons and summaries of the SBOMs produced by different tools, highlighting strengths and weaknesses. -* Extensibility: Add support for new tools or languages with minimal configuration changes. +* **Multi-Tool Support**: Run benchmarks across a diverse set of SBOM generation tools including Trivy, Syft, and sbomify. +* **Cross-Language Compatibility**: Supports multiple programming languages (Python, JavaScript, Java, Go, Rust) and container images. +* **Automated Workflow**: Easily set up and execute benchmarks with minimal manual intervention. +* **Detailed Reports**: Generate detailed comparisons and summaries of the SBOMs produced by different tools, highlighting strengths and weaknesses. +* **Quality Scoring**: Each SBOM is scored using [sbomqs](https://github.com/interlynk-io/sbomqs) to measure SBOM quality. +* **Extensibility**: Add support for new tools or languages with minimal configuration changes. +## Benchmarked Tools + +| Tool | Description | +|------|-------------| +| [Trivy](https://github.com/aquasecurity/trivy) | Comprehensive security scanner with SBOM generation | +| [Syft](https://github.com/anchore/syft) | CLI tool and library for generating SBOMs | +| [sbomify](https://github.com/sbomify/github-action) | SBOM generation with enrichment from package registries | +| [cyclonedx-python](https://github.com/CycloneDX/cyclonedx-python) | Native Python SBOM generator (Python benchmarks only) | +| [sbom4python](https://github.com/anthonyharrison/sbom4python) | Python-specific SBOM generator (Python benchmarks only) | + +## Benchmark Targets + +| Target | Language/Type | Project | Description | +|--------|--------------|---------|-------------| +| [Python](python/) | Python | Django | Python web framework dependencies | +| [JavaScript](javascript/) | JavaScript/TypeScript | workers-sdk | Cloudflare's Wrangler CLI monorepo | +| [Java](java/) | Java/Maven | Keycloak | Enterprise IAM with complex Maven dependencies | +| [Go](go/) | Go | OSV Scanner | Go modules-based security tool | +| [Rust](rust/) | Rust | quiche | Cloudflare's QUIC/HTTP3 implementation | +| [Docker](docker/) | Container | nginx + vim | Container image with added packages | ## Run Details -* [Python](https://github.com/sbomify/sbom-benchmarks/tree/master/python) -* [Docker](https://github.com/sbomify/sbom-benchmarks/tree/master/docker) +Each benchmark runs automatically on push to master and produces: +- SBOMs in both CycloneDX and SPDX formats (where supported) +- Quality scores from sbomqs +- Comparison tables in the GitHub Actions summary + +Click on any badge above to see the latest benchmark results. + +### Detailed Documentation + +* [Python Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/python) - Django requirements.txt +* [JavaScript Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/javascript) - Cloudflare workers-sdk (pnpm-lock.yaml) +* [Java Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/java) - Keycloak (Maven/pom.xml) +* [Go Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/go) - OSV Scanner (go.mod) +* [Rust Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/rust) - Cloudflare quiche (Cargo.lock) +* [Docker Benchmark](https://github.com/sbomify/sbom-benchmarks/tree/master/docker) - nginx container with vim installed + +## Tool Versions + +Current tool versions used in benchmarks: + +| Tool | Version | +|------|---------| +| Trivy | 0.68.2 | +| Syft | 1.39.0 | +| sbomqs | 2.0.2 | +| cyclonedx-bom | 7.2.1 | +| sbom4python | 0.12.4 | diff --git a/go/README.md b/go/README.md new file mode 100644 index 0000000..1675fe4 --- /dev/null +++ b/go/README.md @@ -0,0 +1,39 @@ +# SBOM Generation for Go + +[![Go](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml) + +## Target Project + +This benchmark uses [OSV Scanner](https://github.com/google/osv-scanner), Google's vulnerability scanner that uses the OSV database to find known vulnerabilities in project dependencies. It's a production-grade Go project with a well-structured dependency tree. + +**Version:** 2.3.1 + +## Tools + +Tools from the sbomify [resource list](https://sbomify.com/resources/), specifically: + +* Trivy +* Syft +* sbomify github-action + +## Process + +The benchmark workflow: + +1. Clones the OSV Scanner repository at the specified tag +2. Runs each SBOM generator against the project's `go.mod` and source tree +3. Scores each generated SBOM using sbomqs +4. Produces a comparison table in the workflow summary + +The full process is automated and you can see the exact commands in [go.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/go.yml). + +If you look at the [Go CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/go.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. + +## Why OSV Scanner? + +OSV Scanner was chosen as a benchmark target because: + +- **Go modules**: Clean Go module dependency management via `go.mod` +- **Security tooling**: Ironically, it's a security tool itself - interesting to SBOM a vulnerability scanner +- **Google-maintained**: High-quality codebase with good dependency hygiene +- **Moderate complexity**: Not too simple, not too complex - good middle-ground benchmark diff --git a/java/README.md b/java/README.md new file mode 100644 index 0000000..af647f1 --- /dev/null +++ b/java/README.md @@ -0,0 +1,39 @@ +# SBOM Generation for Java + +[![Java](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml) + +## Target Project + +This benchmark uses [Keycloak](https://github.com/keycloak/keycloak), an Open Source Identity and Access Management solution for modern applications and services. Keycloak is a large-scale Java/Maven project with complex dependency management, making it an excellent benchmark target. + +**Version:** 26.4.7 + +## Tools + +Tools from the sbomify [resource list](https://sbomify.com/resources/), specifically: + +* Trivy +* Syft +* sbomify github-action + +## Process + +The benchmark workflow: + +1. Clones the Keycloak repository at the specified tag +2. Runs each SBOM generator against the project's `pom.xml` and source tree +3. Scores each generated SBOM using sbomqs +4. Produces a comparison table in the workflow summary + +The full process is automated and you can see the exact commands in [java.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/java.yml). + +If you look at the [Java CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/java.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. + +## Why Keycloak? + +Keycloak was chosen as a benchmark target because: + +- **Complex dependency tree**: Hundreds of Maven dependencies across multiple modules +- **Real-world project**: Actively maintained, widely deployed enterprise software +- **Mixed ecosystem**: Combines Java libraries, JavaScript frontend components, and more +- **Multi-module Maven**: Tests SBOM generators' ability to handle complex build structures diff --git a/javascript/README.md b/javascript/README.md new file mode 100644 index 0000000..3cb895c --- /dev/null +++ b/javascript/README.md @@ -0,0 +1,40 @@ +# SBOM Generation for JavaScript + +[![JavaScript](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml) + +## Target Project + +This benchmark uses [workers-sdk](https://github.com/cloudflare/workers-sdk), Cloudflare's monorepo containing Wrangler (the CLI for Cloudflare Workers), Miniflare, and related tooling. It's a large-scale TypeScript/JavaScript project with complex dependency management. + +**Version:** wrangler@3.99.0 + +## Tools + +Tools from the sbomify [resource list](https://sbomify.com/resources/), specifically: + +* Trivy +* Syft +* sbomify github-action + +## Process + +The benchmark workflow: + +1. Clones the workers-sdk repository at the specified tag +2. Runs each SBOM generator against the project's `pnpm-lock.yaml` and source tree +3. Scores each generated SBOM using sbomqs +4. Produces a comparison table in the workflow summary + +The full process is automated and you can see the exact commands in [javascript.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/javascript.yml). + +If you look at the [JavaScript CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/javascript.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. + +## Why workers-sdk? + +workers-sdk was chosen as a benchmark target because: + +- **Monorepo structure**: Multiple packages (Wrangler, Miniflare, C3) in a pnpm workspace +- **Production TypeScript**: Powers Cloudflare's developer tooling used by millions +- **Complex dependencies**: Hundreds of npm packages across multiple workspaces +- **pnpm lockfile**: Tests SBOM generators' support for modern package managers +- **Widely used**: 106k+ dependent repositories on GitHub diff --git a/python/README.md b/python/README.md index 993defd..acec90c 100644 --- a/python/README.md +++ b/python/README.md @@ -1,24 +1,23 @@ # SBOM Generation for Python + [![Python](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml) -## Setup +## Target -Generate `requirements.txt`: +The `requirements.txt` is **intentionally minimal** - it only lists direct dependencies (Django and its immediate deps). This tests each SBOM tool's ability to resolve **transitive dependencies**. -```bash -$ pip install Django -$ pip freeze > requirements.txt -``` +A quality SBOM generator should discover all indirect dependencies, not just what's explicitly listed in the lockfile. ## Tools -Tools from the sbomify [resource list](https://sbomify.com/resources/#Python), but in short they are: +Tools from the sbomify [resource list](https://sbomify.com/resources/#Python): * Trivy * Syft +* sbomify github-action * sbom4python * cyclonedx-python The full process is automated and you can see the exact commands we run in [python.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/python.yml). -If you look at the [Python CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml), you can also see the quality score of the SBOMs (from `sbomqsq`) as well as downloading the actual SBOMs as artifact. +If you look at the [Python CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/python.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. diff --git a/python/requirements.txt b/python/requirements.txt index c3c8b20..744b9a6 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,3 +1,7 @@ +# Intentionally minimal - only direct dependencies listed. +# This tests SBOM tools' ability to resolve transitive dependencies. +# A complete SBOM should include all indirect dependencies (e.g., pytz, typing-extensions, etc.) + asgiref==3.8.1 Django==5.1 sqlparse==0.5.1 diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..78fa6be --- /dev/null +++ b/rust/README.md @@ -0,0 +1,40 @@ +# SBOM Generation for Rust + +[![Rust](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml/badge.svg)](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml) + +## Target Project + +This benchmark uses [quiche](https://github.com/cloudflare/quiche), Cloudflare's implementation of the QUIC transport protocol and HTTP/3. It's a production-grade Rust project used in Cloudflare's infrastructure. + +**Version:** 0.24.5 + +## Tools + +Tools from the sbomify [resource list](https://sbomify.com/resources/), specifically: + +* Trivy +* Syft +* sbomify github-action + +## Process + +The benchmark workflow: + +1. Clones the quiche repository at the specified tag +2. Runs each SBOM generator against the project's `Cargo.lock` and source tree +3. Scores each generated SBOM using sbomqs +4. Produces a comparison table in the workflow summary + +The full process is automated and you can see the exact commands in [rust.yml](https://github.com/sbomify/sbom-benchmarks/blob/master/.github/workflows/rust.yml). + +If you look at the [Rust CI/CD run](https://github.com/sbomify/sbom-benchmarks/actions/workflows/rust.yml), you can see the quality score of the SBOMs (from `sbomqs`) as well as download the actual SBOMs as artifacts. + +## Why quiche? + +quiche was chosen as a benchmark target because: + +- **Production Rust**: Used in Cloudflare's production infrastructure serving millions of requests +- **Complex dependencies**: Multiple crates with native dependencies (BoringSSL) +- **Workspace structure**: Multi-crate Cargo workspace tests SBOM generators' Rust support +- **Well-maintained**: Active development with regular releases +- **Popular**: 11k+ GitHub stars, widely used in the Rust ecosystem