From 993f5a04862953103fb472c64738e5a9aa7ec7ec Mon Sep 17 00:00:00 2001 From: "Jan C. Brammer" Date: Tue, 28 Apr 2026 12:44:30 +0000 Subject: [PATCH 1/5] Test PRs for unit test coverage regression Co-authored-by: Copilot --- .../workflows/test_coverage_regression.yml | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 .github/workflows/test_coverage_regression.yml diff --git a/.github/workflows/test_coverage_regression.yml b/.github/workflows/test_coverage_regression.yml new file mode 100644 index 00000000..a092f2a1 --- /dev/null +++ b/.github/workflows/test_coverage_regression.yml @@ -0,0 +1,187 @@ +name: Test for unit test coverage regression + +on: + pull_request: + branches: [ dev ] + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +concurrency: + group: pr-coverage-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + coverage: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v6 + + - name: Install build dependencies + run: | + sudo ./INCHI-1-TEST/install_build_dependencies.sh + + - name: Generate coverage report + uses: ./.github/actions/generate_coverage_report + + - name: Test for unit test regression + run: | + set -euo pipefail + + cd CMake_build/coverage_build + + BASE_SUMMARY_URL="https://iupac-inchi.github.io/InChI/coverage/summary.txt" + BASE_SUMMARY_FILE="$(mktemp)" + PR_SUMMARY_FILE="coverage_reports/summary.txt" + COVERAGE_REPORT_MD="coverage_reports/coverage_report.md" + + mkdir -p coverage_reports + trap 'rm -f "$BASE_SUMMARY_FILE"' EXIT + + extract_pct() { + local metric="$1" + local file="$2" + + awk -v metric="$metric" ' + $0 ~ "^[[:space:]]*" metric "\\.*:" { + if (match($0, /[0-9]+([.][0-9]+)?%/)) { + value = substr($0, RSTART, RLENGTH - 1) + print value + } + exit + } + ' "$file" + } + + lcov --summary coverage.lcov \ + --branch-coverage \ + --ignore-errors inconsistent > "$PR_SUMMARY_FILE" + + PR_LINES="$(extract_pct lines "$PR_SUMMARY_FILE")" + PR_BRANCHES="$(extract_pct branches "$PR_SUMMARY_FILE")" + + if [[ -z "$PR_LINES" || -z "$PR_BRANCHES" ]]; then + echo "Unable to parse PR coverage metrics from $PR_SUMMARY_FILE" + cat "$PR_SUMMARY_FILE" + exit 1 + fi + + BASE_AVAILABLE=false + BASE_LINES="" + BASE_BRANCHES="" + + if curl -fsSL --max-time 10 "$BASE_SUMMARY_URL" -o "$BASE_SUMMARY_FILE"; then + BASE_LINES="$(extract_pct lines "$BASE_SUMMARY_FILE")" + BASE_BRANCHES="$(extract_pct branches "$BASE_SUMMARY_FILE")" + + if [[ -n "$BASE_LINES" && -n "$BASE_BRANCHES" ]]; then + BASE_AVAILABLE=true + else + echo "Base summary fetched, but required metrics were incomplete." + fi + else + echo "Base summary fetch failed." + fi + + BASE_LINES_DISPLAY="N/A" + BASE_BRANCHES_DISPLAY="N/A" + DIFF_LINES="N/A" + DIFF_BRANCHES="N/A" + + if [[ "$BASE_AVAILABLE" == true ]]; then + BASE_LINES_DISPLAY="$BASE_LINES" + BASE_BRANCHES_DISPLAY="$BASE_BRANCHES" + DIFF_LINES="$(awk -v pr="$PR_LINES" -v base="$BASE_LINES" 'BEGIN {printf "%+.2f", pr-base}')" + DIFF_BRANCHES="$(awk -v pr="$PR_BRANCHES" -v base="$BASE_BRANCHES" 'BEGIN {printf "%+.2f", pr-base}')" + fi + + { + echo "### Coverage Regression Summary" + echo + echo "| Metric | PR (%) | Base (%) | Difference (pp) |" + echo "| --- | ---: | ---: | ---: |" + echo "| Lines | ${PR_LINES} | ${BASE_LINES_DISPLAY} | ${DIFF_LINES} |" + echo "| Branches | ${PR_BRANCHES} | ${BASE_BRANCHES_DISPLAY} | ${DIFF_BRANCHES} |" + echo + echo "Find details on the base coverage at https://iupac-inchi.github.io/InChI/coverage/index.html" + } > "$COVERAGE_REPORT_MD" + + if [[ "$BASE_AVAILABLE" == false ]]; then + echo "Base coverage metrics unavailable. Cannot check for regression." + exit 1 + fi + + FAIL=false + if awk -v pr="$PR_LINES" -v base="$BASE_LINES" 'BEGIN {exit !(pr < base)}'; then + echo "Line coverage regression: ${PR_LINES}% < base ${BASE_LINES}%" + FAIL=true + fi + if awk -v pr="$PR_BRANCHES" -v base="$BASE_BRANCHES" 'BEGIN {exit !(pr < base)}'; then + echo "Branch coverage regression: ${PR_BRANCHES}% < base ${BASE_BRANCHES}%" + FAIL=true + fi + if [[ "$FAIL" == true ]]; then + exit 1 + fi + + echo "Coverage OK: lines ${PR_LINES}% >= base ${BASE_LINES}%, branches ${PR_BRANCHES}% >= base ${BASE_BRANCHES}%" + + - name: Upload coverage reports + uses: actions/upload-artifact@v7 + if: always() + with: + name: coverage-reports-${{ github.run_id }} + path: CMake_build/coverage_build/coverage_reports/ + retention-days: 30 + + - name: Post sticky PR comment (same-repo only) + if: always() && github.event.pull_request.head.repo.full_name == github.repository + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const report = fs.readFileSync('CMake_build/coverage_build/coverage_reports/coverage_report.md', 'utf8'); + + // Build artifact link + const runId = context.runId; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`; + const artifactLink = `Find details on this PR's coverage by downloading [coverage-reports-${runId}](${runUrl}) and opening **html/index.html**`; + + const body = `## Unit Test Coverage Report\n\n${report}\n\n${artifactLink}`; + const identifier = ''; + const bodyWithId = `${identifier}\n${body}`; + + // Search for existing comment with the identifier + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + + const existingComment = comments.find(comment => + comment.body.includes(identifier) + ); + + if (existingComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body: bodyWithId, + }); + console.log('Updated existing comment:', existingComment.id); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: bodyWithId, + }); + console.log('Created new comment'); + } From c2fc7e4b13c4f18e292d54ec0fada0b2d50fe7e1 Mon Sep 17 00:00:00 2001 From: "Jan C. Brammer" Date: Wed, 29 Apr 2026 05:35:57 +0000 Subject: [PATCH 2/5] Use identifier that works for PRs and manual runs --- .github/workflows/test_coverage_regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_coverage_regression.yml b/.github/workflows/test_coverage_regression.yml index a092f2a1..4c43b1d7 100644 --- a/.github/workflows/test_coverage_regression.yml +++ b/.github/workflows/test_coverage_regression.yml @@ -10,7 +10,7 @@ permissions: pull-requests: write concurrency: - group: pr-coverage-${{ github.event.pull_request.number }} + group: pr-coverage-${{ github.run_id }} cancel-in-progress: true jobs: From c6086f3ac9d889924dfcde3e12410d905b569273 Mon Sep 17 00:00:00 2001 From: "Jan C. Brammer" Date: Wed, 29 Apr 2026 05:52:42 +0000 Subject: [PATCH 3/5] Don't fail coverage regression test due to missing metrics Co-authored-by: Copilot --- .../workflows/test_coverage_regression.yml | 57 +++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_coverage_regression.yml b/.github/workflows/test_coverage_regression.yml index 4c43b1d7..5199c18c 100644 --- a/.github/workflows/test_coverage_regression.yml +++ b/.github/workflows/test_coverage_regression.yml @@ -56,17 +56,25 @@ jobs: ' "$file" } - lcov --summary coverage.lcov \ - --branch-coverage \ - --ignore-errors inconsistent > "$PR_SUMMARY_FILE" + PROBLEMS=() - PR_LINES="$(extract_pct lines "$PR_SUMMARY_FILE")" - PR_BRANCHES="$(extract_pct branches "$PR_SUMMARY_FILE")" + PR_AVAILABLE=false + PR_LINES="" + PR_BRANCHES="" - if [[ -z "$PR_LINES" || -z "$PR_BRANCHES" ]]; then - echo "Unable to parse PR coverage metrics from $PR_SUMMARY_FILE" - cat "$PR_SUMMARY_FILE" - exit 1 + if lcov --summary coverage.lcov \ + --branch-coverage \ + --ignore-errors inconsistent > "$PR_SUMMARY_FILE"; then + PR_LINES="$(extract_pct lines "$PR_SUMMARY_FILE")" + PR_BRANCHES="$(extract_pct branches "$PR_SUMMARY_FILE")" + + if [[ -n "$PR_LINES" && -n "$PR_BRANCHES" ]]; then + PR_AVAILABLE=true + else + PROBLEMS+=("Unable to parse PR coverage metrics from $PR_SUMMARY_FILE") + fi + else + PROBLEMS+=("Unable to generate PR coverage summary from coverage.lcov") fi BASE_AVAILABLE=false @@ -80,18 +88,25 @@ jobs: if [[ -n "$BASE_LINES" && -n "$BASE_BRANCHES" ]]; then BASE_AVAILABLE=true else - echo "Base summary fetched, but required metrics were incomplete." + PROBLEMS+=("Base summary fetched, but required metrics were incomplete") fi else - echo "Base summary fetch failed." + PROBLEMS+=("Base summary fetch failed from $BASE_SUMMARY_URL") fi + PR_LINES_DISPLAY="N/A" + PR_BRANCHES_DISPLAY="N/A" BASE_LINES_DISPLAY="N/A" BASE_BRANCHES_DISPLAY="N/A" DIFF_LINES="N/A" DIFF_BRANCHES="N/A" - if [[ "$BASE_AVAILABLE" == true ]]; then + if [[ "$PR_AVAILABLE" == true ]]; then + PR_LINES_DISPLAY="$PR_LINES" + PR_BRANCHES_DISPLAY="$PR_BRANCHES" + fi + + if [[ "$PR_AVAILABLE" == true && "$BASE_AVAILABLE" == true ]]; then BASE_LINES_DISPLAY="$BASE_LINES" BASE_BRANCHES_DISPLAY="$BASE_BRANCHES" DIFF_LINES="$(awk -v pr="$PR_LINES" -v base="$BASE_LINES" 'BEGIN {printf "%+.2f", pr-base}')" @@ -103,15 +118,23 @@ jobs: echo echo "| Metric | PR (%) | Base (%) | Difference (pp) |" echo "| --- | ---: | ---: | ---: |" - echo "| Lines | ${PR_LINES} | ${BASE_LINES_DISPLAY} | ${DIFF_LINES} |" - echo "| Branches | ${PR_BRANCHES} | ${BASE_BRANCHES_DISPLAY} | ${DIFF_BRANCHES} |" + echo "| Lines | ${PR_LINES_DISPLAY} | ${BASE_LINES_DISPLAY} | ${DIFF_LINES} |" + echo "| Branches | ${PR_BRANCHES_DISPLAY} | ${BASE_BRANCHES_DISPLAY} | ${DIFF_BRANCHES} |" + if [[ ${#PROBLEMS[@]} -gt 0 ]]; then + echo + echo "### Problems" + echo + for problem in "${PROBLEMS[@]}"; do + echo "- ${problem}" + done + fi echo echo "Find details on the base coverage at https://iupac-inchi.github.io/InChI/coverage/index.html" } > "$COVERAGE_REPORT_MD" - if [[ "$BASE_AVAILABLE" == false ]]; then - echo "Base coverage metrics unavailable. Cannot check for regression." - exit 1 + if [[ "$PR_AVAILABLE" == false || "$BASE_AVAILABLE" == false ]]; then + echo "Coverage regression check skipped due to missing PR or base metrics." + exit 0 fi FAIL=false From 8830656adf2052ecf315c299afb7233307c519be Mon Sep 17 00:00:00 2001 From: "Jan C. Brammer" Date: Wed, 29 Apr 2026 06:00:35 +0000 Subject: [PATCH 4/5] Refactor script --- .../workflows/test_coverage_regression.yml | 79 +++++++++---------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/.github/workflows/test_coverage_regression.yml b/.github/workflows/test_coverage_regression.yml index 5199c18c..57ed1678 100644 --- a/.github/workflows/test_coverage_regression.yml +++ b/.github/workflows/test_coverage_regression.yml @@ -30,7 +30,6 @@ jobs: - name: Test for unit test regression run: | set -euo pipefail - cd CMake_build/coverage_build BASE_SUMMARY_URL="https://iupac-inchi.github.io/InChI/coverage/summary.txt" @@ -56,61 +55,57 @@ jobs: ' "$file" } - PROBLEMS=() - - PR_AVAILABLE=false - PR_LINES="" - PR_BRANCHES="" + parse_metrics() { + local prefix="$1" + local file="$2" + local parse_error="$3" + local lines branches - if lcov --summary coverage.lcov \ - --branch-coverage \ - --ignore-errors inconsistent > "$PR_SUMMARY_FILE"; then - PR_LINES="$(extract_pct lines "$PR_SUMMARY_FILE")" - PR_BRANCHES="$(extract_pct branches "$PR_SUMMARY_FILE")" + lines="$(extract_pct lines "$file")" + branches="$(extract_pct branches "$file")" - if [[ -n "$PR_LINES" && -n "$PR_BRANCHES" ]]; then - PR_AVAILABLE=true + if [[ -n "$lines" && -n "$branches" ]]; then + printf -v "${prefix}_LINES" '%s' "$lines" + printf -v "${prefix}_BRANCHES" '%s' "$branches" + printf -v "${prefix}_AVAILABLE" 'true' else - PROBLEMS+=("Unable to parse PR coverage metrics from $PR_SUMMARY_FILE") + PROBLEMS+=("$parse_error") fi - else - PROBLEMS+=("Unable to generate PR coverage summary from coverage.lcov") - fi + } + + pct_diff() { awk -v pr="$1" -v base="$2" 'BEGIN {printf "%+.2f", pr-base}'; } + is_lt() { awk -v pr="$1" -v base="$2" 'BEGIN {exit !(pr < base)}'; } + PROBLEMS=() + PR_AVAILABLE=false BASE_AVAILABLE=false + PR_LINES="" + PR_BRANCHES="" BASE_LINES="" BASE_BRANCHES="" - if curl -fsSL --max-time 10 "$BASE_SUMMARY_URL" -o "$BASE_SUMMARY_FILE"; then - BASE_LINES="$(extract_pct lines "$BASE_SUMMARY_FILE")" - BASE_BRANCHES="$(extract_pct branches "$BASE_SUMMARY_FILE")" + if lcov --summary coverage.lcov --branch-coverage --ignore-errors inconsistent > "$PR_SUMMARY_FILE"; then + parse_metrics PR "$PR_SUMMARY_FILE" "Unable to parse PR coverage metrics from $PR_SUMMARY_FILE" + else + PROBLEMS+=("Unable to generate PR coverage summary from coverage.lcov") + fi - if [[ -n "$BASE_LINES" && -n "$BASE_BRANCHES" ]]; then - BASE_AVAILABLE=true - else - PROBLEMS+=("Base summary fetched, but required metrics were incomplete") - fi + if curl -fsSL --max-time 10 "$BASE_SUMMARY_URL" -o "$BASE_SUMMARY_FILE"; then + parse_metrics BASE "$BASE_SUMMARY_FILE" "Base summary fetched, but required metrics were incomplete" else PROBLEMS+=("Base summary fetch failed from $BASE_SUMMARY_URL") fi - PR_LINES_DISPLAY="N/A" - PR_BRANCHES_DISPLAY="N/A" - BASE_LINES_DISPLAY="N/A" - BASE_BRANCHES_DISPLAY="N/A" + PR_LINES_DISPLAY="${PR_LINES:-N/A}" + PR_BRANCHES_DISPLAY="${PR_BRANCHES:-N/A}" + BASE_LINES_DISPLAY="${BASE_LINES:-N/A}" + BASE_BRANCHES_DISPLAY="${BASE_BRANCHES:-N/A}" DIFF_LINES="N/A" DIFF_BRANCHES="N/A" - if [[ "$PR_AVAILABLE" == true ]]; then - PR_LINES_DISPLAY="$PR_LINES" - PR_BRANCHES_DISPLAY="$PR_BRANCHES" - fi - if [[ "$PR_AVAILABLE" == true && "$BASE_AVAILABLE" == true ]]; then - BASE_LINES_DISPLAY="$BASE_LINES" - BASE_BRANCHES_DISPLAY="$BASE_BRANCHES" - DIFF_LINES="$(awk -v pr="$PR_LINES" -v base="$BASE_LINES" 'BEGIN {printf "%+.2f", pr-base}')" - DIFF_BRANCHES="$(awk -v pr="$PR_BRANCHES" -v base="$BASE_BRANCHES" 'BEGIN {printf "%+.2f", pr-base}')" + DIFF_LINES="$(pct_diff "$PR_LINES" "$BASE_LINES")" + DIFF_BRANCHES="$(pct_diff "$PR_BRANCHES" "$BASE_BRANCHES")" fi { @@ -124,9 +119,7 @@ jobs: echo echo "### Problems" echo - for problem in "${PROBLEMS[@]}"; do - echo "- ${problem}" - done + printf -- "- %s\n" "${PROBLEMS[@]}" fi echo echo "Find details on the base coverage at https://iupac-inchi.github.io/InChI/coverage/index.html" @@ -138,11 +131,11 @@ jobs: fi FAIL=false - if awk -v pr="$PR_LINES" -v base="$BASE_LINES" 'BEGIN {exit !(pr < base)}'; then + if is_lt "$PR_LINES" "$BASE_LINES"; then echo "Line coverage regression: ${PR_LINES}% < base ${BASE_LINES}%" FAIL=true fi - if awk -v pr="$PR_BRANCHES" -v base="$BASE_BRANCHES" 'BEGIN {exit !(pr < base)}'; then + if is_lt "$PR_BRANCHES" "$BASE_BRANCHES"; then echo "Branch coverage regression: ${PR_BRANCHES}% < base ${BASE_BRANCHES}%" FAIL=true fi From f23458101f0d812f5fafef66099f2ad3d5cc21a5 Mon Sep 17 00:00:00 2001 From: "Jan C. Brammer" Date: Wed, 29 Apr 2026 06:00:45 +0000 Subject: [PATCH 5/5] Update README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1fa0973c..3236b4c1 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Please write your comments in the header files where possible. The Doxygen documentation syntax is quite powerful: you can include formulas, tables, and diagrams. For details see . -The documentation is [built on every push](.github/workflows/deploy_docs.yml) to the default branch of this repository and hosted at . +The documentation is [built on every push](.github/workflows/deploy_pages.yml) to the default branch of this repository and hosted at . That is, your comments will automatically be rendered to HTML and served as online documentation. ## Code quality @@ -103,6 +103,10 @@ InChI is integrated into [Coverity Scan](https://scan.coverity.com/projects/inch InChI is integrated into [Google OSS-Fuzz](https://github.com/google/oss-fuzz/tree/master/projects/inchi), which runs continuous fuzz testing against the library. Fuzzing-related bugs are tracked at the [OSS-Fuzz issue tracker](https://bugs.chromium.org/p/oss-fuzz/issues/list?q=inchi). +### Code coverage + +The unit test coverage is [evaluated on every push](.github/workflows/deploy_pages.yml) to the default branch of this repository and hosted at . Every PR against the default branch is [tested for unit test coverage regressions](.github/workflows/test_coverage_regression.yml). + ## Contents of this repository ### INCHI-1-DOC