From 12a3f7c9a2906da0229ca60ff3956e24c9eea025 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Sat, 30 May 2026 09:27:50 -0500 Subject: [PATCH 1/4] Adopt tiered coverage gates for SDK package Move threshold enforcement from pytest fail-under to tiered check-coverage.sh (90% total, 95% on intentproof/). Signed-off-by: Nathan Gillett --- .github/workflows/ci.yml | 5 +- .github/workflows/release-pypi.yml | 2 +- README.md | 6 +- pyproject.toml | 2 - scripts/README-coverage-tiers.md | 8 ++ scripts/check-coverage.sh | 116 ++++++++++++++++++++++------- scripts/coverage-tiers.conf | 7 ++ scripts/run-coverage-gate.sh | 13 +++- 8 files changed, 123 insertions(+), 36 deletions(-) create mode 100644 scripts/README-coverage-tiers.md create mode 100644 scripts/coverage-tiers.conf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb353fe..0bdda03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,4 @@ jobs: pip install -e ".[dev]" - name: Run tests with coverage - run: pytest -q - - - name: Enforce coverage threshold - run: bash ./scripts/check-coverage.sh 95 + run: bash ./scripts/run-coverage-gate.sh diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index cff1650..6a24589 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -46,7 +46,7 @@ jobs: run: pytest -q - name: Enforce coverage threshold - run: bash ./scripts/check-coverage.sh 95 + run: bash ./scripts/run-coverage-gate.sh build: name: "IntentProof Release: Build Python Distributions" diff --git a/README.md b/README.md index 814c8c1..8359cdc 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,11 @@ vectors. Run `pytest` locally before publishing. ```bash pytest -bash ./scripts/check-coverage.sh 95 +bash ./scripts/run-coverage-gate.sh ``` -CI enforces at least 95% line coverage on `src/intentproof/` (see -`pyproject.toml` and `scripts/check-coverage.sh`). +Tiered coverage: **90%** total and **95%** on `src/intentproof/` (see +`scripts/README-coverage-tiers.md`). ## Release diff --git a/pyproject.toml b/pyproject.toml index ac00b76..e67696f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ source = ["intentproof"] omit = [] [tool.coverage.report] -fail_under = 95 show_missing = true skip_empty = true @@ -46,5 +45,4 @@ pythonpath = ["src"] addopts = [ "--cov=intentproof", "--cov-report=term-missing", - "--cov-fail-under=95", ] diff --git a/scripts/README-coverage-tiers.md b/scripts/README-coverage-tiers.md new file mode 100644 index 0000000..390caec --- /dev/null +++ b/scripts/README-coverage-tiers.md @@ -0,0 +1,8 @@ +# Tiered coverage policy (intentproof-sdk-python) + +| Tier | Minimum | Scope | +|------|---------|--------| +| **Total** | 90% | `src/intentproof/` | +| **Critical** | 95% | Same — public SDK surface | + +Configuration: `scripts/coverage-tiers.conf`. diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index c0d3ac2..c50b6bf 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -1,41 +1,107 @@ #!/usr/bin/env bash +# Tiered statement coverage using coverage.py JSON output. +# +# Usage: check-coverage.sh [coverage_json] +# Default: coverage.json (run pytest with --cov first) set -euo pipefail -MIN_COVERAGE="${1:-95}" - -COVERAGE=() -if command -v coverage >/dev/null 2>&1; then - COVERAGE=(coverage) -else - for py in python python3; do - if command -v "$py" >/dev/null 2>&1 && "$py" -m coverage --version >/dev/null 2>&1; then - COVERAGE=("$py" -m coverage) - break - fi - done +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CONF="${ROOT}/scripts/coverage-tiers.conf" +COVERAGE_JSON="${1:-coverage.json}" + +if [[ ! -f "$CONF" ]]; then + echo "coverage tiers config not found: $CONF" >&2 + exit 2 +fi + +# shellcheck disable=SC1090 +source "$CONF" + +if [[ -z "${TOTAL_MIN:-}" ]]; then + echo "coverage-tiers.conf must set TOTAL_MIN" >&2 + exit 2 fi -if [[ ${#COVERAGE[@]} -eq 0 ]]; then - echo "coverage CLI not found; install dev deps with: pip install -e \".[dev]\"" >&2 +if [[ ! -f "$COVERAGE_JSON" ]]; then + echo "coverage json not found: $COVERAGE_JSON (run pytest with --cov first)" >&2 exit 2 fi -TOTAL_LINE="$("${COVERAGE[@]}" report --include='src/intentproof/*' | awk '/^TOTAL/{print; exit}')" -if [[ -z "$TOTAL_LINE" ]]; then - echo "unable to read total coverage; run pytest with --cov first" >&2 +rules_file="$(mktemp)" +trap 'rm -f "$rules_file"' EXIT +printf '%s\n' "${CRITICAL_RULES[@]}" >"$rules_file" + +report_threshold() { + local label="$1" covered="$2" total="$3" min="$4" + local pct + pct="$(awk -v c="$covered" -v t="$total" 'BEGIN { printf "%.1f", 100 * c / t }')" + echo "${label}: ${pct}% (${covered}/${total} statements), minimum ${min}%" + if awk -v c="$covered" -v t="$total" -v min="$min" \ + 'BEGIN { exit !(t > 0 && c * 100 >= t * min) }'; then + echo " PASS" + return 0 + fi + echo " FAIL" >&2 + return 1 +} + +stats_for_prefix() { + local prefix="$1" + python3 - "$ROOT/$COVERAGE_JSON" "$prefix" <<'PY' +import json +import sys + +path = sys.argv[1] +prefix = sys.argv[2] +data = json.load(open(path)) +covered = 0 +total = 0 +for file_path, entry in data.get("files", {}).items(): + if prefix not in file_path.replace("\\", "/"): + continue + summary = entry.get("summary", {}) + stmts = int(summary.get("num_statements", 0)) + if stmts == 0: + continue + pct = float(summary.get("percent_covered", 0)) + total += stmts + covered += int(round(stmts * pct / 100.0)) +print(covered, total) +PY +} + +read -r TOTAL_COVERED TOTAL_STMTS <&2 exit 2 fi -TOTAL_PERCENT="$(printf '%s' "$TOTAL_LINE" | awk '{print $NF}' | tr -d '%')" +fail=0 +report_threshold "Total coverage" "$TOTAL_COVERED" "$TOTAL_STMTS" "$TOTAL_MIN" || fail=1 -echo "Total coverage: ${TOTAL_PERCENT}%" -echo "Minimum required: ${MIN_COVERAGE}%" +echo "Critical tiers:" +while IFS= read -r rule; do + [[ -n "$rule" ]] || continue + min="${rule%%:*}" + prefix="${rule#*:}" + read -r c t <= min + 0) }'; then - echo "PASS: coverage threshold met" - exit 0 +if [[ "$fail" -ne 0 ]]; then + echo "FAIL: coverage threshold not met" >&2 + exit 1 fi -echo "FAIL: coverage threshold not met" >&2 -exit 1 +echo "PASS: coverage thresholds met" +exit 0 diff --git a/scripts/coverage-tiers.conf b/scripts/coverage-tiers.conf new file mode 100644 index 0000000..97696ba --- /dev/null +++ b/scripts/coverage-tiers.conf @@ -0,0 +1,7 @@ +# IntentProof tiered coverage (intentproof-sdk-python) + +TOTAL_MIN=90 + +CRITICAL_RULES=( + "95:/intentproof/" +) diff --git a/scripts/run-coverage-gate.sh b/scripts/run-coverage-gate.sh index b20bece..67cae7b 100755 --- a/scripts/run-coverage-gate.sh +++ b/scripts/run-coverage-gate.sh @@ -12,5 +12,16 @@ else PYTHON=python3 fi +COVERAGE=() +if command -v coverage >/dev/null 2>&1; then + COVERAGE=(coverage) +elif "$PYTHON" -m coverage --version >/dev/null 2>&1; then + COVERAGE=("$PYTHON" -m coverage) +else + echo "coverage CLI not found; install dev deps with: pip install -e \".[dev]\"" >&2 + exit 2 +fi + "$PYTHON" -m pytest -q -exec bash ./scripts/check-coverage.sh 95 +"${COVERAGE[@]}" json -o coverage.json +exec bash ./scripts/check-coverage.sh coverage.json From dfb63cfadeb6d39ddc5c07e01f9fcaea303e2acc Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Sat, 30 May 2026 10:11:58 -0500 Subject: [PATCH 2/4] Harden tiered coverage gate and release workflow Signed-off-by: Nathan Gillett --- .github/workflows/release-pypi.yml | 3 -- scripts/check-coverage.sh | 14 ++++--- .../check-coverage_critical_prefix_test.sh | 37 +++++++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) create mode 100755 scripts/check-coverage_critical_prefix_test.sh diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 6a24589..dc5cb91 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -42,9 +42,6 @@ jobs: - name: Install dependencies run: pip install -e ".[dev]" - - name: Run tests with coverage - run: pytest -q - - name: Enforce coverage threshold run: bash ./scripts/run-coverage-gate.sh diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index c50b6bf..e6fc5aa 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -23,8 +23,8 @@ if [[ -z "${TOTAL_MIN:-}" ]]; then exit 2 fi -if [[ ! -f "$COVERAGE_JSON" ]]; then - echo "coverage json not found: $COVERAGE_JSON (run pytest with --cov first)" >&2 +if [[ ! -f "${ROOT}/${COVERAGE_JSON}" ]]; then + echo "coverage json not found: ${ROOT}/${COVERAGE_JSON} (run pytest with --cov first)" >&2 exit 2 fi @@ -64,9 +64,12 @@ for file_path, entry in data.get("files", {}).items(): stmts = int(summary.get("num_statements", 0)) if stmts == 0: continue - pct = float(summary.get("percent_covered", 0)) total += stmts - covered += int(round(stmts * pct / 100.0)) + file_covered = summary.get("covered_lines") + if file_covered is None: + missing = entry.get("missing_lines") or summary.get("missing_lines") or [] + file_covered = stmts - len(missing) + covered += int(file_covered) print(covered, total) PY } @@ -92,7 +95,8 @@ while IFS= read -r rule; do $(stats_for_prefix "$prefix") EOF if [[ "$t" -eq 0 ]]; then - echo " ${prefix} (min ${min}%): no statements in profile, skipped" + echo " ${prefix} (min ${min}%): no statements in profile, FAIL" >&2 + fail=1 continue fi report_threshold " ${prefix}" "$c" "$t" "$min" || fail=1 diff --git a/scripts/check-coverage_critical_prefix_test.sh b/scripts/check-coverage_critical_prefix_test.sh new file mode 100755 index 0000000..e3b05e6 --- /dev/null +++ b/scripts/check-coverage_critical_prefix_test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Regression test: critical tiers must fail when a prefix matches nothing. +set -euo pipefail + +root="$(mktemp -d)" +trap 'rm -rf "$root"' EXIT + +mkdir -p "$root/scripts" +cp "$(dirname "$0")/check-coverage.sh" "$root/scripts/" + +cat >"$root/scripts/coverage-tiers.conf" <<'EOF' +TOTAL_MIN=50 +CRITICAL_RULES=( + "95:src/intentproof/missing/" +) +EOF + +cat >"$root/coverage.json" <<'EOF' +{ + "files": { + "src/intentproof/canon.py": { + "summary": { + "covered_lines": 2, + "num_statements": 2, + "percent_covered": 66.66666666666666 + } + } + } +} +EOF + +if bash "$root/scripts/check-coverage.sh" "$root/coverage.json" >/dev/null 2>&1; then + echo "FAIL: expected gate to fail for missing critical prefix" >&2 + exit 1 +fi + +echo "PASS: missing critical prefix fails the gate" From 2337460a0cc92089d6b94f91ed327c3838efe9cc Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Sat, 30 May 2026 10:20:27 -0500 Subject: [PATCH 3/4] Fix coverage display rounding to match integer gate Signed-off-by: Nathan Gillett --- scripts/check-coverage.sh | 17 ++++++++++++++--- scripts/check-coverage_display_test.sh | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100755 scripts/check-coverage_display_test.sh diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index e6fc5aa..4d7798d 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -32,13 +32,24 @@ rules_file="$(mktemp)" trap 'rm -f "$rules_file"' EXIT printf '%s\n' "${CRITICAL_RULES[@]}" >"$rules_file" +coverage_percent_display() { + awk -v c="$1" -v t="$2" 'BEGIN { + if (t == 0) { print "0.0"; exit } + printf "%.1f", int(1000 * c / t) / 10 + }' +} + +threshold_met() { + awk -v c="$1" -v t="$2" -v min="$3" \ + 'BEGIN { exit !(t > 0 && c * 100 >= t * min) }' +} + report_threshold() { local label="$1" covered="$2" total="$3" min="$4" local pct - pct="$(awk -v c="$covered" -v t="$total" 'BEGIN { printf "%.1f", 100 * c / t }')" + pct="$(coverage_percent_display "$covered" "$total")" echo "${label}: ${pct}% (${covered}/${total} statements), minimum ${min}%" - if awk -v c="$covered" -v t="$total" -v min="$min" \ - 'BEGIN { exit !(t > 0 && c * 100 >= t * min) }'; then + if threshold_met "$covered" "$total" "$min"; then echo " PASS" return 0 fi diff --git a/scripts/check-coverage_display_test.sh b/scripts/check-coverage_display_test.sh new file mode 100755 index 0000000..4c22b30 --- /dev/null +++ b/scripts/check-coverage_display_test.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Regression test: displayed percent must not round up past a failing gate. +set -euo pipefail + +pct="$(awk -v c=9399 -v t=10000 'BEGIN { + if (t == 0) { print "0.0"; exit } + printf "%.1f", int(1000 * c / t) / 10 +}')" + +if [[ "$pct" != "93.9" ]]; then + echo "FAIL: expected truncated display 93.9%, got ${pct}%" >&2 + exit 1 +fi + +if awk -v c=9399 -v t=10000 -v min=94 \ + 'BEGIN { exit !(t > 0 && c * 100 >= t * min) }'; then + echo "FAIL: expected integer gate to fail at 9399/10000 min 94%" >&2 + exit 1 +fi + +echo "PASS: display percent matches integer gate semantics" From b0bc49904f1b351fda83573c9ec88f7e05836390 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Sat, 30 May 2026 10:22:14 -0500 Subject: [PATCH 4/4] Fix coverage path resolution and stats fallback Signed-off-by: Nathan Gillett --- scripts/check-coverage.sh | 34 ++++++++++----- .../check-coverage_critical_prefix_test.sh | 15 ++++++- scripts/check-coverage_path_test.sh | 42 +++++++++++++++++++ 3 files changed, 79 insertions(+), 12 deletions(-) create mode 100755 scripts/check-coverage_path_test.sh diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 4d7798d..fa51007 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -8,7 +8,13 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CONF="${ROOT}/scripts/coverage-tiers.conf" -COVERAGE_JSON="${1:-coverage.json}" +COVERAGE_JSON_ARG="${1:-coverage.json}" + +if [[ "$COVERAGE_JSON_ARG" == /* ]]; then + COVERAGE_JSON_PATH="$COVERAGE_JSON_ARG" +else + COVERAGE_JSON_PATH="${ROOT}/${COVERAGE_JSON_ARG}" +fi if [[ ! -f "$CONF" ]]; then echo "coverage tiers config not found: $CONF" >&2 @@ -23,8 +29,8 @@ if [[ -z "${TOTAL_MIN:-}" ]]; then exit 2 fi -if [[ ! -f "${ROOT}/${COVERAGE_JSON}" ]]; then - echo "coverage json not found: ${ROOT}/${COVERAGE_JSON} (run pytest with --cov first)" >&2 +if [[ ! -f "$COVERAGE_JSON_PATH" ]]; then + echo "coverage json not found: $COVERAGE_JSON_PATH (run pytest with --cov first)" >&2 exit 2 fi @@ -59,7 +65,7 @@ report_threshold() { stats_for_prefix() { local prefix="$1" - python3 - "$ROOT/$COVERAGE_JSON" "$prefix" <<'PY' + python3 - "$COVERAGE_JSON_PATH" "$prefix" <<'PY' import json import sys @@ -68,6 +74,18 @@ prefix = sys.argv[2] data = json.load(open(path)) covered = 0 total = 0 + +def covered_statements(summary, entry, stmts): + if "covered_lines" in summary: + return int(summary["covered_lines"]) + missing_lines = entry.get("missing_lines") + if isinstance(missing_lines, list): + return stmts - len(missing_lines) + missing_count = summary.get("missing_lines") + if isinstance(missing_count, int): + return stmts - missing_count + return 0 + for file_path, entry in data.get("files", {}).items(): if prefix not in file_path.replace("\\", "/"): continue @@ -76,11 +94,7 @@ for file_path, entry in data.get("files", {}).items(): if stmts == 0: continue total += stmts - file_covered = summary.get("covered_lines") - if file_covered is None: - missing = entry.get("missing_lines") or summary.get("missing_lines") or [] - file_covered = stmts - len(missing) - covered += int(file_covered) + covered += covered_statements(summary, entry, stmts) print(covered, total) PY } @@ -90,7 +104,7 @@ $(stats_for_prefix "") EOF if [[ -z "$TOTAL_STMTS" || "$TOTAL_STMTS" -eq 0 ]]; then - echo "unable to read total coverage from $COVERAGE_JSON" >&2 + echo "unable to read total coverage from $COVERAGE_JSON_PATH" >&2 exit 2 fi diff --git a/scripts/check-coverage_critical_prefix_test.sh b/scripts/check-coverage_critical_prefix_test.sh index e3b05e6..1c27645 100755 --- a/scripts/check-coverage_critical_prefix_test.sh +++ b/scripts/check-coverage_critical_prefix_test.sh @@ -29,8 +29,19 @@ cat >"$root/coverage.json" <<'EOF' } EOF -if bash "$root/scripts/check-coverage.sh" "$root/coverage.json" >/dev/null 2>&1; then - echo "FAIL: expected gate to fail for missing critical prefix" >&2 +output="" +code=0 +output="$(cd "$root" && bash scripts/check-coverage.sh coverage.json 2>&1)" || code=$? + +if [[ "$code" -ne 1 ]]; then + echo "FAIL: expected exit 1 for missing critical prefix, got ${code}" >&2 + echo "$output" >&2 + exit 1 +fi + +if ! grep -q "no statements in profile, FAIL" <<<"$output"; then + echo "FAIL: expected critical-prefix failure message, got:" >&2 + echo "$output" >&2 exit 1 fi diff --git a/scripts/check-coverage_path_test.sh b/scripts/check-coverage_path_test.sh new file mode 100755 index 0000000..ce1d3aa --- /dev/null +++ b/scripts/check-coverage_path_test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Regression test: absolute and relative coverage JSON paths resolve correctly. +set -euo pipefail + +root="$(mktemp -d)" +trap 'rm -rf "$root"' EXIT + +mkdir -p "$root/scripts" +cp "$(dirname "$0")/check-coverage.sh" "$root/scripts/" + +cat >"$root/scripts/coverage-tiers.conf" <<'EOF' +TOTAL_MIN=50 +CRITICAL_RULES=( + "95:src/intentproof/" +) +EOF + +cat >"$root/coverage.json" <<'EOF' +{ + "files": { + "src/intentproof/canon.py": { + "summary": { + "covered_lines": 2, + "num_statements": 2, + "percent_covered": 100.0 + } + } + } +} +EOF + +if ! (cd "$root" && bash scripts/check-coverage.sh coverage.json >/dev/null); then + echo "FAIL: relative coverage.json path should pass" >&2 + exit 1 +fi + +if ! bash "$root/scripts/check-coverage.sh" "$root/coverage.json" >/dev/null; then + echo "FAIL: absolute coverage.json path should pass" >&2 + exit 1 +fi + +echo "PASS: coverage JSON path resolution"