diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb353fe..0bdda03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,4 @@ jobs: pip install -e ".[dev]" - name: Run tests with coverage - run: pytest -q - - - name: Enforce coverage threshold - run: bash ./scripts/check-coverage.sh 95 + run: bash ./scripts/run-coverage-gate.sh diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index cff1650..dc5cb91 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -42,11 +42,8 @@ jobs: - name: Install dependencies run: pip install -e ".[dev]" - - name: Run tests with coverage - run: pytest -q - - name: Enforce coverage threshold - run: bash ./scripts/check-coverage.sh 95 + run: bash ./scripts/run-coverage-gate.sh build: name: "IntentProof Release: Build Python Distributions" diff --git a/README.md b/README.md index 814c8c1..8359cdc 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,11 @@ vectors. Run `pytest` locally before publishing. ```bash pytest -bash ./scripts/check-coverage.sh 95 +bash ./scripts/run-coverage-gate.sh ``` -CI enforces at least 95% line coverage on `src/intentproof/` (see -`pyproject.toml` and `scripts/check-coverage.sh`). +Tiered coverage: **90%** total and **95%** on `src/intentproof/` (see +`scripts/README-coverage-tiers.md`). ## Release diff --git a/pyproject.toml b/pyproject.toml index ac00b76..e67696f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ source = ["intentproof"] omit = [] [tool.coverage.report] -fail_under = 95 show_missing = true skip_empty = true @@ -46,5 +45,4 @@ pythonpath = ["src"] addopts = [ "--cov=intentproof", "--cov-report=term-missing", - "--cov-fail-under=95", ] diff --git a/scripts/README-coverage-tiers.md b/scripts/README-coverage-tiers.md new file mode 100644 index 0000000..390caec --- /dev/null +++ b/scripts/README-coverage-tiers.md @@ -0,0 +1,8 @@ +# Tiered coverage policy (intentproof-sdk-python) + +| Tier | Minimum | Scope | +|------|---------|--------| +| **Total** | 90% | `src/intentproof/` | +| **Critical** | 95% | Same — public SDK surface | + +Configuration: `scripts/coverage-tiers.conf`. diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index c0d3ac2..fa51007 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -1,41 +1,136 @@ #!/usr/bin/env bash +# Tiered statement coverage using coverage.py JSON output. +# +# Usage: check-coverage.sh [coverage_json] +# Default: coverage.json (run pytest with --cov first) set -euo pipefail -MIN_COVERAGE="${1:-95}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CONF="${ROOT}/scripts/coverage-tiers.conf" +COVERAGE_JSON_ARG="${1:-coverage.json}" -COVERAGE=() -if command -v coverage >/dev/null 2>&1; then - COVERAGE=(coverage) +if [[ "$COVERAGE_JSON_ARG" == /* ]]; then + COVERAGE_JSON_PATH="$COVERAGE_JSON_ARG" else - for py in python python3; do - if command -v "$py" >/dev/null 2>&1 && "$py" -m coverage --version >/dev/null 2>&1; then - COVERAGE=("$py" -m coverage) - break - fi - done + COVERAGE_JSON_PATH="${ROOT}/${COVERAGE_JSON_ARG}" fi -if [[ ${#COVERAGE[@]} -eq 0 ]]; then - echo "coverage CLI not found; install dev deps with: pip install -e \".[dev]\"" >&2 +if [[ ! -f "$CONF" ]]; then + echo "coverage tiers config not found: $CONF" >&2 exit 2 fi -TOTAL_LINE="$("${COVERAGE[@]}" report --include='src/intentproof/*' | awk '/^TOTAL/{print; exit}')" -if [[ -z "$TOTAL_LINE" ]]; then - echo "unable to read total coverage; run pytest with --cov first" >&2 +# shellcheck disable=SC1090 +source "$CONF" + +if [[ -z "${TOTAL_MIN:-}" ]]; then + echo "coverage-tiers.conf must set TOTAL_MIN" >&2 + exit 2 +fi + +if [[ ! -f "$COVERAGE_JSON_PATH" ]]; then + echo "coverage json not found: $COVERAGE_JSON_PATH (run pytest with --cov first)" >&2 + exit 2 +fi + +rules_file="$(mktemp)" +trap 'rm -f "$rules_file"' EXIT +printf '%s\n' "${CRITICAL_RULES[@]}" >"$rules_file" + +coverage_percent_display() { + awk -v c="$1" -v t="$2" 'BEGIN { + if (t == 0) { print "0.0"; exit } + printf "%.1f", int(1000 * c / t) / 10 + }' +} + +threshold_met() { + awk -v c="$1" -v t="$2" -v min="$3" \ + 'BEGIN { exit !(t > 0 && c * 100 >= t * min) }' +} + +report_threshold() { + local label="$1" covered="$2" total="$3" min="$4" + local pct + pct="$(coverage_percent_display "$covered" "$total")" + echo "${label}: ${pct}% (${covered}/${total} statements), minimum ${min}%" + if threshold_met "$covered" "$total" "$min"; then + echo " PASS" + return 0 + fi + echo " FAIL" >&2 + return 1 +} + +stats_for_prefix() { + local prefix="$1" + python3 - "$COVERAGE_JSON_PATH" "$prefix" <<'PY' +import json +import sys + +path = sys.argv[1] +prefix = sys.argv[2] +data = json.load(open(path)) +covered = 0 +total = 0 + +def covered_statements(summary, entry, stmts): + if "covered_lines" in summary: + return int(summary["covered_lines"]) + missing_lines = entry.get("missing_lines") + if isinstance(missing_lines, list): + return stmts - len(missing_lines) + missing_count = summary.get("missing_lines") + if isinstance(missing_count, int): + return stmts - missing_count + return 0 + +for file_path, entry in data.get("files", {}).items(): + if prefix not in file_path.replace("\\", "/"): + continue + summary = entry.get("summary", {}) + stmts = int(summary.get("num_statements", 0)) + if stmts == 0: + continue + total += stmts + covered += covered_statements(summary, entry, stmts) +print(covered, total) +PY +} + +read -r TOTAL_COVERED TOTAL_STMTS <&2 exit 2 fi -TOTAL_PERCENT="$(printf '%s' "$TOTAL_LINE" | awk '{print $NF}' | tr -d '%')" +fail=0 +report_threshold "Total coverage" "$TOTAL_COVERED" "$TOTAL_STMTS" "$TOTAL_MIN" || fail=1 -echo "Total coverage: ${TOTAL_PERCENT}%" -echo "Minimum required: ${MIN_COVERAGE}%" +echo "Critical tiers:" +while IFS= read -r rule; do + [[ -n "$rule" ]] || continue + min="${rule%%:*}" + prefix="${rule#*:}" + read -r c t <&2 + fail=1 + continue + fi + report_threshold " ${prefix}" "$c" "$t" "$min" || fail=1 +done <"$rules_file" -if awk -v got="$TOTAL_PERCENT" -v min="$MIN_COVERAGE" 'BEGIN { exit !(got + 0 >= min + 0) }'; then - echo "PASS: coverage threshold met" - exit 0 +if [[ "$fail" -ne 0 ]]; then + echo "FAIL: coverage threshold not met" >&2 + exit 1 fi -echo "FAIL: coverage threshold not met" >&2 -exit 1 +echo "PASS: coverage thresholds met" +exit 0 diff --git a/scripts/check-coverage_critical_prefix_test.sh b/scripts/check-coverage_critical_prefix_test.sh new file mode 100755 index 0000000..1c27645 --- /dev/null +++ b/scripts/check-coverage_critical_prefix_test.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Regression test: critical tiers must fail when a prefix matches nothing. +set -euo pipefail + +root="$(mktemp -d)" +trap 'rm -rf "$root"' EXIT + +mkdir -p "$root/scripts" +cp "$(dirname "$0")/check-coverage.sh" "$root/scripts/" + +cat >"$root/scripts/coverage-tiers.conf" <<'EOF' +TOTAL_MIN=50 +CRITICAL_RULES=( + "95:src/intentproof/missing/" +) +EOF + +cat >"$root/coverage.json" <<'EOF' +{ + "files": { + "src/intentproof/canon.py": { + "summary": { + "covered_lines": 2, + "num_statements": 2, + "percent_covered": 66.66666666666666 + } + } + } +} +EOF + +output="" +code=0 +output="$(cd "$root" && bash scripts/check-coverage.sh coverage.json 2>&1)" || code=$? + +if [[ "$code" -ne 1 ]]; then + echo "FAIL: expected exit 1 for missing critical prefix, got ${code}" >&2 + echo "$output" >&2 + exit 1 +fi + +if ! grep -q "no statements in profile, FAIL" <<<"$output"; then + echo "FAIL: expected critical-prefix failure message, got:" >&2 + echo "$output" >&2 + exit 1 +fi + +echo "PASS: missing critical prefix fails the gate" diff --git a/scripts/check-coverage_display_test.sh b/scripts/check-coverage_display_test.sh new file mode 100755 index 0000000..4c22b30 --- /dev/null +++ b/scripts/check-coverage_display_test.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Regression test: displayed percent must not round up past a failing gate. +set -euo pipefail + +pct="$(awk -v c=9399 -v t=10000 'BEGIN { + if (t == 0) { print "0.0"; exit } + printf "%.1f", int(1000 * c / t) / 10 +}')" + +if [[ "$pct" != "93.9" ]]; then + echo "FAIL: expected truncated display 93.9%, got ${pct}%" >&2 + exit 1 +fi + +if awk -v c=9399 -v t=10000 -v min=94 \ + 'BEGIN { exit !(t > 0 && c * 100 >= t * min) }'; then + echo "FAIL: expected integer gate to fail at 9399/10000 min 94%" >&2 + exit 1 +fi + +echo "PASS: display percent matches integer gate semantics" diff --git a/scripts/check-coverage_path_test.sh b/scripts/check-coverage_path_test.sh new file mode 100755 index 0000000..ce1d3aa --- /dev/null +++ b/scripts/check-coverage_path_test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Regression test: absolute and relative coverage JSON paths resolve correctly. +set -euo pipefail + +root="$(mktemp -d)" +trap 'rm -rf "$root"' EXIT + +mkdir -p "$root/scripts" +cp "$(dirname "$0")/check-coverage.sh" "$root/scripts/" + +cat >"$root/scripts/coverage-tiers.conf" <<'EOF' +TOTAL_MIN=50 +CRITICAL_RULES=( + "95:src/intentproof/" +) +EOF + +cat >"$root/coverage.json" <<'EOF' +{ + "files": { + "src/intentproof/canon.py": { + "summary": { + "covered_lines": 2, + "num_statements": 2, + "percent_covered": 100.0 + } + } + } +} +EOF + +if ! (cd "$root" && bash scripts/check-coverage.sh coverage.json >/dev/null); then + echo "FAIL: relative coverage.json path should pass" >&2 + exit 1 +fi + +if ! bash "$root/scripts/check-coverage.sh" "$root/coverage.json" >/dev/null; then + echo "FAIL: absolute coverage.json path should pass" >&2 + exit 1 +fi + +echo "PASS: coverage JSON path resolution" diff --git a/scripts/coverage-tiers.conf b/scripts/coverage-tiers.conf new file mode 100644 index 0000000..97696ba --- /dev/null +++ b/scripts/coverage-tiers.conf @@ -0,0 +1,7 @@ +# IntentProof tiered coverage (intentproof-sdk-python) + +TOTAL_MIN=90 + +CRITICAL_RULES=( + "95:/intentproof/" +) diff --git a/scripts/run-coverage-gate.sh b/scripts/run-coverage-gate.sh index b20bece..67cae7b 100755 --- a/scripts/run-coverage-gate.sh +++ b/scripts/run-coverage-gate.sh @@ -12,5 +12,16 @@ else PYTHON=python3 fi +COVERAGE=() +if command -v coverage >/dev/null 2>&1; then + COVERAGE=(coverage) +elif "$PYTHON" -m coverage --version >/dev/null 2>&1; then + COVERAGE=("$PYTHON" -m coverage) +else + echo "coverage CLI not found; install dev deps with: pip install -e \".[dev]\"" >&2 + exit 2 +fi + "$PYTHON" -m pytest -q -exec bash ./scripts/check-coverage.sh 95 +"${COVERAGE[@]}" json -o coverage.json +exec bash ./scripts/check-coverage.sh coverage.json