diff --git a/.github/scripts/ci_health_report/ci_health_report.py b/.github/scripts/ci_health_report/ci_health_report.py new file mode 100644 index 00000000..23f7bcb6 --- /dev/null +++ b/.github/scripts/ci_health_report/ci_health_report.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +""" +CI Health Report + +Queries completed GitHub Actions workflow runs over a lookback window, +calculates per-job failure rates, and posts a summary to a GitHub issue. + +Required environment variables: + GH_TOKEN - GitHub token with actions:read and issues:write + GH_REPO - Repository in "owner/repo" format + REPORT_ISSUE - Issue number to post the report to + LOOKBACK_DAYS - How many days back to look (default: 30) +""" + +import os +import sys +import time +from datetime import datetime, timedelta, timezone +from collections import defaultdict +import urllib.request +import urllib.error +import json + +COUNTED_CONCLUSIONS = {"success", "failure"} + + +def bucket_count(lookback_days): + """Return the number of trend buckets for a given lookback window. + + Uses natural time units so bucket boundaries are semantically meaningful: + - daily for windows up to 14 days + - weekly for windows up to 90 days + - ~monthly (28-day) for longer windows + """ + if lookback_days <= 14: + return lookback_days + elif lookback_days <= 90: + return lookback_days // 7 + else: + return lookback_days // 28 + + +def trend_indicator(buckets): + """Compare first-half vs second-half failure rate and return an arrow + delta string.""" + mid = len(buckets) // 2 + early, recent = buckets[:mid], buckets[mid:] + e_runs = sum(b["runs"] for b in early) + e_fails = sum(b["failures"] for b in early) + r_runs = sum(b["runs"] for b in recent) + r_fails = sum(b["failures"] for b in recent) + if e_runs == 0 or r_runs == 0: + return "—" + delta = (r_fails / r_runs - e_fails / e_runs) * 100 + if abs(delta) < 1.0: + return f"→ {delta:+.1f}%" + return f"{'↑' if delta > 0 else '↓'} {delta:+.1f}%" + + +def _headers(token): + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def _urlopen(req): + """Execute a request with one automatic retry on rate limit (403/429).""" + try: + with urllib.request.urlopen(req) as resp: + return resp.read() + except urllib.error.HTTPError as e: + if e.code not in (403, 429): + raise RuntimeError(f"GitHub API error {e.code} for {req.full_url}") from e + retry_after = e.headers.get("Retry-After") + reset = e.headers.get("X-RateLimit-Reset") + if retry_after: + wait = int(retry_after) + 5 + elif reset: + wait = max(0, int(reset) - int(time.time())) + 5 # Seconds until reset + 5 + else: + wait = 60 + print(f"Rate limited (HTTP {e.code}). Waiting {wait}s before retry...", file=sys.stderr) + time.sleep(wait) + # Retry once outside the except block so a second failure is handled cleanly. + try: + with urllib.request.urlopen(req) as resp: + return resp.read() + except urllib.error.HTTPError as retry_e: + print( + f"Error: rate limit persists after retry (HTTP {retry_e.code}). Giving up.", + file=sys.stderr, + ) + sys.exit(1) + + +def gh_get(token, path): + """Fetch a single page from the GitHub API and return parsed JSON.""" + url = f"https://api.github.com{path}" + req = urllib.request.Request(url, headers=_headers(token)) + return json.loads(_urlopen(req)) + + +def get_runs(token, repo, since): + """Return all completed workflow runs created on or after `since`.""" + runs = [] + page = 1 + while True: + data = gh_get(token, ( + f"/repos/{repo}/actions/runs" + f"?status=completed&created=>={since}&per_page=100&page={page}" + )) + batch = data.get("workflow_runs", []) + if not batch: + break + runs.extend(batch) + page += 1 + return runs + + +def get_jobs(token, repo, run_id): + """Return all jobs for a workflow run.""" + jobs = [] + page = 1 + while True: + data = gh_get(token, ( + f"/repos/{repo}/actions/runs/{run_id}/jobs" + f"?per_page=100&page={page}" + )) + batch = data.get("jobs", []) + if not batch: + break + jobs.extend(batch) + page += 1 + return jobs + + +def post_comment(token, repo, issue_number, body): + """Post a comment to a GitHub issue.""" + url = f"https://api.github.com/repos/{repo}/issues/{issue_number}/comments" + payload = json.dumps({"body": body}).encode() + req = urllib.request.Request(url, data=payload, headers={ + **_headers(token), + "Content-Type": "application/json", + }) + _urlopen(req) + + +def build_report(stats, lookback_days, top_n, now): + """Build the markdown report string from aggregated job stats.""" + # Sort by failure rate descending for the main table + rows = sorted(stats.items(), key=lambda x: x[1]["failures"] / x[1]["runs"], reverse=True) + + table_lines = [] + for (workflow, job), s in rows: + rate = s["failures"] / s["runs"] * 100 + min_runs = len(s["buckets"]) * 2 + trend = trend_indicator(s["buckets"]) if s["runs"] >= min_runs else "—" + table_lines.append(f"| {workflow} | {job} | {s['runs']} | {s['failures']} | {rate:.1f}% | {trend} |") + + # Top N by absolute failure count + top = sorted(stats.items(), key=lambda x: x[1]["failures"], reverse=True)[:top_n] + top_lines = [] + for rank, ((workflow, job), s) in enumerate(top, start=1): + rate = s["failures"] / s["runs"] * 100 + top_lines.append(f"{rank}. **{workflow} / {job}** — {s['failures']} failures ({rate:.1f}%)") + + total_runs = sum(s["runs"] for s in stats.values()) + total_failures = sum(s["failures"] for s in stats.values()) + overall_rate = (total_failures / total_runs * 100) if total_runs else 0.0 + + timestamp = now.strftime("%Y-%m-%dT%H:%M:%SZ") + + lines = [ + "## CI Health Report", + "", + f"_Last {lookback_days} days — generated {timestamp}_", + "", + "### Job Failure Rates", + "", + "| Workflow | Job | Runs | Failures | Rate | Trend |", + "|----------|-----|------|----------|------|-------|", + *table_lines, + "", + f"_Trend: the {lookback_days}-day window is divided into equal time buckets" + " (daily for ≤ 14 days, weekly for ≤ 90 days, ~monthly beyond that)." + " The failure rate in the first half of those buckets is compared to the second half:" + " ↑ = getting worse, ↓ = improving, → = stable (< 1 pp change)." + " — = fewer than 2 runs per bucket on average; not enough data._", + "", + f"### Top {top_n} Most Failing Jobs", + "", + *top_lines, + "", + "### Summary", + "", + f"- **Total job runs:** {total_runs}", + f"- **Total failures:** {total_failures}", + f"- **Overall failure rate:** {overall_rate:.1f}%", + ] + return "\n".join(lines) + + +def main(): + token = os.environ.get("GH_TOKEN", "") + repo = os.environ.get("GH_REPO", "") + issue_number = os.environ.get("REPORT_ISSUE", "") + lookback_days_str = os.environ.get("LOOKBACK_DAYS", "") + top_jobs_str = os.environ.get("TOP_JOBS", "") + + if not token or not repo or not issue_number or not lookback_days_str or not top_jobs_str: + print("Error: GH_TOKEN, GH_REPO, REPORT_ISSUE, LOOKBACK_DAYS, and TOP_JOBS must all be set.", file=sys.stderr) + sys.exit(1) + + lookback_days = int(lookback_days_str) + top_jobs = int(top_jobs_str) + + now = datetime.now(timezone.utc) + since_dt = now - timedelta(days=lookback_days) + since = since_dt.strftime("%Y-%m-%dT%H:%M:%SZ") + num_buckets = bucket_count(lookback_days) + + print(f"Fetching workflow runs since {since}...") + runs = get_runs(token, repo, since) + print(f"Found {len(runs)} completed runs.") + + if not runs: + print("No runs found. Skipping report.") + return + + # Aggregate: (workflow_name, job_name) -> {runs, failures, buckets} + # Only "success" and "failure" conclusions are counted; skipped/cancelled are excluded. + # Buckets divide the lookback window into equal time slices (oldest → newest) for trend tracking. + stats = defaultdict(lambda: { + "runs": 0, + "failures": 0, + "buckets": [{"runs": 0, "failures": 0} for _ in range(num_buckets)], + }) + window_secs = (now - since_dt).total_seconds() + + for i, run in enumerate(runs, start=1): + print(f" Fetching jobs for run {i}/{len(runs)} (id={run['id']})...") + run_dt = datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) + elapsed = (run_dt - since_dt).total_seconds() # seconds from window start to this run + # clamp: elapsed==window_secs would produce index num_buckets + bucket_idx = min(int(elapsed / window_secs * num_buckets), num_buckets - 1) + bucket_idx = max(0, bucket_idx) # clamp: clock skew can make elapsed slightly negative + + jobs = get_jobs(token, repo, run["id"]) + for job in jobs: + conclusion = job.get("conclusion") + if conclusion not in COUNTED_CONCLUSIONS: + continue + key = (run["name"], job["name"]) + stats[key]["runs"] += 1 + stats[key]["buckets"][bucket_idx]["runs"] += 1 + if conclusion == "failure": + stats[key]["failures"] += 1 + stats[key]["buckets"][bucket_idx]["failures"] += 1 + + if not stats: + print("No job data collected. Skipping report.") + return + + report = build_report(stats, lookback_days, top_jobs, now) + + # Write to GitHub step summary if available + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a") as f: + f.write(report + "\n") + + print(f"Posting report to issue #{issue_number}...") + post_comment(token, repo, issue_number, report) + print("Report generated successfully.") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/ci_health_report/simulate_report.py b/.github/scripts/ci_health_report/simulate_report.py new file mode 100644 index 00000000..910395eb --- /dev/null +++ b/.github/scripts/ci_health_report/simulate_report.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Generates a sample CI health report with synthetic data and writes it to a file. +Usage: python3 simulate_report.py [output.md] +""" + +import sys +from collections import defaultdict +from datetime import datetime, timezone + +from ci_health_report import build_report + +# Each entry: (workflow, job, buckets) +# Buckets run oldest → newest; each is {runs, failures}. +SCENARIOS = [ + # Clearly getting worse — failure rate climbing week over week + ("Tests", "lint", [{"runs": 20, "failures": 1}, + {"runs": 20, "failures": 3}, + {"runs": 20, "failures": 8}, + {"runs": 20, "failures": 14}]), + # Clearly improving — failure rate falling + ("Tests", "unit", [{"runs": 20, "failures": 12}, + {"runs": 20, "failures": 8}, + {"runs": 20, "failures": 3}, + {"runs": 20, "failures": 1}]), + # Flat / stable low failure rate + ("Tests", "build", [{"runs": 20, "failures": 2}, + {"runs": 20, "failures": 2}, + {"runs": 20, "failures": 2}, + {"runs": 20, "failures": 2}]), + # Flat / stable high failure rate + ("Integration", "smoke", [{"runs": 20, "failures": 14}, + {"runs": 20, "failures": 15}, + {"runs": 20, "failures": 13}, + {"runs": 20, "failures": 14}]), + # Spike in the middle, now recovering + ("Integration", "full", [{"runs": 20, "failures": 2}, + {"runs": 20, "failures": 18}, + {"runs": 20, "failures": 18}, + {"runs": 20, "failures": 3}]), + # Sparse — only 2 runs total, should show — + ("Nightly", "deploy", [{"runs": 1, "failures": 1}, + {"runs": 0, "failures": 0}, + {"runs": 0, "failures": 0}, + {"runs": 1, "failures": 0}]), +] + +stats = defaultdict(lambda: {"runs": 0, "failures": 0, "buckets": []}) +for workflow, job, buckets in SCENARIOS: + key = (workflow, job) + stats[key]["runs"] = sum(b["runs"] for b in buckets) + stats[key]["failures"] = sum(b["failures"] for b in buckets) + stats[key]["buckets"] = buckets + +now = datetime(2026, 4, 7, 9, 0, 0, tzinfo=timezone.utc) +report = build_report(stats, lookback_days=30, top_n=5, now=now) + +output = sys.argv[1] if len(sys.argv) > 1 else "sample_report.md" +with open(output, "w") as f: + f.write(report + "\n") + +print(f"Written to {output}") +print() +print(report) diff --git a/.github/scripts/ci_health_report/test_ci_health_report.py b/.github/scripts/ci_health_report/test_ci_health_report.py new file mode 100644 index 00000000..b7de33ac --- /dev/null +++ b/.github/scripts/ci_health_report/test_ci_health_report.py @@ -0,0 +1,114 @@ +import unittest +from unittest.mock import MagicMock, patch +from collections import defaultdict +from datetime import datetime, timezone +import urllib.request +import urllib.error + +import ci_health_report +from ci_health_report import ( + bucket_count, + trend_indicator, + build_report, + main, + _urlopen, +) + + +class _Tests(unittest.TestCase): + + @patch("time.sleep") + @patch("urllib.request.urlopen") + def test_rate_limit_retries_with_wait(self, mock_urlopen, mock_sleep): + """_urlopen sleeps Retry-After + 5s on 429 then retries successfully.""" + import http.client + msg = http.client.HTTPMessage() + msg["Retry-After"] = "10" + resp = MagicMock() + resp.read.return_value = b'{"ok": true}' + resp.__enter__ = lambda s: s + resp.__exit__ = MagicMock(return_value=False) + mock_urlopen.side_effect = [ + urllib.error.HTTPError("https://api.github.com/test", 429, "Too Many Requests", msg, None), + resp, + ] + result = _urlopen(urllib.request.Request("https://api.github.com/test")) + mock_sleep.assert_called_once_with(15) # Retry-After(10) + 5 + self.assertEqual(result, b'{"ok": true}') + + def test_bucket_count(self): + """bucket_count returns daily, weekly, or monthly bucket counts.""" + self.assertEqual(bucket_count(7), 7) # daily + self.assertEqual(bucket_count(14), 14) # daily (boundary) + self.assertEqual(bucket_count(30), 4) # weekly + self.assertEqual(bucket_count(90), 12) # weekly (boundary) + self.assertEqual(bucket_count(91), 3) # monthly + + def test_trend_indicator_increasing(self): + """trend_indicator returns ↑ when recent half has higher failure rate.""" + buckets = [ + {"runs": 10, "failures": 1}, + {"runs": 10, "failures": 1}, + {"runs": 10, "failures": 5}, + {"runs": 10, "failures": 5}, + ] + result = trend_indicator(buckets) + self.assertTrue(result.startswith("↑")) + + def test_trend_indicator_decreasing(self): + """trend_indicator returns ↓ when recent half has lower failure rate.""" + buckets = [ + {"runs": 10, "failures": 5}, + {"runs": 10, "failures": 5}, + {"runs": 10, "failures": 1}, + {"runs": 10, "failures": 1}, + ] + result = trend_indicator(buckets) + self.assertTrue(result.startswith("↓")) + + def test_build_report_trend_shown_when_sufficient_runs(self): + """build_report shows trend arrow when runs >= num_buckets * 2.""" + buckets = [{"runs": 5, "failures": 1}, {"runs": 5, "failures": 4}] # 10 runs, threshold=4 + stats = defaultdict(lambda: {"runs": 0, "failures": 0, "buckets": []}) + stats[("Tests", "build")]["runs"] = 10 + stats[("Tests", "build")]["failures"] = 5 + stats[("Tests", "build")]["buckets"] = buckets + now = datetime(2026, 1, 1, 9, 0, 0, tzinfo=timezone.utc) + report = build_report(stats, 30, 5, now) + self.assertIn("| Workflow | Job | Runs | Failures | Rate | Trend |", report) + self.assertIn("| Tests | build | 10 | 5 | 50.0% | ↑", report) + self.assertIn("**Total job runs:** 10", report) + self.assertIn("**Overall failure rate:** 50.0%", report) + + def test_build_report_trend_suppressed_when_sparse(self): + """build_report shows — for trend when runs < num_buckets * 2.""" + buckets = [{"runs": 1, "failures": 1}, {"runs": 0, "failures": 0}, + {"runs": 0, "failures": 0}, {"runs": 1, "failures": 0}] + stats = defaultdict(lambda: {"runs": 0, "failures": 0, "buckets": []}) + stats[("Nightly", "deploy")]["runs"] = 2 + stats[("Nightly", "deploy")]["failures"] = 1 + stats[("Nightly", "deploy")]["buckets"] = buckets + now = datetime(2026, 1, 1, 9, 0, 0, tzinfo=timezone.utc) + report = build_report(stats, 30, 5, now) + self.assertIn("| Nightly | deploy | 2 | 1 | 50.0% | — |", report) + + @patch("ci_health_report.post_comment") + @patch("ci_health_report.get_jobs") + @patch("ci_health_report.get_runs") + def test_skipped_and_cancelled_not_counted(self, mock_runs, mock_jobs, mock_comment): + """skipped and cancelled conclusions are excluded from run and failure counts.""" + mock_runs.return_value = [{"id": 1, "name": "Tests", "created_at": "2026-01-15T00:00:00Z"}] + mock_jobs.return_value = [ + {"name": "build", "conclusion": "success"}, + {"name": "build", "conclusion": "failure"}, + {"name": "build", "conclusion": "skipped"}, + {"name": "build", "conclusion": "cancelled"}, + ] + with patch.dict("os.environ", {"GH_TOKEN": "tok", "GH_REPO": "o/r", "REPORT_ISSUE": "1", "LOOKBACK_DAYS": "30", "TOP_JOBS": "5"}): + main() + report = mock_comment.call_args[0][3] + self.assertIn("| Tests | build | 2 | 1 |", report) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/ci-health-report.yml b/.github/workflows/ci-health-report.yml new file mode 100644 index 00000000..5c1bb2f2 --- /dev/null +++ b/.github/workflows/ci-health-report.yml @@ -0,0 +1,63 @@ +name: CI Health Report + +on: + push: + paths: + - .github/scripts/ci_health_report/** + pull_request: + paths: + - .github/scripts/ci_health_report/** + schedule: + - cron: "0 9 * * 1" # Every Monday at 09:00 UTC + workflow_dispatch: + inputs: + report-issue: + description: "GitHub issue number to post the report to" + required: false + lookback-days: + description: "How many days back to look (default: 30)" + required: false + default: "30" + top-jobs: + description: "Number of top failing jobs to highlight (default: 5)" + required: false + default: "5" + +env: + REPORT_ISSUE: ${{ inputs.report-issue || vars.CI_HEALTH_REPORT_ISSUE }} + LOOKBACK_DAYS: ${{ inputs.lookback-days || vars.LOOKBACK_DAYS || '30' }} + TOP_JOBS: ${{ inputs.top-jobs || vars.TOP_JOBS || '5' }} + +permissions: + actions: read + issues: write + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Test CI health report script + run: python3 -m unittest test_ci_health_report -v + working-directory: .github/scripts/ci_health_report + + report: + needs: test + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Validate report issue + if: ${{ env.REPORT_ISSUE == '' }} + run: | + echo "::error::REPORT_ISSUE is not set. Configure vars.CI_HEALTH_REPORT_ISSUE or pass report-issue input." + exit 1 + + - name: Checkout + uses: actions/checkout@v4 + + - name: Generate CI Health Report + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + PYTHONUNBUFFERED: "1" + run: python3 .github/scripts/ci_health_report/ci_health_report.py