diff --git a/docs/superpowers/plans/2026-06-16-glook-20-highlights-prompt.md b/docs/superpowers/plans/2026-06-16-glook-20-highlights-prompt.md new file mode 100644 index 0000000..e8499ed --- /dev/null +++ b/docs/superpowers/plans/2026-06-16-glook-20-highlights-prompt.md @@ -0,0 +1,268 @@ +# GLOOK-20: Fix Hallucinated Narrative in Report Highlights + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ground report highlight sentiment in `impact_score` instead of naive metric direction, and add a `_v: 2` cache guard so stale pre-fix results are transparently regenerated. + +**Architecture:** The prompt rewrite and `avgImpact` addition were already validated via a live test during brainstorming — those changes live in the working tree but are not yet committed. This plan formalises them (adds the missing `_v: 2` cache guard, updates tests, updates the snapshot), then commits everything. + +**Tech Stack:** TypeScript, Jest + ts-jest, `loadPrompt` prompt-loader, `report_comparisons` MySQL/SQLite table + +--- + +## File Map + +| File | Change | +|---|---| +| `prompts/report-highlights-system.txt` | Already rewritten (uncommitted) — commit in Task 2 | +| `src/lib/report-highlights/service.ts` | `avgImpact` already added (uncommitted); add `_v: 2` guard in Task 1 | +| `src/lib/__tests__/unit/report-highlights-service.test.ts` | Update cache tests; add stale-cache test; add `avg_impact` assertion | +| `src/lib/__tests__/unit/__snapshots__/` | Snapshot update in Task 2 | + +--- + +### Task 1: `_v: 2` cache guard in service.ts + test updates (TDD) + +**Files:** +- Modify: `src/lib/report-highlights/service.ts` +- Modify: `src/lib/__tests__/unit/report-highlights-service.test.ts` + +**Context:** The cache read/write in `service.ts` currently stores highlights as a bare JSON array and returns it on any cache hit. After this task, it stores `{ _v: 2, highlights: [...] }` and treats any row without `_v === 2` as stale (falls through to regenerate). This mirrors the pattern used for `project-insights/route.ts`. + +The `avgImpact` addition is already in the working tree — do NOT revert it. Leave it as-is and commit it together with the `_v: 2` changes in this task. + +The three existing cache tests need updating; one new stale-cache test is added. + +- [ ] **Step 1: Update the two existing cached-highlights tests to use `_v: 2` format** + +In `src/lib/__tests__/unit/report-highlights-service.test.ts`, find the `describe('cached highlights'` block and update both tests: + +```typescript + describe('cached highlights', () => { + it('returns cached data with cached: true and does NOT call LLM', async () => { + const cachedHighlights = [{ icon: '✅', text: 'Steady state', sentiment: 'neutral' }]; + const cachedRow = { + highlights_json: JSON.stringify({ _v: 2, highlights: cachedHighlights }), + generated_at: '2026-03-16T08:00:00Z', + }; + + mockDbExecute + .mockResolvedValueOnce([[latestReport], null]) // latest + .mockResolvedValueOnce([[prevReport], null]) // prev + .mockResolvedValueOnce([[cachedRow], null]); // cache hit + + const result = await getReportHighlights(); + + expect(result).toEqual({ + available: true, + org: 'acme', + periodDays: 30, + reportDateA: prevReport.created_at, + reportDateB: latestReport.created_at, + highlights: cachedHighlights, + cached: true, + }); + expect(mockGetLLMClient).not.toHaveBeenCalled(); + expect(mockDbExecute).toHaveBeenCalledTimes(3); + }); + + it('handles highlights_json already parsed as object (not string)', async () => { + const cachedHighlights = [{ icon: '📊', text: 'Metrics stable', sentiment: 'neutral' }]; + const cachedRow = { + highlights_json: { _v: 2, highlights: cachedHighlights }, // already parsed + generated_at: '2026-03-16T08:00:00Z', + }; + + mockDbExecute + .mockResolvedValueOnce([[latestReport], null]) + .mockResolvedValueOnce([[prevReport], null]) + .mockResolvedValueOnce([[cachedRow], null]); + + const result = await getReportHighlights(); + + expect(result).toMatchObject({ + available: true, + highlights: cachedHighlights, + cached: true, + }); + expect(mockGetLLMClient).not.toHaveBeenCalled(); + }); + + it('treats stale cache (bare array, no _v) as miss and regenerates via LLM', async () => { + // Old format stored before the _v: 2 migration + const staleRow = { + highlights_json: JSON.stringify([{ icon: '📊', text: 'Old result', sentiment: 'neutral' }]), + generated_at: '2026-03-16T08:00:00Z', + }; + + const mockClient = makeMockLLMClient(); + mockGetLLMClient.mockResolvedValue(mockClient); + + mockDbExecute + .mockResolvedValueOnce([[latestReport], null]) // latest + .mockResolvedValueOnce([[prevReport], null]) // prev + .mockResolvedValueOnce([[staleRow], null]) // cache hit (stale) + .mockResolvedValueOnce([devStatsA, null]) // statsA + .mockResolvedValueOnce([devStatsB, null]) // statsB + .mockResolvedValueOnce([{ affectedRows: 1 }, null]); // INSERT + + const result = await getReportHighlights(); + + // Should NOT return the stale text — must have called LLM + expect(mockGetLLMClient).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ available: true, highlights: llmHighlights, cached: false }); + expect((result as any).highlights[0].text).not.toContain('Old result'); + }); + }); +``` + +- [ ] **Step 2: Update the user message `toContain` assertions to check for `avg_impact`** + +In the `describe('prompt and settings snapshots'` block, find the test `'sends correct user message structure'` and add one assertion after the existing ones: + +```typescript + expect(content).toContain('avg_impact='); +``` + +Also add two new assertions to the system prompt test to check for the formula content (keep the existing `toContain` lines, just add these two): + +```typescript + expect(content).toContain('PERFORMANCE FORMULA:'); + expect(content).toContain('impact_score is the ONLY authoritative measure'); +``` + +- [ ] **Step 3: Run tests to verify they fail as expected** + +```bash +npm test -- --testPathPatterns="report-highlights-service" --no-coverage +``` + +Expected: Multiple FAIL — stale-cache test fails (current code returns cached result instead of regenerating), `avg_impact` assertion fails (not yet in message), system prompt assertions fail (old text). + +- [ ] **Step 4: Add `_v: 2` to the cache read in `service.ts`** + +Find the cache-hit block (lines ~45–58): + +```typescript + if (cached.length > 0) { + const highlights = typeof cached[0].highlights_json === 'string' + ? JSON.parse(cached[0].highlights_json) + : cached[0].highlights_json; + return { + available: true, + org: latest.org, + periodDays: latest.period_days, + reportDateA: prev.created_at, + reportDateB: latest.created_at, + highlights, + cached: true, + }; + } +``` + +Replace with: + +```typescript + if (cached.length > 0) { + const data = typeof cached[0].highlights_json === 'string' + ? JSON.parse(cached[0].highlights_json) + : cached[0].highlights_json; + if (!Array.isArray(data) && data._v === 2) { + const { _v: _, highlights } = data; + return { + available: true, + org: latest.org, + periodDays: latest.period_days, + reportDateA: prev.created_at, + reportDateB: latest.created_at, + highlights, + cached: true, + }; + } + // _v !== 2 or bare array (stale pre-migration format) — fall through to regenerate + } +``` + +- [ ] **Step 5: Wrap the cache write in `_v: 2` in `service.ts`** + +Find the INSERT at the bottom of `getReportHighlights` (it uses `JSON.stringify(highlights)`): + +```typescript + await db.execute( + `INSERT INTO report_comparisons (report_id_a, report_id_b, highlights_json) + VALUES (?, ?, ?) + ON DUPLICATE KEY UPDATE highlights_json = VALUES(highlights_json), generated_at = NOW()`, + [reportIdA, reportIdB, JSON.stringify(highlights)], + ); +``` + +Replace with: + +```typescript + await db.execute( + `INSERT INTO report_comparisons (report_id_a, report_id_b, highlights_json) + VALUES (?, ?, ?) + ON DUPLICATE KEY UPDATE highlights_json = VALUES(highlights_json), generated_at = NOW()`, + [reportIdA, reportIdB, JSON.stringify({ _v: 2, highlights })], + ); +``` + +- [ ] **Step 6: Run tests to verify they pass** + +```bash +npm test -- --testPathPatterns="report-highlights-service" --no-coverage +``` + +Expected: All tests PASS except the two snapshot tests (`highlights-system-prompt` and `highlights-user-message`) — those will fail because the prompt and user message changed. That's correct; the snapshots are updated in Task 2. + +- [ ] **Step 7: Commit service.ts and test changes** + +```bash +git add src/lib/report-highlights/service.ts src/lib/__tests__/unit/report-highlights-service.test.ts +git commit -m "feat(glook-20): _v:2 cache guard + avgImpact in highlights service" +``` + +--- + +### Task 2: Commit prompt rewrite + update snapshot + +**Files:** +- Modify: `prompts/report-highlights-system.txt` (already updated, uncommitted) +- Update: `src/lib/__tests__/unit/__snapshots__/` (via `-u` flag) + +**Context:** The prompt file was rewritten during brainstorming validation and is currently uncommitted. The snapshot tests will be stale — they need to be regenerated with `npm test -- -u`. + +After updating, review the diff to confirm only the expected additions appear: the PERFORMANCE FORMULA section, the RULES FOR SENTIMENT section replacing the old "improvements/regressions" rule, and `avg_impact=` in the user message snapshot. + +- [ ] **Step 1: Update the snapshot** + +```bash +npm test -- --testPathPatterns="report-highlights-service" --no-coverage -u +``` + +Expected: `2 snapshots updated`. Output shows `highlights-system-prompt` and `highlights-user-message` updated. + +- [ ] **Step 2: Review the snapshot diff** + +```bash +git diff src/lib/__tests__/unit/__snapshots__/ +``` + +Confirm the diff shows: +- `highlights-system-prompt`: adds `PERFORMANCE FORMULA:` block, `RULES FOR SENTIMENT:` block, removes old `sentiment: "positive" for improvements` line +- `highlights-user-message`: adds `avg_impact=X.XX` in both TOTALS lines + +- [ ] **Step 3: Run the full test suite** + +```bash +npm test --no-coverage +``` + +Expected: All tests pass. + +- [ ] **Step 4: Commit prompt + snapshot** + +```bash +git add prompts/report-highlights-system.txt src/lib/__tests__/unit/__snapshots__/ +git commit -m "feat(glook-20): anchor highlights sentiment on impact_score formula, update snapshot" +``` diff --git a/docs/superpowers/specs/2026-06-16-glook-20-highlights-prompt-design.md b/docs/superpowers/specs/2026-06-16-glook-20-highlights-prompt-design.md new file mode 100644 index 0000000..bc8a44b --- /dev/null +++ b/docs/superpowers/specs/2026-06-16-glook-20-highlights-prompt-design.md @@ -0,0 +1,91 @@ +# GLOOK-20: Fix Hallucinated Narrative in Home Page Report Highlights + +## Goal + +Stop the report comparison summary from labeling raw metric changes (complexity, commits, AI%) as "improved" or "worse" by grounding all evaluative language in the `impact_score` formula. + +## Problem + +`report-highlights-system.txt` instructs `"positive" for improvements, "warning" for regressions` without defining what "improvement" means. The LLM naively infers direction from raw metric changes — e.g., complexity dropping from 3.1 → 3.0 is labeled "improved" even though complexity is an input to the formula, not independently directional. + +## Architecture + +Two-part fix: (1) rewrite the system prompt to anchor sentiment on `impact_score`; (2) add `avgImpact` to both report totals so the LLM has the aggregate signal; (3) add a `_v: 2` cache guard so stale cached results are not served after deployment. + +--- + +## Section 1: Prompt Rewrite + +**`prompts/report-highlights-system.txt`** — full replacement: + +``` +You are a concise engineering analytics assistant for Glooker (GitHub org analytics). +Compare two reports for the same org and period. Return JSON: +{ "highlights": [{ "icon": "emoji", "text": "one sentence", "sentiment": "positive|neutral|warning" }] } + +PERFORMANCE FORMULA: +impact_score = Commits (×2.0) + PRs (×2.7) + Complexity (×3.5) + PR% (×1.1) + Jira (×0.5) + Reviews (×0.5). Max: 9.3 +impact_score is the ONLY authoritative measure of developer/org performance. Higher is better. + +RULES FOR SENTIMENT: +- "positive": use ONLY when avg_impact or individual impact_score improved. +- "warning": use ONLY when avg_impact or individual impact_score declined. +- "neutral": use for ALL other observations — including changes to commits, PRs, complexity, AI%, lines changed. +- NEVER apply "positive" or "warning" to raw metric changes (commits up, complexity down, AI% up, etc.) unless impact_score confirms the direction. +- Complexity changes are NOT independently directional. Do not call a complexity drop "improved" or a rise "worse". + +OTHER RULES: +- 3-5 bullet highlights max. Be specific — name developers, cite numbers. +- Focus on: biggest movers (rank changes, impact delta), org-wide impact trend, newly active or recently inactive developers. +- Developers missing from the latest report are NOT "departed" — they are simply inactive this period. Never use "departed" or "left". +- If nothing significant changed, return 1 bullet: "Steady state — no major shifts in the leaderboard or metrics." +- Keep each bullet under 20 words. No fluff. +Return ONLY raw JSON. +``` + +--- + +## Section 2: Data Change + +**`src/lib/report-highlights/service.ts`** — add `avgImpact` to both totals: + +```typescript +avgImpact: statsX.length + ? (statsX.reduce((s, d) => s + Number(d.impact_score), 0) / statsX.length).toFixed(2) + : '0', +``` + +Pass it in the user message TOTALS strings: + +``` +TOTALS_A: `... avg_impact=${totalA.avgImpact}` +TOTALS_B: `... avg_impact=${totalB.avgImpact}` +``` + +--- + +## Section 3: Cache Invalidation + +Stored format changes from `JSON.stringify(highlights)` (bare array) to `JSON.stringify({ _v: 2, highlights })`. + +On read, check `data._v === 2`; if missing or wrong, fall through to regenerate. Strip `_v` before returning to callers. + +On write, store `{ _v: 2, highlights }`. + +This ensures any comparison cached with the old prompt is transparently regenerated on the next page load. + +--- + +## Section 4: Snapshot Test + +`prompts/report-highlights-system.txt` is covered by a snapshot test in `src/lib/__tests__/unit/prompts.test.ts`. After editing the file, run `npm test -- --testPathPatterns="prompts" -u` to update the snapshot. Review the diff to confirm only the expected changes. + +--- + +## Files Changed + +| File | Change | +|---|---| +| `prompts/report-highlights-system.txt` | Rewrite — formula context + sentiment rules | +| `src/lib/report-highlights/service.ts` | Add `avgImpact` to totals; add `_v: 2` cache guard | +| `src/lib/__tests__/unit/__snapshots__/prompts.test.ts.snap` | Update snapshot | diff --git a/prompts/report-highlights-system.txt b/prompts/report-highlights-system.txt index a686f9a..f4dc60a 100644 --- a/prompts/report-highlights-system.txt +++ b/prompts/report-highlights-system.txt @@ -2,11 +2,22 @@ You are a concise engineering analytics assistant for Glooker (GitHub org analyt Compare two reports for the same org and period. Return JSON: { "highlights": [{ "icon": "emoji", "text": "one sentence", "sentiment": "positive|neutral|warning" }] } -Rules: +PERFORMANCE FORMULA: +impact_score = Commits (×2.0) + PRs (×2.7) + Complexity (×3.5) + PR% (×1.1) + Jira (×0.5) + Reviews (×0.5). Max: 9.3 +impact_score is the ONLY authoritative measure of developer/org performance. Higher is better. + +RULES FOR SENTIMENT: +- "positive": use ONLY when org avg_impact improved. Individual impact gains can also be "positive" if the org average also rose. +- "warning": use ONLY when org avg_impact declined. Individual impact declines that contradict an improving org avg should be "neutral". +- "neutral": use for ALL other observations — including changes to commits, PRs, complexity, AI%, lines changed. +- NEVER apply "positive" or "warning" to raw metric changes (commits up, complexity down, AI% up, etc.) unless impact_score confirms the direction. +- Complexity changes are NOT independently directional. Do not call a complexity drop "improved" or a rise "worse". +- avg_impact can decrease when new developers join with lower initial scores even if existing developers improved — mention headcount context when relevant. + +OTHER RULES: - 3-5 bullet highlights max. Be specific — name developers, cite numbers. -- Focus on: biggest movers (rank changes, impact delta), org-wide trends (commits, AI%, complexity), newly active or recently inactive developers. -- IMPORTANT: developers missing from the latest report are NOT "departed" — they are simply inactive in this period (vacation, different projects, etc). Never use words like "departed" or "left". +- Focus on: biggest movers (rank changes, impact delta), org-wide impact trend, newly active or recently inactive developers. +- Developers missing from the latest report are NOT "departed" — they are simply inactive this period. Never use "departed" or "left". - If nothing significant changed, return 1 bullet: "Steady state — no major shifts in the leaderboard or metrics." - Keep each bullet under 20 words. No fluff. -- sentiment: "positive" for improvements, "warning" for regressions, "neutral" for informational. -Return ONLY raw JSON. \ No newline at end of file +Return ONLY raw JSON. diff --git a/src/lib/__tests__/unit/__snapshots__/report-highlights-service.test.ts.snap b/src/lib/__tests__/unit/__snapshots__/report-highlights-service.test.ts.snap index ee12812..09e8c37 100644 --- a/src/lib/__tests__/unit/__snapshots__/report-highlights-service.test.ts.snap +++ b/src/lib/__tests__/unit/__snapshots__/report-highlights-service.test.ts.snap @@ -4,14 +4,14 @@ exports[`getReportHighlights prompt and settings snapshots sends correct user me "Org: acme, Period: 30 days PREVIOUS REPORT (2026-02-13T10:00:00Z): - Totals: 2 devs, 30 commits, 7 PRs, avgComplexity=2.8, avgAI=8% - Top 5: @alice (rank #undefined): commits=20, PRs=5, complexity=3.5, impact=80.0, AI%=10 - @bob (rank #undefined): commits=10, PRs=2, complexity=2.0, impact=50.0, AI%=5 + Totals: 2 devs, 30 commits, 7 PRs, avgComplexity=2.8, avgAI=8%, avg_impact=65.00 + Top 5: @alice (rank #1): commits=20, PRs=5, complexity=3.5, impact=80.0, AI%=10 + @bob (rank #2): commits=10, PRs=2, complexity=2.0, impact=50.0, AI%=5 LATEST REPORT (2026-03-15T10:00:00Z): - Totals: 2 devs, 33 commits, 7 PRs, avgComplexity=3.1, avgAI=6% - Top 5: @alice (rank #undefined): commits=25, PRs=6, complexity=3.8, impact=90.0, AI%=12 - @carol (rank #undefined): commits=8, PRs=1, complexity=2.5, impact=40.0, AI%=0 + Totals: 2 devs, 33 commits, 7 PRs, avgComplexity=3.1, avgAI=6%, avg_impact=65.00 + Top 5: @alice (rank #1): commits=25, PRs=6, complexity=3.8, impact=90.0, AI%=12 + @carol (rank #2): commits=8, PRs=1, complexity=2.5, impact=40.0, AI%=0 Top movers: @alice: rank 1→1, impact +10.0 New developers: @carol @@ -23,12 +23,24 @@ exports[`getReportHighlights prompt and settings snapshots sends exact system pr Compare two reports for the same org and period. Return JSON: { "highlights": [{ "icon": "emoji", "text": "one sentence", "sentiment": "positive|neutral|warning" }] } -Rules: +PERFORMANCE FORMULA: +impact_score = Commits (×2.0) + PRs (×2.7) + Complexity (×3.5) + PR% (×1.1) + Jira (×0.5) + Reviews (×0.5). Max: 9.3 +impact_score is the ONLY authoritative measure of developer/org performance. Higher is better. + +RULES FOR SENTIMENT: +- "positive": use ONLY when org avg_impact improved. Individual impact gains can also be "positive" if the org average also rose. +- "warning": use ONLY when org avg_impact declined. Individual impact declines that contradict an improving org avg should be "neutral". +- "neutral": use for ALL other observations — including changes to commits, PRs, complexity, AI%, lines changed. +- NEVER apply "positive" or "warning" to raw metric changes (commits up, complexity down, AI% up, etc.) unless impact_score confirms the direction. +- Complexity changes are NOT independently directional. Do not call a complexity drop "improved" or a rise "worse". +- avg_impact can decrease when new developers join with lower initial scores even if existing developers improved — mention headcount context when relevant. + +OTHER RULES: - 3-5 bullet highlights max. Be specific — name developers, cite numbers. -- Focus on: biggest movers (rank changes, impact delta), org-wide trends (commits, AI%, complexity), newly active or recently inactive developers. -- IMPORTANT: developers missing from the latest report are NOT "departed" — they are simply inactive in this period (vacation, different projects, etc). Never use words like "departed" or "left". +- Focus on: biggest movers (rank changes, impact delta), org-wide impact trend, newly active or recently inactive developers. +- Developers missing from the latest report are NOT "departed" — they are simply inactive this period. Never use "departed" or "left". - If nothing significant changed, return 1 bullet: "Steady state — no major shifts in the leaderboard or metrics." - Keep each bullet under 20 words. No fluff. -- sentiment: "positive" for improvements, "warning" for regressions, "neutral" for informational. -Return ONLY raw JSON." +Return ONLY raw JSON. +" `; diff --git a/src/lib/__tests__/unit/report-highlights-service.test.ts b/src/lib/__tests__/unit/report-highlights-service.test.ts index 80b70e7..a661035 100644 --- a/src/lib/__tests__/unit/report-highlights-service.test.ts +++ b/src/lib/__tests__/unit/report-highlights-service.test.ts @@ -136,7 +136,7 @@ describe('getReportHighlights', () => { it('returns cached data with cached: true and does NOT call LLM', async () => { const cachedHighlights = [{ icon: '✅', text: 'Steady state', sentiment: 'neutral' }]; const cachedRow = { - highlights_json: JSON.stringify(cachedHighlights), + highlights_json: JSON.stringify({ _v: 2, highlights: cachedHighlights }), generated_at: '2026-03-16T08:00:00Z', }; @@ -163,7 +163,7 @@ describe('getReportHighlights', () => { it('handles highlights_json already parsed as object (not string)', async () => { const cachedHighlights = [{ icon: '📊', text: 'Metrics stable', sentiment: 'neutral' }]; const cachedRow = { - highlights_json: cachedHighlights, // already an array, not a string + highlights_json: { _v: 2, highlights: cachedHighlights }, // already parsed generated_at: '2026-03-16T08:00:00Z', }; @@ -181,6 +181,32 @@ describe('getReportHighlights', () => { }); expect(mockGetLLMClient).not.toHaveBeenCalled(); }); + + it('treats stale cache (bare array, no _v) as miss and regenerates via LLM', async () => { + // Old format stored before the _v: 2 migration + const staleRow = { + highlights_json: JSON.stringify([{ icon: '📊', text: 'Old result', sentiment: 'neutral' }]), + generated_at: '2026-03-16T08:00:00Z', + }; + + const mockClient = makeMockLLMClient(); + mockGetLLMClient.mockResolvedValue(mockClient); + + mockDbExecute + .mockResolvedValueOnce([[latestReport], null]) // latest + .mockResolvedValueOnce([[prevReport], null]) // prev + .mockResolvedValueOnce([[staleRow], null]) // cache hit (stale) + .mockResolvedValueOnce([devStatsA, null]) // statsA + .mockResolvedValueOnce([devStatsB, null]) // statsB + .mockResolvedValueOnce([{ affectedRows: 1 }, null]); // INSERT + + const result = await getReportHighlights(); + + // Should NOT return the stale text — must have called LLM + expect(mockGetLLMClient).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ available: true, highlights: llmHighlights, cached: false }); + expect((result as any).highlights[0].text).not.toContain('Old result'); + }); }); describe('fresh generation', () => { @@ -285,8 +311,10 @@ describe('getReportHighlights', () => { expect(content).toContain('You are a concise engineering analytics assistant for Glooker'); expect(content).toContain('Compare two reports'); expect(content).toContain('3-5 bullet highlights max.'); - expect(content).toContain('developers missing from the latest report are NOT "departed"'); + expect(content).toContain('Developers missing from the latest report are NOT "departed"'); expect(content).toContain('Return ONLY raw JSON.'); + expect(content).toContain('PERFORMANCE FORMULA:'); + expect(content).toContain('impact_score is the ONLY authoritative measure'); expect(content).toMatchSnapshot('highlights-system-prompt'); }); @@ -309,6 +337,7 @@ describe('getReportHighlights', () => { expect(content).toContain('@bob'); expect(content).toContain('New developers'); expect(content).toContain('@carol'); + expect(content).toContain('avg_impact='); expect(content).toMatchSnapshot('highlights-user-message'); }); diff --git a/src/lib/report-highlights/service.ts b/src/lib/report-highlights/service.ts index 5089f92..9b72a25 100644 --- a/src/lib/report-highlights/service.ts +++ b/src/lib/report-highlights/service.ts @@ -42,19 +42,30 @@ export async function getReportHighlights() { [reportIdA, reportIdB], ) as [any[], any]; + // HIGHLIGHTS_CACHE_VERSION: bump when the prompt or data shape changes to force + // stale rows to regenerate. All existing rows without this version fall through + // silently (expected one-time burst on first deploy after a version bump). + const HIGHLIGHTS_CACHE_VERSION = 2; + if (cached.length > 0) { - const highlights = typeof cached[0].highlights_json === 'string' - ? JSON.parse(cached[0].highlights_json) - : cached[0].highlights_json; - return { - available: true, - org: latest.org, - periodDays: latest.period_days, - reportDateA: prev.created_at, - reportDateB: latest.created_at, - highlights, - cached: true, - }; + let data: any = null; + try { + data = typeof cached[0].highlights_json === 'string' + ? JSON.parse(cached[0].highlights_json) + : cached[0].highlights_json; + } catch { /* malformed row — fall through to regenerate */ } + if (data && !Array.isArray(data) && data._v === HIGHLIGHTS_CACHE_VERSION) { + return { + available: true, + org: latest.org, + periodDays: latest.period_days, + reportDateA: prev.created_at, + reportDateB: latest.created_at, + highlights: data.highlights, + cached: true, + }; + } + // _v !== HIGHLIGHTS_CACHE_VERSION, bare array, or malformed — fall through to regenerate } // 4. Load dev stats for both reports @@ -89,6 +100,7 @@ export async function getReportHighlights() { prs: statsA.reduce((s: number, d: any) => s + d.total_prs, 0), avgComplexity: statsA.length ? (statsA.reduce((s: number, d: any) => s + Number(d.avg_complexity), 0) / statsA.length).toFixed(1) : '0', avgAi: statsA.length ? Math.round(statsA.reduce((s: number, d: any) => s + d.ai_percentage, 0) / statsA.length) : 0, + avgImpact: statsA.length ? (statsA.reduce((s: number, d: any) => s + Number(d.impact_score), 0) / statsA.length).toFixed(2) : '0', }; const totalB = { devs: statsB.length, @@ -96,6 +108,7 @@ export async function getReportHighlights() { prs: statsB.reduce((s: number, d: any) => s + d.total_prs, 0), avgComplexity: statsB.length ? (statsB.reduce((s: number, d: any) => s + Number(d.avg_complexity), 0) / statsB.length).toFixed(1) : '0', avgAi: statsB.length ? Math.round(statsB.reduce((s: number, d: any) => s + d.ai_percentage, 0) / statsB.length) : 0, + avgImpact: statsB.length ? (statsB.reduce((s: number, d: any) => s + Number(d.impact_score), 0) / statsB.length).toFixed(2) : '0', }; // Top movers (biggest impact score change) @@ -118,11 +131,11 @@ export async function getReportHighlights() { ORG: latest.org, PERIOD_DAYS: String(latest.period_days), PREV_DATE: String(prev.created_at), - TOTALS_A: `${totalA.devs} devs, ${totalA.commits} commits, ${totalA.prs} PRs, avgComplexity=${totalA.avgComplexity}, avgAI=${totalA.avgAi}%`, - TOP5_A: statsA.slice(0, 5).map(formatDev).join('\n '), + TOTALS_A: `${totalA.devs} devs, ${totalA.commits} commits, ${totalA.prs} PRs, avgComplexity=${totalA.avgComplexity}, avgAI=${totalA.avgAi}%, avg_impact=${totalA.avgImpact}`, + TOP5_A: [...mapA.values()].slice(0, 5).map(formatDev).join('\n '), LATEST_DATE: String(latest.created_at), - TOTALS_B: `${totalB.devs} devs, ${totalB.commits} commits, ${totalB.prs} PRs, avgComplexity=${totalB.avgComplexity}, avgAI=${totalB.avgAi}%`, - TOP5_B: statsB.slice(0, 5).map(formatDev).join('\n '), + TOTALS_B: `${totalB.devs} devs, ${totalB.commits} commits, ${totalB.prs} PRs, avgComplexity=${totalB.avgComplexity}, avgAI=${totalB.avgAi}%, avg_impact=${totalB.avgImpact}`, + TOP5_B: [...mapB.values()].slice(0, 5).map(formatDev).join('\n '), MOVERS: movers.map(m => `@${m.login}: rank ${m.rankA}→${m.rankB}, impact ${m.impactDelta > 0 ? '+' : ''}${m.impactDelta.toFixed(1)}`).join(', '), NEW_DEVS_SECTION: newDevsSection, INACTIVE_DEVS_SECTION: inactiveDevsSection, @@ -154,7 +167,7 @@ export async function getReportHighlights() { `INSERT INTO report_comparisons (report_id_a, report_id_b, highlights_json) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE highlights_json = VALUES(highlights_json), generated_at = NOW()`, - [reportIdA, reportIdB, JSON.stringify(highlights)], + [reportIdA, reportIdB, JSON.stringify({ _v: HIGHLIGHTS_CACHE_VERSION, highlights })], ); return {