diff --git a/WORKFLOWS.md b/WORKFLOWS.md index 5395ecc021..5dfa546771 100644 --- a/WORKFLOWS.md +++ b/WORKFLOWS.md @@ -1420,6 +1420,31 @@ flowchart LR --- +### 🧩 Extracted Script Modules (`scripts/agentic/`) + +The inline bash validation logic embedded in `.github/prompts/05-analysis-gate.md` has been extracted into strictly-typed TypeScript modules under `scripts/agentic/`. This bounded context provides: + +| Module | Responsibility | +|--------|---------------| +| `scripts/agentic/artifact-inventory.ts` | Typed definitions for all 23 required artifacts (Families A–D), stub placeholders, evidence patterns, agency lists, dok_id patterns | +| `scripts/agentic/analysis-gate.ts` | Gate validation checks 1–9b: artifact existence, per-document coverage, stub detection, Mermaid validation, Family C/D structure, PIR sidecar, Statskontoret evidence | +| `scripts/agentic/index.ts` | Barrel export — single public surface for downstream consumers | + +**Key types exported:** +- `ArtifactFamily` — `'A' | 'B' | 'C' | 'D' | 'E'` +- `ArtifactDefinition` — filename, family, description, gate metadata +- `GateCheckResult` — per-check pass/fail with message and artifact reference +- `GateValidationResult` — aggregate result with failure count + +**Design principles:** +- Zero `any` types — explicit interfaces for all data structures +- Gate module factored for readability (single-responsibility check functions) +- Co-located tests: `tests/agentic-analysis-gate.test.ts` (76 tests) +- ESLint clean with zero warnings +- No circular dependencies (barrel imports only) + +--- + ## 🔒 Workflow Security Architecture ### Supply Chain Security diff --git a/knip.json b/knip.json index 5b955d59de..e4f22c8673 100644 --- a/knip.json +++ b/knip.json @@ -15,6 +15,7 @@ "scripts/political-intelligence/**/*.ts", "scripts/render-lib/**/*.ts", "scripts/rss/**/*.ts", + "scripts/agentic/**/*.ts", "scripts/shared/**/*.ts", "scripts/sitemap-html/**/*.ts", "scripts/sitemap-xml/**/*.ts", diff --git a/scripts/agentic/analysis-gate.ts b/scripts/agentic/analysis-gate.ts new file mode 100644 index 0000000000..0750b6a79b --- /dev/null +++ b/scripts/agentic/analysis-gate.ts @@ -0,0 +1,1194 @@ +/** + * @module scripts/agentic/analysis-gate + * @description TypeScript implementation of the analysis gate validation + * logic defined in `.github/prompts/05-analysis-gate.md`. + * + * This module extracts a subset of the inline bash gate checks into + * testable, strictly-typed functions. It implements checks 1–9b of the + * prompt specification. Additional prompt-level gates (e.g. check 10 + * full-text outcomes, supplementary/editorial gates) are NOT covered + * here and must be validated separately. + * + * Implemented checks: + * 1. Artifact existence (all 23 files present and non-empty) + * 2. Per-document coverage (Family E vs manifest) + * 3. No stub placeholders (recursive scan) + * 4. Evidence citations in SWOT and significance-scoring + * 5. Mermaid diagrams with colour config + * 6. Pass-2 evidence (mtime or pass1/ snapshot) + * 7. Family C structure checks + * 8. Family D structure checks + * 9. PIR status sidecar validation + * 9b. Statskontoret evidence in implementation-feasibility + * + * @example + * import { validateAnalysisGate } from './analysis-gate.js'; + * const result = await validateAnalysisGate('analysis/daily/2026-05-01/propositions'); + * if (!result.passed) { + * result.checks.filter(c => !c.passed).forEach(c => console.error(c.message)); + * } + * + * @see .github/prompts/05-analysis-gate.md — canonical gate specification + * @see scripts/agentic/artifact-inventory.ts — artifact definitions + * @author Hack23 AB + * @license Apache-2.0 + */ + +import { readFile, stat, readdir } from 'node:fs/promises'; +import { existsSync, statSync } from 'node:fs'; +import { join } from 'node:path'; + +import { + type GateCheckResult, + type GateValidationResult, + REQUIRED_ARTIFACT_FILENAMES, + MERMAID_REQUIRED_ARTIFACTS, + PASS2_REQUIRED_ARTIFACTS, + STUB_PLACEHOLDERS, + DOK_ID_PATTERN, + EVIDENCE_PATTERN, + RECOGNISED_AGENCIES, +} from './artifact-inventory.js'; + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Run all analysis gate checks against an analysis directory. + * + * @param analysisDir - Absolute or relative path to the analysis subfolder + * (e.g. `analysis/daily/2026-05-01/propositions`). + * @returns Aggregate validation result with per-check details. + */ +export async function validateAnalysisGate( + analysisDir: string, +): Promise { + const checks: GateCheckResult[] = []; + + // Check 1 — Artifact existence + checks.push(...checkArtifactExistence(analysisDir)); + + // Check 2 — Per-document coverage + checks.push(...(await checkPerDocumentCoverage(analysisDir))); + + // Check 3 — No stubs + checks.push(...(await checkNoStubs(analysisDir))); + + // Check 4 — Evidence citations + checks.push(...(await checkEvidenceCitations(analysisDir))); + + // Check 5 — Mermaid diagrams + checks.push(...(await checkMermaidDiagrams(analysisDir))); + + // Check 6 — Pass-2 evidence + checks.push(...(await checkPass2Evidence(analysisDir))); + + // Check 7 — Family C structure + checks.push(...(await checkFamilyCStructure(analysisDir))); + + // Check 8 — Family D structure + checks.push(...(await checkFamilyDStructure(analysisDir))); + + // Check 9 — PIR status sidecar + checks.push(...(await checkPirStatus(analysisDir))); + + // Check 9b — Statskontoret evidence + checks.push(...(await checkStatskontoretEvidence(analysisDir))); + + const failureCount = checks.filter((c) => !c.passed).length; + return { + passed: failureCount === 0, + checks, + failureCount, + }; +} + +// --------------------------------------------------------------------------- +// Check 1 — Artifact existence +// --------------------------------------------------------------------------- + +/** + * Verify all 23 required artifacts exist and are non-empty. + */ +export function checkArtifactExistence(analysisDir: string): GateCheckResult[] { + const results: GateCheckResult[] = []; + for (const filename of REQUIRED_ARTIFACT_FILENAMES) { + const filePath = join(analysisDir, filename); + if (!existsSync(filePath)) { + results.push({ + checkId: 'artifact-existence', + passed: false, + message: `Missing artifact: ${filename}`, + artifact: filename, + }); + } else if (statSync(filePath).size === 0) { + results.push({ + checkId: 'artifact-existence', + passed: false, + message: `Empty artifact (zero bytes): ${filename}`, + artifact: filename, + }); + } else { + results.push({ + checkId: 'artifact-existence', + passed: true, + message: `Artifact present: ${filename}`, + artifact: filename, + }); + } + } + return results; +} + +// --------------------------------------------------------------------------- +// Check 2 — Per-document coverage +// --------------------------------------------------------------------------- + +/** + * Extract dok_ids from the data-download-manifest and verify each has + * a corresponding analysis document in the `documents/` subdirectory. + */ +export async function checkPerDocumentCoverage( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + const manifestPath = join(analysisDir, 'data-download-manifest.md'); + + if (!existsSync(manifestPath)) { + return results; // Manifest missing handled by check 1 + } + + const content = await readFile(manifestPath, 'utf-8'); + const dokIds = extractDokIds(content); + + if (dokIds.length === 0) { + results.push({ + checkId: 'per-document-coverage', + passed: false, + message: 'Manifest has no dok_id entries', + }); + return results; + } + + const documentsDir = join(analysisDir, 'documents'); + for (const dokId of dokIds) { + const found = hasDocumentAnalysis(documentsDir, dokId); + results.push({ + checkId: 'per-document-coverage', + passed: found, + message: found + ? `Document analysis found for ${dokId}` + : `documents/${dokId}.md or documents/${dokId}-analysis.md missing (any case)`, + artifact: `documents/${dokId}-analysis.md`, + }); + } + + return results; +} + +/** + * Extract unique dok_ids from markdown content. + */ +export function extractDokIds(content: string): string[] { + const globalPattern = new RegExp(DOK_ID_PATTERN.source, 'g'); + const matches = content.match(globalPattern); + if (!matches) return []; + return [...new Set(matches)]; +} + +/** + * Check if a document analysis file exists and is non-empty (any case variant). + * Mirrors the prompt gate's `-s` check (file exists AND size > 0). + */ +function hasDocumentAnalysis(documentsDir: string, dokId: string): boolean { + const variants = [ + `${dokId}.md`, + `${dokId}-analysis.md`, + `${dokId.toLowerCase()}.md`, + `${dokId.toLowerCase()}-analysis.md`, + ]; + return variants.some((v) => { + const p = join(documentsDir, v); + return existsSync(p) && statSync(p).size > 0; + }); +} + +// --------------------------------------------------------------------------- +// Check 3 — No stubs +// --------------------------------------------------------------------------- + +/** + * Scan all markdown files under the analysis directory (recursive, including + * `documents/` and `pass1/` subdirs) for stub placeholder strings. The + * canonical gate uses a recursive grep over the entire `$ANALYSIS_DIR`. + */ +export async function checkNoStubs(analysisDir: string): Promise { + const results: GateCheckResult[] = []; + + // Recursively collect all .md files under analysisDir + const mdFiles = await collectMdFilesRecursive(analysisDir, ''); + + for (const relPath of mdFiles) { + const filePath = join(analysisDir, relPath); + const content = await readFile(filePath, 'utf-8'); + for (const stub of STUB_PLACEHOLDERS) { + if (content.includes(stub)) { + results.push({ + checkId: 'no-stubs', + passed: false, + message: `Stub placeholder "${stub}" found in ${relPath}`, + artifact: relPath, + }); + } + } + } + + if (results.length === 0) { + results.push({ + checkId: 'no-stubs', + passed: true, + message: 'No stub placeholders detected', + }); + } + + return results; +} + +/** + * Recursively collect all `.md` files under a directory, returning paths + * relative to `baseDir`. Used by the stub scanner to mirror the canonical + * gate's recursive grep over the entire analysis tree. + */ +async function collectMdFilesRecursive( + baseDir: string, + prefix: string, +): Promise { + const results: string[] = []; + const currentDir = prefix ? join(baseDir, prefix) : baseDir; + if (!existsSync(currentDir)) return results; + const entries = await readdir(currentDir, { withFileTypes: true }); + for (const entry of entries) { + const relPath = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + results.push(...(await collectMdFilesRecursive(baseDir, relPath))); + } else if (entry.name.endsWith('.md')) { + results.push(relPath); + } + } + return results; +} + +// --------------------------------------------------------------------------- +// Check 4 — Evidence citations +// --------------------------------------------------------------------------- + +/** + * Verify that swot-analysis.md and significance-scoring.md contain primary- + * source evidence (a dok_id or recognised URL host) in each bullet/table row. + * Mirrors the awk-based gate in `05-analysis-gate.md` (check 4). + */ +export async function checkEvidenceCitations( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + + results.push(...(await checkSwotEvidence(analysisDir))); + results.push(...(await checkSignificanceScoringEvidence(analysisDir))); + + return results; +} + +/** SWOT section headings that trigger per-line evidence enforcement. */ +const SWOT_SECTION_RE = /^###\s+.*(Strengths|Weaknesses|Opportunities|Threats)\b/i; +/** Any heading resets the active SWOT section. */ +const ANY_HEADING_RE = /^#{1,6}\s+/; +/** Bullet lines (- or * style). */ +const BULLET_RE = /^\s*[-*]\s+/; +/** Table row (starts with |). */ +const TABLE_ROW_RE = /^\s*\|/; +/** Separator row (only |, :, -, whitespace). */ +const TABLE_SEP_RE = /^\s*[|:\-\s]+$/; + +/** + * Check swot-analysis.md: every bullet and table row inside a SWOT section + * must contain at least one evidence citation. + */ +async function checkSwotEvidence(analysisDir: string): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'swot-analysis.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + const lines = content.split('\n'); + let currentSection = ''; + let tableRowCount = 0; + + for (const line of lines) { + if (SWOT_SECTION_RE.test(line)) { + currentSection = line.trim(); + tableRowCount = 0; + continue; + } + if (ANY_HEADING_RE.test(line)) { + currentSection = ''; + tableRowCount = 0; + continue; + } + if (!currentSection) continue; + + if (/^\s*$/.test(line)) { + tableRowCount = 0; + continue; + } + + if (BULLET_RE.test(line)) { + if (!EVIDENCE_PATTERN.test(line)) { + results.push({ + checkId: 'evidence-citations', + passed: false, + message: `swot-analysis.md ${currentSection}: bullet missing evidence (dok_id or primary-source URL): ${line.trim()}`, + artifact: 'swot-analysis.md', + }); + } + continue; // bullet lines are never also table rows + } + + if (TABLE_ROW_RE.test(line)) { + if (TABLE_SEP_RE.test(line)) continue; + tableRowCount++; + if (tableRowCount === 1) continue; // skip header row + if (!EVIDENCE_PATTERN.test(line)) { + results.push({ + checkId: 'evidence-citations', + passed: false, + message: `swot-analysis.md ${currentSection}: table row missing evidence (dok_id or primary-source URL): ${line.trim()}`, + artifact: 'swot-analysis.md', + }); + } + } + } + + if (results.length === 0) { + results.push({ + checkId: 'evidence-citations', + passed: true, + message: 'swot-analysis.md: evidence citations present', + artifact: 'swot-analysis.md', + }); + } + + return results; +} + +/** Mermaid structural keywords — these lines are never checked for evidence. */ +const MERMAID_STRUCTURAL_RE = + /^\s*(%%|style\b|classDef\b|class\b|linkStyle\b|subgraph\b|end\b|graph\b|flowchart\b|quadrantChart\b|mindmap\b|timeline\b|journey\b|gantt\b|pie\b|xychart-beta\b|sequenceDiagram\b|stateDiagram(-v2)?\b|erDiagram\b|sankey-beta\b|gitGraph\b|requirementDiagram\b|block-beta\b)/; +/** Mermaid node/label content — lines with bracket/paren/curly-enclosed content indicate node labels. */ +const MERMAID_NODE_RE = /\[[^\]\n]+\]|\([^)\n]+\)|\{[^}\n]+\}/; + +/** + * Check significance-scoring.md: every ranked bullet/list item and table + * row (outside Mermaid) must contain evidence. Mermaid node labels are + * also checked unless they are structural keywords. + */ +async function checkSignificanceScoringEvidence( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'significance-scoring.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + const lines = content.split('\n'); + let inMermaid = false; + let tableRowCount = 0; + + for (const line of lines) { + if (/^```mermaid\s*$/.test(line)) { + inMermaid = true; + tableRowCount = 0; + continue; + } + if (inMermaid && /^```\s*$/.test(line)) { + inMermaid = false; + continue; + } + + if (inMermaid) { + if (MERMAID_STRUCTURAL_RE.test(line)) continue; + if (MERMAID_NODE_RE.test(line) && !EVIDENCE_PATTERN.test(line)) { + results.push({ + checkId: 'evidence-citations', + passed: false, + message: `significance-scoring.md Mermaid node missing evidence (dok_id or primary-source URL): ${line.trim()}`, + artifact: 'significance-scoring.md', + }); + } + continue; + } + + if (/^\s*$/.test(line)) { + tableRowCount = 0; + continue; + } + + // Ranked bullet (- or * or numbered) + if (/^\s*([0-9]+\.\s+|[-*]\s+)/.test(line) && !EVIDENCE_PATTERN.test(line)) { + results.push({ + checkId: 'evidence-citations', + passed: false, + message: `significance-scoring.md ranked item missing evidence (dok_id or primary-source URL): ${line.trim()}`, + artifact: 'significance-scoring.md', + }); + continue; + } + + // Table rows + if (TABLE_ROW_RE.test(line)) { + if (TABLE_SEP_RE.test(line)) continue; + tableRowCount++; + if (tableRowCount === 1) continue; // skip header row + if (!EVIDENCE_PATTERN.test(line)) { + results.push({ + checkId: 'evidence-citations', + passed: false, + message: `significance-scoring.md ranking table row missing evidence (dok_id or primary-source URL): ${line.trim()}`, + artifact: 'significance-scoring.md', + }); + } + } + } + + if (results.length === 0) { + results.push({ + checkId: 'evidence-citations', + passed: true, + message: 'significance-scoring.md: evidence citations present', + artifact: 'significance-scoring.md', + }); + } + + return results; +} + +// --------------------------------------------------------------------------- +// Check 5 — Mermaid diagrams +// --------------------------------------------------------------------------- + +/** + * Verify Mermaid diagrams with colour-coded config exist in required files. + */ +export async function checkMermaidDiagrams( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + + for (const filename of MERMAID_REQUIRED_ARTIFACTS) { + const filePath = join(analysisDir, filename); + if (!existsSync(filePath)) continue; + + const content = await readFile(filePath, 'utf-8'); + + const hasMermaid = /^```mermaid/m.test(content); + if (!hasMermaid) { + results.push({ + checkId: 'mermaid-diagrams', + passed: false, + message: `${filename}: missing Mermaid block`, + artifact: filename, + }); + continue; + } + + const hasColourConfig = + /^\s*style\s+/m.test(content) || + /themeVariables/m.test(content) || + /%%\{\s*init/m.test(content); + + if (!hasColourConfig) { + results.push({ + checkId: 'mermaid-diagrams', + passed: false, + message: `${filename}: missing Mermaid colour-coded config`, + artifact: filename, + }); + } else { + results.push({ + checkId: 'mermaid-diagrams', + passed: true, + message: `${filename}: Mermaid with colour config present`, + artifact: filename, + }); + } + } + + return results; +} + +// --------------------------------------------------------------------------- +// Check 6 — Pass-2 evidence +// --------------------------------------------------------------------------- + +/** + * Minimum mtime delta (ms) from birth time that constitutes evidence of a + * second-pass edit (180 seconds = 3 minutes, matching the bash gate threshold + * in `05-analysis-gate.md §Check 6`). + */ +const PASS2_MTIME_THRESHOLD_MS = 180_000; + +/** + * Verify that Pass-2 iteration was performed on each artifact: either a + * `pass1/` snapshot exists on disk that differs from the current file, OR + * the file's mtime is at least 180 s after its birth time (Linux birth time + * may not be reliable; the `pass1/` check is the preferred mechanism). + */ +export async function checkPass2Evidence( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + const pass1Dir = join(analysisDir, 'pass1'); + + for (const filename of PASS2_REQUIRED_ARTIFACTS) { + const filePath = join(analysisDir, filename); + if (!existsSync(filePath)) continue; + + let pass2Done = false; + + // Primary evidence: a differing pass1/ snapshot + const pass1Path = join(pass1Dir, filename); + if (existsSync(pass1Path)) { + const [current, snapshot] = await Promise.all([ + readFile(filePath, 'utf-8'), + readFile(pass1Path, 'utf-8'), + ]); + if (current !== snapshot) { + pass2Done = true; + } + } + + // Fallback: mtime >= birthtime + 180 s (where birth time is available) + if (!pass2Done) { + const fileStat = await stat(filePath); + const birthtimeMs = fileStat.birthtimeMs; + const mtimeMs = fileStat.mtimeMs; + if (birthtimeMs > 0 && mtimeMs >= birthtimeMs + PASS2_MTIME_THRESHOLD_MS) { + pass2Done = true; + } + } + + if (!pass2Done) { + results.push({ + checkId: 'pass2-evidence', + passed: false, + message: `${filename}: Pass-2 evidence missing (mtime < birth+180s and no differing pass1/ snapshot)`, + artifact: filename, + }); + } else { + results.push({ + checkId: 'pass2-evidence', + passed: true, + message: `${filename}: Pass-2 evidence confirmed`, + artifact: filename, + }); + } + } + + return results; +} + +// --------------------------------------------------------------------------- +// Check 7 — Family C structure +// --------------------------------------------------------------------------- + +/** + * Validate Family C structural requirements. + */ +export async function checkFamilyCStructure( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + + // executive-brief.md: BLUF + Decisions sections + results.push(...(await checkExecutiveBrief(analysisDir))); + + // intelligence-assessment.md: ≥3 Key Judgments + confidence + PIR + results.push(...(await checkIntelligenceAssessment(analysisDir))); + + // scenario-analysis.md: ≥3 scenarios + results.push(...(await checkScenarioAnalysis(analysisDir))); + + // devils-advocate.md: ≥3 hypotheses + results.push(...(await checkDevilsAdvocate(analysisDir))); + + // methodology-reflection.md: ICD 203 or improvements + results.push(...(await checkMethodologyReflection(analysisDir))); + + // comparative-international.md: comparator set or ≥2 rows + results.push(...(await checkComparativeInternational(analysisDir))); + + return results; +} + +/** + * Check executive-brief.md for BLUF and Decisions sections. + */ +async function checkExecutiveBrief(analysisDir: string): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'executive-brief.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + + const hasBluf = /^##\s.*BLUF/m.test(content); + if (!hasBluf) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: "executive-brief.md: missing '## BLUF' section", + artifact: 'executive-brief.md', + }); + } + + const hasDecisions = /^##\s.*(Decision|Decisions\s+This\s+Brief)/m.test(content); + if (!hasDecisions) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: "executive-brief.md: missing 'Decisions' section", + artifact: 'executive-brief.md', + }); + } + + if (hasBluf && hasDecisions) { + results.push({ + checkId: 'family-c-structure', + passed: true, + message: 'executive-brief.md: BLUF and Decisions present', + artifact: 'executive-brief.md', + }); + } + + return results; +} + +/** + * Check intelligence-assessment.md for Key Judgments, confidence labels, PIR. + */ +async function checkIntelligenceAssessment( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'intelligence-assessment.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + + // ≥3 Key Judgments — count distinct matching lines (not raw occurrences) + // to mirror the canonical gate's `grep -cE` behaviour. A line like + // "Key Judgment KJ-1" counts as 1, not 2. + const kjPattern = /(Key\s+Judgment|KJ-?\d+)/; + const kjLines = content.split('\n').filter((line) => kjPattern.test(line)); + const kjCount = kjLines.length; + if (kjCount < 3) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: `intelligence-assessment.md: fewer than 3 Key Judgments (found ${kjCount})`, + artifact: 'intelligence-assessment.md', + }); + } + + // ≥3 confidence labels + const confMatches = content.match( + /\b(VERY\s+HIGH|VERY\s+LOW|HIGH|MEDIUM|LOW)\b/g, + ); + const confCount = confMatches ? confMatches.length : 0; + if (confCount < 3) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: `intelligence-assessment.md: fewer than 3 confidence labels (found ${confCount})`, + artifact: 'intelligence-assessment.md', + }); + } + + // PIR reference + const hasPir = /PIR/i.test(content); + if (!hasPir) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: 'intelligence-assessment.md: no PIR reference', + artifact: 'intelligence-assessment.md', + }); + } + + if (kjCount >= 3 && confCount >= 3 && hasPir) { + results.push({ + checkId: 'family-c-structure', + passed: true, + message: 'intelligence-assessment.md: structure valid', + artifact: 'intelligence-assessment.md', + }); + } + + return results; +} + +/** + * Check scenario-analysis.md for ≥3 distinct scenarios. + */ +async function checkScenarioAnalysis(analysisDir: string): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'scenario-analysis.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + const scenarioMatches = content.match(/^##?\s+.*Scenario/gm); + const count = scenarioMatches ? scenarioMatches.length : 0; + + if (count < 3) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: `scenario-analysis.md: fewer than 3 scenarios (found ${count})`, + artifact: 'scenario-analysis.md', + }); + } else { + results.push({ + checkId: 'family-c-structure', + passed: true, + message: `scenario-analysis.md: ${count} scenarios found`, + artifact: 'scenario-analysis.md', + }); + } + + return results; +} + +/** + * Check devils-advocate.md for ≥3 competing hypotheses. + */ +async function checkDevilsAdvocate(analysisDir: string): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'devils-advocate.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + const hyMatches = content.match( + /^#{2,4}\s*(Hypothesis|H[0-9]+\s*[:.—-])/gm, + ); + const count = hyMatches ? hyMatches.length : 0; + + if (count < 3) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: `devils-advocate.md: fewer than 3 competing hypotheses (found ${count})`, + artifact: 'devils-advocate.md', + }); + } else { + results.push({ + checkId: 'family-c-structure', + passed: true, + message: `devils-advocate.md: ${count} hypotheses found`, + artifact: 'devils-advocate.md', + }); + } + + return results; +} + +/** + * Check methodology-reflection.md for ICD 203 audit or named improvements. + */ +async function checkMethodologyReflection( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'methodology-reflection.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + const hasIcd203 = + /ICD\s+203/i.test(content) || + /Methodology\s+Improvements/i.test(content) || + /Improvement\s+1/i.test(content) || + /^#{2,4}\s+.*Improvements/m.test(content); + + if (!hasIcd203) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: 'methodology-reflection.md: missing ICD 203 audit or Methodology Improvements', + artifact: 'methodology-reflection.md', + }); + } else { + results.push({ + checkId: 'family-c-structure', + passed: true, + message: 'methodology-reflection.md: ICD 203 / improvements present', + artifact: 'methodology-reflection.md', + }); + } + + return results; +} + +/** + * Check comparative-international.md for comparator set or ≥2 rows. + */ +async function checkComparativeInternational( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'comparative-international.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + + // Check for "Comparator set:" line with non-empty value + const COMPARATOR_SET_RE = /^\s*\*{0,2}Comparator set\*{0,2}\s*:/m; + const hasComparatorSet = COMPARATOR_SET_RE.test(content) && + !/^\s*\*{0,2}Comparator set\*{0,2}\s*:\s*[-–—]*\s*$/m.test(content); + + // Count non-header table rows (excluding separator rows) + const tableRows = content.split('\n').filter((line) => { + if (!/^\|/.test(line)) return false; + if (/^\|[\s:-]+(\|[\s:-]+)+\|?\s*$/.test(line)) return false; + if (/^\|\s*(Jurisdiction|Comparator|Country)\s*\|/.test(line)) return false; + return true; + }); + + const hasEnoughRows = tableRows.length >= 2; + + if (!hasComparatorSet && !hasEnoughRows) { + results.push({ + checkId: 'family-c-structure', + passed: false, + message: 'comparative-international.md: missing comparator set or fewer than 2 comparator rows', + artifact: 'comparative-international.md', + }); + } else { + results.push({ + checkId: 'family-c-structure', + passed: true, + message: 'comparative-international.md: comparator data present', + artifact: 'comparative-international.md', + }); + } + + return results; +} + +// --------------------------------------------------------------------------- +// Check 8 — Family D structure +// --------------------------------------------------------------------------- + +/** + * Validate Family D structural requirements. + */ +export async function checkFamilyDStructure( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + + // forward-indicators.md: ≥10 dated indicators + results.push(...(await checkForwardIndicators(analysisDir))); + + // coalition-mathematics.md: seat-count table + results.push(...(await checkCoalitionMathematics(analysisDir))); + + return results; +} + +/** + * Check forward-indicators.md for ≥10 dated indicators. + */ +async function checkForwardIndicators(analysisDir: string): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'forward-indicators.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + // Loose date detection (not strict calendar validation) — matches the + // original bash gate pattern; false positives are acceptable here since + // the goal is to verify the author included date-anchored indicators. + const datePattern = /20[0-9]{2}-[0-1][0-9]-[0-3][0-9]|20[0-9]{2}Q[1-4]|\+[0-9]+\s*(h|d|day|week|month)/g; + const matches = content.match(datePattern); + const count = matches ? matches.length : 0; + + if (count < 10) { + results.push({ + checkId: 'family-d-structure', + passed: false, + message: `forward-indicators.md: fewer than 10 dated indicators (found ${count})`, + artifact: 'forward-indicators.md', + }); + } else { + results.push({ + checkId: 'family-d-structure', + passed: true, + message: `forward-indicators.md: ${count} dated indicators found`, + artifact: 'forward-indicators.md', + }); + } + + return results; +} + +/** + * Check coalition-mathematics.md for seat-count / vote-breakdown table. + */ +async function checkCoalitionMathematics( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'coalition-mathematics.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + const hasTable = /^\|.*(Ja|Nej|Avstår|Frånvarande|Seats|Mandat)/m.test(content); + + if (!hasTable) { + results.push({ + checkId: 'family-d-structure', + passed: false, + message: 'coalition-mathematics.md: missing seat-count / vote-breakdown table', + artifact: 'coalition-mathematics.md', + }); + } else { + results.push({ + checkId: 'family-d-structure', + passed: true, + message: 'coalition-mathematics.md: vote/seat table present', + artifact: 'coalition-mathematics.md', + }); + } + + return results; +} + +// --------------------------------------------------------------------------- +// Check 9 — PIR status sidecar +// --------------------------------------------------------------------------- + +/** PIR status JSON schema structure. */ +interface PirStatusFile { + readonly schema_version?: string; + readonly cycle?: string; + readonly date?: string; + readonly subfolder?: string; + readonly pirs?: readonly PirEntry[]; + readonly generated_at?: string; +} + +/** A single PIR entry in the sidecar file. */ +interface PirEntry { + readonly pir_id?: string; + readonly statement?: string; + readonly status?: string; + readonly confidence?: string; + readonly answer_summary?: string; +} + +const VALID_PIR_STATUSES = new Set([ + 'open', 'answered', 'superseded', 'deferred', 'cancelled', +]); + +const VALID_CONFIDENCE_LEVELS = new Set([ + 'VERY HIGH', 'HIGH', 'MEDIUM', 'LOW', 'VERY LOW', +]); + +const PIR_ID_PATTERN = /^PIR-[A-Za-z0-9]+(-[A-Za-z0-9]+)*$/; + +/** + * Validate pir-status.json exists and has valid structure. + */ +export async function checkPirStatus(analysisDir: string): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'pir-status.json'); + + if (!existsSync(filePath)) { + results.push({ + checkId: 'pir-status', + passed: false, + message: 'pir-status.json missing or empty', + }); + return results; + } + + let data: PirStatusFile; + try { + const raw = await readFile(filePath, 'utf-8'); + data = JSON.parse(raw) as PirStatusFile; + } catch { + results.push({ + checkId: 'pir-status', + passed: false, + message: 'pir-status.json: invalid JSON', + }); + return results; + } + + // Required top-level fields + const requiredFields = ['schema_version', 'cycle', 'date', 'subfolder', 'pirs', 'generated_at'] as const; + for (const field of requiredFields) { + if (!(field in data) || data[field as keyof PirStatusFile] === undefined) { + results.push({ + checkId: 'pir-status', + passed: false, + message: `pir-status.json: missing required field '${field}'`, + }); + } + } + + // schema_version check + if (data.schema_version !== '1.0') { + results.push({ + checkId: 'pir-status', + passed: false, + message: "pir-status.json: schema_version must be '1.0'", + }); + } + + // pirs must be an array + if (!Array.isArray(data.pirs)) { + results.push({ + checkId: 'pir-status', + passed: false, + message: "pir-status.json: 'pirs' field must be a JSON array", + }); + return results; + } + + // subfolder must equal cycle + if (data.subfolder !== data.cycle) { + results.push({ + checkId: 'pir-status', + passed: false, + message: `pir-status.json: subfolder='${data.subfolder}' must equal cycle='${data.cycle}'`, + }); + } + + // Validate each PIR entry + for (const pir of data.pirs) { + const pid = pir.pir_id ?? '(no id)'; + + if (!pir.pir_id || !PIR_ID_PATTERN.test(pir.pir_id)) { + results.push({ + checkId: 'pir-status', + passed: false, + message: `pir-status.json pir=${pid}: invalid pir_id format`, + }); + } + + for (const field of ['statement', 'status', 'confidence'] as const) { + if (!pir[field]) { + results.push({ + checkId: 'pir-status', + passed: false, + message: `pir-status.json pir=${pid}: missing required field "${field}"`, + }); + } + } + + if (pir.status && !VALID_PIR_STATUSES.has(pir.status)) { + results.push({ + checkId: 'pir-status', + passed: false, + message: `pir-status.json pir=${pid}: invalid status '${pir.status}'`, + }); + } + + if (pir.confidence && !VALID_CONFIDENCE_LEVELS.has(pir.confidence)) { + results.push({ + checkId: 'pir-status', + passed: false, + message: `pir-status.json pir=${pid}: invalid confidence '${pir.confidence}'`, + }); + } + + // Conditional: answer_summary required iff status == 'answered'; must + // not be present for any other status (the canonical Python gate enforces + // both directions as a cross-field invariant). + if (pir.status === 'answered' && !pir.answer_summary) { + results.push({ + checkId: 'pir-status', + passed: false, + message: `pir-status.json pir=${pid}: status=answered requires non-empty answer_summary`, + }); + } + if (pir.status !== 'answered' && pir.answer_summary !== undefined) { + results.push({ + checkId: 'pir-status', + passed: false, + message: `pir-status.json pir=${pid}: status=${pir.status} must not carry answer_summary`, + }); + } + } + + if (results.length === 0) { + results.push({ + checkId: 'pir-status', + passed: true, + message: 'pir-status.json: valid', + }); + } + + return results; +} + +// --------------------------------------------------------------------------- +// Check 9b — Statskontoret evidence +// --------------------------------------------------------------------------- + +/** + * When implementation-feasibility.md names a recognised agency, verify + * the Statskontoret relevance row has a URL or 'none found'. + */ +export async function checkStatskontoretEvidence( + analysisDir: string, +): Promise { + const results: GateCheckResult[] = []; + const filePath = join(analysisDir, 'implementation-feasibility.md'); + if (!existsSync(filePath)) return results; + + const content = await readFile(filePath, 'utf-8'); + + // Check if any recognised agency is mentioned + const agencyPattern = new RegExp(RECOGNISED_AGENCIES.join('|'), 'i'); + if (!agencyPattern.test(content)) { + // No agency mentioned, check passes + results.push({ + checkId: 'statskontoret-evidence', + passed: true, + message: 'implementation-feasibility.md: no recognised agency mentioned', + artifact: 'implementation-feasibility.md', + }); + return results; + } + + // Agency mentioned — check for Statskontoret relevance row + const statskontoretRow = + /^\|\s*\*{0,2}Statskontoret relevance\*{0,2}\s*\|\s*([^|]*statskontoret\.se[^|]*|[^|]*none found[^|]*)\|/im; + + if (!statskontoretRow.test(content)) { + results.push({ + checkId: 'statskontoret-evidence', + passed: false, + message: "implementation-feasibility.md: names a recognised agency but Statskontoret relevance row lacks a statskontoret.se URL or 'none found'", + artifact: 'implementation-feasibility.md', + }); + } else { + results.push({ + checkId: 'statskontoret-evidence', + passed: true, + message: 'implementation-feasibility.md: Statskontoret evidence present', + artifact: 'implementation-feasibility.md', + }); + } + + return results; +} diff --git a/scripts/agentic/artifact-inventory.ts b/scripts/agentic/artifact-inventory.ts new file mode 100644 index 0000000000..e140361564 --- /dev/null +++ b/scripts/agentic/artifact-inventory.ts @@ -0,0 +1,212 @@ +/** + * @module scripts/agentic/artifact-inventory + * @description Typed definitions for the 23 required analysis artifacts + * produced by every agentic news workflow run. + * + * The artifact families (A through E) are defined in + * `.github/prompts/04-analysis-pipeline.md` and enforced by + * `.github/prompts/05-analysis-gate.md`. This module provides a single + * source of truth for artifact file names, family membership, and gate + * check metadata — consumable by both the gate validator and downstream + * tooling (aggregate-analysis, render-articles, quality dashboards). + * + * @see .github/prompts/04-analysis-pipeline.md — defines the 23 artifacts + * @see .github/prompts/05-analysis-gate.md — gate checks reference + * @see analysis/methodologies/artifact-catalog.md — canonical catalog + * @author Hack23 AB + * @license Apache-2.0 + */ + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** Artifact family identifier matching the analysis pipeline spec. */ +export type ArtifactFamily = 'A' | 'B' | 'C' | 'D' | 'E'; + +/** A single required artifact in the analysis pipeline. */ +export interface ArtifactDefinition { + /** File name relative to the analysis subfolder. */ + readonly filename: string; + /** Family this artifact belongs to. */ + readonly family: ArtifactFamily; + /** Human-readable description. */ + readonly description: string; + /** Whether this artifact requires a Mermaid diagram with colour config. */ + readonly requiresMermaid: boolean; + /** Whether this artifact is subject to Pass-2 evidence checking. */ + readonly requiresPass2: boolean; +} + +/** Gate check result for a single artifact or validation rule. */ +export interface GateCheckResult { + /** Check identifier (e.g. 'artifact-existence', 'no-stubs'). */ + readonly checkId: string; + /** Whether the check passed. */ + readonly passed: boolean; + /** Human-readable message (populated on failure). */ + readonly message: string; + /** The artifact or file that failed (if applicable). */ + readonly artifact?: string; +} + +/** Aggregate result from running all gate checks. */ +export interface GateValidationResult { + /** Whether all checks passed. */ + readonly passed: boolean; + /** Individual check results. */ + readonly checks: readonly GateCheckResult[]; + /** Count of failures. */ + readonly failureCount: number; +} + +// --------------------------------------------------------------------------- +// Family A — Core Synthesis (9 files) +// --------------------------------------------------------------------------- + +/** Family A artifact file names — Core Synthesis. */ +export const FAMILY_A_ARTIFACTS: readonly ArtifactDefinition[] = Object.freeze([ + { filename: 'README.md', family: 'A', description: 'Folder README with index links', requiresMermaid: false, requiresPass2: true }, + { filename: 'executive-brief.md', family: 'A', description: 'BLUF + 3 decisions supported', requiresMermaid: true, requiresPass2: true }, + { filename: 'synthesis-summary.md', family: 'A', description: 'Lead-story decision + DIW ranking', requiresMermaid: true, requiresPass2: true }, + { filename: 'significance-scoring.md', family: 'A', description: 'DIW scores + sensitivity analysis', requiresMermaid: true, requiresPass2: true }, + { filename: 'classification-results.md', family: 'A', description: '7-dimension classification', requiresMermaid: true, requiresPass2: true }, + { filename: 'swot-analysis.md', family: 'A', description: 'S/W/O/T with evidence + TOWS matrix', requiresMermaid: true, requiresPass2: true }, + { filename: 'risk-assessment.md', family: 'A', description: 'Risk matrix + mitigation strategies', requiresMermaid: true, requiresPass2: true }, + { filename: 'threat-analysis.md', family: 'A', description: 'STRIDE-style threat assessment', requiresMermaid: true, requiresPass2: true }, + { filename: 'stakeholder-perspectives.md', family: 'A', description: 'Stakeholder mapping + positions', requiresMermaid: true, requiresPass2: true }, +]); + +// --------------------------------------------------------------------------- +// Family B — Structural Metadata (2 files) +// --------------------------------------------------------------------------- + +/** Family B artifact file names — Structural Metadata. */ +export const FAMILY_B_ARTIFACTS: readonly ArtifactDefinition[] = Object.freeze([ + { filename: 'data-download-manifest.md', family: 'B', description: 'Download manifest with dok_ids', requiresMermaid: false, requiresPass2: false }, + { filename: 'cross-reference-map.md', family: 'B', description: 'Cross-reference map', requiresMermaid: true, requiresPass2: true }, +]); + +// --------------------------------------------------------------------------- +// Family C — Strategic Extensions (5 files) +// --------------------------------------------------------------------------- + +/** Family C artifact file names — Strategic Extensions. */ +export const FAMILY_C_ARTIFACTS: readonly ArtifactDefinition[] = Object.freeze([ + { filename: 'scenario-analysis.md', family: 'C', description: '≥3 distinct scenarios', requiresMermaid: false, requiresPass2: true }, + { filename: 'comparative-international.md', family: 'C', description: 'Comparator set + rows', requiresMermaid: false, requiresPass2: true }, + { filename: 'devils-advocate.md', family: 'C', description: '≥3 competing hypotheses (ACH)', requiresMermaid: false, requiresPass2: true }, + { filename: 'intelligence-assessment.md', family: 'C', description: '≥3 Key Judgments with confidence', requiresMermaid: false, requiresPass2: true }, + { filename: 'methodology-reflection.md', family: 'C', description: 'ICD 203 audit / improvements', requiresMermaid: false, requiresPass2: true }, +]); + +// --------------------------------------------------------------------------- +// Family D — Electoral & Domain Lenses (7 files) +// --------------------------------------------------------------------------- + +/** Family D artifact file names — Electoral & Domain Lenses. */ +export const FAMILY_D_ARTIFACTS: readonly ArtifactDefinition[] = Object.freeze([ + { filename: 'election-2026-analysis.md', family: 'D', description: 'Election 2026 analysis', requiresMermaid: true, requiresPass2: true }, + { filename: 'voter-segmentation.md', family: 'D', description: 'Voter segmentation analysis', requiresMermaid: true, requiresPass2: true }, + { filename: 'coalition-mathematics.md', family: 'D', description: 'Seat-count / vote-breakdown table', requiresMermaid: true, requiresPass2: true }, + { filename: 'historical-parallels.md', family: 'D', description: 'Historical parallels analysis', requiresMermaid: true, requiresPass2: true }, + { filename: 'media-framing-analysis.md', family: 'D', description: 'Media framing analysis', requiresMermaid: true, requiresPass2: true }, + { filename: 'implementation-feasibility.md', family: 'D', description: 'Implementation feasibility', requiresMermaid: true, requiresPass2: true }, + { filename: 'forward-indicators.md', family: 'D', description: '≥10 dated forward indicators', requiresMermaid: true, requiresPass2: true }, +]); + +// --------------------------------------------------------------------------- +// Aggregate constants +// --------------------------------------------------------------------------- + +/** All 23 required artifacts (Families A + B + C + D). */ +export const ALL_REQUIRED_ARTIFACTS: readonly ArtifactDefinition[] = Object.freeze([ + ...FAMILY_A_ARTIFACTS, + ...FAMILY_B_ARTIFACTS, + ...FAMILY_C_ARTIFACTS, + ...FAMILY_D_ARTIFACTS, +]); + +/** All required artifact file names as a flat array. */ +export const REQUIRED_ARTIFACT_FILENAMES: readonly string[] = Object.freeze( + ALL_REQUIRED_ARTIFACTS.map((a) => a.filename), +); + +/** Artifacts that require Mermaid diagrams with colour config. */ +export const MERMAID_REQUIRED_ARTIFACTS: readonly string[] = Object.freeze( + ALL_REQUIRED_ARTIFACTS.filter((a) => a.requiresMermaid).map((a) => a.filename), +); + +/** Artifacts subject to Pass-2 evidence checking. */ +export const PASS2_REQUIRED_ARTIFACTS: readonly string[] = Object.freeze( + ALL_REQUIRED_ARTIFACTS.filter((a) => a.requiresPass2).map((a) => a.filename), +); + +/** + * Stub placeholder strings that must not appear in committed artifacts. + * Matches check 3 in `05-analysis-gate.md`. + */ +export const STUB_PLACEHOLDERS: readonly string[] = Object.freeze([ + 'AI_MUST_REPLACE', + '[REQUIRED]', + 'TODO:', + 'Lorem ipsum', +]); + +/** + * Recognised Swedish government agencies for the Statskontoret evidence check + * (gate check 9b in `05-analysis-gate.md`). + */ +export const RECOGNISED_AGENCIES: readonly string[] = Object.freeze([ + 'Kriminalvården', + 'Kriminalvård', + 'Polismyndigheten', + 'Försäkringskassan', + 'Skatteverket', + 'Migrationsverket', + 'Arbetsförmedlingen', + 'Socialstyrelsen', + 'Transportstyrelsen', + 'Trafikverket', + 'Naturvårdsverket', + 'Energimyndigheten', +]); + +/** + * Primary-source evidence URL hosts accepted by the evidence citation check + * (gate check 4 in `05-analysis-gate.md`). + */ +export const EVIDENCE_URL_HOSTS: readonly string[] = Object.freeze([ + 'riksdagen.se', + 'regeringen.se', + 'scb.se', + 'statskontoret.se', + 'worldbank.org', + 'api.imf.org', + 'data.imf.org', + 'www.imf.org', +]); + +/** + * Regex pattern matching a valid Riksdag document ID (dok_id). + * Examples: H901FiU1, HD01CU27 + */ +export const DOK_ID_PATTERN = /[Hh][A-Za-z0-9]{3,}[0-9]+/; + +/** + * Escape every character with special meaning in a RegExp so that the + * string is matched literally. This prevents inadvertent pattern + * injection when URL hosts are interpolated into a compiled regex. + */ +function escapeRegexLiteral(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Combined evidence regex — matches either a dok_id or a primary-source URL host. + * All host strings are fully escaped so that special regex chars (dots, etc.) + * are treated as literals. + */ +export const EVIDENCE_PATTERN = new RegExp( + `${DOK_ID_PATTERN.source}|${EVIDENCE_URL_HOSTS.map(escapeRegexLiteral).join('|')}`, +); diff --git a/scripts/agentic/index.ts b/scripts/agentic/index.ts new file mode 100644 index 0000000000..042635773a --- /dev/null +++ b/scripts/agentic/index.ts @@ -0,0 +1,45 @@ +/** + * @module scripts/agentic + * @description Barrel export for the agentic workflow bounded context. + * + * This module exposes typed helpers that extract, validate, and inventory + * the analysis artifacts produced by the 14 agentic news workflows. + * + * @author Hack23 AB + * @license Apache-2.0 + */ + +export { + type ArtifactFamily, + type ArtifactDefinition, + type GateCheckResult, + type GateValidationResult, + FAMILY_A_ARTIFACTS, + FAMILY_B_ARTIFACTS, + FAMILY_C_ARTIFACTS, + FAMILY_D_ARTIFACTS, + ALL_REQUIRED_ARTIFACTS, + REQUIRED_ARTIFACT_FILENAMES, + MERMAID_REQUIRED_ARTIFACTS, + PASS2_REQUIRED_ARTIFACTS, + STUB_PLACEHOLDERS, + RECOGNISED_AGENCIES, + EVIDENCE_URL_HOSTS, + DOK_ID_PATTERN, + EVIDENCE_PATTERN, +} from './artifact-inventory.js'; + +export { + validateAnalysisGate, + checkArtifactExistence, + checkPerDocumentCoverage, + checkNoStubs, + checkEvidenceCitations, + checkMermaidDiagrams, + checkPass2Evidence, + checkFamilyCStructure, + checkFamilyDStructure, + checkPirStatus, + checkStatskontoretEvidence, + extractDokIds, +} from './analysis-gate.js'; diff --git a/tests/agentic-analysis-gate.test.ts b/tests/agentic-analysis-gate.test.ts new file mode 100644 index 0000000000..0436313d17 --- /dev/null +++ b/tests/agentic-analysis-gate.test.ts @@ -0,0 +1,1046 @@ +/** + * @module tests/agentic-analysis-gate + * @description Unit tests for the analysis gate validation logic extracted + * from `.github/prompts/05-analysis-gate.md`. + * + * Tests cover: + * - Artifact inventory type definitions and constants + * - Gate check 1: artifact existence (present and non-empty) + * - Gate check 2: per-document coverage + * - Gate check 3: stub placeholder detection (including documents/ dir) + * - Gate check 4: evidence citations (SWOT and significance-scoring) + * - Gate check 5: Mermaid diagram validation + * - Gate check 6: Pass-2 evidence (pass1/ snapshot or mtime) + * - Gate check 7: Family C structure checks + * - Gate check 8: Family D structure checks + * - Gate check 9: PIR status sidecar validation (incl. answer_summary rule) + * - Gate check 9b: Statskontoret evidence + * - Integration: full gate validation (result.passed === true) + * + * @author Hack23 AB + * @license Apache-2.0 + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdirSync, writeFileSync, rmSync, existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { + ALL_REQUIRED_ARTIFACTS, + FAMILY_A_ARTIFACTS, + FAMILY_B_ARTIFACTS, + FAMILY_C_ARTIFACTS, + FAMILY_D_ARTIFACTS, + REQUIRED_ARTIFACT_FILENAMES, + MERMAID_REQUIRED_ARTIFACTS, + PASS2_REQUIRED_ARTIFACTS, + STUB_PLACEHOLDERS, + RECOGNISED_AGENCIES, + EVIDENCE_URL_HOSTS, + DOK_ID_PATTERN, + EVIDENCE_PATTERN, +} from '../scripts/agentic/artifact-inventory.js'; + +import { + validateAnalysisGate, + checkArtifactExistence, + checkPerDocumentCoverage, + checkNoStubs, + checkEvidenceCitations, + checkMermaidDiagrams, + checkPass2Evidence, + checkFamilyCStructure, + checkFamilyDStructure, + checkPirStatus, + checkStatskontoretEvidence, + extractDokIds, +} from '../scripts/agentic/analysis-gate.js'; + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +let testDir: string; + +function createTestDir(): string { + const dir = join(tmpdir(), `agentic-gate-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + return dir; +} + +function writeArtifact(dir: string, filename: string, content: string): void { + const filePath = join(dir, filename); + writeFileSync(filePath, content, 'utf-8'); +} + +function createMinimalValidAnalysis(dir: string): void { + // Create all 23 required artifacts with minimal valid content + for (const artifact of ALL_REQUIRED_ARTIFACTS) { + let content = `# ${artifact.filename}\n\nMinimal content for testing.\n`; + + // Add Mermaid if required + if (artifact.requiresMermaid) { + content += '\n```mermaid\ngraph TD\n A --> B\n style A fill:#f00\n```\n'; + } + + // Add specific content for structural checks + if (artifact.filename === 'executive-brief.md') { + content = '# Executive Brief\n\n## 🎯 BLUF\n\nBrief summary.\n\n## 🧭 3 Decisions This Brief Supports\n\n1. Decision A\n\n```mermaid\ngraph TD\n A --> B\n style A fill:#f00\n```\n'; + } else if (artifact.filename === 'swot-analysis.md') { + // Check 4: SWOT sections with evidence citations (dok_id per bullet) + content = '# SWOT Analysis\n\n' + + '### Strengths\n- Strong fiscal position H901FiU1 government reform data\n- Parliamentary support HD01CU27 vote record\n\n' + + '### Weaknesses\n- Budget deficit riksdagen.se/sv/dokument-lagar\n- Limited capacity regeringen.se/rattsliga-dokument\n\n' + + '### Opportunities\n- Economic growth scb.se/statistik\n- Policy reform api.imf.org/external/datamapper\n\n' + + '### Threats\n- Geopolitical risks www.imf.org/en/Publications\n- Market volatility data.imf.org/regular.aspx\n\n' + + '```mermaid\ngraph TD\n A --> B\n style A fill:#f00\n```\n'; + } else if (artifact.filename === 'significance-scoring.md') { + // Check 4: ranked bullets with evidence citations + content = '# Significance Scoring\n\n' + + '1. Fiscal policy reform H901FiU1 high impact\n' + + '2. Tax legislation HD01CU27 medium impact\n' + + '3. Social welfare riksdagen.se/sv/ high importance\n\n' + + '| Rank | Item | Evidence |\n|------|------|----------|\n' + + '| 1 | Reform | H901FiU1 |\n' + + '| 2 | Tax | HD01CU27 |\n\n' + + '```mermaid\ngraph TD\n A[H901FiU1] --> B\n style A fill:#f00\n```\n'; + } else if (artifact.filename === 'intelligence-assessment.md') { + content = '# Intelligence Assessment\n\n## Key Judgment KJ-1\nHIGH confidence.\n\n## Key Judgment KJ-2\nMEDIUM confidence.\n\n## Key Judgment KJ-3\nLOW confidence.\n\nReferences PIR-FISCAL-001.\n'; + } else if (artifact.filename === 'scenario-analysis.md') { + content = '# Scenario Analysis\n\n## Scenario 1: Status Quo\n\n## Scenario 2: Reform\n\n## Scenario 3: Crisis\n\n'; + } else if (artifact.filename === 'devils-advocate.md') { + content = '# Devil\'s Advocate\n\n## Hypothesis 1: Government succeeds\n\n## Hypothesis 2: Opposition blocks\n\n## Hypothesis 3: Coalition fractures\n\n'; + } else if (artifact.filename === 'methodology-reflection.md') { + content = '# Methodology Reflection\n\n## ICD 203 Audit\n\nMethodology Improvements applied.\n\n## Improvement 1\n\n'; + } else if (artifact.filename === 'comparative-international.md') { + content = '# Comparative International\n\n**Comparator set**: Denmark, Norway, Finland\n\n| Country | Policy |\n|---------|--------|\n| Denmark | A |\n| Norway | B |\n'; + } else if (artifact.filename === 'forward-indicators.md') { + content = '# Forward Indicators\n\n' + Array.from({ length: 12 }, (_, i) => + `- 2026-06-${String(i + 1).padStart(2, '0')}: Indicator ${i + 1}\n` + ).join('') + '\n```mermaid\ngraph TD\n A --> B\n style A fill:#f00\n```\n'; + } else if (artifact.filename === 'coalition-mathematics.md') { + content = '# Coalition Mathematics\n\n| Party | Seats | Ja | Nej |\n|-------|-------|-----|-----|\n| S | 107 | Yes | |\n\n```mermaid\ngraph TD\n A --> B\n style A fill:#f00\n```\n'; + } else if (artifact.filename === 'data-download-manifest.md') { + content = '# Data Download Manifest\n\nDownloaded documents:\n- H901FiU1\n- HD01CU27\n'; + } + + writeArtifact(dir, artifact.filename, content); + } + + // Create documents directory with per-document analyses + const docsDir = join(dir, 'documents'); + mkdirSync(docsDir, { recursive: true }); + writeFileSync(join(docsDir, 'H901FiU1-analysis.md'), '# H901FiU1 Analysis\n\nDocument analysis content.\n', 'utf-8'); + writeFileSync(join(docsDir, 'HD01CU27-analysis.md'), '# HD01CU27 Analysis\n\nDocument analysis content.\n', 'utf-8'); + + // Create PIR status sidecar + const pirStatus = { + schema_version: '1.0', + cycle: 'propositions', + date: '2026-05-01', + subfolder: 'propositions', + pirs: [ + { + pir_id: 'PIR-FISCAL-001', + statement: 'What is the fiscal impact?', + status: 'open', + confidence: 'MEDIUM', + }, + ], + generated_at: '2026-05-01T10:00:00Z', + }; + writeFileSync(join(dir, 'pir-status.json'), JSON.stringify(pirStatus, null, 2), 'utf-8'); + + // Create pass1/ snapshot directory with content that differs from the current + // artifacts — this satisfies check 6 (Pass-2 evidence) without requiring + // a 180-second mtime window that is not reproducible in a test environment. + const pass1Dir = join(dir, 'pass1'); + mkdirSync(pass1Dir, { recursive: true }); + for (const filename of PASS2_REQUIRED_ARTIFACTS) { + const mainPath = join(dir, filename); + if (existsSync(mainPath)) { + const mainContent = readFileSync(mainPath, 'utf-8'); + // Prepend a pass-1 marker so the snapshot differs from the final version + writeFileSync(join(pass1Dir, filename), `\n${mainContent}`, 'utf-8'); + } + } +} + +// --------------------------------------------------------------------------- +// Tests: Artifact Inventory +// --------------------------------------------------------------------------- + +describe('Artifact Inventory', () => { + it('defines exactly 23 required artifacts', () => { + expect(ALL_REQUIRED_ARTIFACTS).toHaveLength(23); + }); + + it('Family A has 9 artifacts', () => { + expect(FAMILY_A_ARTIFACTS).toHaveLength(9); + }); + + it('Family B has 2 artifacts', () => { + expect(FAMILY_B_ARTIFACTS).toHaveLength(2); + }); + + it('Family C has 5 artifacts', () => { + expect(FAMILY_C_ARTIFACTS).toHaveLength(5); + }); + + it('Family D has 7 artifacts', () => { + expect(FAMILY_D_ARTIFACTS).toHaveLength(7); + }); + + it('all artifact filenames end with .md', () => { + for (const artifact of ALL_REQUIRED_ARTIFACTS) { + expect(artifact.filename).toMatch(/\.md$/); + } + }); + + it('REQUIRED_ARTIFACT_FILENAMES matches ALL_REQUIRED_ARTIFACTS count', () => { + expect(REQUIRED_ARTIFACT_FILENAMES).toHaveLength(23); + }); + + it('MERMAID_REQUIRED_ARTIFACTS is a subset of all artifacts', () => { + for (const filename of MERMAID_REQUIRED_ARTIFACTS) { + expect(REQUIRED_ARTIFACT_FILENAMES).toContain(filename); + } + }); + + it('PASS2_REQUIRED_ARTIFACTS excludes data-download-manifest.md', () => { + expect(PASS2_REQUIRED_ARTIFACTS).not.toContain('data-download-manifest.md'); + }); + + it('STUB_PLACEHOLDERS contains expected markers', () => { + expect(STUB_PLACEHOLDERS).toContain('AI_MUST_REPLACE'); + expect(STUB_PLACEHOLDERS).toContain('[REQUIRED]'); + expect(STUB_PLACEHOLDERS).toContain('TODO:'); + expect(STUB_PLACEHOLDERS).toContain('Lorem ipsum'); + }); + + it('RECOGNISED_AGENCIES contains known Swedish agencies', () => { + expect(RECOGNISED_AGENCIES).toContain('Skatteverket'); + expect(RECOGNISED_AGENCIES).toContain('Polismyndigheten'); + expect(RECOGNISED_AGENCIES.length).toBe(12); + }); + + it('EVIDENCE_URL_HOSTS covers all required primary sources', () => { + expect(EVIDENCE_URL_HOSTS).toContain('riksdagen.se'); + expect(EVIDENCE_URL_HOSTS).toContain('api.imf.org'); + expect(EVIDENCE_URL_HOSTS).toContain('statskontoret.se'); + }); + + it('DOK_ID_PATTERN matches valid Riksdag document IDs', () => { + expect(DOK_ID_PATTERN.test('H901FiU1')).toBe(true); + expect(DOK_ID_PATTERN.test('HD01CU27')).toBe(true); + expect(DOK_ID_PATTERN.test('invalid')).toBe(false); + expect(DOK_ID_PATTERN.test('abc')).toBe(false); + }); + + it('EVIDENCE_PATTERN matches dok_ids and URL hosts', () => { + expect(EVIDENCE_PATTERN.test('H901FiU1')).toBe(true); + expect(EVIDENCE_PATTERN.test('riksdagen.se')).toBe(true); + expect(EVIDENCE_PATTERN.test('api.imf.org')).toBe(true); + expect(EVIDENCE_PATTERN.test('random text')).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: extractDokIds +// --------------------------------------------------------------------------- + +describe('extractDokIds', () => { + it('extracts dok_ids from markdown content', () => { + const content = 'Documents: H901FiU1, HD01CU27, and H901AU10.'; + const ids = extractDokIds(content); + expect(ids).toContain('H901FiU1'); + expect(ids).toContain('HD01CU27'); + expect(ids).toContain('H901AU10'); + }); + + it('deduplicates dok_ids', () => { + const content = 'H901FiU1 appears twice: H901FiU1.'; + const ids = extractDokIds(content); + expect(ids.filter((id) => id === 'H901FiU1')).toHaveLength(1); + }); + + it('returns empty array when no dok_ids found', () => { + const content = 'No document references here.'; + expect(extractDokIds(content)).toHaveLength(0); + }); + + it('handles empty content', () => { + expect(extractDokIds('')).toHaveLength(0); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 1 — Artifact Existence +// --------------------------------------------------------------------------- + +describe('checkArtifactExistence', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('reports all artifacts missing when directory is empty', () => { + const results = checkArtifactExistence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(23); + }); + + it('reports success when all artifacts present', () => { + for (const filename of REQUIRED_ARTIFACT_FILENAMES) { + writeArtifact(testDir, filename, 'content'); + } + const results = checkArtifactExistence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('reports specific missing artifact', () => { + for (const filename of REQUIRED_ARTIFACT_FILENAMES) { + if (filename !== 'swot-analysis.md') { + writeArtifact(testDir, filename, 'content'); + } + } + const results = checkArtifactExistence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(1); + expect(failures[0]?.artifact).toBe('swot-analysis.md'); + }); + + it('reports failure for zero-byte (empty) artifact', () => { + for (const filename of REQUIRED_ARTIFACT_FILENAMES) { + // Write empty content for one artifact, non-empty for others + writeArtifact(testDir, filename, filename === 'README.md' ? '' : 'content'); + } + const results = checkArtifactExistence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(1); + expect(failures[0]?.message).toContain('Empty artifact'); + expect(failures[0]?.artifact).toBe('README.md'); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 2 — Per-document Coverage +// --------------------------------------------------------------------------- + +describe('checkPerDocumentCoverage', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('passes when all dok_ids have analysis files', async () => { + writeArtifact(testDir, 'data-download-manifest.md', 'Docs: H901FiU1, HD01CU27'); + const docsDir = join(testDir, 'documents'); + mkdirSync(docsDir, { recursive: true }); + writeFileSync(join(docsDir, 'H901FiU1-analysis.md'), 'analysis', 'utf-8'); + writeFileSync(join(docsDir, 'HD01CU27-analysis.md'), 'analysis', 'utf-8'); + + const results = await checkPerDocumentCoverage(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('reports missing document analysis', async () => { + writeArtifact(testDir, 'data-download-manifest.md', 'Docs: H901FiU1, HD01CU27'); + const docsDir = join(testDir, 'documents'); + mkdirSync(docsDir, { recursive: true }); + writeFileSync(join(docsDir, 'H901FiU1-analysis.md'), 'analysis', 'utf-8'); + // HD01CU27 intentionally missing + + const results = await checkPerDocumentCoverage(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(1); + expect(failures[0]?.message).toContain('HD01CU27'); + }); + + it('reports failure when manifest has no dok_ids', async () => { + writeArtifact(testDir, 'data-download-manifest.md', 'No documents here.'); + + const results = await checkPerDocumentCoverage(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(1); + expect(failures[0]?.message).toContain('no dok_id entries'); + }); + + it('returns empty when manifest does not exist', async () => { + const results = await checkPerDocumentCoverage(testDir); + expect(results).toHaveLength(0); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 3 — No Stubs +// --------------------------------------------------------------------------- + +describe('checkNoStubs', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('passes when no stubs present', async () => { + writeArtifact(testDir, 'README.md', '# README\n\nClean content.'); + const results = await checkNoStubs(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('detects AI_MUST_REPLACE placeholder', async () => { + writeArtifact(testDir, 'README.md', '# README\n\nAI_MUST_REPLACE this.'); + const results = await checkNoStubs(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(1); + expect(failures[0]?.message).toContain('AI_MUST_REPLACE'); + }); + + it('detects TODO: placeholder', async () => { + writeArtifact(testDir, 'swot-analysis.md', '# SWOT\n\nTODO: add evidence'); + const results = await checkNoStubs(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(1); + expect(failures[0]?.message).toContain('TODO:'); + }); + + it('detects multiple stub types in same file', async () => { + writeArtifact(testDir, 'README.md', 'AI_MUST_REPLACE and [REQUIRED] and TODO: fix'); + const results = await checkNoStubs(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(3); + }); + + it('detects stubs in documents/ directory (Family E)', async () => { + // No stubs in required artifacts + writeArtifact(testDir, 'README.md', '# README\n\nClean content.'); + // But stub in documents/ per-document analysis + const docsDir = join(testDir, 'documents'); + mkdirSync(docsDir, { recursive: true }); + writeFileSync(join(docsDir, 'H901FiU1-analysis.md'), '# Analysis\n\nAI_MUST_REPLACE\n', 'utf-8'); + + const results = await checkNoStubs(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + expect(failures[0]?.artifact).toContain('documents/'); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 4 — Evidence Citations +// --------------------------------------------------------------------------- + +describe('checkEvidenceCitations', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('passes when all SWOT bullets have evidence', async () => { + writeArtifact(testDir, 'swot-analysis.md', + '# SWOT\n\n' + + '### Strengths\n- Strong position H901FiU1 data\n\n' + + '### Weaknesses\n- Deficit HD01CU27 evidence\n\n' + + '### Opportunities\n- Growth riksdagen.se/sv/\n\n' + + '### Threats\n- Risk www.imf.org/en/\n\n' + ); + writeArtifact(testDir, 'significance-scoring.md', '# Scores\n\n1. Item H901FiU1\n'); + const results = await checkEvidenceCitations(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'swot-analysis.md'); + expect(failures).toHaveLength(0); + }); + + it('fails when a SWOT bullet is missing evidence', async () => { + writeArtifact(testDir, 'swot-analysis.md', + '# SWOT\n\n### Strengths\n- Strong position with no citation\n' + ); + writeArtifact(testDir, 'significance-scoring.md', '# Scores\n\n'); + const results = await checkEvidenceCitations(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + expect(failures[0]?.checkId).toBe('evidence-citations'); + }); + + it('passes when all significance-scoring bullets have evidence', async () => { + writeArtifact(testDir, 'swot-analysis.md', '# SWOT\n\n'); + writeArtifact(testDir, 'significance-scoring.md', + '# Significance\n\n1. Reform H901FiU1\n2. Tax HD01CU27\n' + ); + const results = await checkEvidenceCitations(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'significance-scoring.md'); + expect(failures).toHaveLength(0); + }); + + it('fails when significance-scoring bullet is missing evidence', async () => { + writeArtifact(testDir, 'swot-analysis.md', '# SWOT\n\n'); + writeArtifact(testDir, 'significance-scoring.md', + '# Significance\n\n1. Generic ranked item with no citation\n' + ); + const results = await checkEvidenceCitations(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'significance-scoring.md'); + expect(failures.length).toBeGreaterThan(0); + }); + + it('returns empty when files do not exist', async () => { + const results = await checkEvidenceCitations(testDir); + expect(results).toHaveLength(0); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 5 — Mermaid Diagrams +// --------------------------------------------------------------------------- + +describe('checkMermaidDiagrams', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('passes with Mermaid block and style directive', async () => { + writeArtifact(testDir, 'synthesis-summary.md', '# Summary\n\n```mermaid\ngraph TD\n A --> B\n style A fill:#f00\n```\n'); + const results = await checkMermaidDiagrams(testDir); + const passes = results.filter((r) => r.passed); + expect(passes.length).toBeGreaterThan(0); + }); + + it('passes with themeVariables', async () => { + writeArtifact(testDir, 'synthesis-summary.md', '# Summary\n\n```mermaid\n%%{init: {themeVariables: {primaryColor: "#f00"}}}%%\ngraph TD\n A --> B\n```\n'); + const results = await checkMermaidDiagrams(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('fails when Mermaid block missing', async () => { + writeArtifact(testDir, 'synthesis-summary.md', '# Summary\n\nNo diagram here.\n'); + const results = await checkMermaidDiagrams(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + expect(failures[0]?.message).toContain('missing Mermaid block'); + }); + + it('fails when colour config missing', async () => { + writeArtifact(testDir, 'synthesis-summary.md', '# Summary\n\n```mermaid\ngraph TD\n A --> B\n```\n'); + const results = await checkMermaidDiagrams(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + expect(failures[0]?.message).toContain('colour-coded config'); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 6 — Pass-2 Evidence +// --------------------------------------------------------------------------- + +describe('checkPass2Evidence', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('passes when pass1/ snapshot differs from current file', async () => { + writeArtifact(testDir, 'README.md', '# Pass-2 improved content\n\nMore analysis here.\n'); + const pass1Dir = join(testDir, 'pass1'); + mkdirSync(pass1Dir, { recursive: true }); + writeFileSync(join(pass1Dir, 'README.md'), '# Original pass1 draft\n\n', 'utf-8'); + + const results = await checkPass2Evidence(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'README.md'); + expect(failures).toHaveLength(0); + }); + + it('fails when pass1/ snapshot is identical to current file (no improvements)', async () => { + const content = '# Same content in both passes\n\nNothing changed.\n'; + writeArtifact(testDir, 'README.md', content); + const pass1Dir = join(testDir, 'pass1'); + mkdirSync(pass1Dir, { recursive: true }); + writeFileSync(join(pass1Dir, 'README.md'), content, 'utf-8'); + + const results = await checkPass2Evidence(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'README.md'); + expect(failures.length).toBeGreaterThan(0); + expect(failures[0]?.message).toContain('Pass-2 evidence missing'); + }); + + it('skips artifacts that do not exist', async () => { + // No files written — all PASS2 artifacts missing + const results = await checkPass2Evidence(testDir); + // Should return empty (non-existent files are skipped) + expect(results).toHaveLength(0); + }); + + it('returns pass2-evidence checkId on failure', async () => { + const content = '# Same content\n'; + writeArtifact(testDir, 'README.md', content); + const pass1Dir = join(testDir, 'pass1'); + mkdirSync(pass1Dir, { recursive: true }); + writeFileSync(join(pass1Dir, 'README.md'), content, 'utf-8'); + + const results = await checkPass2Evidence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + expect(failures[0]?.checkId).toBe('pass2-evidence'); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 7 — Family C Structure +// --------------------------------------------------------------------------- + +describe('checkFamilyCStructure', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + describe('executive-brief.md', () => { + it('passes with BLUF and Decisions sections', async () => { + writeArtifact(testDir, 'executive-brief.md', + '## 🎯 BLUF\n\nSummary.\n\n## 🧭 Decisions This Brief Supports\n\n1. A\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'executive-brief.md'); + expect(failures).toHaveLength(0); + }); + + it('fails when BLUF missing', async () => { + writeArtifact(testDir, 'executive-brief.md', + '## Introduction\n\n## Decisions This Brief Supports\n\n1. A\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'executive-brief.md'); + expect(failures.length).toBeGreaterThan(0); + }); + }); + + describe('intelligence-assessment.md', () => { + it('passes with 3+ Key Judgments, confidence labels, and PIR', async () => { + writeArtifact(testDir, 'intelligence-assessment.md', + '## Key Judgment KJ-1\nHIGH confidence.\n## Key Judgment KJ-2\nMEDIUM.\n## Key Judgment KJ-3\nLOW.\n\nReferences PIR-FISCAL-001.\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'intelligence-assessment.md'); + expect(failures).toHaveLength(0); + }); + + it('fails with fewer than 3 Key Judgments', async () => { + writeArtifact(testDir, 'intelligence-assessment.md', + '## Assessment\nOnly one KJ-1 here.\nHIGH confidence. MEDIUM. LOW.\nPIR-001\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.message?.includes('Key Judgment')); + expect(failures.length).toBeGreaterThan(0); + }); + }); + + describe('scenario-analysis.md', () => { + it('passes with 3+ scenarios', async () => { + writeArtifact(testDir, 'scenario-analysis.md', + '## Scenario 1\n\n## Scenario 2\n\n## Scenario 3\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'scenario-analysis.md'); + expect(failures).toHaveLength(0); + }); + + it('fails with fewer than 3 scenarios', async () => { + writeArtifact(testDir, 'scenario-analysis.md', + '## Scenario 1\n\n## Scenario 2\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'scenario-analysis.md'); + expect(failures.length).toBeGreaterThan(0); + }); + }); + + describe('devils-advocate.md', () => { + it('passes with 3+ hypotheses', async () => { + writeArtifact(testDir, 'devils-advocate.md', + '## Hypothesis 1: A\n\n## Hypothesis 2: B\n\n## Hypothesis 3: C\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'devils-advocate.md'); + expect(failures).toHaveLength(0); + }); + }); + + describe('methodology-reflection.md', () => { + it('passes with ICD 203 reference', async () => { + writeArtifact(testDir, 'methodology-reflection.md', + '## ICD 203 Audit\n\nCompliance verified.\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'methodology-reflection.md'); + expect(failures).toHaveLength(0); + }); + + it('passes with Methodology Improvements', async () => { + writeArtifact(testDir, 'methodology-reflection.md', + '## Methodology Improvements\n\nImprovement 1: Better sourcing.\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'methodology-reflection.md'); + expect(failures).toHaveLength(0); + }); + }); + + describe('comparative-international.md', () => { + it('passes with comparator set declared', async () => { + writeArtifact(testDir, 'comparative-international.md', + '**Comparator set**: Denmark, Norway, Finland\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'comparative-international.md'); + expect(failures).toHaveLength(0); + }); + + it('passes with 2+ comparator table rows', async () => { + writeArtifact(testDir, 'comparative-international.md', + '| Country | Policy |\n|---------|--------|\n| Denmark | A |\n| Norway | B |\n'); + const results = await checkFamilyCStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'comparative-international.md'); + expect(failures).toHaveLength(0); + }); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 8 — Family D Structure +// --------------------------------------------------------------------------- + +describe('checkFamilyDStructure', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + describe('forward-indicators.md', () => { + it('passes with 10+ dated indicators', async () => { + const dates = Array.from({ length: 12 }, (_, i) => + `- 2026-06-${String(i + 1).padStart(2, '0')}: Event ${i + 1}` + ).join('\n'); + writeArtifact(testDir, 'forward-indicators.md', `# Indicators\n\n${dates}\n`); + const results = await checkFamilyDStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'forward-indicators.md'); + expect(failures).toHaveLength(0); + }); + + it('fails with fewer than 10 dated indicators', async () => { + writeArtifact(testDir, 'forward-indicators.md', '# Indicators\n\n- 2026-06-01: Only one.\n'); + const results = await checkFamilyDStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'forward-indicators.md'); + expect(failures.length).toBeGreaterThan(0); + }); + + it('recognises quarterly date format', async () => { + const quarters = Array.from({ length: 12 }, (_, i) => + `- 2026Q${(i % 4) + 1}: Event ${i + 1}` + ).join('\n'); + writeArtifact(testDir, 'forward-indicators.md', `# Indicators\n\n${quarters}\n`); + const results = await checkFamilyDStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'forward-indicators.md'); + expect(failures).toHaveLength(0); + }); + }); + + describe('coalition-mathematics.md', () => { + it('passes with seat-count table', async () => { + writeArtifact(testDir, 'coalition-mathematics.md', + '| Party | Seats | Ja | Nej |\n|-------|-------|-----|-----|\n| S | 107 | X | |\n'); + const results = await checkFamilyDStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'coalition-mathematics.md'); + expect(failures).toHaveLength(0); + }); + + it('fails without vote-breakdown table', async () => { + writeArtifact(testDir, 'coalition-mathematics.md', '# Coalition\n\nNo table here.\n'); + const results = await checkFamilyDStructure(testDir); + const failures = results.filter((r) => !r.passed && r.artifact === 'coalition-mathematics.md'); + expect(failures.length).toBeGreaterThan(0); + }); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 9 — PIR Status +// --------------------------------------------------------------------------- + +describe('checkPirStatus', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('passes with valid pir-status.json', async () => { + const pir = { + schema_version: '1.0', + cycle: 'propositions', + date: '2026-05-01', + subfolder: 'propositions', + pirs: [{ + pir_id: 'PIR-FISCAL-001', + statement: 'What is the fiscal impact?', + status: 'open', + confidence: 'HIGH', + }], + generated_at: '2026-05-01T10:00:00Z', + }; + writeFileSync(join(testDir, 'pir-status.json'), JSON.stringify(pir), 'utf-8'); + + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('fails when file missing', async () => { + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + }); + + it('fails with invalid JSON', async () => { + writeFileSync(join(testDir, 'pir-status.json'), 'not json', 'utf-8'); + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + }); + + it('fails when schema_version is not 1.0', async () => { + const pir = { + schema_version: '2.0', + cycle: 'propositions', + date: '2026-05-01', + subfolder: 'propositions', + pirs: [], + generated_at: '2026-05-01T10:00:00Z', + }; + writeFileSync(join(testDir, 'pir-status.json'), JSON.stringify(pir), 'utf-8'); + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.some((f) => f.message.includes('schema_version'))).toBe(true); + }); + + it('fails when subfolder != cycle', async () => { + const pir = { + schema_version: '1.0', + cycle: 'propositions', + date: '2026-05-01', + subfolder: 'motions', + pirs: [], + generated_at: '2026-05-01T10:00:00Z', + }; + writeFileSync(join(testDir, 'pir-status.json'), JSON.stringify(pir), 'utf-8'); + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.some((f) => f.message.includes('subfolder'))).toBe(true); + }); + + it('validates PIR entry fields', async () => { + const pir = { + schema_version: '1.0', + cycle: 'propositions', + date: '2026-05-01', + subfolder: 'propositions', + pirs: [{ + pir_id: 'INVALID', + statement: '', + status: 'invalid-status', + confidence: 'INVALID', + }], + generated_at: '2026-05-01T10:00:00Z', + }; + writeFileSync(join(testDir, 'pir-status.json'), JSON.stringify(pir), 'utf-8'); + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + }); + + it('requires answer_summary when status is "answered"', async () => { + const pir = { + schema_version: '1.0', + cycle: 'propositions', + date: '2026-05-01', + subfolder: 'propositions', + pirs: [{ + pir_id: 'PIR-FISCAL-001', + statement: 'Was the budget passed?', + status: 'answered', + confidence: 'HIGH', + // answer_summary intentionally omitted + }], + generated_at: '2026-05-01T10:00:00Z', + }; + writeFileSync(join(testDir, 'pir-status.json'), JSON.stringify(pir), 'utf-8'); + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.some((f) => f.message.includes('answer_summary'))).toBe(true); + }); + + it('passes when status is "answered" and answer_summary is present', async () => { + const pir = { + schema_version: '1.0', + cycle: 'propositions', + date: '2026-05-01', + subfolder: 'propositions', + pirs: [{ + pir_id: 'PIR-FISCAL-001', + statement: 'Was the budget passed?', + status: 'answered', + confidence: 'HIGH', + answer_summary: 'Yes, budget was passed with majority.', + }], + generated_at: '2026-05-01T10:00:00Z', + }; + writeFileSync(join(testDir, 'pir-status.json'), JSON.stringify(pir), 'utf-8'); + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('fails when non-answered PIR carries answer_summary', async () => { + const pir = { + schema_version: '1.0', + cycle: 'propositions', + date: '2026-05-01', + subfolder: 'propositions', + pirs: [{ + pir_id: 'PIR-FISCAL-001', + statement: 'What will the budget be?', + status: 'open', + confidence: 'MEDIUM', + answer_summary: 'Not yet resolved.', // should not be present for 'open' + }], + generated_at: '2026-05-01T10:00:00Z', + }; + writeFileSync(join(testDir, 'pir-status.json'), JSON.stringify(pir), 'utf-8'); + const results = await checkPirStatus(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.some((f) => f.message.includes('must not carry answer_summary'))).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Check 9b — Statskontoret Evidence +// --------------------------------------------------------------------------- + +describe('checkStatskontoretEvidence', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('passes when no recognised agency mentioned', async () => { + writeArtifact(testDir, 'implementation-feasibility.md', '# Implementation\n\nGeneric content.\n'); + const results = await checkStatskontoretEvidence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('passes when agency mentioned with statskontoret.se URL', async () => { + writeArtifact(testDir, 'implementation-feasibility.md', + '# Implementation\n\nSkatteverket is relevant.\n\n| **Statskontoret relevance** | https://www.statskontoret.se/report |\n'); + const results = await checkStatskontoretEvidence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('passes when agency mentioned with "none found"', async () => { + writeArtifact(testDir, 'implementation-feasibility.md', + '# Implementation\n\nPolismyndigheten is relevant.\n\n| **Statskontoret relevance** | none found |\n'); + const results = await checkStatskontoretEvidence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures).toHaveLength(0); + }); + + it('fails when agency mentioned without Statskontoret row', async () => { + writeArtifact(testDir, 'implementation-feasibility.md', + '# Implementation\n\nSkatteverket is relevant.\n\nNo Statskontoret row.\n'); + const results = await checkStatskontoretEvidence(testDir); + const failures = results.filter((r) => !r.passed); + expect(failures.length).toBeGreaterThan(0); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: Integration — Full Gate Validation +// --------------------------------------------------------------------------- + +describe('validateAnalysisGate', () => { + beforeEach(() => { + testDir = createTestDir(); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('fails on empty directory', async () => { + const result = await validateAnalysisGate(testDir); + expect(result.passed).toBe(false); + expect(result.failureCount).toBeGreaterThan(0); + }); + + it('passes with complete valid analysis', async () => { + createMinimalValidAnalysis(testDir); + const result = await validateAnalysisGate(testDir); + // Log failures to aid debugging if the test fails + if (!result.passed) { + for (const f of result.checks.filter((c) => !c.passed)) { + console.error('GATE FAILURE:', f.checkId, f.message); + } + } + expect(result.passed).toBe(true); + expect(result.failureCount).toBe(0); + }); + + it('returns aggregate failure count', async () => { + const result = await validateAnalysisGate(testDir); + expect(result.failureCount).toBe(result.checks.filter((c) => !c.passed).length); + }); +});