From 3c04c62bd7449bed25d42f62035afab878e87f66 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:39:07 +0100 Subject: [PATCH 01/26] Add typecheck script and clear tsc --noEmit baseline - Add npm run typecheck (tsc --noEmit) as the source-of-truth type gate - Narrow exactOptionalPropertyTypes gaps via conditional spreads and optional-with-undefined fields across agent executor, orchestrator, observability, providers, and scorer - Make CheckItem.line/description optional to match runtime reality (reported items carry optional fields; output formatters use a separate Issue type, so display is unaffected) - Confine @ai-sdk/perplexity LanguageModelV1 -> LanguageModel skew to a single cast; ai@6 dropped V1 types and a provider upgrade is a separate, out-of-scope dependency change - No strict compiler options relaxed; src/agent/* left compiling only --- package.json | 1 + src/agent/executor.ts | 36 ++++++++++----------- src/cli/orchestrator.ts | 2 +- src/evaluators/violation-filter.ts | 4 +-- src/observability/factory.ts | 2 +- src/observability/langfuse-observability.ts | 4 +-- src/prompts/schema.ts | 4 +-- src/providers/perplexity-provider.ts | 7 +++- src/providers/vercel-ai-provider.ts | 22 +++++++------ src/scoring/scorer.ts | 5 ++- 10 files changed, 49 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index 74f474d5..d9e16b86 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "scripts": { "dev": "tsx src/index.ts", "build": "tsup", + "typecheck": "tsc --noEmit", "start": "node dist/index.js", "lint": "eslint .", "lint:fix": "eslint . --fix", diff --git a/src/agent/executor.ts b/src/agent/executor.ts index 432c6d64..749a5685 100644 --- a/src/agent/executor.ts +++ b/src/agent/executor.ts @@ -1,7 +1,7 @@ import { readdir, readFile } from 'fs/promises'; import * as os from 'os'; import fg from 'fast-glob'; -import { buildCheckLLMSchema, isJudgeResult, type PromptEvaluationResult } from '../prompts/schema'; +import { buildCheckLLMSchema, isJudgeResult, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; import type { PromptFile } from '../prompts/prompt-loader'; import { Type, Severity } from '../evaluators/types'; import { computeFilterDecision } from '../evaluators/violation-filter'; @@ -14,6 +14,7 @@ import { createReviewSessionStore } from './review-session-store'; import { buildAgentSystemPrompt } from './prompt-builder'; import { AgentToolError } from '../errors'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; +import type { FilePatternConfig } from '../boundaries/file-section-parser'; import { createAgentTools, listAvailableTools, @@ -53,7 +54,7 @@ export interface RunAgentExecutorParams { prompts: PromptFile[]; provider: LLMProvider; workspaceRoot: string; - scanPaths: Array<{ pattern: string; runRules: string[]; overrides: Record }>; + scanPaths: FilePatternConfig[]; outputFormat: OutputFormat; printMode: boolean; sessionHomeDir?: string; @@ -178,7 +179,7 @@ function findingsFromEvents(events: SessionEvent[]): AgentFinding[] { function buildFileRuleMatches( relativeTargets: string[], prompts: PromptFile[], - scanPaths: Array<{ pattern: string; runRules: string[]; overrides: Record }> + scanPaths: FilePatternConfig[] ): Array<{ file: string; ruleSource: string }> { const resolver = new ScanPathResolver(); const availablePacks = Array.from(new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p))); @@ -301,12 +302,13 @@ function resolveVisibleToolContext(params: { } const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); const path = resolvedPath ?? parsed.data.path; + const progressFile = resolvedPath && targetFiles.has(resolvedPath) + ? resolvedPath + : (currentProgressFile ?? defaultProgressFile); return { toolName: 'read_file', path, - progressFile: resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile), + ...(progressFile ? { progressFile } : {}), signature: `read_file:${path}`, }; } @@ -317,10 +319,11 @@ function resolveVisibleToolContext(params: { } const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); const path = resolvedPath ?? parsed.data.path; + const progressFile = currentProgressFile ?? defaultProgressFile; return { toolName: 'list_directory', path, - progressFile: currentProgressFile ?? defaultProgressFile, + ...(progressFile ? { progressFile } : {}), signature: `list_directory:${path}`, }; } @@ -333,14 +336,15 @@ function resolveVisibleToolContext(params: { const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.file); const path = resolvedPath ?? parsed.data.file; const ruleText = parsed.data.reviewInstruction?.trim() || prompt?.body || ''; + const progressFile = resolvedPath && targetFiles.has(resolvedPath) + ? resolvedPath + : (currentProgressFile ?? defaultProgressFile); return { toolName: 'lint', path, ruleName: String(prompt?.meta.name || prompt?.meta.id || 'Rule'), ruleText, - progressFile: resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile), + ...(progressFile ? { progressFile } : {}), signature: `lint:${path}:${normalizeRuleSource(parsed.data.ruleSource)}:${ruleText}`, }; } @@ -390,11 +394,7 @@ type FindingLikeViolation = { suggestion?: string; fix?: string; confidence?: number; - checks?: { - plausible_non_violation?: boolean; - context_supports_violation?: boolean; - rule_supports_claim?: boolean; - }; + checks?: GateChecks; }; async function appendInlineFinding(params: { @@ -516,8 +516,8 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< workspaceRoot, promptBySource, targetFiles, - currentProgressFile, - defaultProgressFile: relativeTargets[0], + ...(currentProgressFile ? { currentProgressFile } : {}), + ...(relativeTargets[0] ? { defaultProgressFile: relativeTargets[0] } : {}), }); if (visibleToolContext?.progressFile) { @@ -798,7 +798,7 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< workspaceRoot, fileRuleMatches, availableTools, - userInstructions, + ...(userInstructions ? { userInstructions } : {}), }), prompt: [ `Workspace root: ${workspaceRoot}`, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f4325095..6ae4acd4 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1194,7 +1194,7 @@ async function evaluateFilesInAgentMode( progressReporter, maxParallelToolCalls: 3, maxRetries: options.agentMaxRetries ?? 10, - userInstructions: options.userInstructionContent, + ...(options.userInstructionContent ? { userInstructions: options.userInstructionContent } : {}), }); let totalErrors = 0; diff --git a/src/evaluators/violation-filter.ts b/src/evaluators/violation-filter.ts index 2d3fc295..c4daa9b5 100644 --- a/src/evaluators/violation-filter.ts +++ b/src/evaluators/violation-filter.ts @@ -56,8 +56,8 @@ export function computeFilterDecision(v: GateViolationLike): FilterDecision { const hasFix = (v.fix ?? "").trim() !== ""; - const hasConfidence = typeof v.confidence === "number"; - const passesConfidence = hasConfidence && v.confidence >= confidenceThreshold; + const confidence = typeof v.confidence === "number" ? v.confidence : undefined; + const passesConfidence = confidence !== undefined && confidence >= confidenceThreshold; if (!passesConfidence) reasons.push(`confidence<${confidenceThreshold}`); const surface = diff --git a/src/observability/factory.ts b/src/observability/factory.ts index 84ec671a..c646fc08 100644 --- a/src/observability/factory.ts +++ b/src/observability/factory.ts @@ -19,6 +19,6 @@ export function createObservability(env: EnvConfig, logger?: Logger): AIObservab publicKey: env.LANGFUSE_PUBLIC_KEY, secretKey: env.LANGFUSE_SECRET_KEY, ...(env.LANGFUSE_BASE_URL ? { baseUrl: env.LANGFUSE_BASE_URL } : {}), - logger, + ...(logger ? { logger } : {}), }); } diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index b8898127..25f4d32c 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -12,8 +12,8 @@ export interface LangfuseObservabilityConfig { } export class LangfuseObservability implements AIObservability { - private sdk?: NodeSDK; - private initPromise?: Promise; + private sdk?: NodeSDK | undefined; + private initPromise?: Promise | undefined; private readonly logger: Logger; constructor(private readonly config: LangfuseObservabilityConfig) { diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index 26c137e1..bd9fd9ee 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -337,8 +337,8 @@ export type JudgeResult = { }; export type CheckItem = { - line: number; - description: string; + line?: number; + description?: string; analysis: string; message?: string; suggestion?: string; diff --git a/src/providers/perplexity-provider.ts b/src/providers/perplexity-provider.ts index c6a05499..432f8c09 100644 --- a/src/providers/perplexity-provider.ts +++ b/src/providers/perplexity-provider.ts @@ -1,4 +1,5 @@ import { generateText } from 'ai'; +import type { LanguageModel } from 'ai'; import { z } from 'zod'; import { createPerplexity } from '@ai-sdk/perplexity'; import type { SearchProvider } from './search-provider'; @@ -46,7 +47,11 @@ export class PerplexitySearchProvider implements SearchProvider { try { const result = await generateText({ - model: this.client('sonar-pro'), + // @ai-sdk/perplexity@1 exposes LanguageModelV1 models, while ai@6's + // generateText types require a LanguageModel (V2/V3). This is a + // third-party SDK version skew, not a repo type hole; resolve by + // upgrading @ai-sdk/perplexity to a V2 release in a follow-up. + model: this.client('sonar-pro') as unknown as LanguageModel, prompt: query, }); diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 689a19c0..5cd833fd 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -25,7 +25,7 @@ export class VercelAIProvider implements LLMProvider { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; - private observability?: AIObservability; + private observability?: AIObservability | undefined; constructor(config: VercelAIConfig, builder?: RequestBuilder) { this.config = { @@ -73,12 +73,14 @@ export class VercelAIProvider implements LLMProvider { } try { + const evaluator = this.extractContextValue(context, 'evaluatorName', 'evaluator'); + const rule = this.extractContextValue(context, 'ruleName', 'rule'); const observabilityOptions = this.getObservabilityOptions({ operation: 'structured-eval', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', - evaluator: this.extractContextValue(context, 'evaluatorName', 'evaluator'), - rule: this.extractContextValue(context, 'ruleName', 'rule'), + ...(evaluator ? { evaluator } : {}), + ...(rule ? { rule } : {}), }); const result = await generateText({ @@ -187,14 +189,14 @@ export class VercelAIProvider implements LLMProvider { } } - return { - usage: result.usage - ? { - inputTokens: result.usage.inputTokens ?? 0, - outputTokens: result.usage.outputTokens ?? 0, + return result.usage + ? { + usage: { + inputTokens: result.usage.inputTokens ?? 0, + outputTokens: result.usage.outputTokens ?? 0, + }, } - : undefined, - }; + : {}; } private getObservabilityOptions(context: AIExecutionContext): Record { diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index ecea0db6..24f0ced5 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -48,7 +48,10 @@ export function calculateCheckScore( ): CheckResult { const strictness = resolveStrictness(options.strictness); - const mappedViolations = violations.map((item) => ({ ...item, criterionName: item.description })); + const mappedViolations = violations.map((item) => ({ + ...item, + ...(item.description !== undefined ? { criterionName: item.description } : {}), + })); // Density Calculation: Violations per 100 words const density = (mappedViolations.length / wordCount) * 100; From e319bba09fb1e07af044a55161da89ebb6510313 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:40:19 +0100 Subject: [PATCH 02/26] Exclude docs/research from eslint - Add docs/research/** to eslint ignores; the directory holds local, untracked throwaway research scripts that are not part of the shipped package and should not be type-checked by lint --- eslint.config.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index 8549cd13..4bec57ff 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -10,7 +10,7 @@ import unicorn from "eslint-plugin-unicorn"; export default defineConfig([ // Ignored + housekeeping - { ignores: ["node_modules", "coverage", "dist", "build"] }, + { ignores: ["node_modules", "coverage", "dist", "build", "docs/research/**"] }, { linterOptions: { reportUnusedDisableDirectives: true } }, // Make resolver settings global From 20e0acc8204f1b67858ab7b31bf3f5b023c2799a Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:41:25 +0100 Subject: [PATCH 03/26] Add vitest.config.ts for stable test resolution - Project had no vitest config; vitest ran on pure defaults - Pin test discovery to tests/**/*.test.ts and inline ora, @langfuse/otel, and @opentelemetry/sdk-node so suites that transitively import agent/observability modules resolve reliably - No change to the 45 suites / 323 tests baseline --- vitest.config.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 vitest.config.ts diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 00000000..b5f7a7b4 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // Tests live under tests/ and follow the *.test.ts convention. + include: ['tests/**/*.test.ts'], + // Inline heavy/ESM-only deps so suites that transitively import the + // agent (ora) or observability (@langfuse/otel, @opentelemetry/sdk-node) + // modules resolve reliably without relying on scattered per-file mocks. + server: { + deps: { + inline: ['ora', '@langfuse/otel', '@opentelemetry/sdk-node'], + }, + }, + }, +}); From 3580e4971c2e6cfa8ce2308d350803c58d43ddba Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:41:59 +0100 Subject: [PATCH 04/26] Add npm run verify aggregate gate - Runs typecheck, lint, and test:run in sequence - Single command for CI and downstream refactor phases to prove the verification baseline --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index d9e16b86..3a73ad4d 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "lint:fix": "eslint . --fix", "test": "vitest", "test:run": "vitest run", - "test:ci": "vitest run --coverage" + "test:ci": "vitest run --coverage", + "verify": "npm run typecheck && npm run lint && npm run test:run" }, "bin": { "vectorlint": "./dist/index.js", From 0fe8700b2e4d2ded0f51ef10e30e1f7fb4f39af5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:47:58 +0100 Subject: [PATCH 05/26] Add typecheck CI job and align test workflow Node version - New typecheck.yml runs tsc --noEmit on push/PR for main and release-docs - Bump test.yml from Node 18 to 20 to satisfy package.json engines>=20.6 - Catches type regressions in CI before merge, not just via ESLint --- .github/workflows/test.yml | 2 +- .github/workflows/typecheck.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/typecheck.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dcf03d6f..06411499 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' cache: 'npm' - name: Install dependencies diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 00000000..73dad374 --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,28 @@ +name: Typecheck + +on: + push: + branches: [main, release-docs] + pull_request: + branches: [main, release-docs] + +jobs: + typecheck: + name: tsc --noEmit + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run typecheck + run: npm run typecheck From 6c271f121b80e05e5e3d7ae2d7b6c02352755abd Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:58:45 +0100 Subject: [PATCH 06/26] Deprecate --mode agent and fall back to standard - README Agent Mode section replaced with under-review notice linking the audit - --mode agent now warns through the injected logger and runs standard evaluation; the agent executor code is retained (unreachable from the CLI) pending Phase 4 removal - Add logger to EvaluationOptions so the deprecation warning routes through the logging abstraction instead of console - Rewrite orchestrator-agent-output tests to assert deprecation + fallback BREAKING: --mode agent no longer runs the autonomous workspace-agent loop --- README.md | 29 +- src/cli/commands.ts | 3 +- src/cli/orchestrator.ts | 9 +- src/cli/types.ts | 2 + tests/orchestrator-agent-output.test.ts | 1153 ++--------------------- 5 files changed, 80 insertions(+), 1116 deletions(-) diff --git a/README.md b/README.md index 859ea51a..f6511e6a 100644 --- a/README.md +++ b/README.md @@ -145,29 +145,16 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## Agent Mode +## Agent Mode (under review) -Agent mode is for reviews that need context from several files such as -documentation drift and cross-file accuracy. - -Run VectorLint in autonomous agent mode: - -```bash -vectorlint doc.md --mode agent -``` - -For machine-parseable output: - -```bash -vectorlint doc.md --mode agent --output json -``` - -To suppress interactive progress in line output: - -```bash -vectorlint doc.md --mode agent --print -``` +The `--mode agent` flag is under active rework. It currently enables an +autonomous cross-file review mode that is being **removed** in favor of a +bounded harness model. See +[`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md) +for the decision and the refactor plan. +Until the refactor lands, `--mode agent` prints a deprecation warning and +falls back to standard mode. Do not build integrations against it. ## Contributing diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 369eda1f..50dbffc8 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -82,7 +82,7 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default) or agent', DEFAULT_REVIEW_MODE) + .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) .option('-p, --print', 'Suppress interactive progress output in agent mode') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') @@ -217,6 +217,7 @@ export function registerMainCommand(program: Command): void { ...(searchProvider ? { searchProvider } : {}), concurrency: config.concurrency, verbose: cliOptions.verbose, + logger: runtimeLogger, debugJson: cliOptions.debugJson, outputFormat: cliOptions.output, mode: cliOptions.mode, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 6ae4acd4..f8df367a 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1168,6 +1168,9 @@ async function buildAgentRuleScores( return results; } +// Retained in quarantine: unreachable from the CLI after --mode agent was +// deprecated (now falls back to standard evaluation). Removed in Phase 4. +// eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine async function evaluateFilesInAgentMode( targets: string[], options: EvaluationOptions, @@ -1322,7 +1325,11 @@ export async function evaluateFiles( } if (mode === AGENT_REVIEW_MODE) { - return evaluateFilesInAgentMode(targets, options, outputFormat, jsonFormatter); + options.logger?.warn( + '--mode agent is deprecated and now falls back to standard mode. ' + + 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', + ); + // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. } for (const file of targets) { diff --git a/src/cli/types.ts b/src/cli/types.ts index a73f55e6..7f95192a 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -10,6 +10,7 @@ import { RdJsonFormatter } from '../output/rdjson-formatter'; import type { PromptEvaluationResult, JudgeResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; +import type { Logger } from '../logging/logger'; export enum OutputFormat { Line = "line", @@ -48,6 +49,7 @@ export interface EvaluationOptions { agentMaxRetries?: number; pricing?: PricingConfig; userInstructionContent?: string; + logger?: Logger; } export interface EvaluationResult { diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts index e08bbdaa..0b61ea63 100644 --- a/tests/orchestrator-agent-output.test.ts +++ b/tests/orchestrator-agent-output.test.ts @@ -2,28 +2,21 @@ import { mkdtempSync, rmSync, writeFileSync } from 'fs'; import * as path from 'path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { evaluateFiles } from '../src/cli/orchestrator'; -import { AGENT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; +import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; import type { PromptFile } from '../src/prompts/prompt-loader'; import type { LLMProvider } from '../src/providers/llm-provider'; +import type { Logger } from '../src/logging/logger'; import { Severity } from '../src/evaluators/types'; -function makePrompt(params?: { - id?: string; - name?: string; - source?: string; - body?: string; -}): PromptFile { - const id = params?.id ?? 'consistency'; - const name = params?.name ?? 'Consistency'; - const source = params?.source ?? 'packs/default/consistency.md'; - const body = params?.body ?? 'Find inconsistent wording'; - +function makePrompt(): PromptFile { + const id = 'consistency'; + const name = 'Consistency'; return { id, filename: `${id}.md`, - fullPath: source, + fullPath: 'packs/default/consistency.md', pack: 'Default', - body, + body: 'Find inconsistent wording', meta: { id: name, name, @@ -33,91 +26,39 @@ function makePrompt(params?: { }; } -function makeProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue found', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; -} +type LoggerSpy = Logger & { warn: ReturnType }; -function makeTopLevelOnlyProvider(): LLMProvider { +function makeLogger(): LoggerSpy { return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Top-level without references', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + } as unknown as LoggerSpy; } -function makeCrossFileTopLevelProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue found', - references: [{ file: 'other.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; +interface StandardProviderSpies { + provider: LLMProvider; + runPromptStructured: ReturnType; + runAgentToolLoop: ReturnType; } -function makeNoFinalizeProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Finding recorded before session ended', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - // intentionally omit finalize_review to simulate missing-finalize scenario - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; +function makeStandardProvider(): StandardProviderSpies { + const runPromptStructured = vi.fn().mockResolvedValue({ + data: { reasoning: 'ok', violations: [] }, + }); + const runAgentToolLoop = vi.fn().mockResolvedValue({ + usage: { inputTokens: 0, outputTokens: 0 }, + }); + const provider = { runPromptStructured, runAgentToolLoop } as unknown as LLMProvider; + return { provider, runPromptStructured, runAgentToolLoop }; } -describe('agent orchestrator output', () => { - const originalIsTTY = Object.getOwnPropertyDescriptor(process.stderr, 'isTTY'); +// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no +// longer reaches the autonomous agent executor: it warns through the injected +// logger and falls back to standard evaluation. The retained agent executor +// code is covered directly by tests/agent/* and removed in Phase 4. +describe('agent mode deprecation', () => { const tempRepos: string[] = []; function createTempRepo(): string { @@ -137,55 +78,15 @@ describe('agent orchestrator output', () => { for (const repo of tempRepos.splice(0, tempRepos.length)) { rmSync(repo, { recursive: true, force: true }); } - if (originalIsTTY) { - Object.defineProperty(process.stderr, 'isTTY', originalIsTTY); - } }); - it('produces agent-mode findings and summary when mode is set to agent', async () => { + it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.totalFiles).toBe(1); - expect(result.totalWarnings).toBeGreaterThan(0); - expect(result.hadOperationalErrors).toBe(false); - }); - - it('includes nested lint usage in the final agent-mode token totals', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { reasoning: 'ok', violations: [] }, - usage: { inputTokens: 7, outputTokens: 3 }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 11, outputTokens: 5 } }; - }, - } as unknown as LLMProvider; + const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); + const logger = makeLogger(); const result = await evaluateFiles([file], { prompts: [makePrompt()], @@ -194,338 +95,31 @@ describe('agent orchestrator output', () => { concurrency: 1, verbose: false, outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, + mode: AGENT_REVIEW_MODE, printMode: true, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.tokenUsage).toEqual({ - totalInputTokens: 18, - totalOutputTokens: 8, + logger, }); - }); - - it('keeps json output shape consistent with formatter-based structure', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const payload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files?: Record; - summary?: { files?: number; errors?: number; warnings?: number }; - metadata?: { version?: string; timestamp?: string }; - }; - - expect(payload.files).toBeDefined(); - expect(payload.summary).toBeDefined(); - expect(typeof payload.summary?.files).toBe('number'); - expect(typeof payload.summary?.errors).toBe('number'); - expect(typeof payload.summary?.warnings).toBe('number'); - expect(payload.metadata?.timestamp).toBeTruthy(); - }); - - it('keeps top-level json keys consistent between standard and agent modes', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const standardPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record; - - vi.mocked(console.log).mockClear(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const agentPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record; - - expect(Object.keys(agentPayload).sort()).toEqual( - Object.keys(standardPayload).sort() + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('deprecated'), ); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), + ); + // Standard evaluation ran; the autonomous executor did not. + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(runPromptStructured).toHaveBeenCalled(); + expect(result.totalFiles).toBe(1); }); - it('emits configured agent progress messages in line output mode', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Reviewing doc.md for Consistency'); - expect(stderrOutput).toContain(' └ Found no issues in doc.md'); - expect(stderrOutput).not.toContain('[vectorlint]'); - expect(stderrOutput).toMatch(/Completed review in \d+s\./); - }); - - it('renders visible tool invocations and results while hiding internal agent tools', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.search_files.execute({ pattern: '**/*.md' }); - await tools.read_file.execute({ path: 'doc.md' }); - await tools.list_directory.execute({ path: '.' }); - await tools.search_content.execute({ pattern: 'bad phrase', path: '.', glob: '**/*.md' }); - await tools.finalize_review.execute({}); - - return { usage: { inputTokens: 6, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Lint("Find inconsistent wording...")'); - expect(stderrOutput).toContain('Found no issues in doc.md'); - expect(stderrOutput).toContain('Read(doc.md)'); - expect(stderrOutput).toContain('Read 1 line from doc.md'); - expect(stderrOutput).toContain('List(.)'); - expect(stderrOutput).toContain('Listed 1 entry in .'); - expect(stderrOutput).not.toContain('SearchFiles('); - expect(stderrOutput).not.toContain('SearchContent('); - expect(stderrOutput).not.toContain('Finalize('); - }); - - it('renders interactive tool lines without a trailing newline', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const firstToolLine = stderrSpy.mock.calls - .map((call) => String(call[0])) - .find((chunk) => chunk.includes(' └ ')); - - expect(firstToolLine).toBeDefined(); - expect(firstToolLine?.endsWith('\n')).toBe(false); - }); - - it('uses in-place progress updates for repeated tool calls instead of appending only plain lines', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/ai-pattern.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/wordiness.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [ - makePrompt({ id: 'ai-pattern', name: 'AI Pattern', source: 'packs/default/ai-pattern.md' }), - makePrompt({ id: 'consistency', name: 'Consistency', source: 'packs/default/consistency.md' }), - makePrompt({ id: 'wordiness', name: 'Wordiness', source: 'packs/default/wordiness.md' }), - ], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('\x1b[1A'); - }); - - it('updates progress rule labels as the active lint rule changes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/ai-pattern.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [ - makePrompt({ id: 'ai-pattern', name: 'AI Pattern', source: 'packs/default/ai-pattern.md' }), - makePrompt({ id: 'consistency', name: 'Consistency', source: 'packs/default/consistency.md' }), - ], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('for AI Pattern'); - expect(stderrOutput).toContain('for Consistency'); - }); - - it('shows visible tool retry status after a visible tool failure', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - + it('does not invoke the agent executor in line output mode', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); - const retriable = path.join(repo, 'retriable.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: 'retriable.md' })).rejects.toThrow(); - writeFileSync(retriable, 'hello\n', 'utf8'); - await tools.read_file.execute({ path: 'retriable.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; + const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); + const logger = makeLogger(); await evaluateFiles([file], { prompts: [makePrompt()], @@ -534,115 +128,23 @@ describe('agent orchestrator output', () => { concurrency: 1, verbose: false, outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, + mode: AGENT_REVIEW_MODE, printMode: false, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Error reading retriable.md'); - expect(stderrOutput).toContain('Retrying Read(retriable.md)...'); - expect(stderrOutput).toContain('Read 1 line from retriable.md'); - }); - - it('shows visible-tool path errors even when path validation fails before file access', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, + logger, }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: '../outside.md' })).rejects.toThrow(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Error reading ../outside.md'); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(runPromptStructured).toHaveBeenCalled(); }); - it('shows quality scores in agent line output and keeps operational failures explicit when finalize_review is missing', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - + it('stays silent in standard mode without a logger', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Bad phrase used', - analysis: 'This wording is inconsistent.', - message: 'Use consistent wording', - suggestion: 'Replace bad phrase', - fix: 'better phrase', - rule_quote: 'Avoid vague wording', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - return { usage: { inputTokens: 6, outputTokens: 3 } }; - }, - } as unknown as LLMProvider; + const { provider, runAgentToolLoop } = makeStandardProvider(); const result = await evaluateFiles([file], { prompts: [makePrompt()], @@ -650,549 +152,14 @@ describe('agent orchestrator output', () => { provider, concurrency: 1, verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.requestFailures).toBe(0); - expect(stdout).toContain('Use consistent wording'); - expect(stdout).toContain('Quality Scores:'); - expect(stderrSpy).toHaveBeenCalled(); - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Review failed after'); - expect(stderrOutput).not.toContain('Completed review in'); - }); - - it('suppresses progress output when print mode is enabled', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('returns machine-parseable json output without progress text contamination', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stdout).not.toContain('Completed review.'); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('appends a new two-line block when agent work moves to the next file', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const fileOne = path.join(repo, 'doc.md'); - const fileTwo = path.join(repo, 'doc2.md'); - writeFileSync(fileOne, 'bad phrase\n', 'utf8'); - writeFileSync(fileTwo, 'another bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.read_file.execute({ path: 'doc.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.read_file.execute({ path: 'doc2.md' }); - await tools.lint.execute({ file: 'doc2.md', ruleSource: 'packs/default/consistency.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 4, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([fileOne, fileTwo], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Reviewing doc.md for Consistency'); - expect(stderrOutput).toContain('Read 1 line from doc.md'); - expect(stderrOutput).toContain('Reviewing doc2.md for Consistency'); - expect(stderrOutput).toContain('Read 1 line from doc2.md'); - const firstFileIndex = stderrOutput.indexOf('Reviewing doc.md for Consistency'); - const secondFileIndex = stderrOutput.indexOf('Reviewing doc2.md for Consistency'); - expect(firstFileIndex).toBeGreaterThan(-1); - expect(secondFileIndex).toBeGreaterThan(firstFileIndex); - expect(stderrOutput).toMatch(/Completed review in \d+s\./); - }); - - it('renders top-level findings without explicit references at 1:1 in line output', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeTopLevelOnlyProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('Top-level without references'); - expect(stdout).toContain('1:1'); - }); - - it('prints lazy file headers for non-target referenced findings in line output', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - const otherFile = path.join(repo, 'other.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - writeFileSync(otherFile, 'other content\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeCrossFileTopLevelProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('other.md'); - expect(stdout).toContain('Cross-file issue found'); - }); - - it('scores against every matched file, including clean files without findings', async () => { - const repo = createTempRepo(); - const firstFile = path.join(repo, 'doc.md'); - const secondFile = path.join(repo, 'other.md'); - writeFileSync(firstFile, 'one two three four five six seven eight nine ten\n', 'utf8'); - writeFileSync(secondFile, 'alpha beta gamma delta epsilon zeta eta theta iota kappa\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'one', - context_before: '', - context_after: '', - description: 'Issue found', - analysis: 'Needs cleanup.', - message: 'Fix the wording', - suggestion: 'Use clearer wording', - fix: 'clear wording', - rule_quote: 'Be precise', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([firstFile, secondFile], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('5.0/10'); - }); - - it('keeps canonical rule identity and warning severity aligned across json and rdjson outputs', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, + mode: DEFAULT_REVIEW_MODE, printMode: true, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const jsonPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files?: Record }>; - }; - - const jsonIssue = - jsonPayload.files?.['doc.md']?.issues?.find( - (issue) => issue.rule === 'Default.Consistency' - ) ?? jsonPayload.files?.['doc.md']?.issues?.[0]; - - vi.mocked(console.log).mockClear(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.RdJson, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const rdjsonPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - diagnostics?: Array<{ code?: { value?: string }; severity?: string }>; - }; - - const rdjsonDiagnostic = - rdjsonPayload.diagnostics?.find( - (diagnostic) => diagnostic.code?.value === 'Default.Consistency' - ) ?? rdjsonPayload.diagnostics?.[0]; - - expect(jsonIssue?.rule).toBe('Default.Consistency'); - expect(jsonIssue?.severity).toBe(Severity.WARNING); - expect(rdjsonDiagnostic?.code?.value).toBe('Default.Consistency'); - expect(rdjsonDiagnostic?.severity).toBe(Severity.WARNING); - }); - - it('keeps rdjson output machine-parseable without interactive progress writes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.RdJson, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi.mocked(console.log).mock.calls.map((call) => String(call[0])).join('\n'); - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('keeps vale-json output machine-parseable without interactive progress writes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.ValeJson, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi.mocked(console.log).mock.calls.map((call) => String(call[0])).join('\n'); - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('surfaces findings recorded before missing finalize while still reporting an operational failure', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeNoFinalizeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.totalWarnings).toBeGreaterThan(0); - expect(result.requestFailures).toBe(0); - expect(result.hadOperationalErrors).toBe(true); - }); - - it('passes userInstructionContent into the agent system prompt', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let capturedSystemPrompt = ''; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const systemPrompt = params.systemPrompt; - capturedSystemPrompt = typeof systemPrompt === 'string' ? systemPrompt : ''; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - userInstructionContent: 'Always enforce concise phrasing.', - } as never); - - expect(capturedSystemPrompt).toContain('User Instructions (from VECTORLINT.md):'); - expect(capturedSystemPrompt).toContain('Always enforce concise phrasing.'); - }); - - it('counts provider tool-loop failures as request failures in agent mode', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop() { - return Promise.reject(new Error('provider request failed')); - }, - } as unknown as LLMProvider; - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.requestFailures).toBe(1); - }); - - it('passes a default agent retry budget to the provider tool loop', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let receivedParams: Record | undefined; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - receivedParams = params; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(receivedParams?.maxRetries).toBe(10); - }); - - it('passes configured agent retry budget to the provider tool loop', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let receivedParams: Record | undefined; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - receivedParams = params; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - agentMaxRetries: 4, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(receivedParams?.maxRetries).toBe(4); + // No deprecation warning and no executor invocation in standard mode. + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(result.totalFiles).toBe(1); }); }); From a2e9f25c8935163285235736ac70ff5926d47b95 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:10:35 +0100 Subject: [PATCH 07/26] Append Phase 1 completion note to harness audit - Record the shifted 2026-07-13 baseline: only tsc --noEmit was failing; lint and test:run were already green, so the audit's lint and four-suite module-resolution failures are stale - Note the durable Phase 1 gates (typecheck, verify, vitest.config, docs/research lint exclusion, Node 20 typecheck/test workflows) - Point to the --mode agent deprecation and standard fallback as the precondition for Phases 2-5; record that no spec or architecture doc is superseded in this appendix --- ...0-vectorlint-harness-architecture-audit.md | 380 ++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md new file mode 100644 index 00000000..51161ce0 --- /dev/null +++ b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md @@ -0,0 +1,380 @@ +# VectorLint Harness Architecture Audit + +Date: 2026-07-10 + +Status: Objective system audit plus product-direction audit. Request changes before shipping the current autonomous workspace-agent surface as a public contract. + +## Product Decision + +VectorLint should not keep the current autonomous workspace-agent mode. + +The retained agent-like capability should be a constrained reader executor: a model or headless agent can read the target content in sections when that is useful for context management, but it should not search the workspace, inspect arbitrary files, perform top-level workspace checks, or rewrite the rule being evaluated. + +The intended execution model is: + +- `direct`: send target content and rule in one structured review call. +- `reader`: give the executor a read-section capability over the target content only. +- `auto`: choose `direct` for normal-sized inputs and `reader` for large inputs or explicit context-management needs. + +This is an execution strategy decision, not a return to VectorLint-as-agent. + +## Product Direction Under Review + +VectorLint should become a programmable, bounded, on-page content review harness that another caller can invoke. + +The caller can be Codex, Claude Code, another coding agent, CI, a local CLI wrapper, or a future service. That caller owns exploration, cross-page reasoning, research, and context gathering. VectorLint owns the constrained review of target content against source-backed rules and returns structured findings, scores, diagnostics, and usage metadata. + +In practical terms: + +- VectorLint reviews the target page or explicitly supplied content. +- External context is caller-supplied, not discovered by VectorLint. +- Rules define review behavior, granularity, severity, and scoring constraints. +- Executors can be API models, headless local agents, or constrained reader executors, but they receive the same review request contract. +- VectorLint should not expose a model-controlled workspace tool loop as its core product surface. + +## Objective System Audit Summary + +Separate from product direction, the current codebase has several system-level liabilities: + +- Build health is weak: `npx tsc --noEmit`, `npm run lint`, and parts of `npm run test:run` fail in the current workspace. +- Runtime contracts are duplicated across hand-written TypeScript, JSON-schema objects, Zod schemas, and formatter-specific shapes. +- Standard mode and agent mode project the same underlying findings differently. +- CLI orchestration owns too many responsibilities: config, file matching, evaluation, scoring, output formatting, debug artifacts, and agent routing. +- Evidence verification and counting are inconsistent enough to affect exit behavior. +- Observability and debug paths can record full prompts, content, and outputs. +- Documentation and implementation disagree on the current agent-mode topology and tool contracts. + +## Architecture Verdict + +The current codebase contains two execution models at once: + +1. The older evaluator path: one structured model call per rule or chunk. +2. The newer autonomous workspace-agent path: VectorLint gives a model tools to read/search the workspace and call lint as a nested tool. + +The second path should not continue as-is. The useful piece is not "agent mode" broadly; it is target-aware reading for context management. The current implementation also duplicates result projection, scoring, evidence validation, output formatting, and configuration behavior from the first path. That is why the architecture feels harder to maintain: the code is not just messy; it is serving two different ownership models. + +The highest-leverage change is to define a neutral review-domain contract and make every executor implement that contract. + +## Proposed Core Contract + +The contract should be centered on review, not providers or agents. + +```ts +interface ReviewRequest { + target: ReviewTarget; + rules: ReviewRule[]; + context?: ReviewContext[]; + budget: ReviewBudget; + outputPolicy: ReviewOutputPolicy; +} + +interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} + +interface ReviewResult { + findings: ReviewFinding[]; + scores: ReviewScore[]; + diagnostics: ReviewDiagnostic[]; + usage?: ReviewUsage; +} +``` + +API models, reasoning models, and headless CLI agents become adapter implementations behind this contract. The CLI becomes orchestration around request creation, result formatting, and exit behavior. + +## Major Findings + +### 1. The Current Agent Surface Is Broader Than The Decided Execution Model + +Evidence: + +- `README.md:148` presents `--mode agent` as autonomous cross-file review mode. +- `src/agent/prompt-builder.ts:43` instructs the model to perform workspace-level checks. +- `src/agent/executor.ts:670` exposes `read_file`. +- `src/agent/executor.ts:715` exposes `search_content`. + +Impact: + +VectorLint still behaves like it owns exploration. That puts it in competition with external coding agents instead of becoming a harness those agents can use. It also overshoots the decided reader-executor model, which only needs target-scoped reading for context management. + +Recommendation: + +Remove the current autonomous workspace-agent surface. Preserve only a constrained reader executor if needed: + +- It may read sections of the target content. +- It may not search the workspace. +- It may not read arbitrary files. +- It may not perform top-level workspace findings. +- It may not override source-backed rules. + +### 2. The Provider Interface Mixes Structured Review With Autonomous Tool Loops + +Evidence: + +- `src/providers/llm-provider.ts:28` requires both `runPromptStructured` and `runAgentToolLoop`. +- `src/providers/vercel-ai-provider.ts:136` implements the agent loop on the same provider. +- `src/agent/executor.ts:796` depends on the provider-level agent loop. + +Impact: + +Every future executor has to pretend to be both a structured model provider and an autonomous agent runner. That is the wrong abstraction for headless CLI support and for a constrained reader executor. + +Recommendation: + +Split the contracts: + +- `StructuredModelClient` for API structured output. +- `HeadlessReviewExecutor` for local CLI agents. +- `ReaderReviewExecutor` for target-scoped read-section execution. +- `ReviewExecutor` as the stable domain-level interface. +- `TelemetrySink` or observability decoration as an optional cross-cutting concern. + +### 3. Runtime Type Contracts Are Not Strong Enough For A Harness + +Evidence: + +- `src/providers/vercel-ai-provider.ts:119` returns `output as T`. +- `src/providers/llm-provider.ts:9` defines tool input and output as `unknown`. +- `src/agent/tools-registry.ts:21` exposes tools without typed output contracts. +- `npx tsc --noEmit` currently fails in core files. + +Impact: + +The code presents types as guarantees, but several are only assertions. A programmable harness needs reliable runtime validation because external callers and headless adapters will depend on these contracts. + +Recommendation: + +Tie structured execution to concrete Zod schemas or canonical parsers. Introduce typed tool contracts only if tools remain. Add a `typecheck` script and wire it into CI and build. + +### 4. Standard Mode And Agent Mode Project Results Differently + +Evidence: + +- `src/cli/orchestrator.ts:626` derives standard check severity from scoring. +- `src/agent/executor.ts:431` stamps agent findings with prompt severity. +- `src/agent/executor.ts:598` flattens judge criteria before recording findings. +- `src/cli/orchestrator.ts:1216` builds agent scores only for line output. +- `src/cli/orchestrator.ts:1237` emits structured output without agent scores or diagnostics. + +Impact: + +The same underlying review can produce different severity, rule identity, score data, JSON shape, and exit behavior depending on mode. That makes VectorLint hard to trust as a machine-facing gate. + +Recommendation: + +Extract one shared result projection pipeline: + +```text +PromptEvaluationResult + -> verified findings + -> rule and criterion identity + -> severity and score + -> diagnostics + -> output formatter +``` + +Every executor should return into this pipeline. + +### 5. The On-Page Boundary Is Not Enforced + +Evidence: + +- `src/agent/executor.ts:585` lets `lint` resolve any file inside `workspaceRoot`. +- `src/agent/executor.ts:670` lets `read_file` read any workspace file. +- `src/agent/executor.ts:715` lets `search_content` scan workspace content. +- `src/agent/executor.ts:134` allows model-supplied `reviewInstruction`. +- `src/agent/executor.ts:583` replaces the source-backed prompt body with the model-supplied override. + +Impact: + +Prompt-injected page content can indirectly steer the model to read non-target files or rewrite the rule being evaluated. That breaks deterministic review semantics and creates privacy and security risk. + +Recommendation: + +For the new harness, enforce a target/context allowlist: + +- Target content is always in scope. +- Caller-supplied context is in scope. +- Workspace reads are out of scope unless the caller explicitly includes those files as context. +- Reader execution can page through target content only. +- Rule bodies are source-backed and caller-authored, not model-authored. + +### 6. Evidence Handling Can Misreport Findings + +Evidence: + +- Standard mode skips unverifiable quotes but can still count original surfaced violation totals in some paths. +- `src/agent/executor.ts:415` attempts to locate evidence. +- `src/agent/executor.ts:426` falls back to the model-provided line when location fails. + +Impact: + +Standard mode can fail a run without printing the issue that caused the failure. Agent mode can surface hallucinated evidence as if it were verified. + +Recommendation: + +Use one evidence verifier. Count only emitted, verified findings. If evidence cannot be located, return a diagnostic and either skip the finding or mark the run operationally failed. + +### 7. Cost And Work Are Not Bounded Enough + +Evidence: + +- `src/providers/vercel-ai-provider.ts:161` defaults agent loops to 1000 steps. +- `src/agent/executor.ts:577` allows repeated nested `lint` calls. +- `src/evaluators/base-evaluator.ts:213` runs one model call per chunk. +- `src/agent/executor.ts:680` and `src/agent/executor.ts:715` glob before applying result caps. + +Impact: + +A single review can multiply cost through agent steps, nested lint calls, rules, chunks, and workspace search. Even a constrained reader executor will be slower than a direct call, so `auto` must have clear thresholds and budgets. + +Recommendation: + +Add explicit budgets: + +- Max target bytes. +- Max caller context bytes. +- Max chunks per rule. +- Max model calls per review. +- Max findings per rule. +- Max wall-clock duration. +- Max headless executor retries. + +### 8. Documentation Still Points At The Old Product + +Evidence: + +- `README.md:148` documents autonomous workspace-agent mode. +- `docs/specs/2026-03-17-agentic-capabilities-design.md:24` frames the roadmap around cross-document agent mode. +- The same spec describes implementation details that are not true now: one agent per rule, final `AgentFindingSchema`, AbortSignal propagation, ripgrep JSON, and Vale JSON omission of agent findings. + +Impact: + +Future contributors will optimize toward the wrong architecture and trust contracts that the code does not implement. + +Recommendation: + +Mark the agentic spec superseded. Replace it with a harness architecture spec that defines the review request, context boundary, executor interface, output contract, and migration plan. + +### 9. Secrets And Sensitive Content Need Better Handling + +Evidence: + +- `src/config/global-config.ts:13` says the global config stores API keys. +- `src/config/global-config.ts:76` creates the config directory with default permissions. +- `src/config/global-config.ts:81` writes the config file with default permissions. +- `src/observability/langfuse-observability.ts:66` records inputs. +- `src/observability/langfuse-observability.ts:67` records outputs. +- `src/providers/vercel-ai-provider.ts:59` can log full prompts and content. + +Impact: + +Provider keys, unreleased docs, caller context, and model outputs can leak through filesystem permissions, telemetry, debug logs, or persisted artifacts. + +Recommendation: + +Use private file modes for config and review artifacts. Make payload telemetry a separate opt-in from metadata telemetry. Add redaction or safe debug modes. + +## Verification Results + +Commands run from `/Users/klinsmann/Projects/TinyRocketLabs/vectorlint`: + +```bash +npm run lint +npm run test:run +npx tsc --noEmit +``` + +Results: + +- `npm run lint` failed before linting because typed ESLint rules were applied to a `.cjs` research script without parser type information. +- `npm run test:run` passed most tests but failed 4 suites during module resolution for `ora` and `@langfuse/otel`. +- `npx tsc --noEmit` failed with contract errors in agent, orchestrator, observability, logging, and provider modules. + +## Recommended Refactor Sequence + +### Phase 1: Stop The Bleeding + +1. Remove or hide the current autonomous workspace-agent mode. +2. Fix `tsc --noEmit`. +3. Fix test module resolution. +4. Fix lint configuration for non-TypeScript research scripts. +5. Add `typecheck` to CI and build validation. + +### Phase 2: Define The Harness Contract + +1. Create a neutral review-domain module. +2. Define `ReviewRequest`, `ReviewRule`, `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, and `ReviewDiagnostic`. +3. Define target/context boundary rules. +4. Define structured output shape once. +5. Define execution strategy: `direct`, `reader`, and `auto`. + +### Phase 3: Share Result Projection + +1. Extract evidence location and verification. +2. Extract filtering. +3. Extract severity and scoring. +4. Extract output formatter inputs. +5. Make standard mode and any executor mode use the same projection path. + +### Phase 4: Replace The Agent Loop With Executors + +1. Keep API model execution as direct execution. +2. Add target-scoped reader execution as another executor. +3. Add headless CLI execution behind the same review contract if still useful. +4. Pass the same review request to each executor. +5. Remove model-controlled rule overrides. +6. Remove workspace read/search tools from core review. + +### Phase 5: Update Documentation + +1. Mark the agentic capabilities spec superseded. +2. Write a new harness architecture spec. +3. Update README and CLI docs. +4. Document migration from autonomous workspace-agent mode to direct/reader/auto execution. + +## Suggested Target Architecture + +```text +CLI / API / External Agent + -> ReviewRequestBuilder + -> ReviewExecutor + -> ApiModelExecutor + -> ReaderExecutor + -> HeadlessCliExecutor + -> ResultProjection + -> EvidenceVerifier + -> ViolationFilter + -> ScoreCalculator + -> Diagnostics + -> OutputFormatter + -> line + -> json + -> rdjson + -> vale-json +``` + +The important inversion is that VectorLint no longer asks, "What tools should this agent use?" It asks, "Given this target, these rules, this context, this execution strategy, and this budget, what findings can be verified?" + +## Final Recommendation + +The decided direction is cleaner than the current architecture. The current pain is not a sign that VectorLint is too complex; it is a sign that the codebase is carrying two ownership models and several duplicated runtime contracts. + +Make VectorLint the harness. Let external agents be agents. Keep reader execution only as a bounded way to manage target-content context. + +--- + +## Appendix: Phase 1 — Completed (2026-07-13) + +Phase 1 of the recommended refactor sequence ("Stop The Bleeding") landed on branch `codex/ci/harness-stop-the-bleeding`. It shifts the verification baseline recorded in "Verification Results" above and addresses Finding 1 at the CLI boundary. + +**Shifted baseline.** The 2026-07-13 pre-work capture differed from the original verification results: `npm run lint` and `npm run test:run` already passed, so the earlier lint failure and the four `ora` / `@langfuse/otel` module-resolution failures were stale. Only `npx tsc --noEmit` was still failing. Phase 1 cleared those remaining type errors (narrowed at boundaries; no strict compiler options relaxed) and added durable gates so the baseline cannot silently regress: + +- `npm run typecheck` (`tsc --noEmit`) and `npm run verify` (typecheck + lint + test:run) scripts. +- `vitest.config.ts` for stable test module resolution. +- `docs/research/**` excluded from linting. +- `.github/workflows/typecheck.yml` and `.github/workflows/test.yml` aligned to Node 20. + +**Current state.** `npm run verify`, `npm run build`, and built-CLI standard review smoke runs are green. Per Finding 1, the autonomous workspace-agent surface is deprecated at the CLI boundary: `--mode agent` emits a deprecation warning and falls back to standard evaluation. `src/agent/*` and the agent-mode helpers are retained as compile-only quarantine, unreachable from the CLI, pending removal in Phase 4. The deprecation notice and pointer to this audit live in the README `## Agent Mode` section and the `--mode` help text. + +This appendix records completion only. Findings 2–9 and the proposed core contract remain owned by Phases 2–5; no spec or architecture document is superseded here. From 888f0271e30b0e0e1fa5d2773ca89f94cc415d58 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:22:49 +0100 Subject: [PATCH 08/26] Add root .vectorlint.ini for reproducible smoke validation - Commit a repository-native .vectorlint.ini using the bundled VectorLint preset (verbatim `vectorlint init` template). - The required validation commands `npm start -- README.md --output line` and `npm start -- README.md --mode agent` previously failed from the committed branch with "Missing configuration file"; they had relied on an untracked, deleted VECTORLINT.md. - Both smoke commands now run from a clean checkout with no untracked setup files; --mode agent still warns and falls back to standard mode. Refs: .agent-runs/2026-07-13-223741-harness-refactor/reports/01-stop-the-bleeding.md --- .vectorlint.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .vectorlint.ini diff --git a/.vectorlint.ini b/.vectorlint.ini new file mode 100644 index 00000000..d5b0d845 --- /dev/null +++ b/.vectorlint.ini @@ -0,0 +1,9 @@ +# VectorLint Configuration +# Global settings +RulesPath= +Concurrency=4 +DefaultSeverity=warning + +# Default rules for all markdown files +[**/*.md] +RunRules=VectorLint From 99f73c22ae728cb3dfb904dd0208b881322ac09b Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:56:45 +0100 Subject: [PATCH 09/26] Add neutral review-domain contract module - Introduce src/review/ with typed + Zod-validated ReviewRequest/Result contracts, boundary helpers, budget defaults/enforcement, model-call selection, ReviewExecutor interface, and a PromptFile request builder - Every external shape has a paired strict Zod schema; legacy scoring-mode, rubric, and model-authored rule-override fields are rejected - modelCall is single | agent | auto; chooseModelCall() resolves auto - On-page boundary (target + caller context only) via buildScope/isInScope with pure lexical URI normalization; no filesystem reads - BudgetExceededError extends the repository VectorlintError base - Purely additive: no changes outside src/review/ and tests/review/ - tests/review/ covers surface, schemas, budget, boundary, results, model-call selection, and the request builder (46 tests) --- src/review/README.md | 42 ++++++ src/review/boundary.ts | 67 ++++++++++ src/review/budget.ts | 83 ++++++++++++ src/review/executor.ts | 53 ++++++++ src/review/index.ts | 15 +++ src/review/request-builder.ts | 88 ++++++++++++ src/review/schemas.ts | 135 +++++++++++++++++++ src/review/types.ts | 192 +++++++++++++++++++++++++++ tests/review/boundary.test.ts | 39 ++++++ tests/review/budget.test.ts | 67 ++++++++++ tests/review/executor.test.ts | 46 +++++++ tests/review/module-surface.test.ts | 34 +++++ tests/review/request-builder.test.ts | 97 ++++++++++++++ tests/review/result.test.ts | 95 +++++++++++++ tests/review/types.test.ts | 74 +++++++++++ 15 files changed, 1127 insertions(+) create mode 100644 src/review/README.md create mode 100644 src/review/boundary.ts create mode 100644 src/review/budget.ts create mode 100644 src/review/executor.ts create mode 100644 src/review/index.ts create mode 100644 src/review/request-builder.ts create mode 100644 src/review/schemas.ts create mode 100644 src/review/types.ts create mode 100644 tests/review/boundary.test.ts create mode 100644 tests/review/budget.test.ts create mode 100644 tests/review/executor.test.ts create mode 100644 tests/review/module-surface.test.ts create mode 100644 tests/review/request-builder.test.ts create mode 100644 tests/review/result.test.ts create mode 100644 tests/review/types.test.ts diff --git a/src/review/README.md b/src/review/README.md new file mode 100644 index 00000000..bc2a120b --- /dev/null +++ b/src/review/README.md @@ -0,0 +1,42 @@ +# `src/review/` — Review-Domain Contract + +The neutral, implementation-neutral contract every executor programs against. +A caller builds a `ReviewRequest`; an executor returns a `ReviewResult`. + +## What lives here + +- `types.ts` — all review-domain interfaces (`ReviewTarget`, `ReviewRule`, + `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, + `ReviewDiagnostic`, `ReviewResult`, `ReviewRequest`, `ReviewModelCall`). +- `schemas.ts` — Zod schemas mirroring every external shape (boundary + validation). All schemas are strict; legacy scoring-mode, rubric, and + model-authored rule-override fields are rejected. +- `budget.ts` — `DEFAULT_REVIEW_BUDGET`, `REVIEW_BUDGET_SCHEMA`, + `enforceBudget()`, and `BudgetExceededError` (extends `VectorlintError`). +- `boundary.ts` — `buildScope()` / `isInScope()` enforcing the on-page boundary. +- `executor.ts` — `ReviewExecutor` interface, `REVIEW_MODEL_CALLS` + (`single | agent | auto`), `chooseModelCall()`. +- `request-builder.ts` — `buildReviewRequest()` bridging `PromptFile` to + `ReviewRequest`. + +## On-page boundary (audit Finding #5) + +- Target content is always in scope. +- Caller-supplied context is in scope. +- Workspace reads are out of scope unless the caller includes them as context. +- Agent model calls page through target content only. +- Rule bodies are source-backed and caller-authored, never model-authored. + +## Execution strategy + +`modelCall: single | agent | auto` selects how the reviewer model is invoked, +not how rules are scored. `single` sends target + rule in one structured call; +`agent` gives the executor a target-scoped read-section capability; `auto` +picks `single` for normal-sized inputs and `agent` for large ones. + +## Wiring status + +Phase 2 (this module) is purely additive and is **not** wired into the CLI. +Phase 3 emits `ReviewResult` via shared finding processing; Phase 4 implements +executors behind `ReviewExecutor` and removes the old agent loop. See +`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`. diff --git a/src/review/boundary.ts b/src/review/boundary.ts new file mode 100644 index 00000000..8dc5bb0c --- /dev/null +++ b/src/review/boundary.ts @@ -0,0 +1,67 @@ +import type { ReviewScope } from './types'; + +const FILE_URI_RE = /^file:\/\/(?[^/]*)(?\/.*)?$/; + +/** + * Lexically resolves '.' and '..' segments in a path string. Pure: performs no + * filesystem reads. The normalization philosophy is adapted from + * src/agent/path-utils.ts (resolveWithinRoot), which is removed in Phase 4; + * copied here so src/review/ has no agent import and no filesystem access. + * + * Symlink canonicalization is intentionally NOT performed inside the contract: + * callers are responsible for canonicalizing real files into target/context + * URIs before building a ReviewRequest. + */ +function resolveDotSegments(input: string): string { + const segments: string[] = []; + for (const segment of input.split('/')) { + if (segment === '' || segment === '.') continue; + if (segment === '..') { + segments.pop(); + continue; + } + segments.push(segment); + } + return (input.startsWith('/') ? '/' : '') + segments.join('/'); +} + +/** + * Normalizes a review URI by resolving '.'/'..' segments in its path. file:// + * URIs are normalized structurally; other (virtual, in-memory) URIs are + * returned as-is after dot resolution of their path-like suffix. + */ +export function normalizeReviewUri(uri: string): string { + const match = FILE_URI_RE.exec(uri); + if (match?.groups) { + const authority = match.groups['authority'] ?? ''; + const rawPath = match.groups['path'] ?? '/'; + return `file://${authority}${resolveDotSegments(rawPath)}`; + } + return uri; +} + +/** + * Builds the on-page boundary scope (audit Finding #5) from the target URI and + * any caller-supplied context URIs. Only these URIs are in scope; arbitrary + * workspace files are out of scope unless the caller explicitly includes them. + */ +export function buildScope(params: { + targetUri: string; + contextUris?: readonly string[]; +}): ReviewScope { + const allowedUris = new Set(); + allowedUris.add(normalizeReviewUri(params.targetUri)); + for (const uri of params.contextUris ?? []) { + allowedUris.add(normalizeReviewUri(uri)); + } + return { allowedUris }; +} + +/** + * Returns true iff `uri` (after normalization) is the target or caller-supplied + * context. Path-traversal segments are resolved before comparison, so a + * traversal-style path cannot masquerade as an in-scope URI. + */ +export function isInScope(scope: ReviewScope, uri: string): boolean { + return scope.allowedUris.has(normalizeReviewUri(uri)); +} diff --git a/src/review/budget.ts b/src/review/budget.ts new file mode 100644 index 00000000..e7d4cd8b --- /dev/null +++ b/src/review/budget.ts @@ -0,0 +1,83 @@ +import { z } from 'zod'; +import type { ReviewBudget } from './types'; +import { VectorlintError } from '../errors'; + +/** + * Sensible default bounds for a single review (audit Finding #7). Frozen so a + * shared default object cannot be accidentally mutated by a caller. + * + * Note: there is intentionally no finding cap (`maxFindingsPerRule`) — + * VectorLint reports every verified issue it finds — and no headless retry + * budget (`maxHeadlessRetries`) because the decided architecture has no + * headless executor. + */ +export const DEFAULT_REVIEW_BUDGET: Readonly = Object.freeze({ + maxTargetBytes: 1_000_000, + maxCallerContextBytes: 500_000, + maxChunksPerRule: 20, + maxModelCallsPerReview: 50, + maxWallClockMs: 5 * 60_000, +}); + +/** Zod schema mirroring {@link ReviewBudget}, applying defaults for omitted fields. */ +export const REVIEW_BUDGET_SCHEMA = z + .object({ + maxTargetBytes: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxTargetBytes), + maxCallerContextBytes: z + .number() + .int() + .positive() + .default(DEFAULT_REVIEW_BUDGET.maxCallerContextBytes), + maxChunksPerRule: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxChunksPerRule), + maxModelCallsPerReview: z + .number() + .int() + .positive() + .default(DEFAULT_REVIEW_BUDGET.maxModelCallsPerReview), + maxWallClockMs: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxWallClockMs), + }) + .strict(); + +/** Current resource usage, checked against a {@link ReviewBudget}. */ +export interface BudgetUsage { + modelCalls: number; + elapsedMs: number; +} + +/** + * Thrown by {@link enforceBudget} when current usage violates a hard limit. + * Extends the repository's canonical error base (VectorlintError). + */ +export class BudgetExceededError extends VectorlintError { + constructor( + message: string, + public readonly limit: keyof ReviewBudget, + public readonly actual: number, + ) { + super(message, 'BUDGET_EXCEEDED'); + this.name = 'BudgetExceededError'; + } +} + +/** + * Throws {@link BudgetExceededError} if current usage violates a hard limit. + * Executors call this before/after each model call and on timeout checks. + * Only runtime counters (model calls, wall clock) are enforced here; size + * limits (target/context bytes, chunks) are enforced at request-build time. + */ +export function enforceBudget(budget: ReviewBudget, usage: BudgetUsage): void { + if (usage.modelCalls > budget.maxModelCallsPerReview) { + throw new BudgetExceededError( + `model calls (${usage.modelCalls}) exceed maxModelCallsPerReview (${budget.maxModelCallsPerReview})`, + 'maxModelCallsPerReview', + usage.modelCalls, + ); + } + if (usage.elapsedMs > budget.maxWallClockMs) { + throw new BudgetExceededError( + `elapsed time (${usage.elapsedMs}ms) exceeds maxWallClockMs (${budget.maxWallClockMs}ms)`, + 'maxWallClockMs', + usage.elapsedMs, + ); + } +} diff --git a/src/review/executor.ts b/src/review/executor.ts new file mode 100644 index 00000000..2eef9011 --- /dev/null +++ b/src/review/executor.ts @@ -0,0 +1,53 @@ +import type { ReviewModelCall, ReviewRequest, ReviewResult } from './types'; + +/** + * Above this target byte size (roughly the existing chunking threshold in + * bytes), `auto` model-call selection prefers the agent executor so the + * reviewer can page through target content for context management. + */ +export const AGENT_MODEL_CALL_BYTE_THRESHOLD = 600_000; + +/** + * The single source of truth for reviewer model-call strategies. Kept in sync + * with the {@link ReviewModelCall} union via `satisfies`. + */ +export const REVIEW_MODEL_CALLS = ['single', 'agent', 'auto'] as const satisfies readonly ReviewModelCall[]; + +/** + * The stable domain-level interface every executor implements. Single and + * agent executors are implementations behind this contract (audit Finding #2). + * A headless/autonomous workspace executor is explicitly NOT part of this + * contract. + */ +export interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} + +/** + * Agent-call capability: a bounded way to page through target content for + * context management. NOT a workspace tool (audit Product Decision). An agent + * executor receives this capability scoped to the target only. + */ +export interface ReviewTargetReadCapability { + /** Read a 1-based [startLine, endLine] window of the target content. */ + readTargetSection( + startLine: number, + endLine: number, + ): Promise<{ startLine: number; endLine: number; content: string }>; +} + +/** + * Resolves 'auto' to 'single' or 'agent'. Single for normal-sized inputs; + * agent for large inputs or multi-rule runs that benefit from paging. + */ +export function chooseModelCall( + modelCall: ReviewModelCall, + signal: { targetBytes: number; rules: number }, +): 'single' | 'agent' { + if (modelCall === 'single') return 'single'; + if (modelCall === 'agent') return 'agent'; + if (signal.targetBytes > AGENT_MODEL_CALL_BYTE_THRESHOLD || signal.rules > 5) { + return 'agent'; + } + return 'single'; +} diff --git a/src/review/index.ts b/src/review/index.ts new file mode 100644 index 00000000..c44dbe66 --- /dev/null +++ b/src/review/index.ts @@ -0,0 +1,15 @@ +/** + * Neutral review-domain contract for the VectorLint harness. + * + * Everything an executor needs to review target content against + * source-backed rules flows through this module. The CLI, headless adapters, + * and external callers all build a ReviewRequest and receive a ReviewResult. + * + * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + */ +export * from './types'; +export * from './schemas'; +export * from './budget'; +export * from './boundary'; +export * from './executor'; +export * from './request-builder'; diff --git a/src/review/request-builder.ts b/src/review/request-builder.ts new file mode 100644 index 00000000..72500596 --- /dev/null +++ b/src/review/request-builder.ts @@ -0,0 +1,88 @@ +import type { PromptFile } from '../schemas/prompt-schemas'; +import type { + ReviewBudget, + ReviewContext, + ReviewModelCall, + ReviewOutputPolicy, + ReviewRequest, + ReviewRule, + ReviewTarget, +} from './types'; +import { DEFAULT_REVIEW_BUDGET } from './budget'; +import { ValidationError } from '../errors'; + +/** Default output policy: include usage metadata, never record payloads. */ +export const DEFAULT_REVIEW_OUTPUT_POLICY: Readonly = Object.freeze({ + includeUsage: true, + recordPayloadTelemetry: false, +}); + +/** Optional overrides applied by {@link buildReviewRequest}. */ +export interface ReviewRequestBuilderConfig { + budget?: ReviewBudget; + outputPolicy?: ReviewOutputPolicy; + modelCall?: ReviewModelCall; +} + +export interface BuildReviewRequestParams { + target: ReviewTarget; + /** Existing prompt files to convert into source-backed ReviewRules. */ + prompts: readonly PromptFile[]; + /** Caller-supplied, in-scope context, passed through unchanged. */ + context?: ReviewContext[]; + config?: ReviewRequestBuilderConfig; +} + +/** Builds the stable ReviewRule id formatted Pack.Rule (mirrors src/agent/rule-id.ts). */ +function toReviewRuleId(prompt: PromptFile): string { + const pack = prompt.pack || 'Default'; + return `${pack}.${prompt.meta.id}`; +} + +/** + * Converts a single PromptFile into a source-backed ReviewRule, mapping only + * fields that belong in the neutral contract. Legacy `meta.type`, `evaluator`, + * `criteria`, and judge/rubric fields are intentionally NOT copied. + */ +function toReviewRule(prompt: PromptFile): ReviewRule { + const rule: ReviewRule = { + id: toReviewRuleId(prompt), + source: prompt.fullPath, + body: prompt.body, + }; + if (prompt.meta.name !== undefined) { + rule.name = prompt.meta.name; + } + // Severity is a string enum whose values ('error' | 'warning') match + // ReviewSeverity exactly, so it maps cleanly without transformation. + if (prompt.meta.severity !== undefined) { + rule.severity = prompt.meta.severity; + } + return rule; +} + +/** + * Builds a {@link ReviewRequest} from existing {@link PromptFile}s plus a + * target. This is the conservative bridge Phase 4 uses to convert the current + * prompt pipeline into the review contract without changing how prompts load. + * Throws a {@link ValidationError} when no prompts are supplied. + */ +export function buildReviewRequest(params: BuildReviewRequestParams): ReviewRequest { + const { target, prompts, context } = params; + if (prompts.length === 0) { + throw new ValidationError('buildReviewRequest requires at least one prompt.'); + } + + const config = params.config; + const request: ReviewRequest = { + target, + rules: prompts.map(toReviewRule), + budget: config?.budget ?? DEFAULT_REVIEW_BUDGET, + outputPolicy: config?.outputPolicy ?? DEFAULT_REVIEW_OUTPUT_POLICY, + modelCall: config?.modelCall ?? 'auto', + }; + if (context !== undefined) { + request.context = context; + } + return request; +} diff --git a/src/review/schemas.ts b/src/review/schemas.ts new file mode 100644 index 00000000..a1132cd3 --- /dev/null +++ b/src/review/schemas.ts @@ -0,0 +1,135 @@ +import { z } from 'zod'; +import { REVIEW_BUDGET_SCHEMA } from './budget'; + +/** + * Zod schemas mirroring src/review/types.ts (boundary validation). + * + * Every external review-domain shape has a paired schema here so callers and + * external adapters can validate untrusted input at the system boundary + * (audit Finding #3). Schemas are intentionally strict: unknown keys are + * rejected so legacy scoring-mode, rubric, and model-authored rule-override + * fields cannot leak into the new contract. + */ + +export const REVIEW_SEVERITY_SCHEMA = z.enum(['error', 'warning']); + +export const REVIEW_VIOLATION_CONDITION_SCHEMA = z + .object({ + id: z.string().min(1), + description: z.string().min(1), + }) + .strict(); + +export const REVIEW_TARGET_SCHEMA = z + .object({ + uri: z.string().min(1), + content: z.string(), + contentType: z.string().min(1), + byteLength: z.number().int().nonnegative().optional(), + }) + .strict(); + +export const REVIEW_RULE_SCHEMA = z + .object({ + id: z.string().min(1), + source: z.string().min(1), + body: z.string(), + name: z.string().optional(), + severity: REVIEW_SEVERITY_SCHEMA.default('warning'), + violationConditions: z.array(REVIEW_VIOLATION_CONDITION_SCHEMA).optional(), + }) + .strict(); + +export const REVIEW_CONTEXT_SCHEMA = z + .object({ + label: z.string().min(1), + content: z.string(), + relation: z.string().optional(), + uri: z.string().optional(), + }) + .strict(); + +export const REVIEW_OUTPUT_POLICY_SCHEMA = z + .object({ + includeUsage: z.boolean(), + recordPayloadTelemetry: z.boolean(), + }) + .strict(); + +export const REVIEW_DIAGNOSTIC_LEVEL_SCHEMA = z.enum(['info', 'warn', 'error']); + +export const REVIEW_SCORE_COMPONENT_SCHEMA = z + .object({ + id: z.string().min(1), + scoreText: z.string(), + score: z.number(), + weight: z.number().optional(), + }) + .strict(); + +export const REVIEW_SCORE_SCHEMA = z + .object({ + ruleId: z.string().min(1), + score: z.number(), + scoreText: z.string(), + severity: REVIEW_SEVERITY_SCHEMA, + findingCount: z.number().int().nonnegative().optional(), + components: z.array(REVIEW_SCORE_COMPONENT_SCHEMA).optional(), + }) + .strict(); + +export const REVIEW_FINDING_SCHEMA = z + .object({ + ruleId: z.string().min(1), + ruleSource: z.string().min(1), + severity: REVIEW_SEVERITY_SCHEMA, + message: z.string(), + line: z.number().int().positive(), + column: z.number().int().positive(), + match: z.string(), + analysis: z.string().optional(), + suggestion: z.string().optional(), + fix: z.string().optional(), + }) + .strict(); + +export const REVIEW_DIAGNOSTIC_SCHEMA = z + .object({ + level: REVIEW_DIAGNOSTIC_LEVEL_SCHEMA, + code: z.string().min(1), + message: z.string(), + ruleId: z.string().optional(), + context: z.record(z.string(), z.unknown()).optional(), + }) + .strict(); + +export const REVIEW_USAGE_SCHEMA = z + .object({ + inputTokens: z.number().int().nonnegative().optional(), + outputTokens: z.number().int().nonnegative().optional(), + modelCalls: z.number().int().nonnegative(), + costUsd: z.number().nonnegative().optional(), + wallClockMs: z.number().int().nonnegative().optional(), + }) + .strict(); + +export const REVIEW_RESULT_SCHEMA = z + .object({ + findings: z.array(REVIEW_FINDING_SCHEMA), + scores: z.array(REVIEW_SCORE_SCHEMA), + diagnostics: z.array(REVIEW_DIAGNOSTIC_SCHEMA), + usage: REVIEW_USAGE_SCHEMA.optional(), + hadOperationalErrors: z.boolean().optional(), + }) + .strict(); + +export const REVIEW_REQUEST_SCHEMA = z + .object({ + target: REVIEW_TARGET_SCHEMA, + rules: z.array(REVIEW_RULE_SCHEMA).min(1), + context: z.array(REVIEW_CONTEXT_SCHEMA).optional(), + budget: REVIEW_BUDGET_SCHEMA, + outputPolicy: REVIEW_OUTPUT_POLICY_SCHEMA, + modelCall: z.enum(['single', 'agent', 'auto']), + }) + .strict(); diff --git a/src/review/types.ts b/src/review/types.ts new file mode 100644 index 00000000..97e22039 --- /dev/null +++ b/src/review/types.ts @@ -0,0 +1,192 @@ +/** + * Neutral review-domain contract types for the VectorLint harness. + * + * Everything an executor needs to review target content against + * source-backed rules flows through this module. Callers build a + * {@link ReviewRequest} and executors return a {@link ReviewResult}. + * + * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + * + * This module is implementation-neutral: it deliberately exposes no legacy + * scoring-mode, rubric, or model-authored rule-override surface. `modelCall` + * selects how the reviewer model is invoked, not how rules are scored. + */ + +/** Finding/Rule severity. */ +export type ReviewSeverity = 'error' | 'warning'; + +/** Objective Via Negativa condition that counts as a violation when present. */ +export interface ReviewViolationCondition { + id: string; + description: string; +} + +/** + * Target content under review. The on-page boundary (audit Finding #5) is + * enforced against this: executors may only read sections of `content`. + */ +export interface ReviewTarget { + /** Stable absolute URI (file:// or virtual scheme for in-memory content). */ + uri: string; + /** Full target content. May be paged by an agent executor. */ + content: string; + /** MIME-ish content type, e.g. 'text/markdown'. */ + contentType: string; + /** Optional byte length hint; computed from content if absent. */ + byteLength?: number; +} + +/** + * A source-backed, caller-authored rule. Model-authored rule overrides are + * explicitly disallowed (audit Finding #1, #5). + */ +export interface ReviewRule { + /** Stable id, formatted Pack.Rule[.Criterion] in output. */ + id: string; + /** Canonical source path (where the rule body came from). */ + source: string; + /** The rule prompt body. Source-backed; never model-authored. */ + body: string; + /** Human-readable name. */ + name?: string; + /** 'error' | 'warning'. Defaults to 'warning'. */ + severity?: ReviewSeverity; + /** Optional structured Via Negativa violation conditions. */ + violationConditions?: ReviewViolationCondition[]; +} + +/** + * Caller-supplied context. Explicitly in scope (audit boundary rule): + * VectorLint does NOT discover workspace files as context on its own; the + * caller owns exploration and context gathering. + */ +export interface ReviewContext { + /** Short label used in diagnostics/tracing. */ + label: string; + /** The context content itself. */ + content: string; + /** How this context relates to the target, e.g. 'reference' | 'glossary'. */ + relation?: string; + /** Optional source URI for provenance. */ + uri?: string; +} + +/** + * The on-page boundary scope: the normalized URIs an executor may read. + * Built by {@link buildScope} (see boundary.ts) and checked by + * {@link isInScope}. + */ +export interface ReviewScope { + /** Normalized absolute URIs that are in scope (target + caller context). */ + readonly allowedUris: ReadonlySet; +} + +/** + * Hard bounds on a single review (audit Finding #7). These limit work, not + * output. Executors MUST check these and surface a ReviewDiagnostic (or fail + * the run) when exceeded. + */ +export interface ReviewBudget { + maxTargetBytes: number; + maxCallerContextBytes: number; + maxChunksPerRule: number; + maxModelCallsPerReview: number; + /** Maximum elapsed review time in milliseconds. This is the run timeout. */ + maxWallClockMs: number; +} + +/** + * Controls optional output/telemetry behavior. Diagnostics are always part of + * ReviewResult; this policy does not hide operational warnings. Audit + * Finding #9: payload telemetry must be a separate opt-in from metadata + * telemetry. + */ +export interface ReviewOutputPolicy { + /** Include usage/cost in the result. */ + includeUsage: boolean; + /** Opt-in: record prompt/content payloads to telemetry. Default false. */ + recordPayloadTelemetry: boolean; +} + +/** A verified finding anchored in the target content. */ +export interface ReviewFinding { + ruleId: string; + ruleSource: string; + severity: ReviewSeverity; + message: string; + /** 1-based line in the target content. */ + line: number; + /** 1-based column. */ + column: number; + /** Verified anchored text. Unverified finding evidence is a diagnostic. */ + match: string; + analysis?: string; + suggestion?: string; + fix?: string; +} + +/** A per-rule score produced through shared finding processing. */ +export interface ReviewScore { + ruleId: string; + score: number; + /** Human-readable score, e.g. "8.0/10". */ + scoreText: string; + severity: ReviewSeverity; + /** Count of verified findings that contributed to this score, if applicable. */ + findingCount?: number; + components?: ReviewScoreComponent[]; +} + +export interface ReviewScoreComponent { + id: string; + scoreText: string; + score: number; + weight?: number; +} + +export type ReviewDiagnosticLevel = 'info' | 'warn' | 'error'; + +/** An operational or finding-processing note. Always part of ReviewResult. */ +export interface ReviewDiagnostic { + level: ReviewDiagnosticLevel; + /** Stable machine code, e.g. 'finding-evidence-not-locatable'. */ + code: string; + message: string; + ruleId?: string; + /** Extra machine-readable detail (model-call count, evidence preview, ...). */ + context?: Record; +} + +/** Aggregated resource usage for a review run. */ +export interface ReviewUsage { + inputTokens?: number; + outputTokens?: number; + modelCalls: number; + costUsd?: number; + wallClockMs?: number; +} + +/** The output from any executor, produced through shared finding processing. */ +export interface ReviewResult { + findings: ReviewFinding[]; + scores: ReviewScore[]; + diagnostics: ReviewDiagnostic[]; + usage?: ReviewUsage; + /** True if the run hit an operational error but still returned partial results. */ + hadOperationalErrors?: boolean; +} + +/** How to call the reviewer model. 'auto' resolves via chooseModelCall(). */ +export type ReviewModelCall = 'single' | 'agent' | 'auto'; + +/** The complete input to any executor. */ +export interface ReviewRequest { + target: ReviewTarget; + rules: ReviewRule[]; + /** Caller-supplied, in-scope context (never workspace-discovered). */ + context?: ReviewContext[]; + budget: ReviewBudget; + outputPolicy: ReviewOutputPolicy; + /** Reviewer model call shape. 'auto' resolves via chooseModelCall(). */ + modelCall: ReviewModelCall; +} diff --git a/tests/review/boundary.test.ts b/tests/review/boundary.test.ts new file mode 100644 index 00000000..5ff92dad --- /dev/null +++ b/tests/review/boundary.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { buildScope, isInScope, normalizeReviewUri } from '../../src/review'; + +describe('review boundary', () => { + const scope = buildScope({ + targetUri: 'file:///repo/docs/guide.md', + contextUris: ['file:///repo/docs/glossary.md'], + }); + + it('target is always in scope', () => { + expect(isInScope(scope, 'file:///repo/docs/guide.md')).toBe(true); + }); + + it('caller-supplied context uri is in scope', () => { + expect(isInScope(scope, 'file:///repo/docs/glossary.md')).toBe(true); + }); + + it('arbitrary workspace file is NOT in scope', () => { + expect(isInScope(scope, 'file:///repo/src/index.ts')).toBe(false); + }); + + it('rejects path traversal that escapes the target', () => { + expect(isInScope(scope, 'file:///repo/docs/../src/index.ts')).toBe(false); + }); + + it('normalizes traversal that resolves back onto an in-scope uri', () => { + expect(isInScope(scope, 'file:///repo/docs/sub/../guide.md')).toBe(true); + }); +}); + +describe('normalizeReviewUri', () => { + it('resolves dot segments in file uris', () => { + expect(normalizeReviewUri('file:///repo/a/../b/./c.md')).toBe('file:///repo/b/c.md'); + }); + + it('preserves authority', () => { + expect(normalizeReviewUri('file://host/x/../y.md')).toBe('file://host/y.md'); + }); +}); diff --git a/tests/review/budget.test.ts b/tests/review/budget.test.ts new file mode 100644 index 00000000..a63bcc24 --- /dev/null +++ b/tests/review/budget.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import { + BudgetExceededError, + DEFAULT_REVIEW_BUDGET, + REVIEW_BUDGET_SCHEMA, + enforceBudget, +} from '../../src/review'; +import { VectorlintError } from '../../src/errors'; + +describe('review budget defaults', () => { + it('provides sensible positive defaults', () => { + expect(DEFAULT_REVIEW_BUDGET.maxModelCallsPerReview).toBeGreaterThan(0); + expect(DEFAULT_REVIEW_BUDGET.maxWallClockMs).toBeGreaterThan(0); + expect(DEFAULT_REVIEW_BUDGET.maxTargetBytes).toBeGreaterThan(0); + }); + + it('does not cap findings or headless retries', () => { + expect('maxFindingsPerRule' in DEFAULT_REVIEW_BUDGET).toBe(false); + expect('maxHeadlessRetries' in DEFAULT_REVIEW_BUDGET).toBe(false); + }); +}); + +describe('REVIEW_BUDGET_SCHEMA', () => { + it('applies defaults for omitted fields', () => { + const parsed = REVIEW_BUDGET_SCHEMA.parse({ maxModelCallsPerReview: 5 }); + expect(parsed.maxWallClockMs).toBe(DEFAULT_REVIEW_BUDGET.maxWallClockMs); + expect(parsed.maxModelCallsPerReview).toBe(5); + }); + + it('rejects non-positive limits', () => { + expect(() => REVIEW_BUDGET_SCHEMA.parse({ maxModelCallsPerReview: 0 })).toThrow(); + }); +}); + +describe('enforceBudget', () => { + it('is silent within limits', () => { + expect(() => + enforceBudget(DEFAULT_REVIEW_BUDGET, { modelCalls: 1, elapsedMs: 1000 }), + ).not.toThrow(); + }); + + it('throws BudgetExceededError when model calls exceed the limit', () => { + expect(() => + enforceBudget( + { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 3 }, + { modelCalls: 4, elapsedMs: 0 }, + ), + ).toThrow(BudgetExceededError); + }); + + it('throws BudgetExceededError when wall clock exceeds the limit', () => { + expect(() => + enforceBudget( + { ...DEFAULT_REVIEW_BUDGET, maxWallClockMs: 100 }, + { modelCalls: 0, elapsedMs: 101 }, + ), + ).toThrow(BudgetExceededError); + }); + + it('extends the repository error base', () => { + const err = new BudgetExceededError('boom', 'maxModelCallsPerReview', 9); + expect(err).toBeInstanceOf(VectorlintError); + expect(err.limit).toBe('maxModelCallsPerReview'); + expect(err.actual).toBe(9); + expect(err.code).toBe('BUDGET_EXCEEDED'); + }); +}); diff --git a/tests/review/executor.test.ts b/tests/review/executor.test.ts new file mode 100644 index 00000000..c15e670b --- /dev/null +++ b/tests/review/executor.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { + REVIEW_MODEL_CALLS, + chooseModelCall, + type ReviewExecutor, + type ReviewRequest, +} from '../../src/review'; + +describe('REVIEW_MODEL_CALLS', () => { + it('exposes single | agent | auto', () => { + expect(REVIEW_MODEL_CALLS).toEqual(['single', 'agent', 'auto']); + }); +}); + +describe('chooseModelCall', () => { + it('respects an explicit single call', () => { + expect(chooseModelCall('single', { targetBytes: 2_000_000, rules: 10 })).toBe('single'); + }); + + it('respects an explicit agent call', () => { + expect(chooseModelCall('agent', { targetBytes: 10, rules: 1 })).toBe('agent'); + }); + + it('returns single for small inputs under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 10_000, rules: 1 })).toBe('single'); + }); + + it('returns agent for large inputs under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 2_000_000, rules: 1 })).toBe('agent'); + }); + + it('returns agent for many rules under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 1_000, rules: 6 })).toBe('agent'); + }); +}); + +describe('ReviewExecutor contract', () => { + it('can be implemented and run() returns a ReviewResult', async () => { + const fake: ReviewExecutor = { + run: () => Promise.resolve({ findings: [], scores: [], diagnostics: [] }), + }; + const result = await fake.run({} as unknown as ReviewRequest); + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + }); +}); diff --git a/tests/review/module-surface.test.ts b/tests/review/module-surface.test.ts new file mode 100644 index 00000000..6c03ad30 --- /dev/null +++ b/tests/review/module-surface.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import * as review from '../../src/review'; + +describe('src/review public surface', () => { + it('exports the core contract values and functions', () => { + // Runtime presence checks for values; types are checked by tsc. + expect(review.REVIEW_MODEL_CALLS).toBeDefined(); + expect(review.DEFAULT_REVIEW_BUDGET).toBeDefined(); + expect(typeof review.chooseModelCall).toBe('function'); + expect(typeof review.isInScope).toBe('function'); + expect(typeof review.buildReviewRequest).toBe('function'); + }); + + it('exposes the boundary, budget, and error helpers', () => { + expect(typeof review.buildScope).toBe('function'); + expect(typeof review.enforceBudget).toBe('function'); + expect(typeof review.normalizeReviewUri).toBe('function'); + expect(review.BudgetExceededError).toBeDefined(); + }); + + it('exposes the Zod schemas for every external shape', () => { + expect(review.REVIEW_TARGET_SCHEMA).toBeDefined(); + expect(review.REVIEW_RULE_SCHEMA).toBeDefined(); + expect(review.REVIEW_CONTEXT_SCHEMA).toBeDefined(); + expect(review.REVIEW_BUDGET_SCHEMA).toBeDefined(); + expect(review.REVIEW_OUTPUT_POLICY_SCHEMA).toBeDefined(); + expect(review.REVIEW_FINDING_SCHEMA).toBeDefined(); + expect(review.REVIEW_SCORE_SCHEMA).toBeDefined(); + expect(review.REVIEW_DIAGNOSTIC_SCHEMA).toBeDefined(); + expect(review.REVIEW_USAGE_SCHEMA).toBeDefined(); + expect(review.REVIEW_RESULT_SCHEMA).toBeDefined(); + expect(review.REVIEW_REQUEST_SCHEMA).toBeDefined(); + }); +}); diff --git a/tests/review/request-builder.test.ts b/tests/review/request-builder.test.ts new file mode 100644 index 00000000..618efa03 --- /dev/null +++ b/tests/review/request-builder.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_REVIEW_BUDGET, + REVIEW_RULE_SCHEMA, + buildReviewRequest, +} from '../../src/review'; +import { ValidationError } from '../../src/errors'; +import { Severity } from '../../src/evaluators/types'; +import type { PromptFile } from '../../src/schemas/prompt-schemas'; +import type { ReviewContext, ReviewTarget } from '../../src/review'; + +function makePrompt(overrides: Partial = {}): PromptFile { + return { + id: 'pseudo-advice', + filename: 'pseudo-advice.md', + fullPath: '/repo/presets/VectorLint/pseudo-advice.md', + meta: { + id: 'PseudoAdvice', + name: 'Pseudo Advice', + severity: Severity.WARNING, + type: 'check', + criteria: [{ id: 'Vague', name: 'Vague advice' }], + }, + body: 'Check for pseudo advice.', + pack: 'VectorLint', + ...overrides, + }; +} + +describe('buildReviewRequest', () => { + const target: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: '# Guide', + contentType: 'text/markdown', + }; + + it('maps one PromptFile to one ReviewRule with default budget and auto modelCall', () => { + const request = buildReviewRequest({ target, prompts: [makePrompt()] }); + expect(request.modelCall).toBe('auto'); + expect(request.budget).toEqual(DEFAULT_REVIEW_BUDGET); + expect(request.rules).toHaveLength(1); + + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule?.id).toBe('VectorLint.PseudoAdvice'); + expect(rule?.source).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(rule?.body).toBe('Check for pseudo advice.'); + expect(rule?.severity).toBe('warning'); + expect(rule?.name).toBe('Pseudo Advice'); + // Strict parse proves no legacy evaluator/criteria/type fields leaked. + expect(() => REVIEW_RULE_SCHEMA.parse(rule)).not.toThrow(); + }); + + it('does not copy legacy meta.type, evaluator, or criteria fields', () => { + const request = buildReviewRequest({ target, prompts: [makePrompt()] }); + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule && 'type' in rule).toBe(false); + expect(rule && 'evaluator' in rule).toBe(false); + expect(rule && 'criteria' in rule).toBe(false); + }); + + it('throws a ValidationError when no prompts are supplied', () => { + expect(() => buildReviewRequest({ target, prompts: [] })).toThrow(ValidationError); + }); + + it('passes caller-supplied context through unchanged', () => { + const context: ReviewContext[] = [ + { label: 'glossary', content: 'term a means ...', relation: 'reference' }, + ]; + const request = buildReviewRequest({ target, prompts: [makePrompt()], context }); + expect(request.context).toBe(context); + }); + + it('honors an explicit modelCall override', () => { + const request = buildReviewRequest({ + target, + prompts: [makePrompt()], + config: { modelCall: 'agent' }, + }); + expect(request.modelCall).toBe('agent'); + }); + + it('omits severity on the rule when the prompt has none', () => { + const request = buildReviewRequest({ + target, + prompts: [ + makePrompt({ + meta: { id: 'PseudoAdvice', name: 'Pseudo Advice' }, + }), + ], + }); + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule && 'severity' in rule).toBe(false); + }); +}); diff --git a/tests/review/result.test.ts b/tests/review/result.test.ts new file mode 100644 index 00000000..3be3767a --- /dev/null +++ b/tests/review/result.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest'; +import { + REVIEW_DIAGNOSTIC_SCHEMA, + REVIEW_FINDING_SCHEMA, + REVIEW_RESULT_SCHEMA, +} from '../../src/review'; + +describe('ReviewFinding schema', () => { + it('parses a complete finding', () => { + const finding = REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'VectorLint.Consistency', + ruleSource: '/repo/presets/VectorLint/consistency.md', + severity: 'warning', + message: 'Vague advice.', + line: 5, + column: 1, + match: 'consider leveraging', + analysis: 'too vague', + suggestion: 'be specific', + fix: 'consider using concrete terms', + }); + expect(finding.line).toBe(5); + expect(finding.match).toBe('consider leveraging'); + expect(finding.severity).toBe('warning'); + }); + + it('rejects a finding with an unknown severity', () => { + expect(() => + REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'P.R', + ruleSource: '/x.md', + severity: 'critical', + message: 'x', + line: 1, + column: 1, + match: 'x', + }), + ).toThrow(); + }); + + it('rejects a finding missing required anchored match evidence', () => { + expect(() => + REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'P.R', + ruleSource: '/x.md', + severity: 'warning', + message: 'x', + line: 1, + column: 1, + }), + ).toThrow(); + }); +}); + +describe('ReviewDiagnostic schema', () => { + it('parses a diagnostic with level/code/message', () => { + const diag = REVIEW_DIAGNOSTIC_SCHEMA.parse({ + level: 'warn', + code: 'finding-evidence-not-locatable', + message: 'could not anchor the finding', + }); + expect(diag.level).toBe('warn'); + }); + + it('rejects an unknown level', () => { + expect(() => + REVIEW_DIAGNOSTIC_SCHEMA.parse({ level: 'fatal', code: 'x', message: 'y' }), + ).toThrow(); + }); +}); + +describe('ReviewResult schema', () => { + it('accepts findings/scores/diagnostics and defaults usage to undefined', () => { + const result = REVIEW_RESULT_SCHEMA.parse({ findings: [], scores: [], diagnostics: [] }); + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + expect(result.usage).toBeUndefined(); + }); + + it('keeps diagnostics mandatory', () => { + expect(() => + REVIEW_RESULT_SCHEMA.parse({ findings: [], scores: [] }), + ).toThrow(); + }); + + it('accepts usage when provided', () => { + const result = REVIEW_RESULT_SCHEMA.parse({ + findings: [], + scores: [], + diagnostics: [], + usage: { modelCalls: 2, inputTokens: 10, outputTokens: 5 }, + }); + expect(result.usage?.modelCalls).toBe(2); + }); +}); diff --git a/tests/review/types.test.ts b/tests/review/types.test.ts new file mode 100644 index 00000000..d3bee632 --- /dev/null +++ b/tests/review/types.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import { REVIEW_CONTEXT_SCHEMA, REVIEW_RULE_SCHEMA, REVIEW_TARGET_SCHEMA } from '../../src/review'; + +describe('ReviewTarget schema', () => { + it('parses a valid target', () => { + const target = REVIEW_TARGET_SCHEMA.parse({ + uri: 'file:///repo/docs/guide.md', + content: '# Guide\n\nBody text.', + contentType: 'text/markdown', + }); + expect(target.uri).toBe('file:///repo/docs/guide.md'); + }); + + it('allows empty content (streaming target)', () => { + expect(() => + REVIEW_TARGET_SCHEMA.parse({ uri: 'file:///x', content: '', contentType: 'text/plain' }), + ).not.toThrow(); + }); + + it('rejects a target missing a uri', () => { + expect(() => + REVIEW_TARGET_SCHEMA.parse({ content: 'x', contentType: 'text/plain' }), + ).toThrow(); + }); +}); + +describe('ReviewRule schema', () => { + it('parses a rule with required fields and defaults severity to warning', () => { + const rule = REVIEW_RULE_SCHEMA.parse({ + id: 'Consistency', + source: 'VectorLint/consistency.md', + body: 'Check internal consistency.', + violationConditions: [ + { id: 'Contradiction', description: 'The target makes mutually inconsistent claims.' }, + ], + }); + expect(rule.severity).toBe('warning'); + expect(rule.violationConditions?.[0]?.id).toBe('Contradiction'); + }); + + it('rejects legacy evaluator and judge criteria fields', () => { + expect(() => + REVIEW_RULE_SCHEMA.parse({ + id: 'Tone', + source: 'VectorLint/tone.md', + body: 'Judge whether the tone is good.', + evaluator: 'judge', + criteria: [{ id: 'Tone', name: 'Tone quality' }], + }), + ).toThrow(); + }); + + it('rejects an unknown severity', () => { + expect(() => + REVIEW_RULE_SCHEMA.parse({ + id: 'Tone', + source: 'VectorLint/tone.md', + body: 'x', + severity: 'critical', + }), + ).toThrow(); + }); +}); + +describe('ReviewContext schema', () => { + it('parses caller-supplied scoped content', () => { + const ctx = REVIEW_CONTEXT_SCHEMA.parse({ + label: 'related-glossary', + content: 'Term A means ...', + relation: 'reference', + }); + expect(ctx.label).toBe('related-glossary'); + }); +}); From e4259e7865a1604a4091d6432bcc04c516b9d2ee Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:11:06 +0100 Subject: [PATCH 10/26] Extract single finding evidence verifier - Add src/findings/finding-evidence-verifier.ts wrapping locateQuotedText from src/output/location.ts as the single source of truth (audit #6). - Anchored quotes return verified line/column/match; unanchored quotes return a finding-evidence-not-locatable warn diagnostic and no finding. - Never falls back to a model-provided line number the way the old agent path did. - Cover anchored, unanchored, no-line-hint, and truncation cases. --- src/findings/finding-evidence-verifier.ts | 84 +++++++++++++++++++ .../finding-evidence-verifier.test.ts | 69 +++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 src/findings/finding-evidence-verifier.ts create mode 100644 tests/findings/finding-evidence-verifier.test.ts diff --git a/src/findings/finding-evidence-verifier.ts b/src/findings/finding-evidence-verifier.ts new file mode 100644 index 00000000..7a6ffb6a --- /dev/null +++ b/src/findings/finding-evidence-verifier.ts @@ -0,0 +1,84 @@ +import { locateQuotedText, type LocationWithMatch } from '../output/location'; + +/** Stable machine code emitted when finding evidence cannot be anchored. */ +export const FINDING_EVIDENCE_NOT_LOCATABLE = 'finding-evidence-not-locatable' as const; + +/** Evidence markers a model returns for a candidate finding. */ +export interface FindingEvidenceInput { + quoted_text: string; + context_before?: string; + context_after?: string; + /** Optional 1-based line hint from the model; never used as a fallback. */ + line?: number; +} + +/** Located coordinates for a verified finding, anchored in target content. */ +export interface VerifiedFindingCoords { + /** 1-based line in the target content. */ + line: number; + /** 1-based column. */ + column: number; + /** Verified anchored text. */ + match: string; +} + +/** Operational note produced when evidence cannot be verified. */ +export interface FindingEvidenceDiagnostic { + code: string; + level: 'warn' | 'error'; + message: string; +} + +/** The outcome of verifying a single candidate finding's evidence. */ +export interface FindingEvidenceVerification { + verified: boolean; + finding?: VerifiedFindingCoords; + diagnostic?: FindingEvidenceDiagnostic; +} + +/** + * The single source of truth for finding evidence verification + * (audit Finding #6). + * + * Wraps the existing `locateQuotedText` fuzzy matcher; it never reimplements + * matching. On a successful anchor it returns a verified finding with the + * located line/column/match. When the quoted text cannot be anchored it + * returns a `finding-evidence-not-locatable` warn diagnostic and NO finding — + * it never falls back to a model-provided line number the way the old agent + * path did. + */ +export function verifyFindingEvidence( + content: string, + findingEvidence: FindingEvidenceInput, +): FindingEvidenceVerification { + const location: LocationWithMatch | null = locateQuotedText( + content, + { + quoted_text: findingEvidence.quoted_text || '', + context_before: findingEvidence.context_before || '', + context_after: findingEvidence.context_after || '', + }, + 80, + findingEvidence.line, + ); + + if (!location) { + return { + verified: false, + diagnostic: { + code: FINDING_EVIDENCE_NOT_LOCATABLE, + level: 'warn', + message: `Could not locate quoted text "${(findingEvidence.quoted_text || '').slice(0, 60)}" in target content.`, + }, + }; + } + + return { + verified: true, + finding: { + line: location.line, + column: location.column, + match: location.match, + }, + }; +} diff --git a/tests/findings/finding-evidence-verifier.test.ts b/tests/findings/finding-evidence-verifier.test.ts new file mode 100644 index 00000000..64d38209 --- /dev/null +++ b/tests/findings/finding-evidence-verifier.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from 'vitest'; +import { + verifyFindingEvidence, + FINDING_EVIDENCE_NOT_LOCATABLE, +} from '../../src/findings/finding-evidence-verifier'; + +const CONTENT = 'line one\nline two has the quote\nline three'; + +describe('verifyFindingEvidence', () => { + it('locates an exact quote and returns a verified finding', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'the quote', + context_before: '', + context_after: '', + line: 2, + }); + expect(result.verified).toBe(true); + expect(result.finding?.line).toBe(2); + expect(result.finding?.match).toBe('the quote'); + expect(result.diagnostic).toBeUndefined(); + }); + + it('returns a warn diagnostic (not a fallback finding) when evidence cannot be located', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'this does not exist anywhere', + context_before: '', + context_after: '', + line: 99, + }); + expect(result.verified).toBe(false); + expect(result.finding).toBeUndefined(); + expect(result.diagnostic?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); + expect(result.diagnostic?.level).toBe('warn'); + }); + + it('never returns a finding whose line came from the model when the quote did not anchor', () => { + // The old agent path fell back to the model-provided line (here 1) and + // surfaced the raw quoted_text as a verified match. Assert we do not. + // (Use a lexically distant quote that locateQuotedText cannot anchor.) + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'definitely not present xyzzy', + context_before: '', + context_after: '', + line: 1, + }); + expect(result.verified).toBe(false); + expect(result.finding).toBeUndefined(); + }); + + it('still anchors a quote when no line hint is provided', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'line three', + }); + expect(result.verified).toBe(true); + expect(result.finding?.line).toBe(3); + expect(result.finding?.column).toBe(1); + }); + + it('produces a diagnostic that mentions the (truncated) quoted text', () => { + const longQuote = 'x'.repeat(120); + const result = verifyFindingEvidence(CONTENT, { + quoted_text: longQuote, + }); + expect(result.verified).toBe(false); + expect(result.diagnostic?.message).toContain('x'.repeat(60)); + // Truncated, not the full 120 chars. + expect(result.diagnostic?.message).not.toContain('x'.repeat(61)); + }); +}); From 25ec49aa43ec112c56785ef41a20207ebb4c9c88 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:13:50 +0100 Subject: [PATCH 11/26] Add shared finding severity and scoring helpers - Add src/findings/types.ts with the FindingProcessingInput contract and strict Zod schemas; no modelCall/mode/agent/evaluator/judge/rubric fields, so legacy judge/evaluator-shaped input is rejected. - Add src/findings/severity.ts: resolveSeverity derives severity from the calculateCheckScore result (one path, no agent stamping); buildRuleId matches Pack.Rule[.Criterion] naming without importing src/agent/. - Add src/findings/scorer.ts as a thin wrapper over calculateCheckScore; the verified finding count drives the score. - Cover schema parse/reject, severity, rule-id, criterion resolution, and score numerics against calculateCheckScore. --- src/findings/scorer.ts | 60 ++++++++++++ src/findings/severity.ts | 52 +++++++++++ src/findings/types.ts | 160 ++++++++++++++++++++++++++++++++ tests/findings/scorer.test.ts | 94 +++++++++++++++++++ tests/findings/severity.test.ts | 73 +++++++++++++++ tests/findings/types.test.ts | 145 +++++++++++++++++++++++++++++ 6 files changed, 584 insertions(+) create mode 100644 src/findings/scorer.ts create mode 100644 src/findings/severity.ts create mode 100644 src/findings/types.ts create mode 100644 tests/findings/scorer.test.ts create mode 100644 tests/findings/severity.test.ts create mode 100644 tests/findings/types.test.ts diff --git a/src/findings/scorer.ts b/src/findings/scorer.ts new file mode 100644 index 00000000..b5fb93fa --- /dev/null +++ b/src/findings/scorer.ts @@ -0,0 +1,60 @@ +import { calculateCheckScore, type CheckScoringOptions } from '../scoring'; +import type { CheckResult } from '../prompts/schema'; +import { resolveSeverity } from './severity'; +import type { RawViolation, RuleSeverity } from './types'; + +/** Optional scoring knobs mirroring {@link CheckScoringOptions}. */ +export interface ScoreCheckOptions { + strictness?: CheckScoringOptions['strictness']; + promptSeverity?: RuleSeverity; +} + +/** + * The count/density score result for a rule's verified findings. Mirrors the + * fields the processor needs to assemble a `ReviewScore`. The numeric values + * come straight from {@link calculateCheckScore}; this layer never + * reimplements the scoring math (audit Finding #4). + */ +export interface ScoredCheck { + score: number; + scoreText: string; + severity: RuleSeverity; + /** Count of verified findings that drove the score. */ + findingCount: number; + /** The raw check result, used to resolve severity. */ + scored: CheckResult; +} + +/** + * Scores a rule's verified findings using the existing check density formula. + * + * The score is driven by the **verified** finding count, not the raw candidate + * count (audit Finding #6): callers must pass only findings whose evidence has + * been anchored. This is a thin adapter over {@link calculateCheckScore}; it + * only normalizes the result into the shared score shape. + */ +export function scoreCheck(params: { + verifiedViolations: RawViolation[]; + wordCount: number; + strictness?: CheckScoringOptions['strictness']; + promptSeverity?: RuleSeverity; +}): ScoredCheck { + const scored = calculateCheckScore( + params.verifiedViolations, + params.wordCount, + { + ...(params.strictness !== undefined ? { strictness: params.strictness } : {}), + ...(params.promptSeverity !== undefined + ? { promptSeverity: params.promptSeverity } + : {}), + }, + ); + + return { + score: scored.final_score, + scoreText: `${scored.final_score.toFixed(1)}/10`, + severity: resolveSeverity({ scored }), + findingCount: scored.violation_count, + scored, + }; +} diff --git a/src/findings/severity.ts b/src/findings/severity.ts new file mode 100644 index 00000000..cca534f0 --- /dev/null +++ b/src/findings/severity.ts @@ -0,0 +1,52 @@ +import type { CheckResult } from '../prompts/schema'; +import type { FindingsCriterion, RuleSeverity } from './types'; + +/** Input to {@link resolveSeverity}. */ +export interface SeverityInput { + scored: CheckResult; +} + +/** + * The one severity resolution path for objective violation checks + * (audit Finding #4). Severity is derived from the existing count/density + * score result (`CheckResult.severity`); there is no agent-specific or + * mode-specific severity stamping here. + */ +export function resolveSeverity(input: SeverityInput): RuleSeverity { + return input.scored.severity; +} + +/** + * Builds the hierarchical output rule id matching the current standard + * orchestrator naming (`buildRuleName`): `Pack.Rule` or, when a violation is + * attributed to a criterion, `Pack.Rule.Criterion`. This is a faithful port of + * that behavior; it does not import from `src/agent/`. + */ +export function buildRuleId( + pack: string, + ruleId: string, + criterionId?: string, +): string { + const parts = [pack, ruleId]; + if (criterionId) { + parts.push(criterionId); + } + return parts.join('.'); +} + +/** + * Resolves a criterion id from the rule's declared criteria by matching a + * `criterionName` a violation carries to a declared criterion's `name`. + * Returns `undefined` when the violation is not attributed to a declared + * criterion (the common case for objective check violations). + */ +export function resolveCriterionId( + criteria: readonly FindingsCriterion[] | undefined, + criterionName: string | undefined, +): string | undefined { + if (!criterionName || !criteria) { + return undefined; + } + const match = criteria.find((c) => c.name === criterionName); + return match?.id; +} diff --git a/src/findings/types.ts b/src/findings/types.ts new file mode 100644 index 00000000..9a6360c6 --- /dev/null +++ b/src/findings/types.ts @@ -0,0 +1,160 @@ +import { z } from 'zod'; +import { Severity } from '../evaluators/types'; + +/** + * Input contract for shared finding processing (Phase 3, audit Finding #6). + * + * This module is the boundary between "what a model returned" (raw candidate + * violations) and "what the formatters receive" (`ReviewResult`). It is + * intentionally independent of model call (`single` | `agent`): there is no + * `modelCall`, `mode`, `agent`, `evaluator`, `judge`, or rubric field here. + * + * Following the `src/review/` pattern, the domain types below are explicit + * interfaces (so their optional properties stay assignable under + * `exactOptionalPropertyTypes` to `CheckItem`/`GateViolationLike`), and the + * Zod schemas mirror them for boundary validation. The schemas are strict so + * legacy `evaluator`/`judge`/`modelCall`/`mode`/`agent`/rubric payloads are + * rejected. + */ + +/** + * The two severity levels objective violation checks can resolve to. Mirrors + * `ReviewSeverity`; values are identical (`'error'` | `'warning'`). + */ +export type RuleSeverity = typeof Severity.ERROR | typeof Severity.WARNING; + +/** Optional strictness knob for check density scoring. */ +export type Strictness = number | 'lenient' | 'standard' | 'strict'; + +/** + * The six boolean evidence gates a model returns per violation. Structurally + * identical to `GateChecks` in `src/prompts/schema.ts`. + */ +export interface GateChecks { + rule_supports_claim: boolean; + evidence_exact: boolean; + context_supports_violation: boolean; + plausible_non_violation: boolean; + fix_is_drop_in: boolean; + fix_preserves_meaning: boolean; +} + +/** + * A raw candidate finding returned by a reviewer model, before evidence + * verification or filtering. `analysis` is the only required descriptive + * field; everything else is optional to tolerate partial model output while + * still failing closed at the filter and verifier stages. + */ +export interface RawViolation { + line?: number; + quoted_text?: string; + context_before?: string; + context_after?: string; + description?: string; + /** Optional criterion attribution used to build `Pack.Rule.Criterion` ids. */ + criterionName?: string; + analysis: string; + message?: string; + suggestion?: string; + fix?: string; + rule_quote?: string; + confidence?: number; + checks?: GateChecks; +} + +/** + * Criterion metadata used ONLY to name output rule ids + * (`Pack.Rule.Criterion`). Weight and target are judge/rubric concerns and are + * deliberately absent so the finding-processing contract cannot leak them. + */ +export interface FindingsCriterion { + id: string; + name: string; +} + +/** + * The narrow slice of prompt metadata that finding processing needs: severity + * for scoring, strictness for density, and criteria for rule-id naming. It + * excludes `evaluator`, `type`, `evaluateAs`, `target`, and other legacy + * fields that do not affect objective violation-check processing. + */ +export interface PromptMetaForFindings { + severity?: RuleSeverity; + strictness?: Strictness; + criteria?: FindingsCriterion[]; +} + +/** + * The full finding-processing input. There is deliberately no `outputFormat`: + * the processor emits a format-agnostic `ReviewResult`; the caller routes it + * to a formatter. + */ +export interface FindingProcessingInput { + pack: string; + ruleId: string; + ruleSource: string; + candidateFindings: RawViolation[]; + wordCount: number; + promptMeta: PromptMetaForFindings; + targetContent: string; +} + +// --- Boundary validation schemas (mirror the interfaces above) --------------- + +export const GATE_CHECKS_SCHEMA = z + .object({ + rule_supports_claim: z.boolean(), + evidence_exact: z.boolean(), + context_supports_violation: z.boolean(), + plausible_non_violation: z.boolean(), + fix_is_drop_in: z.boolean(), + fix_preserves_meaning: z.boolean(), + }) + .strict(); + +export const RAW_VIOLATION_SCHEMA = z + .object({ + line: z.number().optional(), + quoted_text: z.string().optional(), + context_before: z.string().optional(), + context_after: z.string().optional(), + description: z.string().optional(), + criterionName: z.string().optional(), + analysis: z.string(), + message: z.string().optional(), + suggestion: z.string().optional(), + fix: z.string().optional(), + rule_quote: z.string().optional(), + confidence: z.number().optional(), + checks: GATE_CHECKS_SCHEMA.optional(), + }) + .strict(); + +export const FINDINGS_CRITERION_SCHEMA = z + .object({ + id: z.string().min(1), + name: z.string().min(1), + }) + .strict(); + +export const PROMPT_META_FOR_FINDINGS_SCHEMA = z + .object({ + severity: z.nativeEnum(Severity).optional(), + strictness: z + .union([z.number().positive(), z.enum(['lenient', 'standard', 'strict'])]) + .optional(), + criteria: z.array(FINDINGS_CRITERION_SCHEMA).optional(), + }) + .strict(); + +export const FINDING_PROCESSING_INPUT_SCHEMA = z + .object({ + pack: z.string().min(1), + ruleId: z.string().min(1), + ruleSource: z.string().min(1), + candidateFindings: z.array(RAW_VIOLATION_SCHEMA), + wordCount: z.number().nonnegative(), + promptMeta: PROMPT_META_FOR_FINDINGS_SCHEMA, + targetContent: z.string(), + }) + .strict(); diff --git a/tests/findings/scorer.test.ts b/tests/findings/scorer.test.ts new file mode 100644 index 00000000..7e8147b0 --- /dev/null +++ b/tests/findings/scorer.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; +import { calculateCheckScore } from '../../src/scoring'; +import { Severity } from '../../src/evaluators/types'; +import { scoreCheck } from '../../src/findings/scorer'; +import type { RawViolation } from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function violation(overrides: Partial = {}): RawViolation { + return { + analysis: 'Issue', + message: 'Issue message', + fix: 'Fix', + rule_quote: 'Rule quote', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + ...overrides, + }; +} + +describe('scoreCheck', () => { + it('returns the same numeric result as calculateCheckScore for the same density', () => { + const verified = [violation(), violation()]; + const result = scoreCheck({ + verifiedViolations: verified, + wordCount: 100, + strictness: 'standard', + promptSeverity: Severity.WARNING, + }); + + const direct = calculateCheckScore(verified, 100, { + strictness: 'standard', + promptSeverity: Severity.WARNING, + }); + + expect(result.score).toBe(direct.final_score); + expect(result.scoreText).toBe(`${direct.final_score.toFixed(1)}/10`); + expect(result.severity).toBe(direct.severity); + expect(result.findingCount).toBe(verified.length); + expect(result.findingCount).toBe(direct.violation_count); + }); + + it('is driven by the verified finding count, not a raw candidate count', () => { + // 1 verified violation over 100 words at standard strictness: + // density 1 -> 100 - 1*10 = 90 -> 9.0/10 + const one = scoreCheck({ + verifiedViolations: [violation()], + wordCount: 100, + strictness: 'standard', + }); + expect(one.score).toBe(9.0); + expect(one.scoreText).toBe('9.0/10'); + + // 2 verified violations -> density 2 -> 100 - 2*10 = 80 -> 8.0/10 + const two = scoreCheck({ + verifiedViolations: [violation(), violation()], + wordCount: 100, + strictness: 'standard', + }); + expect(two.score).toBe(8.0); + expect(two.findingCount).toBe(2); + }); + + it('resolves error severity from the density score when score is low and prompt severity is error', () => { + // 20 violations over 100 words at standard strictness: + // density 20 -> 100 - 20*10 = -100 -> clamp 0 -> 0.0/10 -> prompt severity + const result = scoreCheck({ + verifiedViolations: Array.from({ length: 20 }, () => violation()), + wordCount: 100, + strictness: 'standard', + promptSeverity: Severity.ERROR, + }); + expect(result.score).toBe(0); + expect(result.severity).toBe(Severity.ERROR); + }); + + it('forwards strictness and prompt severity options to calculateCheckScore', () => { + const verified = [violation()]; + const result = scoreCheck({ + verifiedViolations: verified, + wordCount: 100, + strictness: 'strict', + }); + const direct = calculateCheckScore(verified, 100, { strictness: 'strict' }); + expect(result.score).toBe(direct.final_score); + }); +}); diff --git a/tests/findings/severity.test.ts b/tests/findings/severity.test.ts new file mode 100644 index 00000000..46e0c93a --- /dev/null +++ b/tests/findings/severity.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect } from 'vitest'; +import { + buildRuleId, + resolveCriterionId, + resolveSeverity, +} from '../../src/findings/severity'; +import { EvaluationType, Severity } from '../../src/evaluators/types'; +import type { CheckResult } from '../../src/prompts/schema'; + +function makeCheckResult(severity: Severity): CheckResult { + return { + type: EvaluationType.CHECK, + final_score: 5, + percentage: 50, + violation_count: 1, + items: [], + severity, + message: 'Found 1 issue', + violations: [], + }; +} + +describe('resolveSeverity', () => { + it('returns the density-derived severity from the check score result', () => { + expect(resolveSeverity({ scored: makeCheckResult(Severity.WARNING) })).toBe( + Severity.WARNING, + ); + expect(resolveSeverity({ scored: makeCheckResult(Severity.ERROR) })).toBe( + Severity.ERROR, + ); + }); +}); + +describe('buildRuleId', () => { + it('builds Pack.Rule when no criterion is given', () => { + expect(buildRuleId('VectorLint', 'AiPattern')).toBe('VectorLint.AiPattern'); + }); + + it('builds Pack.Rule.Criterion when a criterion id is given', () => { + expect(buildRuleId('VectorLint', 'AiPattern', 'Hedging')).toBe( + 'VectorLint.AiPattern.Hedging', + ); + }); + + it('ignores an empty criterion id', () => { + expect(buildRuleId('VectorLint', 'AiPattern', '')).toBe( + 'VectorLint.AiPattern', + ); + }); +}); + +describe('resolveCriterionId', () => { + const criteria = [ + { id: 'Hedging', name: 'Hedge words' }, + { id: 'Filler', name: 'Filler phrases' }, + ]; + + it('maps a criterion name to its declared id', () => { + expect(resolveCriterionId(criteria, 'Hedge words')).toBe('Hedging'); + }); + + it('returns undefined when the name does not match a criterion', () => { + expect(resolveCriterionId(criteria, 'Unknown')).toBeUndefined(); + }); + + it('returns undefined when no criterion name is provided', () => { + expect(resolveCriterionId(criteria, undefined)).toBeUndefined(); + }); + + it('returns undefined when no criteria are declared', () => { + expect(resolveCriterionId(undefined, 'Hedge words')).toBeUndefined(); + }); +}); diff --git a/tests/findings/types.test.ts b/tests/findings/types.test.ts new file mode 100644 index 00000000..4309ead6 --- /dev/null +++ b/tests/findings/types.test.ts @@ -0,0 +1,145 @@ +import { describe, it, expect } from 'vitest'; +import { + FINDING_PROCESSING_INPUT_SCHEMA, + PROMPT_META_FOR_FINDINGS_SCHEMA, + RAW_VIOLATION_SCHEMA, +} from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function validInput() { + return { + pack: 'VectorLint', + ruleId: 'AiPattern', + ruleSource: '/repo/presets/VectorLint/ai-pattern.md', + candidateFindings: [ + { + line: 1, + quoted_text: 'leverage synergies', + context_before: '', + context_after: '', + description: 'cliché phrase', + analysis: 'vague corporate cliché', + message: 'Avoid cliché phrase', + suggestion: 'Use concrete verbs', + fix: 'work together', + rule_quote: 'Avoid clichés', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + }, + ], + wordCount: 100, + promptMeta: { + severity: 'warning' as const, + strictness: 'standard' as const, + criteria: [{ id: 'Hedging', name: 'Hedge words' }], + }, + targetContent: 'leverage synergies everywhere', + }; +} + +describe('RAW_VIOLATION_SCHEMA', () => { + it('parses a representative candidate finding', () => { + const parsed = RAW_VIOLATION_SCHEMA.parse(validInput().candidateFindings[0]); + expect(parsed.analysis).toBe('vague corporate cliché'); + expect(parsed.checks?.evidence_exact).toBe(true); + }); + + it('requires analysis', () => { + expect(() => + RAW_VIOLATION_SCHEMA.parse({ message: 'no analysis field' }), + ).toThrow(); + }); +}); + +describe('PROMPT_META_FOR_FINDINGS_SCHEMA', () => { + it('rejects legacy evaluator/type/evaluateAs/target fields', () => { + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ + severity: 'warning', + evaluator: 'base', + }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ type: 'judge' }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ evaluateAs: 'document' }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ target: { regex: 'x' } }), + ).toThrow(); + }); + + it('rejects rubric weight on criteria', () => { + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ + criteria: [{ id: 'Hedging', name: 'Hedge words', weight: 2 }], + }), + ).toThrow(); + }); +}); + +describe('FINDING_PROCESSING_INPUT_SCHEMA', () => { + it('parses a representative supported finding-processing input', () => { + const parsed = FINDING_PROCESSING_INPUT_SCHEMA.parse(validInput()); + expect(parsed.pack).toBe('VectorLint'); + expect(parsed.candidateFindings).toHaveLength(1); + expect(parsed.promptMeta.criteria?.[0]?.id).toBe('Hedging'); + }); + + it('has no modelCall/mode/agent field and rejects them', () => { + for (const legacyKey of ['modelCall', 'mode', 'agent'] as const) { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), [legacyKey]: 'agent' }), + ).toThrow(); + } + }); + + it('rejects legacy judge/evaluator/rubric-shaped top-level input', () => { + // Legacy judge result shape: criteria array with scores at the top level. + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + pack: 'VectorLint', + ruleId: 'Tone', + ruleSource: '/x.md', + wordCount: 100, + targetContent: 'x', + promptMeta: {}, + evaluator: 'judge', + criteria: [{ name: 'Tone', score: 2, violations: [] }], + }), + ).toThrow(); + }); + + it('rejects a legacy evaluator field nested under promptMeta', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + ...validInput(), + promptMeta: { ...validInput().promptMeta, evaluator: 'base' }, + }), + ).toThrow(); + }); + + it('rejects malformed candidate findings', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + ...validInput(), + candidateFindings: [{ message: 'no analysis field' }], + }), + ).toThrow(); + }); + + it('rejects an unknown top-level key', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), judge: true }), + ).toThrow(); + }); +}); From 20d42ff7d9441f3269d6fdea53f527ead80342b8 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:15:37 +0100 Subject: [PATCH 12/26] Implement shared finding processor contract - Add src/findings/processor.ts: processFindings runs the single pipeline (filter -> verify -> dedupe -> score -> severity -> diagnostics) and returns the Phase 2 ReviewResult from src/review/types.ts. - Verified finding count, not raw candidate count, drives the score; unanchored evidence becomes a warn diagnostic and emits no finding; hadOperationalErrors is true only for error-level diagnostics. - Add src/findings/index.ts barrel export. - Golden processor test asserts findings/score match the standard check path; cover diagnostics, filtering, verified-count fix, dedupe, rule-id attribution, and ReviewResult contract validation. --- src/findings/index.ts | 48 +++++ src/findings/processor.ts | 167 ++++++++++++++++ tests/findings/module-surface.test.ts | 27 +++ tests/findings/processor.test.ts | 278 ++++++++++++++++++++++++++ 4 files changed, 520 insertions(+) create mode 100644 src/findings/index.ts create mode 100644 src/findings/processor.ts create mode 100644 tests/findings/module-surface.test.ts create mode 100644 tests/findings/processor.test.ts diff --git a/src/findings/index.ts b/src/findings/index.ts new file mode 100644 index 00000000..095b878c --- /dev/null +++ b/src/findings/index.ts @@ -0,0 +1,48 @@ +/** + * Shared finding-processing layer (Phase 3). + * + * Transforms raw candidate violation findings into a formatter-ready + * `ReviewResult` through one pipeline: evidence verification, filtering, + * count/density scoring, severity resolution, and diagnostics. + * + * This module is independent of model call (`single` | `agent`) and contains + * no legacy `judge`, `evaluator`, rubric, or autonomous-loop output + * compatibility code. See + * docs/plans/2026-07-10-phase-3-share-result-projection.md. + */ +export { + FINDING_EVIDENCE_NOT_LOCATABLE, + verifyFindingEvidence, + type FindingEvidenceDiagnostic, + type FindingEvidenceInput, + type FindingEvidenceVerification, + type VerifiedFindingCoords, +} from './finding-evidence-verifier'; + +export { + buildRuleId, + resolveCriterionId, + resolveSeverity, + type SeverityInput, +} from './severity'; + +export { scoreCheck, type ScoreCheckOptions, type ScoredCheck } from './scorer'; + +export { + processFindings, +} from './processor'; + +export { + FINDING_PROCESSING_INPUT_SCHEMA, + FINDINGS_CRITERION_SCHEMA, + GATE_CHECKS_SCHEMA, + PROMPT_META_FOR_FINDINGS_SCHEMA, + RAW_VIOLATION_SCHEMA, + type FindingsCriterion, + type FindingProcessingInput, + type GateChecks, + type PromptMetaForFindings, + type RawViolation, + type RuleSeverity, + type Strictness, +} from './types'; diff --git a/src/findings/processor.ts b/src/findings/processor.ts new file mode 100644 index 00000000..45a1c7ae --- /dev/null +++ b/src/findings/processor.ts @@ -0,0 +1,167 @@ +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewResult, + ReviewScore, +} from '../review/types'; +import { computeFilterDecision } from '../evaluators/violation-filter'; +import { + verifyFindingEvidence, + FINDING_EVIDENCE_NOT_LOCATABLE, + type VerifiedFindingCoords, +} from './finding-evidence-verifier'; +import { scoreCheck } from './scorer'; +import { buildRuleId, resolveCriterionId } from './severity'; +import type { FindingProcessingInput, RawViolation } from './types'; + +interface VerifiedEntry { + raw: RawViolation; + coords: VerifiedFindingCoords; +} + +/** + * The single finding-processing pipeline for objective Via Negativa violation + * checks (Phase 3). Transforms raw candidate findings into a formatter-ready + * {@link ReviewResult}. + * + * Order (audit lines 167-174): + * 1. filter candidates by the evidence gate (`computeFilterDecision`); + * 2. verify each surfaced finding's evidence (anchored quote or diagnostic); + * 3. deduplicate verified findings by `(quoted_text, line)`; + * 4. score by verified finding count/density; + * 5. resolve severity from the density score; + * 6. assemble findings, score, and diagnostics. + * + * The verified finding count — not the raw candidate count — drives the score + * (audit Finding #6). Unanchored evidence becomes a + * `finding-evidence-not-locatable` warn diagnostic and emits no finding. + * `hadOperationalErrors` is `true` only when a diagnostic is `error` level. + */ +export function processFindings(input: FindingProcessingInput): ReviewResult { + const surfaced = filterCandidates(input.candidateFindings); + const { verified, evidenceDiagnostics } = verifyAndDedupe( + surfaced, + input.targetContent, + ); + + const scored = scoreCheck({ + verifiedViolations: verified.map((entry) => entry.raw), + wordCount: input.wordCount, + ...(input.promptMeta.strictness !== undefined + ? { strictness: input.promptMeta.strictness } + : {}), + ...(input.promptMeta.severity !== undefined + ? { promptSeverity: input.promptMeta.severity } + : {}), + }); + + const severity = scored.severity; + const scoreRuleId = buildRuleId(input.pack, input.ruleId); + + const findings: ReviewFinding[] = verified.map(({ raw, coords }) => { + const criterionId = resolveCriterionId( + input.promptMeta.criteria, + raw.criterionName, + ); + const ruleId = buildRuleId(input.pack, input.ruleId, criterionId); + const message = (raw.message || raw.analysis || '').trim(); + return { + ruleId, + ruleSource: input.ruleSource, + severity, + message, + line: coords.line, + column: coords.column, + match: coords.match, + ...(raw.analysis ? { analysis: raw.analysis } : {}), + ...(raw.suggestion ? { suggestion: raw.suggestion } : {}), + ...(raw.fix ? { fix: raw.fix } : {}), + }; + }); + + const scores: ReviewScore[] = [ + { + ruleId: scoreRuleId, + score: scored.score, + scoreText: scored.scoreText, + severity, + findingCount: verified.length, + }, + ]; + + const diagnostics: ReviewDiagnostic[] = [...evidenceDiagnostics]; + const hadOperationalErrors = diagnostics.some( + (diagnostic) => diagnostic.level === 'error', + ); + + return { findings, scores, diagnostics, hadOperationalErrors }; +} + +/** + * Drops candidate findings that fail the evidence gate. Filtered candidates + * contribute neither findings nor diagnostics, matching the current standard + * orchestrator behavior. + */ +function filterCandidates(candidates: readonly RawViolation[]): RawViolation[] { + const surfaced: RawViolation[] = []; + for (const candidate of candidates) { + const decision = computeFilterDecision(candidate); + if (decision.surface) { + surfaced.push(candidate); + } + } + return surfaced; +} + +/** + * Verifies each surfaced finding's evidence and deduplicates verified findings + * by `(quoted_text, line)`. Unanchored findings become warn diagnostics and are + * not emitted. + */ +function verifyAndDedupe( + surfaced: readonly RawViolation[], + targetContent: string, +): { verified: VerifiedEntry[]; evidenceDiagnostics: ReviewDiagnostic[] } { + const verified: VerifiedEntry[] = []; + const evidenceDiagnostics: ReviewDiagnostic[] = []; + const seen = new Set(); + + for (const candidate of surfaced) { + const verification = verifyFindingEvidence(targetContent, { + quoted_text: candidate.quoted_text || '', + ...(candidate.context_before !== undefined + ? { context_before: candidate.context_before } + : {}), + ...(candidate.context_after !== undefined + ? { context_after: candidate.context_after } + : {}), + ...(candidate.line !== undefined ? { line: candidate.line } : {}), + }); + + if (!verification.verified || !verification.finding) { + evidenceDiagnostics.push({ + level: 'warn', + code: FINDING_EVIDENCE_NOT_LOCATABLE, + message: + verification.diagnostic?.message ?? + 'Could not locate finding evidence in target content.', + }); + continue; + } + + const coords = verification.finding; + const dedupeKey = candidate.quoted_text + ? `${candidate.quoted_text}:${coords.line}` + : null; + if (dedupeKey) { + if (seen.has(dedupeKey)) { + continue; + } + seen.add(dedupeKey); + } + + verified.push({ raw: candidate, coords }); + } + + return { verified, evidenceDiagnostics }; +} diff --git a/tests/findings/module-surface.test.ts b/tests/findings/module-surface.test.ts new file mode 100644 index 00000000..a0e5c481 --- /dev/null +++ b/tests/findings/module-surface.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import * as findings from '../../src/findings'; + +describe('src/findings public surface', () => { + it('exports the processing entry point and helpers', () => { + expect(typeof findings.processFindings).toBe('function'); + expect(typeof findings.verifyFindingEvidence).toBe('function'); + expect(typeof findings.resolveSeverity).toBe('function'); + expect(typeof findings.buildRuleId).toBe('function'); + expect(typeof findings.resolveCriterionId).toBe('function'); + expect(typeof findings.scoreCheck).toBe('function'); + }); + + it('exports the stable diagnostic code constant', () => { + expect(findings.FINDING_EVIDENCE_NOT_LOCATABLE).toBe( + 'finding-evidence-not-locatable', + ); + }); + + it('exports the Zod boundary schemas', () => { + expect(findings.FINDING_PROCESSING_INPUT_SCHEMA).toBeDefined(); + expect(findings.RAW_VIOLATION_SCHEMA).toBeDefined(); + expect(findings.GATE_CHECKS_SCHEMA).toBeDefined(); + expect(findings.FINDINGS_CRITERION_SCHEMA).toBeDefined(); + expect(findings.PROMPT_META_FOR_FINDINGS_SCHEMA).toBeDefined(); + }); +}); diff --git a/tests/findings/processor.test.ts b/tests/findings/processor.test.ts new file mode 100644 index 00000000..90f9641d --- /dev/null +++ b/tests/findings/processor.test.ts @@ -0,0 +1,278 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { processFindings } from '../../src/findings/processor'; +import { FINDING_EVIDENCE_NOT_LOCATABLE } from '../../src/findings/finding-evidence-verifier'; +import { REVIEW_RESULT_SCHEMA } from '../../src/review'; +import type { + FindingProcessingInput, + RawViolation, +} from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function violation(overrides: Partial = {}): RawViolation { + return { + line: 1, + quoted_text: 'Alpha text', + context_before: '', + context_after: '', + analysis: 'Issue 1', + message: 'Issue 1', + suggestion: 'Suggestion 1', + fix: 'Fix 1', + rule_quote: 'Rule quote', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + ...overrides, + }; +} + +function baseInput( + candidateFindings: RawViolation[], + overrides: Partial = {}, +): FindingProcessingInput { + return { + pack: 'TestPack', + ruleId: 'CheckPrompt', + ruleSource: '/repo/prompts/CheckPrompt.md', + candidateFindings, + wordCount: 100, + promptMeta: { severity: 'warning', strictness: 'standard' }, + targetContent: 'Alpha text\nBeta text\n', + ...overrides, + }; +} + +describe('processFindings', () => { + const originalThreshold = process.env.CONFIDENCE_THRESHOLD; + + beforeEach(() => { + // Default threshold (0.75) unless a test opts in. + delete process.env.CONFIDENCE_THRESHOLD; + }); + + afterEach(() => { + if (originalThreshold === undefined) { + delete process.env.CONFIDENCE_THRESHOLD; + } else { + process.env.CONFIDENCE_THRESHOLD = originalThreshold; + } + }); + + describe('golden objective-check output', () => { + beforeEach(() => { + // Surface every gate-passing candidate regardless of confidence. + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('produces byte-for-byte findings/score matching the standard orchestrator path', () => { + const input = baseInput([ + violation(), + violation({ + line: 2, + quoted_text: 'Beta text', + analysis: 'Issue 2', + message: 'Issue 2', + suggestion: 'Suggestion 2', + fix: 'Fix 2', + }), + ]); + + const result = processFindings(input); + + // Score: density 2/100 at standard strictness (10) -> 100 - 20 -> 8.0/10. + expect(result.scores).toEqual([ + { + ruleId: 'TestPack.CheckPrompt', + score: 8.0, + scoreText: '8.0/10', + severity: 'warning', + findingCount: 2, + }, + ]); + + // Findings: Pack.Rule (check violations carry no criterion name), + // anchored lines/columns/match. + expect(result.findings).toHaveLength(2); + expect(result.findings[0]).toMatchObject({ + ruleId: 'TestPack.CheckPrompt', + ruleSource: '/repo/prompts/CheckPrompt.md', + severity: 'warning', + message: 'Issue 1', + line: 1, + column: 1, + match: 'Alpha text', + suggestion: 'Suggestion 1', + fix: 'Fix 1', + }); + expect(result.findings[1]).toMatchObject({ + ruleId: 'TestPack.CheckPrompt', + severity: 'warning', + message: 'Issue 2', + line: 2, + column: 1, + match: 'Beta text', + }); + + expect(result.diagnostics).toEqual([]); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('returns a ReviewResult that validates against the review contract', () => { + const result = processFindings(baseInput([violation()])); + expect(() => REVIEW_RESULT_SCHEMA.parse(result)).not.toThrow(); + }); + }); + + describe('evidence verification (audit Finding #6)', () => { + beforeEach(() => { + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('turns unanchored evidence into a warn diagnostic and emits no finding', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'this quote is not in the content' }), + ]), + ); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); + expect(result.diagnostics[0]?.level).toBe('warn'); + // Score reflects 0 verified findings -> perfect 10.0/10. + expect(result.scores[0]?.score).toBe(10.0); + expect(result.scores[0]?.findingCount).toBe(0); + }); + + it('counts only verified findings toward the score, not raw candidate count', () => { + // One anchored + one unanchored candidate. Verified count = 1. + // Density 1/100 -> 100 - 10 -> 9.0/10 (the intentional fix). + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text' }), + violation({ quoted_text: 'nowhere to be found' }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.score).toBe(9.0); + expect(result.scores[0]?.findingCount).toBe(1); + expect(result.diagnostics).toHaveLength(1); + }); + + it('does not set hadOperationalErrors for warn-level evidence diagnostics', () => { + const result = processFindings( + baseInput([violation({ quoted_text: 'missing' })]), + ); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('deduplicates verified findings by quoted_text and line', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', line: 1 }), + violation({ quoted_text: 'Alpha text', line: 1, message: 'dup' }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.findingCount).toBe(1); + }); + }); + + describe('candidate filtering', () => { + it('drops candidates that fail computeFilterDecision without a diagnostic', () => { + // Default threshold 0.75: confidence 0.2 is filtered out. + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', confidence: 0.2 }), + ]), + ); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + // No verified findings -> 10.0/10. + expect(result.scores[0]?.findingCount).toBe(0); + }); + + it('filtered candidates do not contribute to the score count', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', confidence: 0.9 }), + violation({ quoted_text: 'Beta text', line: 2, confidence: 0.2 }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.findingCount).toBe(1); + expect(result.scores[0]?.score).toBe(9.0); + }); + }); + + describe('rule id building', () => { + beforeEach(() => { + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('attributes findings to Pack.Rule when violations carry no criterion name', () => { + const result = processFindings(baseInput([violation({ quoted_text: 'Alpha text' })])); + expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt'); + expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + }); + + it('attributes findings to Pack.Rule.Criterion when a violation names a declared criterion', () => { + const result = processFindings( + baseInput( + [ + violation({ + quoted_text: 'Alpha text', + criterionName: 'Hedge words', + }), + ], + { + promptMeta: { + severity: 'warning', + strictness: 'standard', + criteria: [{ id: 'Hedging', name: 'Hedge words' }], + }, + }, + ), + ); + expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt.Hedging'); + // The score stays at Pack.Rule (no criterion), matching the orchestrator. + expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + }); + }); + + describe('severity resolution', () => { + it('resolves error severity from the density score and stamps it on findings', () => { + // 20 verified violations over 100 words -> 0.0/10 -> prompt severity error. + const result = processFindings( + baseInput( + Array.from({ length: 20 }, (_, i) => + violation({ + quoted_text: `Alpha text${i}`, + analysis: `Issue ${i}`, + message: `Issue ${i}`, + }), + ), + { + targetContent: Array.from({ length: 20 }, (_, i) => `Alpha text${i}`).join('\n') + '\n', + promptMeta: { severity: 'error', strictness: 'standard' }, + }, + ), + ); + + expect(result.scores[0]?.severity).toBe('error'); + expect(result.findings.every((f) => f.severity === 'error')).toBe(true); + }); + }); +}); From 671da9cd5e1cfae9863970274e338c541fc63893 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 03:24:02 +0100 Subject: [PATCH 13/26] Route standard check evaluation through shared processor - Build FindingProcessingInput and call processFindings in the standard objective-check branch of routePromptResult - Feed returned findings/scores/diagnostics through the existing line/json/rdjson/vale sinks and quality-score output - Remove inline filter/score/severity/quote-location logic from the check path; computeFilterDecision kept only for the debug-run artifact - Verified finding count now drives counts and score (audit Finding #6); unanchored quotes become warn diagnostics and no longer flag operational errors - Add orchestrator regression tests covering finding/score output, JSON/Vale shape, and exit behavior for supported objective-check reviews --- src/cli/orchestrator.ts | 120 ++++--- tests/orchestrator-check-processor.test.ts | 356 +++++++++++++++++++++ 2 files changed, 413 insertions(+), 63 deletions(-) create mode 100644 tests/orchestrator-check-processor.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f8df367a..a8e24436 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -12,6 +12,7 @@ import { printFileHeader, printIssueRow, printEvaluationSummaries, type Evaluati import { checkTarget } from '../prompts/target'; import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; +import { processFindings } from '../findings'; import { createEvaluator } from '../evaluators/index'; import { Type, Severity } from '../evaluators/types'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; @@ -616,79 +617,72 @@ function routePromptResult( let promptErrors = 0; let promptWarnings = 0; - // Handle Check Result + // Handle Check Result — routed through the shared finding processor + // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, + // filters, deduplicates, scores, and resolves severity; the orchestrator + // only feeds the returned ReviewResult to the existing formatter sinks. if (!isJudgeResult(result)) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - - // Score calculated from surfaced violations only — matches what user sees - const scored = calculateCheckScore( - surfacedViolations, - result.word_count, - { - strictness: promptFile.meta.strictness, - promptSeverity: promptFile.meta.severity, - } - ); - const severity = scored.severity; - // Group violations by criterionName - const violationsByCriterion = new Map< - string | undefined, - typeof surfacedViolations - >(); - for (const v of surfacedViolations) { - const criterionName = v.criterionName; - if (!violationsByCriterion.has(criterionName)) { - violationsByCriterion.set(criterionName, []); - } - violationsByCriterion.get(criterionName)!.push(v); - } - - // Report violations grouped by criterion - let totalErrors = 0; - let totalWarnings = 0; + const reviewResult = processFindings({ + pack: promptFile.pack, + ruleId: promptId, + ruleSource: promptFile.fullPath, + candidateFindings: result.violations, + wordCount: result.word_count, + promptMeta: { + ...(meta.severity !== undefined ? { severity: meta.severity } : {}), + ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), + ...(meta.criteria ? { criteria: meta.criteria } : {}), + }, + targetContent: content, + }); - for (const [criterionName, violations] of violationsByCriterion) { - // Find criterion ID from meta - let criterionId: string | undefined; - if (criterionName && meta.criteria) { - const criterion = meta.criteria.find(c => c.name === criterionName); - criterionId = criterion?.id; - } + // processFindings always returns exactly one score entry (processor contract). + const ruleScore = reviewResult.scores[0]!; + const severity = + ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - const ruleName = buildRuleName(promptFile.pack, promptId, criterionId); - - if (violations.length > 0) { - const violationResult = locateAndReportViolations({ - violations, - content, - relFile, - severity, - ruleName, - scoreText: '', - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; + // Report only verified findings through the existing line/json/rdjson/vale sink. + for (const finding of reviewResult.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity, + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); + } - if (severity === Severity.ERROR) { - totalErrors += violations.length; - } else { - totalWarnings += violations.length; - } + // Diagnostics (e.g. finding-evidence-not-locatable) surface in verbose mode, + // consistent with prior operational reporting. They are warn-level and do not + // flag the run as operationally failed (audit Finding #6). + if (verbose) { + for (const diagnostic of reviewResult.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } } - // Create scoreEntry for Quality Scores display + // Verified finding count drives the counts (audit Finding #6). + const findingCount = reviewResult.findings.length; + const totalErrors = severity === Severity.ERROR ? findingCount : 0; + const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; + const scoreEntry: EvaluationSummary = { - id: buildRuleName(promptFile.pack, promptId, undefined), - scoreText: `${scored.final_score.toFixed(1)}/10`, - score: scored.final_score, + id: ruleScore.ruleId, + scoreText: ruleScore.scoreText, + score: ruleScore.score, }; if (debugJson) { + const { decisions, surfacedViolations } = getViolationFilterResults( + result.violations + ); const runId = randomUUID(); const model = getModelInfoFromEnv(); @@ -721,7 +715,7 @@ function routePromptResult( return { errors: totalErrors, warnings: totalWarnings, - hadOperationalErrors, + hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, scoreEntries: [scoreEntry], }; diff --git a/tests/orchestrator-check-processor.test.ts b/tests/orchestrator-check-processor.test.ts new file mode 100644 index 00000000..eaaa1529 --- /dev/null +++ b/tests/orchestrator-check-processor.test.ts @@ -0,0 +1,356 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { evaluateFiles } from "../src/cli/orchestrator"; +import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; +import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { Result } from "../src/output/json-formatter"; +import type { ValeOutput } from "../src/schemas/vale-responses"; +import type { PromptFile } from "../src/prompts/prompt-loader"; +import type { RawCheckResult } from "../src/prompts/schema"; + +const { EVALUATE_MOCK } = vi.hoisted(() => ({ + EVALUATE_MOCK: vi.fn(), +})); + +type CheckViolation = RawCheckResult["violations"][number]; + +vi.mock("../src/evaluators/index", () => ({ + createEvaluator: vi.fn(() => ({ + evaluate: EVALUATE_MOCK, + })), +})); + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +} as const; + +function createPrompt(meta: PromptFile["meta"]): PromptFile { + return { + id: meta.id, + filename: `${meta.id}.md`, + fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), + meta, + body: "Prompt body", + pack: "TestPack", + }; +} + +function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Line, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-check-proc-")); + const filePath = path.join(dir, "input.md"); + writeFileSync(filePath, content); + return filePath; +} + +/** + * Builds a check violation that passes the confidence gate by default. Real + * model output always carries `message`; the helper sets it explicitly so the + * test reflects production behavior rather than the analysis fallback. + */ +function makeCheckViolation( + overrides: Partial = {} +): CheckViolation { + return { + line: 1, + analysis: "Issue 1", + message: "Issue 1", + suggestion: "Suggestion 1", + fix: "Fix 1", + quoted_text: "Alpha text", + context_before: "", + context_after: "", + rule_quote: "Rule quote", + checks: FULLY_SUPPORTED_CHECKS, + confidence: 0.9, + ...overrides, + }; +} + +function makeCheckResult(params: { + violations: CheckViolation[]; + wordCount?: number; +}): RawCheckResult { + return { + type: EvaluationType.CHECK, + violations: params.violations, + word_count: params.wordCount ?? 100, + }; +} + +describe("standard check evaluation via the shared finding processor", () => { + const originalThreshold = process.env.CONFIDENCE_THRESHOLD; + + beforeEach(() => { + EVALUATE_MOCK.mockReset(); + delete process.env.CONFIDENCE_THRESHOLD; + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + afterEach(() => { + if (originalThreshold === undefined) { + delete process.env.CONFIDENCE_THRESHOLD; + } else { + process.env.CONFIDENCE_THRESHOLD = originalThreshold; + } + vi.restoreAllMocks(); + }); + + it("reports fully locatable findings, score, and counts unchanged", async () => { + const targetFile = createTempFile("Alpha text\nBeta text\n"); + const prompt = createPrompt({ + id: "CheckPrompt", + name: "Check Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ message: "First issue", analysis: "First issue" }), + makeCheckViolation({ + line: 2, + quoted_text: "Beta text", + message: "Second issue", + analysis: "Second issue", + suggestion: "Suggestion 2", + fix: "Fix 2", + }), + ], + wordCount: 100, + }) + ); + + const logCalls: string[] = []; + vi.spyOn(console, "log").mockImplementation((...args) => { + logCalls.push(args.map(String).join(" ")); + }); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + // Both quotes anchor: 2 verified findings over 100 words -> 8.0/10. + expect(run.totalWarnings).toBe(2); + expect(run.totalErrors).toBe(0); + expect(run.hadOperationalErrors).toBe(false); + + const scoreLine = logCalls.find((l) => l.includes("/10")); + expect(scoreLine).toBeDefined(); + expect(scoreLine).toContain("8.0/10"); + // The line summary prints the last segment of the Pack.Rule score id. + expect(scoreLine).toContain("CheckPrompt"); + }); + + it("counts only verified findings and excludes unanchored quotes from the score", async () => { + // One anchored + one unanchored quote. Both pass the confidence gate, so + // only the verifier distinguishes them. Verified count = 1 -> 9.0/10. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CountingFixPrompt", + name: "Counting Fix Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ quoted_text: "Alpha text" }), + makeCheckViolation({ + line: 9, + quoted_text: "this quote is not anywhere in the content", + message: "Ghost issue", + analysis: "Ghost issue", + }), + ], + wordCount: 100, + }) + ); + + const logCalls: string[] = []; + vi.spyOn(console, "log").mockImplementation((...args) => { + logCalls.push(args.map(String).join(" ")); + }); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + // Intentional Phase 3 fix: the unanchored quote is a diagnostic, not a + // finding; it contributes neither to the count nor the score. + expect(run.totalWarnings).toBe(1); + expect(run.hadOperationalErrors).toBe(false); + + const scoreLine = logCalls.find((l) => l.includes("/10")); + expect(scoreLine).toBeDefined(); + expect(scoreLine).toContain("9.0/10"); + }); + + it("does not flag severity errors when no finding can be verified", async () => { + // severity=error, but the only quote cannot anchor -> 0 verified findings -> + // perfect 10.0/10 -> severity resolves to warning -> no error exit signal. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "ErrorNoVerifiedPrompt", + name: "Error No Verified Prompt", + type: "check", + severity: Severity.ERROR, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ + line: 9, + quoted_text: "nowhere to be found", + message: "Ghost", + analysis: "Ghost", + }), + ], + wordCount: 100, + }) + ); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(run.totalErrors).toBe(0); + expect(run.hadSeverityErrors).toBe(false); + expect(run.hadOperationalErrors).toBe(false); + }); + + it("flags severity errors when verified error-severity findings exist", async () => { + // 1 verified finding over 10 words -> density 10 -> score 0.0 < 10 -> + // prompt severity error applies. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "ErrorVerifiedPrompt", + name: "Error Verified Prompt", + type: "check", + severity: Severity.ERROR, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [makeCheckViolation({ quoted_text: "Alpha text" })], + wordCount: 10, + }) + ); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(run.totalErrors).toBe(1); + expect(run.hadSeverityErrors).toBe(true); + }); + + it("emits verified findings with anchored location through the JSON sink", async () => { + const targetFile = createTempFile("Alpha text\nBeta text\n"); + const prompt = createPrompt({ + id: "CheckJsonPrompt", + name: "Check JSON Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ message: "First", analysis: "First" }), + makeCheckViolation({ + line: 2, + quoted_text: "Beta text", + message: "Second", + analysis: "Second", + }), + ], + wordCount: 100, + }) + ); + + await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.Json, + }); + + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as Result; + const issues = Object.values(parsed.files).flatMap((file) => file.issues); + + expect(issues).toHaveLength(2); + expect(issues[0]).toMatchObject({ + line: 1, + column: 1, + severity: Severity.WARNING, + message: "First", + rule: "TestPack.CheckJsonPrompt", + match: "Alpha text", + }); + expect(issues[1]).toMatchObject({ + line: 2, + message: "Second", + match: "Beta text", + }); + }); + + it("omits unanchored quotes from the Vale JSON output", async () => { + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CheckValePrompt", + name: "Check Vale Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ quoted_text: "Alpha text" }), + makeCheckViolation({ + line: 9, + quoted_text: "missing quote", + message: "Ghost", + analysis: "Ghost", + }), + ], + wordCount: 100, + }) + ); + + await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.ValeJson, + }); + + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as ValeOutput; + const issues = Object.values(parsed).flat(); + + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + Check: "TestPack.CheckValePrompt", + Line: 1, + Match: "Alpha text", + Severity: "warning", + }); + }); +}); From 9fe6440c7cc768ec0da72cb2a7fd2d73330b51d5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 07:30:36 +0100 Subject: [PATCH 14/26] Reject judge prompt type at the loader boundary - PROMPT_META_SCHEMA now rejects `type: judge` and its deprecated alias `subjective` via a superRefine, so judge-typed prompts fail to load with a descriptive warning instead of reaching evaluation. - superRefine (not refine) keeps the inferred union including "judge" so the legacy BaseEvaluator type-checks unchanged until Phase 4 deletes that path. - Add loader tests covering judge/subjective rejection and that check still loads; subjective and semi-objective are documented as deprecated aliases. --- src/schemas/prompt-schemas.ts | 26 ++++++++++--- tests/prompt-loader-validation.test.ts | 51 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/schemas/prompt-schemas.ts b/src/schemas/prompt-schemas.ts index f8689645..2dadd85e 100644 --- a/src/schemas/prompt-schemas.ts +++ b/src/schemas/prompt-schemas.ts @@ -27,12 +27,15 @@ export const PROMPT_CRITERION_SCHEMA = z.object({ * - 'technical-accuracy': specialized evaluator with claim extraction + search * * Evaluation type: - * - 'judge': 1-4 scores per criterion, normalized to 1-10 * - 'check': density-based scoring (errors per 100 words) + * - 'semi-objective': deprecated alias for 'check' * - * Deprecated aliases (still supported): - * - 'subjective' → 'judge' - * - 'semi-objective' → 'check' + * 'judge' and its deprecated alias 'subjective' are no longer supported + * (Phase 3): subjective rubric scoring is not a future-facing review type. + * The enum still admits them so the Zod error is descriptive and the inferred + * type stays compatible with the legacy BaseEvaluator until Phase 4 deletes + * that path; a superRefine rejects them at this boundary so judge-typed prompts + * fail to load. * * Strictness factor for check scoring: * - Determines penalty weight per 1% error density. @@ -44,11 +47,24 @@ export const PROMPT_META_SCHEMA = z.object({ type: z .enum(["judge", "check", "subjective", "semi-objective"]) .transform((val) => { - // Map deprecated values to new canonical values + // Map deprecated values to their canonical forms. if (val === "subjective") return "judge" as const; if (val === "semi-objective") return "check" as const; return val; }) + .superRefine((val, ctx) => { + // Judge/rubric reviews are not a future-facing review type (Phase 3). + // superRefine (not refine) so the inferred union still includes 'judge' + // and stays compatible with the legacy BaseEvaluator until Phase 4 deletes + // that path; the value is rejected at this boundary regardless. + if (val === "judge") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "judge evaluation is no longer supported; use check (Via Negativa) rules", + }); + } + }) .optional(), id: z.string(), name: z.string(), diff --git a/tests/prompt-loader-validation.test.ts b/tests/prompt-loader-validation.test.ts index 094ad7cb..fc34cc1d 100644 --- a/tests/prompt-loader-validation.test.ts +++ b/tests/prompt-loader-validation.test.ts @@ -130,4 +130,55 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); }); + + describe('Judge boundary rejection (Phase 3)', () => { + it('rejects judge-typed prompts at the loader boundary', () => { + createPrompt('judge.md', `--- +evaluator: base +id: Judge +name: Judge Prompt +type: judge +criteria: + - name: Clarity + id: Clarity +--- +Judge content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(prompts).toHaveLength(0); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings[0]).toContain('judge.md'); + expect(warnings[0]).toMatch(/judge/i); + }); + + it('rejects subjective-typed prompts (deprecated judge alias)', () => { + createPrompt('subjective.md', `--- +evaluator: base +id: Subjective +name: Subjective Prompt +type: subjective +--- +Subjective content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(prompts).toHaveLength(0); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings[0]).toContain('subjective.md'); + }); + + it('still loads check-typed prompts', () => { + createPrompt('check.md', `--- +evaluator: base +id: Check +name: Check Prompt +type: check +--- +Check content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(warnings).toHaveLength(0); + expect(prompts).toHaveLength(1); + expect(prompts[0].meta.type).toBe('check'); + }); + }); }); From d36d06dd139439667ac765b4adfe1e1162a76e42 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 07:30:54 +0100 Subject: [PATCH 15/26] Remove legacy judge review path - Drop the judge branch of routePromptResult and the now-dead criterion extraction/reporting helpers (extractAndReportCriterion, validateCriteriaCompleteness, validateScores, locateAndReportViolations, buildRuleName); subjective rubric scoring is not a future-facing review type. - A JudgeResult that still reaches the orchestrator is refused explicitly (operational error, no findings) rather than misprojected as a check result; no adapter is added for old judge metadata. - Standard objective-check orchestration still routes through processFindings unchanged. - Replace the judge-filtering orchestrator test with a judge-rejection test. --- src/cli/orchestrator.ts | 545 +-------------------------- tests/orchestrator-filtering.test.ts | 41 +- 2 files changed, 28 insertions(+), 558 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index a8e24436..4d09e917 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -6,10 +6,9 @@ import * as os from 'os'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; -import { JsonFormatter, type Issue, type ScoreComponent } from '../output/json-formatter'; +import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { checkTarget } from '../prompts/target'; import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; import { processFindings } from '../findings'; @@ -21,8 +20,7 @@ import { runAgentExecutor, type AgentExecutorResult, type AgentFinding } from '. import { AgentProgressReporter, shouldEmitAgentProgress } from '../agent/progress'; import type { EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessViolationsParams, - ProcessCriterionParams, ProcessCriterionResult, ValidationParams, ProcessPromptResultParams, + ReportIssueParams, ProcessPromptResultParams, RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, RunPromptEvaluationResultSuccess } from './types'; @@ -33,7 +31,6 @@ import { import { calculateCheckScore } from '../scoring'; import { countWords } from '../chunking/utils'; import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; -import { locateQuotedText } from "../output/location"; import { computeFilterDecision, type FilterDecision, @@ -64,23 +61,6 @@ function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } -/* - * Constructs a hierarchical rule name following the pattern: - * - With criterion: PackName.RuleId.CriterionId - * - Without criterion: PackName.RuleId - */ -function buildRuleName( - packName: string, - ruleId: string, - criterionId: string | undefined -): string { - const parts = [packName, ruleId]; - if (criterionId) { - parts.push(criterionId); - } - return parts.join('.'); -} - /* * Generic concurrency runner that executes workers in parallel up to a specified limit. * Preserves result order matching input order. @@ -174,119 +154,6 @@ function reportIssue(params: ReportIssueParams): void { } } -/* - * Locates and reports each violation using pre/post evidence markers. - * If location matching fails (missing markers, content mismatch), logs warning - * and continues processing. Returns hadOperationalErrors=true if any violations - * couldn't be located, signaling text matching issues vs. content quality issues. - */ -function locateAndReportViolations(params: ProcessViolationsParams): { - hadOperationalErrors: boolean; -} { - const { - violations, - content, - relFile, - severity, - ruleName, - scoreText, - outputFormat, - jsonFormatter, - verbose, - } = params; - - let hadOperationalErrors = false; - - // Locate all violations and filter out those that can't be verified - // Then de-duplicate by (quoted_text, line) - const seen = new Set(); - const verifiedViolations: Array<{ - v: (typeof violations)[0]; - line: number; - column: number; - matchedText: string; - rowSummary: string; - }> = []; - - for (const v of violations) { - if (!v) continue; - - const rowSummary = (v.message || "").trim(); - - try { - const locWithMatch = locateQuotedText( - content, - { - quoted_text: v.quoted_text || "", - context_before: v.context_before || "", - context_after: v.context_after || "", - }, - 80, - v.line - ); - - if (!locWithMatch) { - // Can't verify this quote exists - skip it entirely - if (verbose) { - console.warn( - `[vectorlint] Skipping unverifiable quote: "${v.quoted_text}"` - ); - } - hadOperationalErrors = true; - continue; - } - - const line = locWithMatch.line; - const column = locWithMatch.column; - const matchedText = locWithMatch.match || ""; - - // De-duplicate by (quoted_text, line) - skip if quoted_text is empty - const dedupeKey = v.quoted_text ? `${v.quoted_text}:${line}` : null; - if (dedupeKey && seen.has(dedupeKey)) { - continue; // Skip duplicate - } - if (dedupeKey) { - seen.add(dedupeKey); - } - - verifiedViolations.push({ v, line, column, matchedText, rowSummary }); - } catch (e: unknown) { - const err = handleUnknownError(e, "Locating evidence"); - if (verbose) { - console.warn(`[vectorlint] Error locating evidence: ${err.message}`); - } - hadOperationalErrors = true; - } - } - - // Report only verified, unique violations - for (const { - v, - line, - column, - matchedText, - rowSummary, - } of verifiedViolations) { - reportIssue({ - file: relFile, - line, - column, - severity, - summary: rowSummary, - ruleName, - outputFormat, - jsonFormatter, - ...(v.analysis !== undefined && { analysis: v.analysis }), - ...(v.suggestion !== undefined && { suggestion: v.suggestion }), - ...(v.fix !== undefined && { fix: v.fix }), - scoreText, - match: matchedText, - }); - } - - return { hadOperationalErrors }; -} - function getViolationFilterResults< TViolation extends Parameters[0] >( @@ -304,297 +171,10 @@ function getViolationFilterResults< } /* - * Extracts pre-calculated scores from a subjective evaluation criterion and reports surfaced violations. - * Violations that do not pass computeFilterDecision are not reported. - * Returns error/warning counts, score entry for Quality Scores, and score components for JSON. - */ -function extractAndReportCriterion( - params: ProcessCriterionParams -): ProcessCriterionResult { - const { - exp, - result, - content, - relFile, - packName, - promptId, - meta, - outputFormat, - jsonFormatter, - verbose, - } = params; - let hadOperationalErrors = false; - let hadSeverityErrors = false; - - const nameKey = String(exp.name || exp.id || ""); - const criterionId = exp.id - ? String(exp.id) - : exp.name - ? String(exp.name) - .replace(/[^A-Za-z0-9]+/g, " ") - .split(" ") - .filter(Boolean) - .map((s) => s.charAt(0).toUpperCase() + s.slice(1)) - .join("") - : ""; - const ruleName = buildRuleName(packName, promptId, criterionId); - - const weightNum = exp.weight || 1; - const maxScore = weightNum; - - // Target gating (deterministic precheck) - const metaTargetSpec = meta.target; - const expTargetSpec = exp.target; - const targetCheck = checkTarget(content, metaTargetSpec, expTargetSpec); - const missingTarget = targetCheck.missing; - - if (missingTarget) { - hadSeverityErrors = true; - const summary = "target not found"; - const suggestion = - targetCheck.suggestion || - expTargetSpec?.suggestion || - metaTargetSpec?.suggestion || - "Add the required target section."; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity: Severity.ERROR, - summary, - ruleName, - outputFormat, - jsonFormatter, - suggestion, - scoreText: "nil", - match: "", - }); - return { - errors: 1, - warnings: 0, - userScore: 0, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText: "0.0/10", score: 0.0 }, - scoreComponent: { - criterion: nameKey, - rawScore: 0, - maxScore: 4, - weightedScore: 0, - weightedMaxScore: weightNum, - normalizedScore: 0, - normalizedMaxScore: 10, - }, - }; - } - - const got = result.criteria.find( - (c) => c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase() - ); - if (!got) { - return { - errors: 0, - warnings: 0, - userScore: 0, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText: "-", score: 0.0 }, - scoreComponent: { - criterion: nameKey, - rawScore: 0, - maxScore: 4, - weightedScore: 0, - weightedMaxScore: weightNum, - normalizedScore: 0, - normalizedMaxScore: 10, - }, - }; - } - - const score = Number(got.score); - - // Use pre-calculated values from evaluator - const rawWeighted = got.weighted_points; - const normalizedScore = got.normalized_score; - const userScore = rawWeighted; - const violations = got.violations; - const { surfacedViolations } = getViolationFilterResults(violations); - - // Display normalized score (1-10) in CLI output - const scoreText = `${normalizedScore.toFixed(1)}/10`; - - // Determine severity based on violations - // If there are violations, use evaluator's scoring to determine severity - // Score <= 1 = error, score = 2 = warning, score > 2 = no severity needed (but we still create scoreEntry) - let errors = 0; - let warnings = 0; - let severity: Severity | undefined; - - if (surfacedViolations.length > 0) { - // Determine severity from score for violations - if (score <= 1) { - severity = Severity.ERROR; - hadSeverityErrors = true; - errors = surfacedViolations.length; - } else if (score === 2) { - severity = Severity.WARNING; - warnings = surfacedViolations.length; - } else { - // Score > 2 but has violations - this is informational - // Use WARNING as default for informational violations - severity = Severity.WARNING; - warnings = surfacedViolations.length; - } - - // Report surfaced violations only - const violationResult = locateAndReportViolations({ - violations: surfacedViolations as Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - analysis?: string; - suggestion?: string; - }>, - content, - relFile, - severity, - ruleName, - scoreText, - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - hadOperationalErrors = - hadOperationalErrors || violationResult.hadOperationalErrors; - } else if (score <= 2) { - // No violations but low score - report with summary - severity = score <= 1 ? Severity.ERROR : Severity.WARNING; - if (severity === Severity.ERROR) { - hadSeverityErrors = true; - errors = 1; - } else { - warnings = 1; - } - - const sum = got.summary.trim(); - const words = sum.split(/\s+/).filter(Boolean); - const limited = words.slice(0, 15).join(" "); - const summaryText = limited || "No findings"; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity, - summary: summaryText, - ruleName, - outputFormat, - jsonFormatter, - scoreText, - match: "", - }); - } - - return { - errors, - warnings, - userScore, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText, score: normalizedScore }, - scoreComponent: { - criterion: nameKey, - rawScore: score, - maxScore: 4, - weightedScore: rawWeighted, - weightedMaxScore: weightNum, - normalizedScore: normalizedScore, - normalizedMaxScore: 10, - }, - }; -} - -/* - * Validates that all expected criteria are present in the result. - */ -function validateCriteriaCompleteness(params: ValidationParams): boolean { - const { meta, result } = params; - let hadErrors = false; - - const expectedNames = new Set( - (meta.criteria || []).map((c) => String(c.name || c.id || "")) - ); - const returnedNames = new Set( - result.criteria.map((c: { name: string }) => c.name) - ); - - // Create normalized maps for case-insensitive lookup - const expectedNormalized = new Set(); - const expectedOriginalMap = new Map(); - for (const name of expectedNames) { - const norm = name.toLowerCase(); - expectedNormalized.add(norm); - expectedOriginalMap.set(norm, name); - } - - const returnedNormalized = new Set(); - for (const name of returnedNames) { - returnedNormalized.add(name.toLowerCase()); - } - - for (const norm of expectedNormalized) { - if (!returnedNormalized.has(norm)) { - console.error( - `Missing criterion in model output: ${expectedOriginalMap.get(norm)}` - ); - hadErrors = true; - } - } - - for (const name of returnedNames) { - if (!expectedNormalized.has(name.toLowerCase())) { - console.warn( - `[vectorlint] Extra criterion returned by model (ignored): ${name}` - ); - } - } - - return hadErrors; -} - -/* - * Validates that all criterion scores are within valid range. - */ -function validateScores(params: ValidationParams): boolean { - const { meta, result } = params; - let hadErrors = false; - - for (const exp of meta.criteria || []) { - const nameKey = String(exp.name || exp.id || ""); - const got = result.criteria.find( - (c) => - c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase() - ); - if (!got) continue; - - const score = Number(got.score); - if (!Number.isFinite(score) || score < 0 || score > 4) { - console.error(`Invalid score for ${nameKey}: ${score}`); - hadErrors = true; - } - } - - return hadErrors; -} - -/* - * Routes evaluation results through check or judge processing paths. - * Check: Reports violations, creates scoreEntry using final_score. - * Judge: Iterates through criteria, validates scores, creates scoreEntry per criterion. - * Both paths generate scoreEntries for Quality Scores display. + * Routes an evaluation result through the shared finding processor. + * Check results are verified, filtered, scored, and reported; judge/rubric + * results are rejected (Phase 3) because subjective scoring is not a + * future-facing review type. */ function routePromptResult( params: ProcessPromptResultParams @@ -612,11 +192,6 @@ function routePromptResult( const meta = promptFile.meta; const promptId = (meta.id || "").toString(); - let hadOperationalErrors = false; - let hadSeverityErrors = false; - let promptErrors = 0; - let promptWarnings = 0; - // Handle Check Result — routed through the shared finding processor // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, // filters, deduplicates, scores, and resolves severity; the orchestrator @@ -721,105 +296,21 @@ function routePromptResult( }; } - // Handle Judge Result - // Validate criterion completeness and scores - hadOperationalErrors = - validateCriteriaCompleteness({ meta, result }) || hadOperationalErrors; - hadOperationalErrors = - validateScores({ meta, result }) || hadOperationalErrors; - - // Reset promptErrors and promptWarnings for subjective results - promptErrors = 0; - promptWarnings = 0; - const criterionScores: EvaluationSummary[] = []; - const scoreComponents: ScoreComponent[] = []; - - // Iterate through each criterion - for (const exp of meta.criteria || []) { - const criterionResult = extractAndReportCriterion({ - exp, - result, - content, - relFile, - packName: promptFile.pack, - promptId, - promptFilename: promptFile.filename, - meta, - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - - promptErrors += criterionResult.errors; - promptWarnings += criterionResult.warnings; - hadOperationalErrors = - hadOperationalErrors || criterionResult.hadOperationalErrors; - hadSeverityErrors = hadSeverityErrors || criterionResult.hadSeverityErrors; - criterionScores.push(criterionResult.scoreEntry); - - if (criterionResult.scoreComponent) { - scoreComponents.push(criterionResult.scoreComponent); - } - } - - if (outputFormat === OutputFormat.Json && scoreComponents.length > 0) { - (jsonFormatter as JsonFormatter | RdJsonFormatter).addEvaluationScore( - relFile, - { - id: promptId || promptFile.filename.replace(/\.md$/, ""), - scores: scoreComponents, - } + // Judge/rubric reviews are not a future-facing review type (Phase 3). The + // prompt-meta boundary rejects `type: judge`, so a JudgeResult here means a + // caller bypassed that boundary. Refuse it explicitly rather than + // misprojecting it as a check result. No adapter is added for judge metadata. + if (verbose) { + console.warn( + '[vectorlint] Judge/rubric review results are no longer supported; skipping prompt.' ); } - - if (debugJson) { - const runId = randomUUID(); - const flat = result.criteria.flatMap((c) => - (c.violations || []).map((v, i) => ({ - criterion: c.name, - index: i, - violation: v, - decision: computeFilterDecision(v), - })) - ); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - evaluation_type: "judge", - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: flat.map((x) => ({ - criterion: x.criterion, - index: x.index, - surface: x.decision.surface, - reasons: x.decision.reasons, - })), - surfaced_violations: flat.filter((x) => x.decision.surface).map((x) => ({ - criterion: x.criterion, - violation: x.violation, - })), - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } - return { - errors: promptErrors, - warnings: promptWarnings, - hadOperationalErrors, - hadSeverityErrors, - scoreEntries: criterionScores, + errors: 0, + warnings: 0, + hadOperationalErrors: true, + hadSeverityErrors: false, + scoreEntries: [], }; } diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts index 73e56119..b4d2cf89 100644 --- a/tests/orchestrator-filtering.test.ts +++ b/tests/orchestrator-filtering.test.ts @@ -307,7 +307,11 @@ describe("CLI violation filtering", () => { expect(scoreLine).not.toContain("8.0/10"); }); - it("filters low-confidence judge violations from CLI counts by default", async () => { + it("rejects judge/rubric results instead of projecting them as check findings", async () => { + // Judge/rubric reviews are not a future-facing review type (Phase 3). The + // orchestrator refuses a JudgeResult rather than running the old criterion + // extraction/reporting path: no findings are projected and the run is + // flagged with an operational error. const targetFile = createTempFile("Alpha text\nBeta text\n"); const prompt = createPrompt({ id: "JudgePrompt", @@ -323,42 +327,17 @@ describe("CLI violation filtering", () => { makeJudgeViolation({ line: 2, quoted_text: "Beta text", - description: "Issue 2", - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", confidence: 0.2, }), ]) ); - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalWarnings).toBe(1); + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeJudgeResult([ - makeJudgeViolation(), - makeJudgeViolation({ - line: 2, - quoted_text: "Beta text", - description: "Issue 2", - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - confidence: 0.2, - }), - ]) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalWarnings).toBe(2); + expect(run.totalErrors).toBe(0); + expect(run.totalWarnings).toBe(0); + expect(run.hadOperationalErrors).toBe(true); + expect(run.hadSeverityErrors).toBe(false); }); it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { From d4ea9a7764f682e1288a09303f7fa94d0c20c599 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:24:20 +0100 Subject: [PATCH 16/26] Remove orphaned orchestrator param types from CLI contract - Drop ProcessViolationsParams, ProcessCriterionParams, ProcessCriterionResult, and ValidationParams from src/cli/types.ts. They were orphaned when the orchestrator's inline violation/criterion processing moved to src/findings (Tasks 2-3) and the judge path was removed, and are referenced nowhere else. - Remove the now-unused imports PromptMeta, PromptCriterionSpec, ScoreComponent, and JudgeResult. Severity and PromptEvaluationResult are retained (still used by ReportIssueParams / ProcessPromptResultParams). --- src/cli/types.ts | 42 ++---------------------------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/src/cli/types.ts b/src/cli/types.ts index 7f95192a..a083f2e2 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,13 +1,12 @@ import type { PromptFile } from '../prompts/prompt-loader'; import type { LLMProvider } from '../providers/llm-provider'; import type { SearchProvider } from '../providers/search-provider'; -import type { PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; -import { JsonFormatter, type ScoreComponent } from '../output/json-formatter'; +import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { PromptEvaluationResult, JudgeResult } from '../prompts/schema'; +import type { PromptEvaluationResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; @@ -95,43 +94,6 @@ export interface ReportIssueParams { match?: string; } -export interface ProcessViolationsParams extends EvaluationContext { - violations: Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - message?: string; - analysis?: string; - suggestion?: string; - fix?: string; - }>; - severity: Severity; - ruleName: string; - scoreText: string; -} - -export interface ProcessCriterionParams extends EvaluationContext { - exp: PromptCriterionSpec; - result: JudgeResult; - packName: string; - promptId: string; - promptFilename: string; - meta: PromptMeta; -} - -export interface ProcessCriterionResult extends ErrorTrackingResult { - userScore: number; - maxScore: number; - scoreEntry: { id: string; scoreText: string; score?: number }; - scoreComponent?: ScoreComponent; -} - -export interface ValidationParams { - meta: PromptMeta; - result: JudgeResult; -} - export interface ProcessPromptResultParams extends EvaluationContext { promptFile: PromptFile; result: PromptEvaluationResult; From 15ec7a3e146df0820d39b9680e06b0369d5dd33e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:24:33 +0100 Subject: [PATCH 17/26] Document finding-processing behavior and clean stale judge docs - Add tests/findings/README.md mapping the behavior moved into src/findings (filtering, evidence verification, density scoring, rule-id/severity, formatter routing) and the intentionally removed judge/rubric and agent behavior, with agent tests marked out of scope for Phase 3 (Phase 3 Task 8). - Stop describing type: judge/subjective as a valid future-facing prompt type in CREATING_RULES.md, docs/frontmatter-fields.mdx, and docs/style-guide.mdx; note that judge/subjective are now rejected at load. - Drop the stale rubric-based scoring feature blurb from README.md. --- CREATING_RULES.md | 197 +++--------------------------------- README.md | 3 +- docs/frontmatter-fields.mdx | 80 ++------------- docs/style-guide.mdx | 113 ++------------------- tests/findings/README.md | 83 +++++++++++++++ 5 files changed, 117 insertions(+), 359 deletions(-) create mode 100644 tests/findings/README.md diff --git a/CREATING_RULES.md b/CREATING_RULES.md index ec90954f..7b800c9d 100644 --- a/CREATING_RULES.md +++ b/CREATING_RULES.md @@ -8,7 +8,6 @@ A comprehensive guide to creating powerful, reusable content evaluations using V - [Rule Anatomy](#rule-anatomy) - [Evaluation Modes](#evaluation-modes) - [Check Rules](#check-rules) -- [Judge Rules](#judge-rules) - [Target Specification](#target-specification) - [Configuration Reference](#configuration-reference) - [Best Practices](#best-practices) @@ -25,7 +24,7 @@ VectorLint rules are Markdown files with YAML frontmatter that define how your c - **Rule = Prompt file** (`.md` file organized in rule packs) - **Pack** = Subdirectory containing related rules (typically named after a company/style guide) - **Criteria** = Individual quality checks within a rule -- **Score** = LLM-assigned rating (1-4 scale for judge, density-based for check) +- **Score** = Density-based quality score derived from violation count - **Severity** = How failures are reported (`error` or `warning`) @@ -74,27 +73,20 @@ project/ ## Evaluation Modes -VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in two distinct modes, determined by the `type` field: +VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in one mode, determined by the `type` field: | Mode | `type` | Use Case | Scoring | Output | | ---------- | ------- | ------------------------------------- | --------------------------- | ----------------------- | | **Check** | `check` | Pass/fail checks, counting violations | 10 points - 1 per violation | List of specific issues | -| **Judge** | `judge` | Multi-dimensional quality scoring | 0-4 scale per criterion | Weighted average score | -### When to Use Each +The deprecated `type: judge` (alias `subjective`) is no longer supported; a rule using it is rejected at load. -**Use Check when:** +### When to Use Check - You need to find specific errors (e.g., "Find all grammar mistakes") - The check is binary (Pass/Fail) for each item - You want a list of specific violations to fix -**Use Judge when:** - -- You're measuring quality on a spectrum (e.g., "How engaging is this?") -- You have multiple dimensions (Clarity, Tone, Depth) -- You need weighted importance (some criteria matter more) - --- ## Check Rules @@ -141,71 +133,6 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes --- -## Judge Rules - -Judge rules use weighted criteria and a 1-4 rubric for sophisticated quality measurement. - -### Structure - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator - -severity: error -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 ---- - -You are a headline evaluator... [Your detailed instructions] - -## RUBRIC - -# Value Communication - -### Excellent - -Specific, immediately appealing benefit - -### Good - -Clear benefit but less specific impact - -... -``` - -### The 1-4 Scoring Scale - -VectorLint uses a **1-4 scale** for all judge criteria, which is then normalized to a 1-10 scale: - -| LLM Rating | Meaning | Normalized Score | -| :--------- | :-------- | :--------------- | -| **4** | Excellent | **10.0** | -| **3** | Good | **7.0** | -| **2** | Fair | **4.0** | -| **1** | Poor | **1.0** | - -### Score Calculation - -1. **Normalization**: We map the 1-4 rating to a 1-10 score using the formula: `1 + ((Rating - 1) / 3) * 9`. -2. **Weighted Average**: The final score is the weighted average of all normalized criterion scores. - -**Example:** - -- Criterion: "Value Communication" (weight=12) -- Rating: 3 (Good) -> Normalized: 7.0 -- Weighted Points: 7.0 \* 12 = 84 points - ---- - ## Target Specification The `target` field allows you to: @@ -243,27 +170,17 @@ target: ### Frontmatter Fields -| Field | Type | Required | Description | -| ------------- | ------------- | -------- | ------------------------------------------------------------------ | -| `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | -| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | -| `type` | string | No | Mode: `judge` or `check` (default: `check`) | -| `id` | string | **Yes** | Unique identifier (used in error reporting) | -| `name` | string | **Yes** | Human-readable name | +| Field | Type | Required | Description | +| ------------- | ------------- | -------- | -------------------------------------------------------------- | +| `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | +| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | +| `type` | string | No | Mode: `check` (default: `check`). `judge`/`subjective` rejected at load | +| `id` | string | **Yes** | Unique identifier (used in error reporting) | +| `name` | string | **Yes** | Human-readable name | | `severity` | string | No | `error` or `warning` (default: `warning`) | | `evaluateAs` | string | No | `document` or `chunk` - whether to evaluate content as a whole or in chunks (default: `chunk`) | | `target` | object | No | Content matching specification | -| `criteria` | array | **Yes\*** | List of evaluation criteria (\*required for judge) | - -### Criterion Fields - -| Field | Type | Required | Description | -| -------- | ------ | -------- | ------------------------------------------ | -| `name` | string | **Yes** | Human-readable criterion name | -| `id` | string | **Yes** | Unique identifier (PascalCase recommended) | -| `weight` | number | No | Importance weight (default: 1) | -| `target` | object | No | Criterion-specific content matching | --- @@ -288,25 +205,10 @@ You are a headline evaluator for developer blog posts. Assess whether the headli 2. Uses natural, conversational language (avoid buzzwords) 3. Creates curiosity without being clickbait -For each criterion, provide a score (0-4) and specific examples from the text. -``` - -### 2. **Use Meaningful Weights (Subjective)** - -Scale weights to reflect real-world importance: - -```yaml -criteria: - # Technical accuracy is critical - - name: Technical Accuracy - weight: 40 - - # Readability is important - - name: Readability - weight: 30 +For each violation, quote the exact text and suggest a concrete fix. ``` -### 3. **Provide Context in Prompts** +### 2. **Provide Context in Prompts** Help the LLM understand your domain: @@ -338,79 +240,6 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes Report any errors found with specific examples. ``` -### Example 2: Headline Evaluator (Judge) - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: Headline -name: Headline Evaluator - -severity: error -target: - regex: '^#\s+(.+)$' - flags: "mu" - group: 1 - required: true - suggestion: Add an H1 headline for the article. -criteria: - - name: Value Communication - id: ValueCommunication - weight: 10 - - name: Language Authenticity - id: LanguageAuthenticity - weight: 5 ---- - -You are a headline evaluator. Assess the H1 headline for: - -1. **Value Communication** (10 points): Does it clearly state what the reader gains? -2. **Language Authenticity** (5 points): Does it use natural, conversational language? - -## RUBRIC - -# Value Communication - -### Excellent - -Specific, immediately appealing benefit clearly stated - -... -``` - -### Example 3: AI Pattern Detector (Judge) - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: AIPatterns -name: AI Pattern Detector - -severity: warning -criteria: - - name: Language Authenticity - id: LanguageAuthenticity - weight: 40 - - name: Structural Naturalness - id: StructuralNaturalness - weight: 30 ---- - -Detect AI-generated writing patterns in this content. - -## INSTRUCTION - -Scan for common AI patterns: - -1. **Buzzwords**: leverage, synergy, elevate -2. **Formulaic transitions**: Moreover, Furthermore - ... -``` - ## Resources - [VectorLint README](./README.md) - Installation and basic usage diff --git a/README.md b/README.md index f6511e6a..662ac7f0 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,9 @@ If you can write a prompt for it, you can lint it with VectorLint. ## Quality Scores -VectorLint scores your content using error density and a rubric-based system, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. +VectorLint scores your content using error density, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. - **Density-Based Scoring:** For errors that can be counted, scores are calculated based on **error density (errors per 100 words)**, making quality assessment fair across documents of any length. -- **Rubric-Based Scoring:** For more nuanced quality standards, like flow and completeness, scores are graded on a 1-4 rubric system and then normalized to a **1-10 scale**. ## How VectorLint Reduces False Positives diff --git a/docs/frontmatter-fields.mdx b/docs/frontmatter-fields.mdx index 82e7d93c..d840d7b9 100644 --- a/docs/frontmatter-fields.mdx +++ b/docs/frontmatter-fields.mdx @@ -27,14 +27,12 @@ Your LLM prompt goes here. | Field | Default | Description | |---|---|---| -| `specVersion` | _(none)_ | Rule spec version. Set to `1.0.0` for rules using the full judge schema. | +| `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | | `evaluator` | `base` | Evaluator type. Use `base` for standard rules. Use `technical-accuracy` for rules that verify factual claims against live web search (requires a search provider). | -| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` scores content against weighted criteria on a 1–4 rubric. | +| `type` | `check` | Evaluation mode. `check` finds specific, countable violations and scores them by error density. `judge` (and its deprecated `subjective` alias) is no longer supported — a rule with `type: judge` or `type: subjective` is rejected at load. | | `severity` | `warning` | How a failing result is reported. `error` causes a non-zero exit code; `warning` reports the issue without failing the run. | | `evaluateAs` | `chunk` | Whether to evaluate content in sections (`chunk`) or as a single unit (`document`). Chunking is applied automatically to documents over 600 words. Set to `document` to disable chunking for a specific rule. | | `target` | _(none)_ | Regex specification to extract and evaluate a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | -| `criteria` | _(none)_ | Required when `type` is `judge`. Defines the scoring dimensions and their weights. See [Criteria fields](#criteria-fields) below. | - ## Target fields The `target` field narrows evaluation to a specific part of the document. It is an object with the following sub-fields: @@ -58,40 +56,16 @@ target: suggestion: Add an H1 headline for the article. ``` -## Criteria fields - -The `criteria` field is an array of criterion objects used with `type: judge`. Each criterion defines one scoring dimension. - -| Field | Type | Required | Description | -|---|---|---|---| -| `name` | string | Yes | Human-readable criterion name shown in CLI output. | -| `id` | string | Yes | Unique identifier for this criterion. PascalCase recommended (e.g., `TechnicalAccuracy`). Referenced in the Markdown body rubric as `# CriterionName `. | -| `weight` | number | No | Relative importance of this criterion in the weighted average. Higher values increase the criterion's influence on the final score. Defaults to `1`. | -| `target` | object | No | Criterion-specific content matching. Uses the same sub-fields as the top-level `target`. Overrides the rule-level target for this criterion only. | - -**Example — judge rule with two weighted criteria:** - -```yaml -criteria: - - name: Technical Accuracy - id: TechnicalAccuracy - weight: 40 - - name: Readability - id: Readability - weight: 30 -``` - ## Complete example -The following rule uses all commonly-used fields. It targets the H1 headline, judges it against two criteria, and requires the headline to exist before evaluation runs. +The following rule uses the most commonly-used fields. It targets the H1 headline and requires the headline to exist before evaluation runs. ```markdown --- -specVersion: 1.0.0 evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator +type: check +id: HeadlineQuality +name: Headline Quality severity: error evaluateAs: document target: @@ -100,44 +74,10 @@ target: group: 1 required: true suggestion: Add an H1 headline for the article. -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 --- -You are a headline evaluator for developer blog posts. - -## RUBRIC - -# Value Communication - -### Excellent -Specific, immediately appealing benefit clearly stated. - -### Good -Clear benefit but less specific impact. - -### Fair -Vague benefit implied but not stated. - -### Poor -No apparent benefit to the reader. - -# Curiosity Gap - -### Excellent -Creates genuine intrigue without being misleading. - -### Good -Mildly interesting, reader may continue. - -### Fair -Neutral — no curiosity created. - -### Poor -Actively off-putting or confusing. +You are a headline reviewer for developer blog posts. Evaluate the H1 headline +and report specific, fixable violations — for example: a vague benefit, a +missing subject, or passive voice. For each violation, quote the exact text and +suggest a concrete rewrite. ``` diff --git a/docs/style-guide.mdx b/docs/style-guide.mdx index 1fcaccb1..3dfe0c20 100644 --- a/docs/style-guide.mdx +++ b/docs/style-guide.mdx @@ -79,21 +79,19 @@ Your detailed instructions for the LLM go here. | Field | Default | Description | |-------|---------|-------------| | `evaluator` | `base` | Evaluator type. Use `base` for most rules; `technical-accuracy` for fact-checking rules that need search. | -| `type` | `check` | Evaluation mode: `check` or `judge`. See below. | +| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` (and its `subjective` alias) is no longer supported and is rejected at load. | | `severity` | `warning` | How failures are reported: `error` or `warning`. | | `evaluateAs` | `chunk` | Whether to evaluate content as a whole (`document`) or in sections (`chunk`). | | `target` | _(none)_ | Regex to target a specific part of the content (e.g., only the H1 headline). | -| `criteria` | _(none)_ | Required for `judge` rules. Defines the scoring dimensions. | | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | ## Evaluation Modes -VectorLint supports two evaluation modes, chosen with the `type` field. +VectorLint uses a single evaluation mode, set with the `type` field. | Mode | `type` value | Best for | |------|-------------|----------| | Check | `check` | Finding specific, countable violations (grammar errors, banned terms) | -| Judge | `judge` | Measuring quality on a spectrum (clarity, tone, completeness) | ### Check rules @@ -122,69 +120,6 @@ Report any errors found with specific examples. | `standard` (4–7) | ~10 points | | `strict` (8–10) | ~20 points | -### Judge rules - -The LLM scores content against multiple weighted criteria using a 1–4 rubric. VectorLint normalizes each score to a 1–10 scale and computes a weighted average. - -| LLM rating | Meaning | Normalized | -|------------|---------|------------| -| 4 | Excellent | 10.0 | -| 3 | Good | 7.0 | -| 2 | Fair | 4.0 | -| 1 | Poor | 1.0 | - -Define criteria in the frontmatter and expand each into a rubric section in the Markdown body: - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator -severity: error -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 ---- - -You are a headline evaluator for developer blog posts. - -## RUBRIC - -# Value Communication - -### Excellent -Specific, immediately appealing benefit clearly stated. - -### Good -Clear benefit but less specific impact. - -### Fair -Vague benefit implied but not stated. - -### Poor -No apparent benefit to the reader. - -# Curiosity Gap - -### Excellent -Creates genuine intrigue without being misleading. - -### Good -Mildly interesting, reader may continue. - -### Fair -Neutral — no curiosity created. - -### Poor -Actively off-putting or confusing. -``` - ## Targeting Specific Content The `target` field lets you evaluate a specific portion of a document — for example, only the H1 headline — rather than the full content. @@ -227,51 +162,23 @@ RunRules=Acme ## Examples -### AI Pattern Detector (Judge) +### AI Pattern Detector (Check) ```markdown --- -specVersion: 1.0.0 evaluator: base -type: judge +type: check id: AIPatterns name: AI Pattern Detector severity: warning -criteria: - - name: Language Authenticity - id: LanguageAuthenticity - weight: 40 - - name: Structural Naturalness - id: StructuralNaturalness - weight: 30 --- -Detect AI-generated writing patterns in this content. - -## INSTRUCTION - -Scan for common AI patterns: - -1. **Overused buzzwords**: leverage, synergy, elevate, unlock, empower -2. **Formulaic transitions**: Moreover, Furthermore, In conclusion -3. **Hollow openings**: "In today's fast-paced world..." -4. **Excessive hedging**: "It's worth noting that...", "It's important to mention..." - -## RUBRIC - -# Language Authenticity - -### Excellent -Natural, human voice throughout. No AI patterns detected. - -### Good -Mostly natural. One or two minor patterns present. - -### Fair -Several AI patterns reduce authenticity. - -### Poor -Pervasive AI patterns throughout. +Detect AI-generated writing patterns in this content. Report each occurrence as a +violation: overused buzzwords (leverage, synergy, elevate, unlock, empower), +formulaic transitions (Moreover, Furthermore, In conclusion), hollow openings +("In today's fast-paced world..."), and excessive hedging ("It's worth noting +that..."). For each violation, quote the exact text and suggest a concrete, +human-sounding rewrite. ``` ## Next Steps diff --git a/tests/findings/README.md b/tests/findings/README.md new file mode 100644 index 00000000..f48cc664 --- /dev/null +++ b/tests/findings/README.md @@ -0,0 +1,83 @@ +# Finding Processing — Test Coverage & Behavior Mapping + +This directory owns the regression coverage for the **shared objective +violation-check finding-processing pipeline** (`src/findings/`). It documents +which behaviors moved out of `src/cli/orchestrator.ts` (and adjacent output +tests) into `src/findings/`, which behaviors were **intentionally removed**, and +which behaviors are **out of scope** for this phase. + +The pipeline (`processFindings`) composes a single path: + +```text +candidate findings ──▶ verify evidence ──▶ filter ──▶ dedupe ──▶ score ──▶ severity ──▶ ReviewResult +``` + +It is independent of model call (`single` / `agent` / `auto`) and carries no +`judge`, `evaluator`, rubric, or autonomous-agent compatibility path. + +--- + +## Behavior-preservation mapping + +| Behavior (origin) | Preserved by | Notes | +|---|---|---| +| Violation filtering / surfacing (`orchestrator.ts` `computeFilterDecision`) | `processor.test.ts` — *drops candidates that fail computeFilterDecision without a diagnostic*; *filtered candidates do not contribute to the score count* | Filter logic itself stays in `src/debug/violation-filter.ts`; the processor is a caller. `orchestrator-filtering.test.ts` still guards the end-to-end CLI surfacing. | +| Fuzzy finding-evidence location (`locateQuotedText`) | `finding-evidence-verifier.test.ts` — exact-quote location, warn diagnostic on miss, **no model-line fallback**, anchors without a line hint, diagnostic names the quoted text | One verifier (audit Finding #6). The agent path's model-line fallback is **not** preserved (see Removed). | +| Verified-finding counting & density scoring (`calculateCheckScore`) | `scorer.test.ts` — same numeric result as `calculateCheckScore`; score driven by verified count, not raw candidate count; resolves error severity; forwards strictness/severity | `scorer.ts` is a thin wrapper; no reimplemented math. | +| Severity + rule-id attribution | `severity.test.ts` — `resolveSeverity`, `buildRuleId` (`Pack.Rule` / `Pack.Rule.Criterion`), `resolveCriterionId`; `processor.test.ts` — stamps severity, attributes to pack rule / criterion | One severity path; one rule-id builder (audit Finding #4). | +| Golden check output (byte-for-byte) | `processor.test.ts` — *produces byte-for-byte findings/score matching the standard orchestrator path* | Proves no regression before the orchestrator was rewired (Task 2). | +| Unanchored-evidence diagnostics & exit behavior | `processor.test.ts` — *turns unanchored evidence into a warn diagnostic and emits no finding*; *counts only verified findings toward the score*; *does not set hadOperationalErrors for warn-level evidence diagnostics* | Intentional Phase 3 fix: unanchored quotes no longer flag the run as operationally failed. | +| Deduplication | `processor.test.ts` — *deduplicates verified findings by quoted_text and line* | | +| Contract / input boundary | `types.test.ts` — `FINDING_PROCESSING_INPUT_SCHEMA`, `RAW_VIOLATION_SCHEMA`, `PROMPT_META_FOR_FINDINGS_SCHEMA` parse supported input and reject legacy `evaluator`/`judge`/`rubric weight`/`agent`/`modelCall`/`mode`; `module-surface.test.ts` — public surface + diagnostic code constant | Strict Zod at the boundary; no `any`. | +| Formatter routing (line / JSON / RDJSON / Vale) | `orchestrator-check-processor.test.ts` — locatable findings + score + counts unchanged; verified findings through the JSON sink; unanchored quotes omitted from Vale JSON. `orchestrator-filtering.test.ts` — no dummy JSON/Vale issues when nothing is surfaced | End-to-end coverage of the rewired check path feeding existing formatters from `ReviewResult`. | + +--- + +## Intentionally removed behavior (no replacement fixture) + +These were legacy paths that the Phase 3 contract no longer supports. They are +covered by **rejection tests**, not behavior-parity tests. + +| Removed behavior | Rejection coverage | +|---|---| +| Subjective `judge` / rubric criterion evaluation | `prompt-loader-validation.test.ts` — *Judge boundary rejection (Phase 3)* rejects `type: judge` and the `subjective` alias at the loader boundary; `orchestrator-filtering.test.ts` — *rejects judge/rubric results instead of projecting them as check findings* | +| Legacy `evaluator` / `judge` / rubric criteria in the review contract | `review/types.test.ts` — *rejects legacy evaluator and judge criteria fields*; `findings/types.test.ts` — input contract rejects judge/evaluator/rubric shapes | +| Agent finding-evidence model-line fallback | `finding-evidence-verifier.test.ts` — *never returns a finding whose line came from the model when the quote did not anchor* (the removed behavior is asserted as absent) | + +Judge criterion output behavior (1–4 rubric, weighted average) has **no** +replacement fixture in this phase. Objective rule coverage belongs to the +objective-check test inventory, not a ported rubric. + +--- + +## Out of scope for Phase 3 (Phase 4 territory) + +These are **not** behavior-preservation targets for this phase and are +intentionally left untouched: + +- `tests/agent/*`, the orchestrator test covering legacy agent output, and + `tests/providers/*agent-loop*.test.ts` — the old autonomous agent surface. + Phase 4 deletes or replaces these with agent-model-call executor tests. +- `BaseEvaluator`'s internal judge mode (`runJudgeEvaluation`), + `buildJudgeLLMSchema` (`src/prompts/schema.ts`), the judge scoring helpers in + `src/scoring/`, `tests/scoring-types.test.ts`, and the judge cases in + `tests/prompt-schema.test.ts` — evaluator internals that are now **unreachable + from the CLI** (judge prompts fail to load) but are scheduled for deletion in + Phase 4 (executors / provider interface / agent deletion). + +Phase 3 added **no** compatibility code for the legacy agent surface — no +agent tool-loop wiring, no review-instruction handler, and no agent result +projection — to `src/findings/`. + +--- + +## Files + +| File | Covers | +|---|---| +| `finding-evidence-verifier.test.ts` | Single evidence verifier; no model-line fallback. | +| `severity.test.ts` | `resolveSeverity`, `buildRuleId`, `resolveCriterionId`. | +| `scorer.test.ts` | `scoreCheck` numerics vs `calculateCheckScore`; verified-count scoring. | +| `types.test.ts` | Zod input contract: parses supported input, rejects legacy shapes. | +| `processor.test.ts` | `processFindings` golden output, diagnostics, counting fix, dedupe, rule-id attribution, `ReviewResult` validation. | +| `module-surface.test.ts` | Public barrel surface + stable diagnostic code constant. | From 4fa07011105f3f2b84df3ae22d41fd774e27749e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:50:55 +0100 Subject: [PATCH 18/26] Sanitize prompt criteria at the findings boundary - Map meta.criteria to { id, name } before processFindings so legacy PromptCriterionSpec rubric fields (weight, target) cannot cross the standard check orchestration -> findings contract - Add tests/orchestrator-finding-criteria.test.ts mocking processFindings to assert weight/target are stripped on the production path - prompt-loader criteria support is unchanged; criterion id/name still drive output rule-id attribution --- src/cli/orchestrator.ts | 6 +- tests/orchestrator-finding-criteria.test.ts | 126 ++++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 tests/orchestrator-finding-criteria.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 4d09e917..1e61a47d 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -206,7 +206,11 @@ function routePromptResult( promptMeta: { ...(meta.severity !== undefined ? { severity: meta.severity } : {}), ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - ...(meta.criteria ? { criteria: meta.criteria } : {}), + // Sanitize to the findings contract ({ id, name }): meta.criteria is + // PromptCriterionSpec[], which can carry legacy rubric weight/target. + ...(meta.criteria + ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } + : {}), }, targetContent: content, }); diff --git a/tests/orchestrator-finding-criteria.test.ts b/tests/orchestrator-finding-criteria.test.ts new file mode 100644 index 00000000..dc0b1ad8 --- /dev/null +++ b/tests/orchestrator-finding-criteria.test.ts @@ -0,0 +1,126 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { evaluateFiles } from "../src/cli/orchestrator"; +import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; +import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { PromptFile } from "../src/prompts/prompt-loader"; +import type { RawCheckResult } from "../src/prompts/schema"; +import type { ReviewResult } from "../src/review/types"; + +const { EVALUATE_MOCK, PROCESS_FINDINGS_MOCK } = vi.hoisted(() => ({ + EVALUATE_MOCK: vi.fn(), + PROCESS_FINDINGS_MOCK: vi.fn<(input: unknown) => ReviewResult>(), +})); + +// createEvaluator returns a controllable evaluator so evaluateFiles reaches +// routePromptResult -> processFindings without a real model call. +vi.mock("../src/evaluators/index", () => ({ + createEvaluator: vi.fn(() => ({ + evaluate: EVALUATE_MOCK, + })), +})); + +// Replace only processFindings (the findings boundary); keep every other +// findings export real so unrelated imports stay intact. +vi.mock("../src/findings", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, processFindings: PROCESS_FINDINGS_MOCK }; +}); + +function createPrompt(meta: PromptFile["meta"]): PromptFile { + return { + id: meta.id, + filename: `${meta.id}.md`, + fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), + meta, + body: "Prompt body", + pack: "TestPack", + }; +} + +function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Line, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-criteria-")); + const filePath = path.join(dir, "input.md"); + writeFileSync(filePath, content); + return filePath; +} + +function makeCheckResult(): RawCheckResult { + return { + type: EvaluationType.CHECK, + violations: [], + word_count: 100, + }; +} + +describe("standard check orchestration sanitizes criteria at the findings boundary", () => { + beforeEach(() => { + EVALUATE_MOCK.mockReset(); + PROCESS_FINDINGS_MOCK.mockReset(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("drops rubric weight/target from criteria before reaching processFindings", async () => { + const targetFile = createTempFile("Alpha text\n"); + // PromptCriterionSpec admits legacy rubric fields (weight, target). The + // orchestrator must strip them so only { id, name } crosses the boundary. + const prompt = createPrompt({ + id: "SanitizePrompt", + name: "Sanitize Prompt", + type: "check", + severity: Severity.WARNING, + criteria: [ + { id: "Hedging", name: "Hedge words", weight: 3, target: { regex: "x" } }, + ], + }); + + EVALUATE_MOCK.mockResolvedValue(makeCheckResult()); + PROCESS_FINDINGS_MOCK.mockReturnValue({ + findings: [], + scores: [ + { + ruleId: "TestPack.SanitizePrompt", + score: 10, + scoreText: "10.0/10", + severity: "warning", + }, + ], + diagnostics: [], + }); + + await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(PROCESS_FINDINGS_MOCK).toHaveBeenCalledTimes(1); + const input = PROCESS_FINDINGS_MOCK.mock.calls[0]![0] as { + promptMeta: { criteria?: Array> }; + }; + + // toEqual fails if any extra (weight/target) key survives the mapping. + expect(input.promptMeta.criteria).toEqual([{ id: "Hedging", name: "Hedge words" }]); + for (const criterion of input.promptMeta.criteria ?? []) { + expect(criterion).not.toHaveProperty("weight"); + expect(criterion).not.toHaveProperty("target"); + } + }); +}); From 699f3557e31ec86c0ac62d64ce8270e15f31221f Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 09:23:27 +0100 Subject: [PATCH 19/26] Define structured and tool-calling model clients - Add StructuredModelClient (runPromptStructured + LLMResult/TokenUsage) as the permanent single structured-output capability - Add ToolCallingModelClient: bounded, executor-owned tool transport (runWithTools) that names no workspace/agent-finding concepts - Make LLMProvider extend StructuredModelClient; keep runAgentToolLoop only as a temporary compile bridge for src/agent/ (deleted in a later task) - VercelAIProvider implements both capabilities via a bounded runWithTools transport (caller-owned tools + caller-set step budget), no product loop - Extend AIExecutionContext.operation with a transport-shaped 'tool-calling' label - Add provider contract tests asserting the new surfaces and the absence of legacy workspace-agent concepts Refs: docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md (Finding #2; Product Decision) --- src/observability/ai-observability.ts | 2 +- src/providers/index.ts | 8 +- src/providers/llm-provider.ts | 23 ++-- src/providers/structured-model-client.ts | 28 +++++ src/providers/tool-calling-model-client.ts | 53 ++++++++ src/providers/vercel-ai-provider.ts | 119 +++++++++++++++++- .../providers/structured-model-client.test.ts | 67 ++++++++++ .../tool-calling-model-client.test.ts | 95 ++++++++++++++ 8 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 src/providers/structured-model-client.ts create mode 100644 src/providers/tool-calling-model-client.ts create mode 100644 tests/providers/structured-model-client.test.ts create mode 100644 tests/providers/tool-calling-model-client.test.ts diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index 5369c639..e16a3c9d 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -1,5 +1,5 @@ export interface AIExecutionContext { - operation: 'structured-eval' | 'agent-tool-loop'; + operation: 'structured-eval' | 'tool-calling' | 'agent-tool-loop'; provider: string; model: string; evaluator?: string; diff --git a/src/providers/index.ts b/src/providers/index.ts index 4bdb538c..2c02f6e0 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,4 +1,10 @@ -export { LLMProvider, type LLMResult } from './llm-provider'; +export { LLMProvider } from './llm-provider'; +export { type StructuredModelClient, type LLMResult } from './structured-model-client'; +export { + type ToolCallingModelClient, + type ToolCallDefinition, + type ToolCallRunOptions, +} from './tool-calling-model-client'; export { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; export { SearchProvider } from './search-provider'; export { PerplexitySearchProvider, type PerplexitySearchConfig } from './perplexity-provider'; diff --git a/src/providers/llm-provider.ts b/src/providers/llm-provider.ts index f87304dd..635d5ecd 100644 --- a/src/providers/llm-provider.ts +++ b/src/providers/llm-provider.ts @@ -1,10 +1,12 @@ import type { TokenUsage } from './token-usage'; -import type { EvalContext } from './request-builder'; +import type { StructuredModelClient } from './structured-model-client'; -export interface LLMResult { - data: T; - usage?: TokenUsage; -} +/** + * `LLMResult` now lives on the permanent structured-output capability. It is + * re-exported here so existing deep imports (`from './llm-provider'`) keep + * compiling until the legacy provider surface is removed in a later task. + */ +export type { LLMResult } from './structured-model-client'; export interface AgentToolDefinition { description: string; @@ -25,7 +27,14 @@ export interface AgentToolLoopResult { usage?: TokenUsage; } -export interface LLMProvider { - runPromptStructured(content: string, promptText: string, schema: { name: string; schema: Record }, context?: EvalContext): Promise>; +/** + * Temporary compile bridge, preserved only until later tasks remove the + * autonomous agent surface. It extends the permanent + * {@link StructuredModelClient} (single structured output) with the legacy + * `runAgentToolLoop` method still consumed by `src/agent/`. Do not expand this + * surface; `runAgentToolLoop` and the `Agent*` types above are deleted once the + * agent tree is removed (audit Finding #2; Product Decision). + */ +export interface LLMProvider extends StructuredModelClient { runAgentToolLoop(params: AgentToolLoopParams): Promise; } diff --git a/src/providers/structured-model-client.ts b/src/providers/structured-model-client.ts new file mode 100644 index 00000000..99b1a1e0 --- /dev/null +++ b/src/providers/structured-model-client.ts @@ -0,0 +1,28 @@ +import type { TokenUsage } from './token-usage'; +import type { EvalContext } from './request-builder'; + +/** + * Result of a structured model call: validated output plus optional usage. + */ +export interface LLMResult { + data: T; + usage?: TokenUsage; +} + +/** + * Permanent structured-output model capability (audit Finding #2). + * + * A {@link StructuredModelClient} makes a single structured model call and + * returns validated output. It owns no tool surface and no autonomous agent + * loop; that product-level autonomy is removed from the provider surface + * (audit Product Decision). The single structured-call executor builds on this + * capability. + */ +export interface StructuredModelClient { + runPromptStructured( + content: string, + promptText: string, + schema: { name: string; schema: Record }, + context?: EvalContext, + ): Promise>; +} diff --git a/src/providers/tool-calling-model-client.ts b/src/providers/tool-calling-model-client.ts new file mode 100644 index 00000000..2139c8f2 --- /dev/null +++ b/src/providers/tool-calling-model-client.ts @@ -0,0 +1,53 @@ +import type { ZodType } from 'zod'; +import type { LLMResult } from './structured-model-client'; + +/** + * An executor-owned tool definition supplied to the bounded tool-calling + * transport. The provider is transport only: it does not define or name + * product tools. Descriptions and parameter schemas come from the caller (the + * agent executor), keyed by the caller-supplied tool name, never from the + * provider (audit Finding #2). The object key in the tools map is the tool + * name, matching the existing provider tool-mapping convention. + */ +export interface ToolCallDefinition { + description: string; + parameters: ZodType; + execute: (input: unknown) => Promise; +} + +/** + * Bounded execution metadata for a single tool-calling run. The caller + * (executor) sets the bounds from the review budget (audit Finding #7); the + * transport honors them. It does not invent its own product-level loop. + */ +export interface ToolCallRunOptions { + /** Maximum number of model steps (tool-call rounds) before the run stops. */ + maxSteps?: number; + /** Provider retry budget for transient failures. */ + maxRetries?: number; + /** Maximum concurrent tool executions per model step. Defaults to 1. */ + maxParallelToolCalls?: number; + /** Optional abort signal for cooperative cancellation. */ + signal?: AbortSignal; +} + +/** + * Permanent bounded tool-calling model capability (audit Finding #2). + * + * A {@link ToolCallingModelClient} performs a single bounded generation that + * may execute caller-supplied tools and then emits structured output. The tool + * map is executor-owned and transport-shaped: this interface deliberately + * names no product, workspace, or agent-finding concept (audit Product + * Decision; Finding #1). The autonomous product loop is removed; bounded + * orchestration lives in the executor, which supplies the only tool(s) it + * permits (for example a target-scoped reader). + */ +export interface ToolCallingModelClient { + runWithTools(params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }): Promise>; +} diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 5cd833fd..ed314001 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -3,11 +3,20 @@ import type { LanguageModel } from 'ai'; import { z } from 'zod'; import pLimit from 'p-limit'; import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from './llm-provider'; +import type { ToolCallDefinition, ToolCallRunOptions, ToolCallingModelClient } from './tool-calling-model-client'; import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; import type { AIExecutionContext, AIObservability } from '../observability/ai-observability'; import { handleUnknownError } from '../errors'; +/** + * Conservative default step cap for a single bounded tool-calling run when the + * caller does not supply one. Executors should pass an explicit `maxSteps` + * derived from the review budget (audit Finding #7) rather than relying on + * this default. + */ +const DEFAULT_TOOL_CALLING_MAX_STEPS = 10; + export interface VercelAIConfig { model: LanguageModel; providerName?: string; @@ -21,7 +30,7 @@ export interface VercelAIConfig { observability?: AIObservability; } -export class VercelAIProvider implements LLMProvider { +export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; @@ -135,6 +144,114 @@ export class VercelAIProvider implements LLMProvider { } } + /** + * Bounded tool-calling transport (audit Finding #2). Performs a single + * bounded generation that may execute caller-supplied tools and then emits + * structured output. The tool map, step budget, and schema are all + * executor-owned; the provider defines no product tools and runs no + * autonomous product loop (audit Product Decision). + */ + async runWithTools(params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }): Promise> { + const options = params.options ?? {}; + const maxParallel = options.maxParallelToolCalls ?? 1; + const limit = pLimit(maxParallel); + + const zodSchema = this.jsonSchemaToZod(params.schema); + + const mappedTools = Object.fromEntries( + Object.entries(params.tools).map(([name, definition]) => [ + name, + tool({ + description: definition.description, + inputSchema: definition.parameters, + execute: (input: unknown) => limit(() => definition.execute(input)), + }), + ]), + ); + + if (this.config.debug) { + this.logger.debug('[vectorlint] Sending tool-calling request via Vercel AI SDK', { + model: this.config.model, + toolNames: Object.keys(params.tools), + maxSteps: options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS, + }); + } + + try { + const result = await generateText({ + model: this.config.model, + system: params.systemPrompt, + prompt: params.prompt, + ...(this.config.temperature !== undefined && { temperature: this.config.temperature }), + ...(this.config.maxTokens !== undefined && { maxTokens: this.config.maxTokens }), + ...(options.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), + ...(options.signal !== undefined ? { abortSignal: options.signal } : {}), + ...this.getObservabilityOptions({ + operation: 'tool-calling', + provider: this.config.providerName ?? 'unknown', + model: this.config.modelName ?? 'unknown', + }), + stopWhen: stepCountIs(options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS), + providerOptions: { + openai: { + parallelToolCalls: maxParallel > 1, + }, + }, + tools: mappedTools, + output: Output.object({ + schema: zodSchema, + }), + }); + + if (this.config.debug && result.usage) { + this.logger.debug('[vectorlint] tool-calling LLM response meta', { + usage: { + input_tokens: result.usage.inputTokens, + output_tokens: result.usage.outputTokens, + total_tokens: result.usage.totalTokens, + }, + finish_reason: result.finishReason, + steps: result.steps.length, + }); + } + + const usage = result.usage + ? { + inputTokens: result.usage.inputTokens ?? 0, + outputTokens: result.usage.outputTokens ?? 0, + } + : undefined; + + const output: unknown = result.output; + if (output === undefined || output === null) { + throw new Error( + `LLM returned no structured output. Raw text: ${result.text?.slice(0, 500) ?? '(empty)'}` + ); + } + + const llmResult: LLMResult = { data: output as T }; + if (usage) { + llmResult.usage = usage; + } + return llmResult; + } catch (e: unknown) { + if (NoObjectGeneratedError.isInstance(e)) { + const rawText = e instanceof Error && 'text' in e ? String(e.text) : 'unknown'; + throw new Error( + `LLM failed to generate valid structured output. Raw text: ${rawText}` + ); + } + const err = e instanceof Error ? e : new Error(String(e)); + throw new Error(`Vercel AI SDK tool-calling call failed: ${err.message}`); + } + } + runAgentToolLoop = async (params: AgentToolLoopParams): Promise => { const maxParallel = params.maxParallelToolCalls ?? 1; const limit = pLimit(maxParallel); diff --git a/tests/providers/structured-model-client.test.ts b/tests/providers/structured-model-client.test.ts new file mode 100644 index 00000000..7fa6b61d --- /dev/null +++ b/tests/providers/structured-model-client.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import type { LanguageModel } from 'ai'; +import type { + LLMResult, + StructuredModelClient, +} from '../../src/providers/structured-model-client'; +import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; + +const SCHEMA = { name: 'Schema', schema: { type: 'object' } }; + +describe('StructuredModelClient contract', () => { + it('exposes runPromptStructured and returns an LLMResult', async () => { + const client: StructuredModelClient = { + runPromptStructured: () => Promise.resolve({ data: { ok: true } }), + }; + + expect(typeof client.runPromptStructured).toBe('function'); + + const result = await client.runPromptStructured('content', 'prompt', SCHEMA); + expect(result.data).toEqual({ ok: true }); + expect(result.usage).toBeUndefined(); + }); + + it('carries optional token usage on LLMResult', async () => { + const client: StructuredModelClient = { + runPromptStructured: () => + Promise.resolve({ data: 1, usage: { inputTokens: 3, outputTokens: 4 } }), + }; + + const result: LLMResult = await client.runPromptStructured( + 'content', + 'prompt', + SCHEMA, + ); + + expect(result.data).toBe(1); + expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 4 }); + }); + + it('does not declare an autonomous agent-loop method', () => { + // Compile-time guard: the capability surface is exactly runPromptStructured. + // If a second member were added, this conditional type would resolve to + // `false` and the assignment below would fail tsc. + type StructuredKeys = keyof StructuredModelClient; + const onlyRunPromptStructured: StructuredKeys extends 'runPromptStructured' ? true : false = true; + expect(onlyRunPromptStructured).toBe(true); + + // Runtime guard: a minimal implementation exposes only runPromptStructured. + const client: StructuredModelClient = { + runPromptStructured: () => Promise.resolve({ data: null }), + }; + expect( + (client as unknown as Record).runAgentToolLoop, + ).toBeUndefined(); + }); + + it('is satisfied by VercelAIProvider', () => { + const provider = new VercelAIProvider({ + model: {} as unknown as LanguageModel, + }); + + // Type-level assignability is enforced by this assignment (tsc fails if + // VercelAIProvider does not satisfy StructuredModelClient). + const client: StructuredModelClient = provider; + expect(typeof client.runPromptStructured).toBe('function'); + }); +}); diff --git a/tests/providers/tool-calling-model-client.test.ts b/tests/providers/tool-calling-model-client.test.ts new file mode 100644 index 00000000..ccf84f68 --- /dev/null +++ b/tests/providers/tool-calling-model-client.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest'; +import type { LanguageModel } from 'ai'; +import { z } from 'zod'; +import type { + ToolCallDefinition, + ToolCallRunOptions, + ToolCallingModelClient, +} from '../../src/providers/tool-calling-model-client'; +import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; + +const SCHEMA = { name: 'Findings', schema: { type: 'object' } }; + +describe('ToolCallingModelClient contract', () => { + it('exposes a single bounded runWithTools method', () => { + // Compile-time guard: the capability surface is exactly runWithTools. + type ToolCallingKeys = keyof ToolCallingModelClient; + const onlyRunWithTools: ToolCallingKeys extends 'runWithTools' ? true : false = true; + expect(onlyRunWithTools).toBe(true); + + const client: ToolCallingModelClient = { + runWithTools: () => Promise.resolve({ data: { findings: [] } }), + }; + expect(typeof client.runWithTools).toBe('function'); + }); + + it('returns structured output plus optional usage', async () => { + const client: ToolCallingModelClient = { + runWithTools: () => + Promise.resolve({ + data: { findings: [{ line: 1 }] }, + usage: { inputTokens: 5, outputTokens: 6 }, + }), + }; + + const result = await client.runWithTools({ + systemPrompt: 'system', + prompt: 'prompt', + tools: { read: makeTool() }, + schema: SCHEMA, + options: { maxSteps: 3, maxParallelToolCalls: 1 } satisfies ToolCallRunOptions, + }); + + expect(result.data).toEqual({ findings: [{ line: 1 }] }); + expect(result.usage).toEqual({ inputTokens: 5, outputTokens: 6 }); + }); + + it('accepts executor-owned tool definitions with typed parameters', () => { + const parameters = z.object({ startLine: z.number().int().positive() }); + const definition: ToolCallDefinition = { + description: 'paged read', + parameters, + execute: (input) => Promise.resolve(input), + }; + + expect(definition.parameters).toBe(parameters); + expect(typeof definition.execute).toBe('function'); + }); + + it('does not name workspace-agent or product-finding concepts on the capability surface', () => { + const client: ToolCallingModelClient = { + runWithTools: () => Promise.resolve({ data: {} }), + }; + const keys = Object.keys(client as Record); + + const forbidden = [ + 'read_file', + 'search_content', + 'report_finding', + 'reviewInstruction', + 'runAgentToolLoop', + ]; + for (const concept of forbidden) { + expect(keys).not.toContain(concept); + } + }); + + it('is satisfied by VercelAIProvider', () => { + const provider = new VercelAIProvider({ + model: {} as unknown as LanguageModel, + }); + + // Type-level assignability is enforced by this assignment (tsc fails if + // VercelAIProvider does not satisfy ToolCallingModelClient). + const client: ToolCallingModelClient = provider; + expect(typeof client.runWithTools).toBe('function'); + }); +}); + +function makeTool(): ToolCallDefinition { + return { + description: 'a bounded executor-owned tool', + parameters: z.object({ value: z.number().int().positive() }), + execute: () => Promise.resolve({ ok: true }), + }; +} From 74957bcb6528d35988c11f66705d4875ade1f503 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 09:54:05 +0100 Subject: [PATCH 20/26] Implement SingleModelCallExecutor for single modelCall - Add SingleModelCallExecutor behind the Phase 2 ReviewExecutor contract, reviewing source-backed rules with one structured model call per (rule, chunk) through an injected StructuredModelClient. - Build rule prompts verbatim from request.rules[i].body; no tool surface and no reviewInstruction. - Reuse buildCheckLLMSchema, RecursiveChunker, mergeViolations, prependLineNumbers, and processFindings instead of parallel projection. - Enforce ReviewBudget.maxModelCallsPerReview via enforceBudget before each call; on exhaustion record an error diagnostic + hadOperationalErrors and return partial results rather than throwing past the contract. - Record usage (modelCalls, wallClockMs always; input/output tokens gated on outputPolicy.includeUsage) consistent with ReviewResult.usage. - Add focused tests: one call per rule, projected findings/scores/diagnostics, unanchored-evidence diagnostic, budget exhaustion, no tool calls, and large-target chunking/merging. --- src/executors/single-model-call-executor.ts | 227 ++++++++++++++++ .../single-model-call-executor.test.ts | 248 ++++++++++++++++++ 2 files changed, 475 insertions(+) create mode 100644 src/executors/single-model-call-executor.ts create mode 100644 tests/executors/single-model-call-executor.test.ts diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts new file mode 100644 index 00000000..de3957cb --- /dev/null +++ b/src/executors/single-model-call-executor.ts @@ -0,0 +1,227 @@ +import path from 'path'; + +import type { ReviewExecutor } from '../review/executor'; +import { BudgetExceededError, enforceBudget } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewRequest, + ReviewResult, + ReviewRule, + ReviewScore, + ReviewUsage, +} from '../review/types'; +import type { EvalContext } from '../providers/request-builder'; +import type { StructuredModelClient } from '../providers/structured-model-client'; +import { Severity } from '../evaluators/types'; +import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; +import { countWords, mergeViolations, RecursiveChunker, type Chunk } from '../chunking'; +import { prependLineNumbers } from '../output/line-numbering'; +import { processFindings } from '../findings'; + +/** + * Word-count threshold above which the single-call executor chunks the target + * before reviewing it. Mirrors the check evaluator's chunking threshold so the + * single-call path preserves the existing chunk/merge behavior for large + * documents. + */ +const CHUNKING_WORD_THRESHOLD = 600; +const MAX_CHUNK_WORDS = 500; + +/** + * Stable diagnostic code recorded when a run stops because the model-call + * budget was exhausted before every rule could be reviewed. + */ +const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; + +/** + * Mutable run-wide counters shared across rules and chunks. + */ +interface RunCounters { + modelCalls: number; + inputTokens: number; + outputTokens: number; +} + +/** + * The single modelCall {@link ReviewExecutor} (audit Finding #2). + * + * Reviews target content against source-backed rules with one structured model + * call per (rule, chunk) through an injected {@link StructuredModelClient}. It + * owns no tool surface and no autonomous loop: rule prompts come verbatim from + * {@link ReviewRule.body}, candidate findings flow through the shared + * {@link processFindings} pipeline, and the model-call budget is enforced via + * the review budget module before every call. + * + * This is the bounded, transport-only review strategy selected by + * `--model-call single` (and by `auto` for normal-sized inputs). + */ +export class SingleModelCallExecutor implements ReviewExecutor { + constructor(private readonly client: StructuredModelClient) {} + + async run(request: ReviewRequest): Promise { + const schema = buildCheckLLMSchema(); + const context = this.buildContext(request.target.uri); + + const findings: ReviewFinding[] = []; + const scores: ReviewScore[] = []; + const diagnostics: ReviewDiagnostic[] = []; + let hadOperationalErrors = false; + + const counters: RunCounters = { modelCalls: 0, inputTokens: 0, outputTokens: 0 }; + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + + try { + for (const rule of request.rules) { + const ruleResult = await this.reviewRule( + request, + rule, + schema, + context, + counters, + elapsedMs, + ); + findings.push(...ruleResult.findings); + scores.push(...ruleResult.scores); + diagnostics.push(...ruleResult.diagnostics); + if (ruleResult.hadOperationalErrors) { + hadOperationalErrors = true; + } + } + } catch (error: unknown) { + if (error instanceof BudgetExceededError) { + // Surface the existing budget error pattern as an operational failure: + // the run returns partial results plus an error diagnostic rather than + // throwing past the ReviewExecutor contract. + hadOperationalErrors = true; + diagnostics.push({ + level: 'error', + code: REVIEW_BUDGET_EXCEEDED_CODE, + message: error.message, + context: { limit: error.limit, actual: error.actual }, + }); + } else { + throw error; + } + } + + return { + findings, + scores, + diagnostics, + hadOperationalErrors, + usage: this.buildUsage(request, counters, elapsedMs()), + }; + } + + /** + * Reviews a single rule: chunks the line-numbered target, makes one + * structured model call per chunk, merges violations across chunks, and + * projects the merged candidates through {@link processFindings}. + */ + private async reviewRule( + request: ReviewRequest, + rule: ReviewRule, + schema: ReturnType, + context: EvalContext, + counters: RunCounters, + elapsedMs: () => number, + ): Promise { + const numberedContent = prependLineNumbers(request.target.content); + const wordCount = countWords(request.target.content) || 1; + const chunks = this.chunkTarget(numberedContent, wordCount, request.budget.maxChunksPerRule); + + const chunkViolations: CheckLLMResult['violations'][] = []; + for (const chunk of chunks) { + // Enforce the model-call budget before committing to another call. The + // prospective count (calls made so far plus this one) lets enforceBudget + // reject the call that would push the run over maxModelCallsPerReview. + enforceBudget(request.budget, { + modelCalls: counters.modelCalls + 1, + elapsedMs: elapsedMs(), + }); + + const { data, usage } = await this.client.runPromptStructured( + chunk.content, + rule.body, + schema, + context, + ); + counters.modelCalls += 1; + if (usage) { + counters.inputTokens += usage.inputTokens; + counters.outputTokens += usage.outputTokens; + } + chunkViolations.push(data.violations); + } + + const { pack, ruleId } = splitRuleId(rule.id); + const mergedViolations = mergeViolations(chunkViolations); + + return processFindings({ + pack, + ruleId, + ruleSource: rule.source, + candidateFindings: mergedViolations, + wordCount, + // Map the review contract's plain severity union onto the finding + // processor's Severity enum at this boundary. + promptMeta: { + ...(rule.severity !== undefined + ? { severity: rule.severity === 'error' ? Severity.ERROR : Severity.WARNING } + : {}), + }, + targetContent: request.target.content, + }); + } + + /** + * Chunks line-numbered target content for context management. Small targets + * review as a single chunk; larger targets are split recursively and capped + * at the review budget's per-rule chunk limit. + */ + private chunkTarget( + numberedContent: string, + wordCount: number, + maxChunks: number, + ): Chunk[] { + if (wordCount <= CHUNKING_WORD_THRESHOLD) { + return [{ content: numberedContent, index: 0 }]; + } + const chunker = new RecursiveChunker(); + return chunker.chunk(numberedContent, { maxChunkSize: MAX_CHUNK_WORDS }).slice(0, maxChunks); + } + + private buildContext(uri: string): EvalContext { + const ext = path.extname(uri); + return ext ? { fileType: ext } : {}; + } + + private buildUsage( + request: ReviewRequest, + counters: RunCounters, + wallClockMs: number, + ): ReviewUsage { + const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; + if (request.outputPolicy.includeUsage) { + usage.inputTokens = counters.inputTokens; + usage.outputTokens = counters.outputTokens; + } + return usage; + } +} + +/** + * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. + * The review contract carries the composite id, while {@link processFindings} + * rebuilds the same id from the parts via `buildRuleId`. Splits on the first + * dot; pack names are single path segments and rule ids are PascalCase. + */ +function splitRuleId(id: string): { pack: string; ruleId: string } { + const dot = id.indexOf('.'); + if (dot === -1) { + return { pack: id, ruleId: id }; + } + return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; +} diff --git a/tests/executors/single-model-call-executor.test.ts b/tests/executors/single-model-call-executor.test.ts new file mode 100644 index 00000000..5172f9b0 --- /dev/null +++ b/tests/executors/single-model-call-executor.test.ts @@ -0,0 +1,248 @@ +import { describe, expect, it } from 'vitest'; + +import { SingleModelCallExecutor } from '../../src/executors/single-model-call-executor'; +import { DEFAULT_REVIEW_BUDGET } from '../../src/review'; +import type { + ReviewRequest, + ReviewRule, + ReviewTarget, +} from '../../src/review'; +import type { LLMResult, StructuredModelClient } from '../../src/providers/structured-model-client'; +import type { TokenUsage } from '../../src/providers/token-usage'; +import type { CheckLLMResult } from '../../src/prompts/schema'; + +const SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const SUPPORTED_NOTES = { + rule_supports_claim: 'yes', + evidence_exact: 'yes', + context_supports_violation: 'yes', + plausible_non_violation: 'no', + fix_is_drop_in: 'yes', + fix_preserves_meaning: 'yes', +}; + +type ModelViolation = CheckLLMResult['violations'][number]; + +function modelViolation(overrides: Partial = {}): ModelViolation { + return { + line: 1, + quoted_text: 'vague text', + context_before: '', + context_after: '', + description: 'desc', + analysis: 'analysis', + message: 'message', + suggestion: 'suggestion', + fix: 'fix', + rule_quote: 'rule quote', + checks: SUPPORTED_CHECKS, + check_notes: SUPPORTED_NOTES, + confidence: 0.9, + ...overrides, + }; +} + +type FakeStructuredClient = StructuredModelClient & { + runWithTools: () => Promise; + structuredCalls: number; + toolCalls: number; + lastPromptText?: string; +}; + +function makeFakeClient( + respond: (content: string, promptText: string) => { data: CheckLLMResult; usage?: TokenUsage }, +): FakeStructuredClient { + const client = { + structuredCalls: 0, + toolCalls: 0, + lastPromptText: undefined as string | undefined, + runPromptStructured: ( + content: string, + promptText: string, + ): Promise> => { + client.structuredCalls += 1; + client.lastPromptText = promptText; + const { data, usage } = respond(content, promptText); + const result: LLMResult = { data: data as unknown as T }; + if (usage) { + result.usage = usage; + } + return Promise.resolve(result); + }, + runWithTools: (): Promise => { + client.toolCalls += 1; + return Promise.reject( + new Error('SingleModelCallExecutor must not call runWithTools'), + ); + }, + }; + return client; +} + +const TARGET: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: 'vague text here\nsecond line is fine\n', + contentType: 'text/markdown', +}; + +function makeRule(overrides: Partial = {}): ReviewRule { + return { + id: 'VectorLint.PseudoAdvice', + source: '/repo/presets/VectorLint/pseudo-advice.md', + body: 'Flag vague advice.', + ...overrides, + }; +} + +function makeRequest(rules: ReviewRule[], overrides: Partial = {}): ReviewRequest { + return { + target: TARGET, + rules, + budget: { ...DEFAULT_REVIEW_BUDGET }, + outputPolicy: { includeUsage: true, recordPayloadTelemetry: false }, + modelCall: 'single', + ...overrides, + }; +} + +describe('SingleModelCallExecutor', () => { + it('makes one structured call per rule and projects findings/scores through processFindings', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text', message: 'Too vague.' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run(makeRequest([makeRule()])); + + // One structured call, with the rule body as the source-backed prompt. + expect(client.structuredCalls).toBe(1); + expect(client.lastPromptText).toBe('Flag vague advice.'); + + // The finding is verified + anchored through the shared processor. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.findings[0]?.ruleSource).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(result.findings[0]?.line).toBeGreaterThanOrEqual(1); + expect(result.findings[0]?.match).toBe('vague text'); + + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.scores[0]?.findingCount).toBe(1); + + expect(result.usage?.modelCalls).toBe(1); + }); + + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'quantum entanglement device' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run(makeRequest([makeRule()])); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe('finding-evidence-not-locatable'); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('reviews each rule independently and aggregates usage across calls', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text' })], + }, + usage: { inputTokens: 10, outputTokens: 5 }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest([ + makeRule({ id: 'VectorLint.RuleA', source: '/repo/a.md' }), + makeRule({ id: 'VectorLint.RuleB', source: '/repo/b.md' }), + ]), + ); + + expect(client.structuredCalls).toBe(2); + expect(result.scores).toHaveLength(2); + expect(result.scores.map((s) => s.ruleId).sort()).toEqual(['VectorLint.RuleA', 'VectorLint.RuleB']); + expect(result.usage?.modelCalls).toBe(2); + expect(result.usage?.inputTokens).toBe(20); + expect(result.usage?.outputTokens).toBe(10); + }); + + it('stops and records an operational error when the model-call budget is exhausted', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest( + [makeRule({ id: 'VectorLint.RuleA' }), makeRule({ id: 'VectorLint.RuleB' })], + { budget: { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 1 } }, + ), + ); + + // Only the first rule's single call fits within maxModelCallsPerReview = 1. + expect(client.structuredCalls).toBe(1); + expect(result.usage?.modelCalls).toBe(1); + expect(result.hadOperationalErrors).toBe(true); + expect(result.diagnostics.some((d) => d.code === 'review-budget-exceeded')).toBe(true); + // The completed rule is still projected; the budget-exceeded rule is not. + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.RuleA'); + }); + + it('never invokes the tool-calling surface', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + await executor.run(makeRequest([makeRule()])); + + expect(client.toolCalls).toBe(0); + expect(client.structuredCalls).toBe(1); + }); + + it('chunks large targets into multiple model calls and merges per-chunk violations', async () => { + // > 600 words triggers recursive chunking. + const targetContent = Array.from({ length: 200 }, (_, i) => `vague text line ${i}`).join('\n'); + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text line 0' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest([makeRule()], { + target: { uri: 'file:///repo/docs/big.md', content: targetContent, contentType: 'text/markdown' }, + }), + ); + + expect(client.structuredCalls).toBeGreaterThan(1); + expect(result.usage?.modelCalls).toBe(client.structuredCalls); + expect(result.scores).toHaveLength(1); + // Identical violations across chunks dedupe to a single anchored finding. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + }); +}); From 8bf8e86c4012b41680bc24be3d7b98e0887ad166 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 11:10:34 +0100 Subject: [PATCH 21/26] Implement AgentModelCallExecutor with target-only read tool - Add AgentModelCallExecutor implementing ReviewExecutor via one bounded ToolCallingModelClient.runWithTools call per rule - Expose exactly one executor-owned tool, read_target_section, through TargetReadCapability bound to in-memory request.target.content (no filesystem or non-target URI access) - Map out-of-range/invalid ranges to model-visible error results so the bounded run continues instead of aborting - Enforce ReviewBudget.maxModelCallsPerReview via enforceBudget before each run, surfacing exhaustion as an error diagnostic with partial results - Pass request.rules[i].body verbatim as the source-backed prompt; no reviewInstruction is accepted or introduced - Project results through src/findings/processor.ts so anchored findings, scores, and diagnostics match the shared pipeline - Extract shared executor helpers (splitRuleId, buildEvalContext, buildReviewUsage, toFindingSeverity, budgetExceededDiagnostic, RunCounters) into src/executors/shared.ts and reuse from the single executor to remove duplication - Add focused tests for the agent executor and target-read adapter --- src/executors/agent-model-call-executor.ts | 172 +++++++++++ src/executors/shared.ts | 96 ++++++ src/executors/single-model-call-executor.ts | 87 ++---- .../target-read-capability-adapter.ts | 162 ++++++++++ .../agent-model-call-executor.test.ts | 277 ++++++++++++++++++ .../target-read-capability-adapter.test.ts | 108 +++++++ 6 files changed, 833 insertions(+), 69 deletions(-) create mode 100644 src/executors/agent-model-call-executor.ts create mode 100644 src/executors/shared.ts create mode 100644 src/executors/target-read-capability-adapter.ts create mode 100644 tests/executors/agent-model-call-executor.test.ts create mode 100644 tests/executors/target-read-capability-adapter.test.ts diff --git a/src/executors/agent-model-call-executor.ts b/src/executors/agent-model-call-executor.ts new file mode 100644 index 00000000..1b09f388 --- /dev/null +++ b/src/executors/agent-model-call-executor.ts @@ -0,0 +1,172 @@ +import type { ToolCallDefinition, ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { EvalContext, RequestBuilder } from '../providers/request-builder'; +import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; +import { countWords } from '../chunking'; +import { processFindings } from '../findings'; +import { enforceBudget } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewRequest, + ReviewResult, + ReviewRule, + ReviewScore, +} from '../review/types'; +import type { ReviewExecutor } from '../review/executor'; +import { TargetReadCapability, buildReadTargetSectionTool } from './target-read-capability-adapter'; +import { + budgetExceededDiagnostic, + buildEvalContext, + buildReviewUsage, + splitRuleId, + toFindingSeverity, + type RunCounters, +} from './shared'; + +/** + * The agent `modelCall` {@link ReviewExecutor} (audit Product Decision; Finding #2). + * + * Reviews target content against source-backed rules through a single bounded + * tool-calling run per rule via an injected {@link ToolCallingModelClient}. The + * only executor-owned tool exposed to the model is `read_target_section`, + * which pages through the in-memory `request.target.content` (the on-page + * boundary, audit Finding #5). Rule prompts come verbatim from + * {@link ReviewRule.body} — no model-supplied rule override is introduced — + * candidate findings flow through the shared {@link processFindings} pipeline, + * and the model-call budget is enforced via the review budget module before + * every call. + * + * This is the bounded, target-only review strategy selected by + * `--model-call agent` (and by `auto` for large inputs). + */ +export class AgentModelCallExecutor implements ReviewExecutor { + constructor( + private readonly client: ToolCallingModelClient, + private readonly builder: RequestBuilder, + ) {} + + async run(request: ReviewRequest): Promise { + const schema = buildCheckLLMSchema(); + const capability = new TargetReadCapability(request.target.content); + // Exactly one executor-owned tool is exposed: target-section paging. + const tools = buildReadTargetSectionTool(capability); + const context = buildEvalContext(request.target.uri); + + const findings: ReviewFinding[] = []; + const scores: ReviewScore[] = []; + const diagnostics: ReviewDiagnostic[] = []; + let hadOperationalErrors = false; + + const counters: RunCounters = { modelCalls: 0, inputTokens: 0, outputTokens: 0 }; + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + + try { + for (const rule of request.rules) { + const ruleResult = await this.reviewRule( + request, + rule, + schema, + tools, + context, + capability.lineCount, + counters, + elapsedMs, + ); + findings.push(...ruleResult.findings); + scores.push(...ruleResult.scores); + diagnostics.push(...ruleResult.diagnostics); + if (ruleResult.hadOperationalErrors) { + hadOperationalErrors = true; + } + } + } catch (error: unknown) { + const diagnostic = budgetExceededDiagnostic(error); + if (diagnostic) { + hadOperationalErrors = true; + diagnostics.push(diagnostic); + } else { + throw error; + } + } + + return { + findings, + scores, + diagnostics, + hadOperationalErrors, + usage: buildReviewUsage(request, counters, elapsedMs()), + }; + } + + /** + * Reviews a single rule: makes one bounded tool-calling run that lets the + * model page through the target via `read_target_section`, then projects the + * returned violations through {@link processFindings}. + */ + private async reviewRule( + request: ReviewRequest, + rule: ReviewRule, + schema: ReturnType, + tools: Record, + context: EvalContext, + targetLineCount: number, + counters: RunCounters, + elapsedMs: () => number, + ): Promise { + // Enforce the model-call budget before committing to the run. The + // prospective count (calls made so far plus this one) lets enforceBudget + // reject the run that would push the review over maxModelCallsPerReview. + enforceBudget(request.budget, { + modelCalls: counters.modelCalls + 1, + elapsedMs: elapsedMs(), + }); + + const { data, usage } = await this.client.runWithTools({ + // The source-backed rule body, wrapped with the directive/user + // instructions exactly as the single-call path does. No model-supplied + // rule override is introduced. + systemPrompt: this.builder.buildPromptBodyForStructured(rule.body, context), + prompt: this.buildTargetPrompt(request.target.uri, targetLineCount), + tools, + schema, + options: { + // Each run is bounded by the budget's per-rule section limit: the + // model may page through at most maxChunksPerRule sections before + // emitting its structured finding set. + maxSteps: request.budget.maxChunksPerRule, + maxParallelToolCalls: 1, + }, + }); + + counters.modelCalls += 1; + if (usage) { + counters.inputTokens += usage.inputTokens; + counters.outputTokens += usage.outputTokens; + } + + const { pack, ruleId } = splitRuleId(rule.id); + const wordCount = countWords(request.target.content) || 1; + + return processFindings({ + pack, + ruleId, + ruleSource: rule.source, + candidateFindings: data.violations, + wordCount, + promptMeta: { + ...(rule.severity !== undefined ? { severity: toFindingSeverity(rule.severity) } : {}), + }, + targetContent: request.target.content, + }); + } + + private buildTargetPrompt(uri: string, lineCount: number): string { + return [ + `Target URI: ${uri}`, + `Target lines: ${lineCount}`, + '', + 'Page through the target with the read_target_section tool, then report verified violations in the structured output.', + ].join('\n'); + } +} diff --git a/src/executors/shared.ts b/src/executors/shared.ts new file mode 100644 index 00000000..e9e24ef9 --- /dev/null +++ b/src/executors/shared.ts @@ -0,0 +1,96 @@ +import path from 'path'; + +import { Severity } from '../evaluators/types'; +import type { EvalContext } from '../providers/request-builder'; +import { BudgetExceededError } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewRequest, + ReviewSeverity, + ReviewUsage, +} from '../review/types'; + +/** + * Stable diagnostic code recorded when a run stops because the model-call + * budget was exhausted before every rule could be reviewed. + */ +export const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; + +/** + * Mutable run-wide counters shared across rules (and chunks/sections) by both + * the single and agent model-call executors. + */ +export interface RunCounters { + modelCalls: number; + inputTokens: number; + outputTokens: number; +} + +/** + * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. + * The review contract carries the composite id, while {@link processFindings} + * rebuilds the same id from the parts via `buildRuleId`. Splits on the first + * dot; pack names are single path segments and rule ids are PascalCase. + */ +export function splitRuleId(id: string): { pack: string; ruleId: string } { + const dot = id.indexOf('.'); + if (dot === -1) { + return { pack: id, ruleId: id }; + } + return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; +} + +/** + * Builds the provider {@link EvalContext} (file-type hint) for a target URI. + * Shared by both executors so structured and tool-calling calls receive the + * same file-type context for directive substitution. + */ +export function buildEvalContext(uri: string): EvalContext { + const ext = path.extname(uri); + return ext ? { fileType: ext } : {}; +} + +/** + * Maps the review contract's plain {@link ReviewSeverity} union onto the + * finding processor's {@link Severity} enum at the executor boundary (no + * unsafe cast). + */ +export function toFindingSeverity(severity: ReviewSeverity): Severity { + return severity === 'error' ? Severity.ERROR : Severity.WARNING; +} + +/** + * Aggregates run-wide counters into a {@link ReviewUsage}, including token + * counts only when the output policy opts in to usage reporting. + */ +export function buildReviewUsage( + request: ReviewRequest, + counters: RunCounters, + wallClockMs: number, +): ReviewUsage { + const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; + if (request.outputPolicy.includeUsage) { + usage.inputTokens = counters.inputTokens; + usage.outputTokens = counters.outputTokens; + } + return usage; +} + +/** + * Returns a `review-budget-exceeded` error diagnostic when `error` is a + * {@link BudgetExceededError}; otherwise returns `undefined` so the caller can + * rethrow non-budget errors unchanged. Shared by both executors' run loops so + * budget exhaustion surfaces as a partial-result operational failure instead + * of throwing past the {@link ReviewExecutor} contract. + */ +export function budgetExceededDiagnostic(error: unknown): ReviewDiagnostic | undefined { + if (!(error instanceof BudgetExceededError)) { + return undefined; + } + return { + level: 'error', + code: REVIEW_BUDGET_EXCEEDED_CODE, + message: error.message, + context: { limit: error.limit, actual: error.actual }, + }; +} diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts index de3957cb..3809cbb3 100644 --- a/src/executors/single-model-call-executor.ts +++ b/src/executors/single-model-call-executor.ts @@ -1,7 +1,5 @@ -import path from 'path'; - import type { ReviewExecutor } from '../review/executor'; -import { BudgetExceededError, enforceBudget } from '../review/budget'; +import { enforceBudget } from '../review/budget'; import type { ReviewDiagnostic, ReviewFinding, @@ -9,15 +7,21 @@ import type { ReviewResult, ReviewRule, ReviewScore, - ReviewUsage, } from '../review/types'; import type { EvalContext } from '../providers/request-builder'; import type { StructuredModelClient } from '../providers/structured-model-client'; -import { Severity } from '../evaluators/types'; import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; import { countWords, mergeViolations, RecursiveChunker, type Chunk } from '../chunking'; import { prependLineNumbers } from '../output/line-numbering'; import { processFindings } from '../findings'; +import { + budgetExceededDiagnostic, + buildEvalContext, + buildReviewUsage, + splitRuleId, + toFindingSeverity, + type RunCounters, +} from './shared'; /** * Word-count threshold above which the single-call executor chunks the target @@ -28,21 +32,6 @@ import { processFindings } from '../findings'; const CHUNKING_WORD_THRESHOLD = 600; const MAX_CHUNK_WORDS = 500; -/** - * Stable diagnostic code recorded when a run stops because the model-call - * budget was exhausted before every rule could be reviewed. - */ -const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; - -/** - * Mutable run-wide counters shared across rules and chunks. - */ -interface RunCounters { - modelCalls: number; - inputTokens: number; - outputTokens: number; -} - /** * The single modelCall {@link ReviewExecutor} (audit Finding #2). * @@ -61,7 +50,7 @@ export class SingleModelCallExecutor implements ReviewExecutor { async run(request: ReviewRequest): Promise { const schema = buildCheckLLMSchema(); - const context = this.buildContext(request.target.uri); + const context = buildEvalContext(request.target.uri); const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; @@ -90,17 +79,13 @@ export class SingleModelCallExecutor implements ReviewExecutor { } } } catch (error: unknown) { - if (error instanceof BudgetExceededError) { - // Surface the existing budget error pattern as an operational failure: - // the run returns partial results plus an error diagnostic rather than - // throwing past the ReviewExecutor contract. + // Surface budget exhaustion as an operational failure: the run returns + // partial results plus an error diagnostic rather than throwing past the + // ReviewExecutor contract. Non-budget errors propagate unchanged. + const diagnostic = budgetExceededDiagnostic(error); + if (diagnostic) { hadOperationalErrors = true; - diagnostics.push({ - level: 'error', - code: REVIEW_BUDGET_EXCEEDED_CODE, - message: error.message, - context: { limit: error.limit, actual: error.actual }, - }); + diagnostics.push(diagnostic); } else { throw error; } @@ -111,7 +96,7 @@ export class SingleModelCallExecutor implements ReviewExecutor { scores, diagnostics, hadOperationalErrors, - usage: this.buildUsage(request, counters, elapsedMs()), + usage: buildReviewUsage(request, counters, elapsedMs()), }; } @@ -165,12 +150,8 @@ export class SingleModelCallExecutor implements ReviewExecutor { ruleSource: rule.source, candidateFindings: mergedViolations, wordCount, - // Map the review contract's plain severity union onto the finding - // processor's Severity enum at this boundary. promptMeta: { - ...(rule.severity !== undefined - ? { severity: rule.severity === 'error' ? Severity.ERROR : Severity.WARNING } - : {}), + ...(rule.severity !== undefined ? { severity: toFindingSeverity(rule.severity) } : {}), }, targetContent: request.target.content, }); @@ -192,36 +173,4 @@ export class SingleModelCallExecutor implements ReviewExecutor { const chunker = new RecursiveChunker(); return chunker.chunk(numberedContent, { maxChunkSize: MAX_CHUNK_WORDS }).slice(0, maxChunks); } - - private buildContext(uri: string): EvalContext { - const ext = path.extname(uri); - return ext ? { fileType: ext } : {}; - } - - private buildUsage( - request: ReviewRequest, - counters: RunCounters, - wallClockMs: number, - ): ReviewUsage { - const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; - if (request.outputPolicy.includeUsage) { - usage.inputTokens = counters.inputTokens; - usage.outputTokens = counters.outputTokens; - } - return usage; - } -} - -/** - * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. - * The review contract carries the composite id, while {@link processFindings} - * rebuilds the same id from the parts via `buildRuleId`. Splits on the first - * dot; pack names are single path segments and rule ids are PascalCase. - */ -function splitRuleId(id: string): { pack: string; ruleId: string } { - const dot = id.indexOf('.'); - if (dot === -1) { - return { pack: id, ruleId: id }; - } - return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; } diff --git a/src/executors/target-read-capability-adapter.ts b/src/executors/target-read-capability-adapter.ts new file mode 100644 index 00000000..041c286a --- /dev/null +++ b/src/executors/target-read-capability-adapter.ts @@ -0,0 +1,162 @@ +import { z } from 'zod'; + +import { VectorlintError } from '../errors'; +import type { ToolCallDefinition } from '../providers/tool-calling-model-client'; +import type { ReviewTargetReadCapability } from '../review/executor'; + +/** + * The single executor-owned tool name exposed to the bounded tool-calling + * transport (audit Product Decision): page through the target under review by + * 1-based line range. This is the only agent-like capability that survives the + * Phase 4 agent removal. + */ +export const READ_TARGET_SECTION_TOOL_NAME = 'read_target_section'; + +/** + * Zod schema for {@link TargetReadCapability.readTargetSection} arguments: + * 1-based positive integers. The tool adapter parses model input against this + * before slicing, so malformed arguments become model-visible errors. + */ +export const READ_TARGET_SECTION_PARAMETERS = z + .object({ + startLine: z.number().int().positive(), + endLine: z.number().int().positive(), + }) + .strict(); + +const READ_TARGET_SECTION_DESCRIPTION = [ + 'Read a 1-based inclusive [startLine, endLine] window of the target content under review.', + 'Returns the window with original line numbers prepended so findings can cite exact lines.', + 'A window outside [1, targetLineCount], or with startLine > endLine, returns an error result', + 'describing the valid range instead of aborting the review.', +].join(' '); + +/** + * Thrown by {@link TargetReadCapability.readTargetSection} when the requested + * window is outside the target's valid line range. The tool adapter translates + * this into a model-visible error result so the bounded run continues. + */ +export class TargetSectionRangeError extends VectorlintError { + constructor( + message: string, + public readonly lineCount: number, + ) { + super(message, 'TARGET_SECTION_RANGE'); + this.name = 'TargetSectionRangeError'; + } +} + +/** Success result of a {@link TargetReadCapability.readTargetSection} call. */ +export interface TargetSectionResult { + startLine: number; + endLine: number; + /** The requested window, each line prefixed with its 1-based line number. */ + content: string; +} + +/** Model-visible error result returned in place of a window for invalid calls. */ +export interface TargetSectionErrorResult { + error: string; + /** Total lines in the target, so the model can retry with a valid range. */ + lineCount: number; +} + +/** + * The on-page boundary adapter (audit Finding #5): a target-only + * {@link ReviewTargetReadCapability} bound to the in-memory + * `ReviewTarget.content`. It performs NO filesystem access and reads no URI + * other than the target it was constructed from. The agent executor exposes + * this single capability to the model via {@link buildReadTargetSectionTool}. + */ +export class TargetReadCapability implements ReviewTargetReadCapability { + private readonly lines: readonly string[]; + + constructor(content: string) { + // Mirrors prependLineNumbers: a trailing newline yields a final empty line. + this.lines = content.split('\n'); + } + + /** Number of addressable lines in the target. */ + get lineCount(): number { + return this.lines.length; + } + + readTargetSection(startLine: number, endLine: number): Promise { + this.assertValidRange(startLine, endLine); + const slice = this.lines.slice(startLine - 1, endLine); + const content = slice.map((line, index) => `${startLine + index}\t${line}`).join('\n'); + return Promise.resolve({ startLine, endLine, content }); + } + + private assertValidRange(startLine: number, endLine: number): void { + const lineCount = this.lineCount; + if (!Number.isInteger(startLine) || !Number.isInteger(endLine)) { + throw new TargetSectionRangeError( + `read_target_section requires integer line numbers; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine < 1 || endLine < 1) { + throw new TargetSectionRangeError( + `read_target_section requires 1-based positive line numbers; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine > endLine) { + throw new TargetSectionRangeError( + `read_target_section requires startLine <= endLine; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine > lineCount) { + throw new TargetSectionRangeError( + `read_target_section startLine=${startLine} is beyond the target's ${lineCount} line(s). Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (endLine > lineCount) { + throw new TargetSectionRangeError( + `read_target_section endLine=${endLine} is beyond the target's ${lineCount} line(s). Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + } +} + +/** + * Builds the single executor-owned tool map exposed to the + * {@link ToolCallingModelClient}: exactly one entry, `read_target_section`, + * bound to the target-only {@link TargetReadCapability}. Argument-parse and + * range failures are returned as model-visible error results (never thrown + * past the tool boundary) so the bounded review run continues. + */ +export function buildReadTargetSectionTool( + capability: TargetReadCapability, +): Record { + return { + [READ_TARGET_SECTION_TOOL_NAME]: { + description: READ_TARGET_SECTION_DESCRIPTION, + parameters: READ_TARGET_SECTION_PARAMETERS, + execute: async ( + input: unknown, + ): Promise => { + const parsed = READ_TARGET_SECTION_PARAMETERS.safeParse(input); + if (!parsed.success) { + return { + error: `Invalid read_target_section arguments: ${parsed.error.message}.`, + lineCount: capability.lineCount, + }; + } + const { startLine, endLine } = parsed.data; + try { + return await capability.readTargetSection(startLine, endLine); + } catch (error: unknown) { + if (error instanceof TargetSectionRangeError) { + return { error: error.message, lineCount: error.lineCount }; + } + throw error; + } + }, + }, + }; +} diff --git a/tests/executors/agent-model-call-executor.test.ts b/tests/executors/agent-model-call-executor.test.ts new file mode 100644 index 00000000..9ade0f1a --- /dev/null +++ b/tests/executors/agent-model-call-executor.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, it } from 'vitest'; + +import { AgentModelCallExecutor } from '../../src/executors/agent-model-call-executor'; +import { DEFAULT_REVIEW_BUDGET } from '../../src/review'; +import type { ReviewRequest, ReviewRule, ReviewTarget } from '../../src/review'; +import { DefaultRequestBuilder } from '../../src/providers/request-builder'; +import type { ToolCallDefinition, ToolCallingModelClient, ToolCallRunOptions } from '../../src/providers/tool-calling-model-client'; +import type { LLMResult } from '../../src/providers/structured-model-client'; +import type { TokenUsage } from '../../src/providers/token-usage'; +import type { CheckLLMResult } from '../../src/prompts/schema'; +import type { TargetSectionErrorResult, TargetSectionResult } from '../../src/executors/target-read-capability-adapter'; + +const SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const SUPPORTED_NOTES = { + rule_supports_claim: 'yes', + evidence_exact: 'yes', + context_supports_violation: 'yes', + plausible_non_violation: 'no', + fix_is_drop_in: 'yes', + fix_preserves_meaning: 'yes', +}; + +type ModelViolation = CheckLLMResult['violations'][number]; + +function modelViolation(overrides: Partial = {}): ModelViolation { + return { + line: 1, + quoted_text: 'vague text', + context_before: '', + context_after: '', + description: 'desc', + analysis: 'analysis', + message: 'message', + suggestion: 'suggestion', + fix: 'fix', + rule_quote: 'rule quote', + checks: SUPPORTED_CHECKS, + check_notes: SUPPORTED_NOTES, + confidence: 0.9, + ...overrides, + }; +} + +interface CapturedCall { + systemPrompt: string; + prompt: string; + tools: Record; + options?: ToolCallRunOptions; +} + +interface FakeToolClient extends ToolCallingModelClient { + calls: number; + captured: CapturedCall[]; +} + +function makeFakeClient( + respond: () => { data: CheckLLMResult; usage?: TokenUsage }, +): FakeToolClient { + const fake = { + calls: 0, + captured: [] as CapturedCall[], + runWithTools: ( + params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }, + ): Promise> => { + fake.calls += 1; + fake.captured.push({ + systemPrompt: params.systemPrompt, + prompt: params.prompt, + tools: params.tools, + ...(params.options !== undefined ? { options: params.options } : {}), + }); + const { data, usage } = respond(); + const result: LLMResult = { data: data as unknown as T }; + if (usage) { + result.usage = usage; + } + return Promise.resolve(result); + }, + }; + return fake; +} + +const TARGET: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: 'vague text here\nsecond line is fine\nthird line\n', + contentType: 'text/markdown', +}; + +function makeRule(overrides: Partial = {}): ReviewRule { + return { + id: 'VectorLint.PseudoAdvice', + source: '/repo/presets/VectorLint/pseudo-advice.md', + body: 'Flag vague advice.', + ...overrides, + }; +} + +function makeRequest(rules: ReviewRule[], overrides: Partial = {}): ReviewRequest { + return { + target: TARGET, + rules, + budget: { ...DEFAULT_REVIEW_BUDGET }, + outputPolicy: { includeUsage: true, recordPayloadTelemetry: false }, + modelCall: 'agent', + ...overrides, + }; +} + +describe('AgentModelCallExecutor', () => { + it('exposes exactly one tool (read_target_section) and projects findings/scores through processFindings', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text', message: 'Too vague.' })], + }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run(makeRequest([makeRule()])); + + // Exactly one bounded model run, exposing exactly one tool. + expect(client.calls).toBe(1); + expect(Object.keys(client.captured[0]!.tools)).toEqual(['read_target_section']); + + // The finding is verified + anchored through the shared processor. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.findings[0]?.ruleSource).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(result.findings[0]?.match).toBe('vague text'); + + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.scores[0]?.findingCount).toBe(1); + + expect(result.usage?.modelCalls).toBe(1); + }); + + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'quantum entanglement device' })], + }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run(makeRequest([makeRule()])); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe('finding-evidence-not-locatable'); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('passes the source-backed rule body verbatim as the system prompt (no model-supplied rule override)', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule({ body: 'Flag vague advice.' })])); + + // With an empty directive/user-instructions builder, the system prompt is + // the rule body verbatim — no model-supplied rule override is introduced. + expect(client.captured[0]!.systemPrompt).toBe('Flag vague advice.'); + // The agent instruction names the only tool and the target. + expect(client.captured[0]!.prompt).toContain('read_target_section'); + expect(client.captured[0]!.prompt).toContain('file:///repo/docs/guide.md'); + }); + + it('stops and records an operational error when the model-call budget is exhausted', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run( + makeRequest( + [makeRule({ id: 'VectorLint.RuleA' }), makeRule({ id: 'VectorLint.RuleB' })], + { budget: { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 1 } }, + ), + ); + + // Only the first rule's run fits within maxModelCallsPerReview = 1. + expect(client.calls).toBe(1); + expect(result.usage?.modelCalls).toBe(1); + expect(result.hadOperationalErrors).toBe(true); + expect(result.diagnostics.some((d) => d.code === 'review-budget-exceeded')).toBe(true); + // The completed rule is still projected; the budget-exceeded rule is not. + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.RuleA'); + }); + + it('reviews each rule independently and aggregates usage across runs', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text' })], + }, + usage: { inputTokens: 10, outputTokens: 5 }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run( + makeRequest([ + makeRule({ id: 'VectorLint.RuleA', source: '/repo/a.md' }), + makeRule({ id: 'VectorLint.RuleB', source: '/repo/b.md' }), + ]), + ); + + expect(client.calls).toBe(2); + expect(result.scores.map((s) => s.ruleId).sort()).toEqual(['VectorLint.RuleA', 'VectorLint.RuleB']); + expect(result.usage?.modelCalls).toBe(2); + expect(result.usage?.inputTokens).toBe(20); + expect(result.usage?.outputTokens).toBe(10); + }); + + it('exposes a read_target_section that slices only request.target.content', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule()])); + + const tool = client.captured[0]!.tools['read_target_section']!; + const result = (await tool.execute({ startLine: 2, endLine: 3 })) as TargetSectionResult; + + // The window matches the in-memory target lines with their real line numbers. + expect(result).toEqual({ + startLine: 2, + endLine: 3, + content: '2\tsecond line is fine\n3\tthird line', + }); + }); + + it('returns a model-visible error for out-of-range windows without aborting the run', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + // The run completes and returns a normal ReviewResult despite the model + // (simulated below) requesting an out-of-range section. + const result = await executor.run(makeRequest([makeRule()])); + expect(result.hadOperationalErrors).toBe(false); + + const tool = client.captured[0]!.tools['read_target_section']!; + const errorResult = (await tool.execute({ startLine: 1, endLine: 999 })) as TargetSectionErrorResult; + + expect(errorResult.error).toContain('999'); + expect(errorResult.lineCount).toBe(4); + }); + + it('bounds each run by the budget maxChunksPerRule step limit', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run( + makeRequest([makeRule()], { budget: { ...DEFAULT_REVIEW_BUDGET, maxChunksPerRule: 7 } }), + ); + + expect(client.captured[0]!.options?.maxSteps).toBe(7); + expect(client.captured[0]!.options?.maxParallelToolCalls).toBe(1); + }); +}); diff --git a/tests/executors/target-read-capability-adapter.test.ts b/tests/executors/target-read-capability-adapter.test.ts new file mode 100644 index 00000000..c46016af --- /dev/null +++ b/tests/executors/target-read-capability-adapter.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest'; + +import { + buildReadTargetSectionTool, + READ_TARGET_SECTION_TOOL_NAME, + TargetReadCapability, + type TargetSectionErrorResult, + type TargetSectionResult, +} from '../../src/executors/target-read-capability-adapter'; + +const CONTENT = 'alpha line\nbeta line\ngamma line\n'; + +describe('TargetReadCapability', () => { + it('counts lines including the trailing empty line (mirrors prependLineNumbers)', () => { + expect(new TargetReadCapability(CONTENT).lineCount).toBe(4); + expect(new TargetReadCapability('only one').lineCount).toBe(1); + }); + + it('slices only the in-memory target content with original line numbers', async () => { + const capability = new TargetReadCapability(CONTENT); + + const first = await capability.readTargetSection(1, 2); + expect(first).toEqual({ + startLine: 1, + endLine: 2, + content: '1\talpha line\n2\tbeta line', + }); + + const tail = await capability.readTargetSection(3, 4); + expect(tail).toEqual({ + startLine: 3, + endLine: 4, + content: '3\tgamma line\n4\t', + }); + }); +}); + +describe('buildReadTargetSectionTool', () => { + function toolFor(content: string) { + return buildReadTargetSectionTool(new TargetReadCapability(content)); + } + + it('exposes exactly one tool named read_target_section', () => { + const tools = toolFor(CONTENT); + expect(Object.keys(tools)).toEqual([READ_TARGET_SECTION_TOOL_NAME]); + }); + + it('returns numbered target windows for valid ranges (target-only slicing)', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 1, + endLine: 3, + })) as TargetSectionResult; + + expect(result).toEqual({ + startLine: 1, + endLine: 3, + content: '1\talpha line\n2\tbeta line\n3\tgamma line', + }); + }); + + it('returns a model-visible error (never throws) when endLine exceeds the target', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 1, + endLine: 99, + })) as TargetSectionErrorResult; + + expect(result.error).toContain('99'); + expect(result.lineCount).toBe(4); + }); + + it('returns a model-visible error when startLine is past the target', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 10, + endLine: 12, + })) as TargetSectionErrorResult; + + expect(result.error).toContain('10'); + expect(result.lineCount).toBe(4); + }); + + it('returns a model-visible error for an inverted range', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 3, + endLine: 1, + })) as TargetSectionErrorResult; + + expect(result.error).toMatch(/startLine <= endLine/); + }); + + it('returns a model-visible error for malformed arguments', async () => { + const tools = toolFor(CONTENT); + const execute = tools[READ_TARGET_SECTION_TOOL_NAME]!.execute; + + const missing = (await execute({ startLine: 1 })) as TargetSectionErrorResult; + expect(missing.error).toMatch(/Invalid read_target_section arguments/); + expect(missing.lineCount).toBe(4); + + const negative = (await execute({ startLine: 0, endLine: 2 })) as TargetSectionErrorResult; + expect(negative.error).toMatch(/Invalid read_target_section arguments/); + + const floats = (await execute({ startLine: 1.5, endLine: 2 })) as TargetSectionErrorResult; + expect(floats.error).toMatch(/Invalid read_target_section arguments/); + }); +}); From 8bfb47de8264fd7e75440347d64e3c87e2b2f4fb Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:18:00 +0100 Subject: [PATCH 22/26] Wire --model-call executors into the CLI and remove the agent surface (core) --- src/agent/executor.ts | 843 ------------------ src/agent/path-utils.ts | 73 -- src/agent/progress.ts | 282 ------ src/agent/prompt-builder.ts | 57 -- src/agent/review-session-store.ts | 94 -- src/agent/rule-id.ts | 11 - src/agent/tools-registry.ts | 80 -- src/agent/types.ts | 134 --- src/cli/commands.ts | 22 +- src/cli/orchestrator.ts | 772 ++++------------ src/cli/types.ts | 65 +- src/executors/index.ts | 38 + src/providers/llm-provider.ts | 42 +- src/providers/provider-factory.ts | 5 +- src/providers/vercel-ai-provider.ts | 67 +- src/schemas/cli-schemas.ts | 5 +- tests/agent/agent-executor.test.ts | 841 ----------------- tests/agent/progress.test.ts | 90 -- tests/agent/prompt-builder.test.ts | 51 -- tests/agent/review-session-store.test.ts | 142 --- tests/agent/types-contract.test.ts | 83 -- tests/main-command-observability.test.ts | 6 +- tests/orchestrator-agent-output.test.ts | 165 ---- tests/orchestrator-check-processor.test.ts | 356 -------- tests/orchestrator-filtering.test.ts | 412 --------- tests/orchestrator-finding-criteria.test.ts | 126 --- tests/providers/llm-provider-contract.test.ts | 14 - .../providers/structured-model-client.test.ts | 6 +- .../tool-calling-model-client.test.ts | 2 - .../vercel-ai-provider-agent-loop.test.ts | 314 ------- 30 files changed, 263 insertions(+), 4935 deletions(-) delete mode 100644 src/agent/executor.ts delete mode 100644 src/agent/path-utils.ts delete mode 100644 src/agent/progress.ts delete mode 100644 src/agent/prompt-builder.ts delete mode 100644 src/agent/review-session-store.ts delete mode 100644 src/agent/rule-id.ts delete mode 100644 src/agent/tools-registry.ts delete mode 100644 src/agent/types.ts create mode 100644 src/executors/index.ts delete mode 100644 tests/agent/agent-executor.test.ts delete mode 100644 tests/agent/progress.test.ts delete mode 100644 tests/agent/prompt-builder.test.ts delete mode 100644 tests/agent/review-session-store.test.ts delete mode 100644 tests/agent/types-contract.test.ts delete mode 100644 tests/orchestrator-agent-output.test.ts delete mode 100644 tests/orchestrator-check-processor.test.ts delete mode 100644 tests/orchestrator-filtering.test.ts delete mode 100644 tests/orchestrator-finding-criteria.test.ts delete mode 100644 tests/providers/llm-provider-contract.test.ts delete mode 100644 tests/providers/vercel-ai-provider-agent-loop.test.ts diff --git a/src/agent/executor.ts b/src/agent/executor.ts deleted file mode 100644 index 749a5685..00000000 --- a/src/agent/executor.ts +++ /dev/null @@ -1,843 +0,0 @@ -import { readdir, readFile } from 'fs/promises'; -import * as os from 'os'; -import fg from 'fast-glob'; -import { buildCheckLLMSchema, isJudgeResult, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; -import type { PromptFile } from '../prompts/prompt-loader'; -import { Type, Severity } from '../evaluators/types'; -import { computeFilterDecision } from '../evaluators/violation-filter'; -import { locateQuotedText } from '../output/location'; -import type { AgentToolLoopResult, LLMProvider } from '../providers/llm-provider'; -import type { TokenUsage } from '../providers/token-usage'; -import type { OutputFormat } from '../cli/types'; -import { createEvaluator } from '../evaluators'; -import { createReviewSessionStore } from './review-session-store'; -import { buildAgentSystemPrompt } from './prompt-builder'; -import { AgentToolError } from '../errors'; -import { ScanPathResolver } from '../boundaries/scan-path-resolver'; -import type { FilePatternConfig } from '../boundaries/file-section-parser'; -import { - createAgentTools, - listAvailableTools, - type AgentToolHandler, - type AgentToolName, -} from './tools-registry'; -import { - LINT_TOOL_INPUT_SCHEMA, - FINALIZE_REVIEW_INPUT_SCHEMA, - LIST_DIRECTORY_INPUT_SCHEMA, - READ_FILE_INPUT_SCHEMA, - SEARCH_CONTENT_INPUT_SCHEMA, - SEARCH_FILES_INPUT_SCHEMA, - TOP_LEVEL_REPORT_INPUT_SCHEMA, - SESSION_EVENT_TYPE, - type SessionEvent, -} from './types'; -import { resolveGlobPatternWithinRoot, resolveWithinRoot, toRelativePathFromRoot } from './path-utils'; -import type { AgentProgressReporter, VisibleToolName, VisibleToolProgress } from './progress'; -import { buildRuleId, normalizeRuleSource } from './rule-id'; -export interface AgentFinding { - file: string; - line: number; - column: number; - severity: Severity; - message: string; - ruleId: string; - ruleSource: string; - analysis?: string; - suggestion?: string; - fix?: string; - match?: string; -} - -export interface RunAgentExecutorParams { - targets: string[]; - prompts: PromptFile[]; - provider: LLMProvider; - workspaceRoot: string; - scanPaths: FilePatternConfig[]; - outputFormat: OutputFormat; - printMode: boolean; - sessionHomeDir?: string; - progressReporter?: AgentProgressReporter; - maxSteps?: number; - maxRetries?: number; - maxParallelToolCalls?: number; - userInstructions?: string; -} - -export interface AgentExecutorResult { - findings: AgentFinding[]; - events: SessionEvent[]; - fileRuleMatches: Array<{ file: string; ruleSource: string }>; - requestFailures: number; - hadOperationalErrors: boolean; - errorMessage?: string; - usage?: AgentToolLoopResult['usage']; -} - -const MAX_SEARCH_FILE_RESULTS = 500; -const MAX_CONTENT_MATCH_RESULTS = 200; -const VISIBLE_TOOL_NAMES = new Set(['read_file', 'list_directory', 'lint']); - -function mergeTokenUsage(left?: TokenUsage, right?: TokenUsage): TokenUsage | undefined { - if (!left && !right) { - return undefined; - } - - return { - inputTokens: (left?.inputTokens ?? 0) + (right?.inputTokens ?? 0), - outputTokens: (left?.outputTokens ?? 0) + (right?.outputTokens ?? 0), - }; -} - -function buildUnknownRuleSourceError(ruleSource: string, validSources: string[]): Error { - const validHint = validSources.length > 0 ? validSources.join(', ') : '(none)'; - return new AgentToolError( - `Unknown ruleSource "${ruleSource}". Valid sources: ${validHint}`, - 'UNKNOWN_RULE_SOURCE' - ); -} - -function fallbackMessage(result: PromptEvaluationResult): string { - if (!isJudgeResult(result) && result.reasoning) { - return result.reasoning; - } - return 'Potential issue detected'; -} - -function severityFromPrompt(prompt: PromptFile): Severity { - return prompt.meta.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING; -} - -function resolvePromptBySource( - ruleSource: string, - promptBySource: Map -): PromptFile | undefined { - const normalized = normalizeRuleSource(ruleSource); - return promptBySource.get(normalized); -} - -function resolveTargetForTopLevel( - workspaceRoot: string, - targets: string[], - file?: string -): string { - if (file && file.trim().length > 0) { - const resolved = resolveWithinRoot(workspaceRoot, file); - return toRelativePathFromRoot(workspaceRoot, resolved); - } - if (targets.length > 0) { - return toRelativePathFromRoot(workspaceRoot, targets[0]!); - } - return '.'; -} - -function withReviewInstructionOverride(prompt: PromptFile, reviewInstruction?: string): PromptFile { - const instruction = reviewInstruction?.trim(); - if (!instruction) { - return prompt; - } - return { - ...prompt, - body: instruction, - }; -} - -function findingsFromEvents(events: SessionEvent[]): AgentFinding[] { - const findings: AgentFinding[] = []; - - for (const event of events) { - if ( - event.eventType !== SESSION_EVENT_TYPE.FindingRecordedInline && - event.eventType !== SESSION_EVENT_TYPE.FindingRecordedTopLevel - ) { - continue; - } - - const payload = event.payload; - const file = payload.file ?? '.'; - const line = payload.line ?? 1; - findings.push({ - file, - line, - column: payload.column ?? 1, - severity: payload.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING, - message: payload.message, - ruleId: payload.ruleId ?? payload.ruleSource, - ruleSource: payload.ruleSource, - ...('analysis' in payload && payload.analysis ? { analysis: payload.analysis } : {}), - ...('suggestion' in payload && payload.suggestion ? { suggestion: payload.suggestion } : {}), - ...('fix' in payload && payload.fix ? { fix: payload.fix } : {}), - ...('match' in payload && payload.match ? { match: payload.match } : {}), - }); - } - - return findings; -} - -// Build the concrete matched file-to-rule pairs that the agent should review for this run. -function buildFileRuleMatches( - relativeTargets: string[], - prompts: PromptFile[], - scanPaths: FilePatternConfig[] -): Array<{ file: string; ruleSource: string }> { - const resolver = new ScanPathResolver(); - const availablePacks = Array.from(new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p))); - const matches: Array<{ file: string; ruleSource: string }> = []; - - for (const relFile of relativeTargets) { - const resolution = - scanPaths.length > 0 - ? resolver.resolveConfiguration(relFile, scanPaths, availablePacks) - : { packs: availablePacks, overrides: {} }; - const matchedPrompts = prompts.filter((prompt) => { - if (prompt.pack === '') return true; - if (!prompt.pack) return false; - if (scanPaths.length > 0 && !resolution.packs.includes(prompt.pack)) return false; - if (!prompt.meta?.id) return true; - const disableKey = `${prompt.pack}.${prompt.meta.id}`; - const overrideValue = resolution.overrides[disableKey]; - return typeof overrideValue !== 'string' || overrideValue.toLowerCase() !== 'disabled'; - }); - - for (const prompt of matchedPrompts) { - matches.push({ file: relFile, ruleSource: normalizeRuleSource(prompt.fullPath) }); - } - } - - return matches; -} - -function summarizeToolOutput(toolName: AgentToolName, output: unknown): unknown { - if (typeof output !== 'object' || output === null) { - return output; - } - - const candidate = output as Record; - - switch (toolName) { - case 'read_file': { - const content = typeof candidate.content === 'string' ? candidate.content : ''; - return { - ...(typeof candidate.path === 'string' ? { path: candidate.path } : {}), - contentLength: content.length, - }; - } - case 'search_content': { - const matches = Array.isArray(candidate.matches) ? candidate.matches : []; - return { - matchCount: matches.length, - ...(candidate.truncated === true ? { truncated: true } : {}), - }; - } - case 'search_files': { - const matches = Array.isArray(candidate.matches) ? candidate.matches : []; - return { - matchCount: matches.length, - ...(candidate.truncated === true ? { truncated: true } : {}), - }; - } - case 'list_directory': { - const entries = Array.isArray(candidate.entries) ? candidate.entries : []; - return { - ...(typeof candidate.path === 'string' ? { path: candidate.path } : {}), - entryCount: entries.length, - }; - } - default: - return output; - } -} - -interface VisibleToolContext extends VisibleToolProgress { - signature: string; - progressFile?: string; -} - -function countLines(content: string): number { - if (content.length === 0) { - return 0; - } - return content.split('\n').filter((_, index, lines) => { - return !(index === lines.length - 1 && lines[index] === ''); - }).length; -} - -function tryResolveRelativePath(workspaceRoot: string, rawPath: string): string | undefined { - try { - return toRelativePathFromRoot(workspaceRoot, resolveWithinRoot(workspaceRoot, rawPath)); - } catch { - return undefined; - } -} - -function resolveVisibleToolContext(params: { - toolName: AgentToolName; - input: unknown; - workspaceRoot: string; - promptBySource: Map; - targetFiles: Set; - currentProgressFile?: string; - defaultProgressFile?: string; -}): VisibleToolContext | undefined { - const { - toolName, - input, - workspaceRoot, - promptBySource, - targetFiles, - currentProgressFile, - defaultProgressFile, - } = params; - - if (!VISIBLE_TOOL_NAMES.has(toolName as VisibleToolName)) { - return undefined; - } - - switch (toolName) { - case 'read_file': { - const parsed = READ_FILE_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); - const path = resolvedPath ?? parsed.data.path; - const progressFile = resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile); - return { - toolName: 'read_file', - path, - ...(progressFile ? { progressFile } : {}), - signature: `read_file:${path}`, - }; - } - case 'list_directory': { - const parsed = LIST_DIRECTORY_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); - const path = resolvedPath ?? parsed.data.path; - const progressFile = currentProgressFile ?? defaultProgressFile; - return { - toolName: 'list_directory', - path, - ...(progressFile ? { progressFile } : {}), - signature: `list_directory:${path}`, - }; - } - case 'lint': { - const parsed = LINT_TOOL_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const prompt = resolvePromptBySource(parsed.data.ruleSource, promptBySource); - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.file); - const path = resolvedPath ?? parsed.data.file; - const ruleText = parsed.data.reviewInstruction?.trim() || prompt?.body || ''; - const progressFile = resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile); - return { - toolName: 'lint', - path, - ruleName: String(prompt?.meta.name || prompt?.meta.id || 'Rule'), - ruleText, - ...(progressFile ? { progressFile } : {}), - signature: `lint:${path}:${normalizeRuleSource(parsed.data.ruleSource)}:${ruleText}`, - }; - } - } -} - -function buildVisibleToolSuccessState( - context: VisibleToolContext, - output: unknown -): VisibleToolProgress { - switch (context.toolName) { - case 'read_file': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - const content = typeof record.content === 'string' ? record.content : ''; - return { - ...context, - lineCount: countLines(content), - }; - } - case 'list_directory': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - const entries = Array.isArray(record.entries) ? record.entries : []; - return { - ...context, - entryCount: entries.length, - }; - } - case 'lint': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - return { - ...context, - findingsCount: - typeof record.findingsRecorded === 'number' ? Math.max(0, Math.trunc(record.findingsRecorded)) : 0, - }; - } - } -} - -type FindingLikeViolation = { - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - description?: string; - analysis?: string; - message?: string; - suggestion?: string; - fix?: string; - confidence?: number; - checks?: GateChecks; -}; - -async function appendInlineFinding(params: { - violation: FindingLikeViolation; - result: PromptEvaluationResult; - content: string; - relFile: string; - prompt: PromptFile; - ruleSource: string; - store: Awaited>; -}): Promise { - const { violation, result, content, relFile, prompt, ruleSource, store } = params; - const filterDecision = computeFilterDecision(violation); - if (!filterDecision.surface) { - return false; - } - - const location = locateQuotedText( - content, - { - quoted_text: violation.quoted_text || '', - context_before: violation.context_before || '', - context_after: violation.context_after || '', - }, - 80, - violation.line - ); - - const line = location?.line ?? Math.max(1, Math.trunc(violation.line ?? 1)); - const column = location?.column ?? 1; - const match = location?.match ?? violation.quoted_text ?? ''; - const message = (violation.message || violation.description || fallbackMessage(result)).trim(); - - const finding: AgentFinding = { - file: relFile, - line, - column, - severity: severityFromPrompt(prompt), - message, - ruleId: buildRuleId(prompt), - ruleSource: normalizeRuleSource(ruleSource), - ...(violation.analysis ? { analysis: violation.analysis } : {}), - ...(violation.suggestion ? { suggestion: violation.suggestion } : {}), - ...(violation.fix ? { fix: violation.fix } : {}), - ...(match ? { match } : {}), - }; - - await store.append({ - eventType: SESSION_EVENT_TYPE.FindingRecordedInline, - payload: { - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - ruleId: finding.ruleId, - ruleSource: finding.ruleSource, - message: finding.message, - ...(finding.analysis ? { analysis: finding.analysis } : {}), - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(finding.fix ? { fix: finding.fix } : {}), - ...(finding.match ? { match: finding.match } : {}), - }, - }); - - return true; -} - -export async function runAgentExecutor(params: RunAgentExecutorParams): Promise { - const { - targets, - prompts, - provider, - workspaceRoot, - scanPaths, - sessionHomeDir = os.homedir(), - progressReporter, - maxSteps, - maxRetries, - maxParallelToolCalls, - userInstructions, - } = params; - - const promptBySource = new Map(); - for (const prompt of prompts) { - promptBySource.set(normalizeRuleSource(prompt.fullPath), prompt); - } - const validSources = Array.from(promptBySource.keys()).sort(); - - const store = await createReviewSessionStore({ homeDir: sessionHomeDir }); - let findingsCount = 0; - const relativeTargets = targets.map((target) => - toRelativePathFromRoot(workspaceRoot, resolveWithinRoot(workspaceRoot, target)) - ); - const fileRuleMatches = buildFileRuleMatches(relativeTargets, prompts, scanPaths); - const defaultRuleName = String(prompts[0]?.meta.name || prompts[0]?.meta.id || 'Rule'); - const targetFiles = new Set(relativeTargets); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { - cwd: workspaceRoot, - targets: relativeTargets, - }, - }); - - let finalized = false; - let currentProgressFile: string | undefined; - const failedVisibleToolSignatures = new Set(); - let nestedToolUsage: TokenUsage | undefined; - - async function runTool( - toolName: AgentToolName, - input: unknown, - handler: AgentToolHandler - ): Promise { - const visibleToolContext = resolveVisibleToolContext({ - toolName, - input, - workspaceRoot, - promptBySource, - targetFiles, - ...(currentProgressFile ? { currentProgressFile } : {}), - ...(relativeTargets[0] ? { defaultProgressFile: relativeTargets[0] } : {}), - }); - - if (visibleToolContext?.progressFile) { - const nextRuleName = visibleToolContext.ruleName ?? defaultRuleName; - if (currentProgressFile !== visibleToolContext.progressFile) { - progressReporter?.startFile(visibleToolContext.progressFile, nextRuleName); - currentProgressFile = visibleToolContext.progressFile; - } else if (visibleToolContext.ruleName) { - progressReporter?.updateRule(visibleToolContext.ruleName); - } - } - - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallStarted, - payload: { toolName, input }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolStart({ - ...visibleToolContext, - retrying: failedVisibleToolSignatures.has(visibleToolContext.signature), - }); - } - - try { - const output = await handler(input); - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { - toolName, - ok: true, - output: summarizeToolOutput(toolName, output), - }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolSuccess(buildVisibleToolSuccessState(visibleToolContext, output)); - failedVisibleToolSignatures.delete(visibleToolContext.signature); - } - return output; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { - toolName, - ok: false, - error: message, - }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolError(visibleToolContext); - failedVisibleToolSignatures.add(visibleToolContext.signature); - } - throw error; - } - } - - async function lintToolHandler(input: unknown): Promise { - const parsed = LINT_TOOL_INPUT_SCHEMA.parse(input); - const prompt = resolvePromptBySource(parsed.ruleSource, promptBySource); - if (!prompt) { - throw buildUnknownRuleSourceError(parsed.ruleSource, validSources); - } - const promptForCall = withReviewInstructionOverride(prompt, parsed.reviewInstruction); - - const absoluteFile = resolveWithinRoot(workspaceRoot, parsed.file); - const relFile = toRelativePathFromRoot(workspaceRoot, absoluteFile); - const content = await readFile(absoluteFile, 'utf8'); - - const evaluator = createEvaluator( - promptForCall.meta.evaluator || Type.BASE, - provider, - promptForCall - ); - const result = await evaluator.evaluate(relFile, content); - nestedToolUsage = mergeTokenUsage(nestedToolUsage, result.usage); - - let findingsRecorded = 0; - const violations = isJudgeResult(result) - ? result.criteria.flatMap((criterion) => criterion.violations) - : result.violations; - for (const violation of violations) { - const wasRecorded = await appendInlineFinding({ - violation, - result, - content, - relFile, - prompt, - ruleSource: parsed.ruleSource, - store, - }); - if (wasRecorded) { - findingsCount += 1; - findingsRecorded += 1; - } - } - - return { - ok: true, - findingsRecorded, - schema: buildCheckLLMSchema().name, - }; - } - - async function reportFindingToolHandler(input: unknown): Promise { - const parsed = TOP_LEVEL_REPORT_INPUT_SCHEMA.parse(input); - const prompt = resolvePromptBySource(parsed.ruleSource, promptBySource); - if (!prompt) { - throw buildUnknownRuleSourceError(parsed.ruleSource, validSources); - } - - const references = parsed.references && parsed.references.length > 0 - ? parsed.references - : [{ file: resolveTargetForTopLevel(workspaceRoot, targets), startLine: 1, endLine: 1 }]; - - let findingsRecorded = 0; - for (const reference of references) { - const relFile = resolveTargetForTopLevel(workspaceRoot, targets, reference.file); - const finding: AgentFinding = { - file: relFile, - line: Math.max(1, Math.trunc(reference.startLine)), - column: 1, - severity: severityFromPrompt(prompt), - message: parsed.message, - ruleId: buildRuleId(prompt), - ruleSource: normalizeRuleSource(parsed.ruleSource), - ...(parsed.suggestion ? { suggestion: parsed.suggestion } : {}), - }; - - findingsCount += 1; - findingsRecorded += 1; - await store.append({ - eventType: SESSION_EVENT_TYPE.FindingRecordedTopLevel, - payload: { - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - ruleId: finding.ruleId, - ruleSource: finding.ruleSource, - message: finding.message, - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(parsed.references ? { references: parsed.references } : {}), - }, - }); - } - - return { ok: true, findingsRecorded }; - } - - async function readFileToolHandler(input: unknown): Promise { - const parsed = READ_FILE_INPUT_SCHEMA.parse(input); - const absolutePath = resolveWithinRoot(workspaceRoot, parsed.path); - const content = await readFile(absolutePath, 'utf8'); - return { - path: toRelativePathFromRoot(workspaceRoot, absolutePath), - content, - }; - } - - async function searchFilesToolHandler(input: unknown): Promise { - const parsed = SEARCH_FILES_INPUT_SCHEMA.parse(input); - const scope = resolveGlobPatternWithinRoot(workspaceRoot, parsed.pattern); - const allMatches = await fg(scope.pattern, { - cwd: scope.cwd, - dot: false, - onlyFiles: true, - absolute: true, - }); - - const truncated = allMatches.length > MAX_SEARCH_FILE_RESULTS; - const matches = allMatches - .slice(0, MAX_SEARCH_FILE_RESULTS) - .map((match) => toRelativePathFromRoot(workspaceRoot, match)) - .sort((a, b) => a.localeCompare(b)); - - return { matches, ...(truncated ? { truncated: true } : {}) }; - } - - async function listDirectoryToolHandler(input: unknown): Promise { - const parsed = LIST_DIRECTORY_INPUT_SCHEMA.parse(input); - const absolutePath = resolveWithinRoot(workspaceRoot, parsed.path); - const entries = await readdir(absolutePath, { withFileTypes: true }); - - return { - path: toRelativePathFromRoot(workspaceRoot, absolutePath), - entries: entries - .map((entry) => ({ - name: entry.name, - type: entry.isDirectory() ? 'directory' : 'file', - })) - .sort((left, right) => left.name.localeCompare(right.name)), - }; - } - - async function searchContentToolHandler(input: unknown): Promise { - const parsed = SEARCH_CONTENT_INPUT_SCHEMA.parse(input); - const absoluteSearchRoot = resolveWithinRoot(workspaceRoot, parsed.path || '.'); - const globScope = resolveGlobPatternWithinRoot(absoluteSearchRoot, parsed.glob || '**/*'); - const files = await fg(globScope.pattern, { - cwd: globScope.cwd, - dot: false, - onlyFiles: true, - absolute: true, - }); - - const matches: Array<{ file: string; line: number; text: string }> = []; - let truncated = false; - - outer: for (const filePath of files) { - let content = ''; - try { - content = await readFile(filePath, 'utf8'); - } catch { - continue; - } - - const lines = content.split('\n'); - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]!; - if (line.includes(parsed.pattern)) { - if (matches.length >= MAX_CONTENT_MATCH_RESULTS) { - truncated = true; - break outer; - } - matches.push({ - file: toRelativePathFromRoot(workspaceRoot, filePath), - line: index + 1, - text: line, - }); - } - } - } - - return { matches, ...(truncated ? { truncated: true } : {}) }; - } - - async function finalizeReviewToolHandler(input: unknown): Promise { - const parsed = FINALIZE_REVIEW_INPUT_SCHEMA.parse(input); - if (finalized) { - throw new AgentToolError( - 'finalize_review can only be called once per session.', - 'FINALIZE_REVIEW_ALREADY_CALLED' - ); - } - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { - totalFindings: findingsCount, - ...(parsed.summary ? { summary: parsed.summary } : {}), - }, - }); - finalized = true; - return { ok: true }; - } - - const tools = createAgentTools({ - runTool, - handlers: { - lint: lintToolHandler, - report_finding: reportFindingToolHandler, - read_file: readFileToolHandler, - search_files: searchFilesToolHandler, - list_directory: listDirectoryToolHandler, - search_content: searchContentToolHandler, - finalize_review: finalizeReviewToolHandler, - }, - }); - const availableTools = listAvailableTools(tools); - - let usage: AgentToolLoopResult['usage'] | undefined; - let requestFailures = 0; - let hadOperationalErrors = false; - let errorMessage: string | undefined; - - try { - const result = await provider.runAgentToolLoop({ - systemPrompt: buildAgentSystemPrompt({ - workspaceRoot, - fileRuleMatches, - availableTools, - ...(userInstructions ? { userInstructions } : {}), - }), - prompt: [ - `Workspace root: ${workspaceRoot}`, - `Targets: ${relativeTargets.join(', ')}`, - `Available ruleSources: ${validSources.join(', ')}`, - ].join('\n'), - tools, - ...(maxSteps !== undefined ? { maxSteps } : {}), - ...(maxRetries !== undefined ? { maxRetries } : {}), - ...(maxParallelToolCalls !== undefined ? { maxParallelToolCalls } : {}), - }); - usage = result.usage; - } catch (error) { - requestFailures += 1; - hadOperationalErrors = true; - errorMessage = error instanceof Error ? error.message : String(error); - } - - const events = await store.replay(); - const hasFinalizedEvent = events.some((event) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized); - - if (!hasFinalizedEvent) { - hadOperationalErrors = true; - if (!errorMessage) { - errorMessage = 'Agent run ended without finalize_review.'; - } - } - - const findings = findingsFromEvents(events); - const aggregatedUsage = mergeTokenUsage(usage, nestedToolUsage); - progressReporter?.finishRun(hadOperationalErrors ? 'failed' : 'completed'); - - return { - findings, - events, - fileRuleMatches, - requestFailures, - hadOperationalErrors, - ...(errorMessage ? { errorMessage } : {}), - ...(aggregatedUsage ? { usage: aggregatedUsage } : {}), - }; -} diff --git a/src/agent/path-utils.ts b/src/agent/path-utils.ts deleted file mode 100644 index 55378366..00000000 --- a/src/agent/path-utils.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { realpathSync } from 'fs'; -import * as path from 'path'; -import { AgentToolError } from '../errors'; - -function isGlobSegment(segment: string): boolean { - return /[*?[\]{}()!+@]/.test(segment); -} - -export function resolveWithinRoot(root: string, inputPath: string): string { - const rootResolved = path.resolve(root); - const targetResolved = path.resolve(rootResolved, inputPath); - - // Lexical check: fast reject for obvious traversal (e.g. ../../etc) - const relative = path.relative(rootResolved, targetResolved); - const outsideRoot = relative.startsWith('..') || path.isAbsolute(relative); - if (outsideRoot) { - throw new AgentToolError( - `Path "${inputPath}" is outside workspace root bounds.`, - 'PATH_OUTSIDE_WORKSPACE_ROOT' - ); - } - - // Symlink check: resolve symlinks on both sides before comparing, so that - // a symlink inside the repo pointing to an external path is caught. - try { - const realRoot = realpathSync(rootResolved); - const realTarget = realpathSync(targetResolved); - const realRelative = path.relative(realRoot, realTarget); - if (realRelative.startsWith('..') || path.isAbsolute(realRelative)) { - throw new AgentToolError( - `Path "${inputPath}" escapes workspace root via symlink.`, - 'PATH_ESCAPES_WORKSPACE_ROOT_SYMLINK' - ); - } - } catch (err) { - // ENOENT means the path does not yet exist — no symlinks to follow, - // so the lexical check above is sufficient. Re-throw any other error. - if (!(err instanceof Error && 'code' in err && (err as { code: string }).code === 'ENOENT')) { - throw err; - } - } - - return targetResolved; -} - -export function resolveGlobPatternWithinRoot(root: string, pattern: string): { cwd: string; pattern: string } { - const normalizedPattern = pattern.replace(/\\/g, '/').trim(); - const segments = normalizedPattern.split('/').filter((segment) => segment.length > 0); - const firstGlobIndex = segments.findIndex((segment) => isGlobSegment(segment)); - - if (firstGlobIndex === -1) { - const literalPath = resolveWithinRoot(root, normalizedPattern || '.'); - return { - cwd: path.dirname(literalPath), - pattern: path.basename(literalPath), - }; - } - - const baseSegments = segments.slice(0, firstGlobIndex); - const globSegments = segments.slice(firstGlobIndex); - const basePath = baseSegments.length > 0 ? baseSegments.join('/') : '.'; - - return { - cwd: resolveWithinRoot(root, basePath), - pattern: globSegments.join('/'), - }; -} - -export function toRelativePathFromRoot(root: string, absolutePath: string): string { - const rootResolved = path.resolve(root); - const absoluteResolved = path.resolve(absolutePath); - return path.relative(rootResolved, absoluteResolved).replace(/\\/g, '/'); -} diff --git a/src/agent/progress.ts b/src/agent/progress.ts deleted file mode 100644 index dff0f64d..00000000 --- a/src/agent/progress.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { OutputFormat } from '../cli/types'; -import * as readline from 'node:readline'; -import ora, { type Ora } from 'ora'; - -const TOOL_PREFIX = ' └ '; -const MAX_TOOL_LINE_LENGTH = 140; - -export type VisibleToolName = 'read_file' | 'list_directory' | 'lint'; - -export interface VisibleToolProgress { - toolName: VisibleToolName; - path?: string; - ruleName?: string; - ruleText?: string; - lineCount?: number; - entryCount?: number; - findingsCount?: number; -} - -type RunStatus = 'completed' | 'failed'; - -export class AgentProgressReporter { - private readonly enabled: boolean; - private readonly spinner: Ora | undefined; - private readonly runStartedAt = Date.now(); - private activeFile: string | undefined; - private activeRuleName: string | undefined; - private activeLine = TOOL_PREFIX; - - constructor(enabled: boolean) { - this.enabled = enabled; - if (enabled) { - this.spinner = ora({ - spinner: 'dots', - stream: createOraStream(process.stderr), - isEnabled: true, - discardStdin: false, - color: false, - }); - } - } - - private writeLine(message: string): void { - if (!this.enabled) { - return; - } - process.stderr.write(`${message}\n`); - } - - private currentBlockText(): string { - return `Reviewing ${this.activeFile ?? ''} for ${this.activeRuleName ?? 'Rule'}\n${this.activeLine}`; - } - - private renderCurrent(): void { - if (!this.enabled || !this.activeFile || !this.spinner) { - return; - } - - const nextText = this.currentBlockText(); - if (!this.spinner.isSpinning) { - this.spinner.start(nextText); - return; - } - this.spinner.text = nextText; - this.spinner.render(); - } - - startFile(file: string, ruleName: string): void { - if (!this.enabled || !this.spinner) { - return; - } - - if (this.spinner.isSpinning && this.activeFile && this.activeFile !== file) { - this.spinner.stopAndPersist({ - symbol: '', - text: this.currentBlockText(), - }); - } - - this.activeFile = file; - this.activeRuleName = ruleName; - this.activeLine = TOOL_PREFIX; - this.renderCurrent(); - } - - updateRule(ruleName: string): void { - if (!this.enabled || !this.activeFile || this.activeRuleName === ruleName) { - return; - } - - this.activeRuleName = ruleName; - this.renderCurrent(); - } - - showVisibleToolStart(params: VisibleToolProgress & { retrying?: boolean }): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = params.retrying - ? formatRetryingLine(params) - : formatInvocationLine(params); - this.renderCurrent(); - } - - showVisibleToolSuccess(params: VisibleToolProgress): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = formatSuccessLine(params); - this.renderCurrent(); - } - - showVisibleToolError(params: VisibleToolProgress): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = formatErrorLine(params); - this.renderCurrent(); - } - - finishRun(status: RunStatus = 'completed'): void { - if (!this.enabled) { - return; - } - - if (this.spinner?.isSpinning && this.activeFile) { - this.spinner.stopAndPersist({ - symbol: '', - text: this.currentBlockText(), - }); - } - this.writeLine(formatRunFooter(status, this.runStartedAt)); - this.activeFile = undefined; - this.activeRuleName = undefined; - this.activeLine = TOOL_PREFIX; - } -} - -function formatInvocationLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Read(${sanitizePath(params.path)})`); - case 'list_directory': - return formatToolLine(`List(${sanitizePath(params.path)})`); - case 'lint': - return formatToolLine(`Lint("${formatRuleSnippet(params.ruleText)}")`); - } -} - -function formatRetryingLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Retrying Read(${sanitizePath(params.path)})...`); - case 'list_directory': - return formatToolLine(`Retrying List(${sanitizePath(params.path)})...`); - case 'lint': - return formatToolLine(`Retrying Lint("${formatRuleSnippet(params.ruleText)}")...`); - } -} - -function formatSuccessLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine( - `Read ${formatCount(params.lineCount ?? 0, 'line')} from ${sanitizePath(params.path)}` - ); - case 'list_directory': - return formatToolLine( - `Listed ${formatCount(params.entryCount ?? 0, 'entry')} in ${sanitizePath(params.path)}` - ); - case 'lint': - if ((params.findingsCount ?? 0) === 0) { - return formatToolLine(`Found no issues in ${sanitizePath(params.path)}`); - } - return formatToolLine( - `Found ${formatCount(params.findingsCount ?? 0, 'issue')} in ${sanitizePath(params.path)}` - ); - } -} - -function formatErrorLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Error reading ${sanitizePath(params.path)}`); - case 'list_directory': - return formatToolLine(`Error listing ${sanitizePath(params.path)}`); - case 'lint': - return formatToolLine(`Error linting ${sanitizePath(params.path)}`); - } -} - -function formatToolLine(message: string): string { - return truncate(`${TOOL_PREFIX}${message}`, MAX_TOOL_LINE_LENGTH); -} - -function formatRuleSnippet(ruleText: string | undefined): string { - const sanitized = sanitizeInline(ruleText || ''); - if (sanitized.length === 0) { - return '...'; - } - return truncate(`${sanitized}...`, 52); -} - -function sanitizePath(path: string | undefined): string { - const value = sanitizeInline(path || '.'); - return value.length > 0 ? value : '.'; -} - -function formatCount(count: number, noun: string): string { - return `${count} ${noun}${count === 1 ? '' : 's'}`; -} - -function sanitizeInline(value: string): string { - return value.replace(/\s+/g, ' ').trim(); -} - -function truncate(value: string, maxLength: number): string { - if (value.length <= maxLength) { - return value; - } - return `${value.slice(0, Math.max(0, maxLength - 3))}...`; -} - -function formatElapsed(startedAt: number): string { - const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000)); - const hours = Math.floor(elapsedSeconds / 3600); - const minutes = Math.floor((elapsedSeconds % 3600) / 60); - const seconds = elapsedSeconds % 60; - - if (hours > 0) { - return `${hours}h ${minutes}m ${seconds}s`; - } - if (minutes > 0) { - return `${minutes}m ${seconds}s`; - } - return `${seconds}s`; -} - -function formatRunFooter(status: RunStatus, startedAt: number): string { - const elapsed = formatElapsed(startedAt); - return status === 'failed' - ? `Review failed after ${elapsed}.` - : `Completed review in ${elapsed}.`; -} - -function createOraStream(stream: NodeJS.WriteStream): NodeJS.WriteStream { - return { - ...stream, - isTTY: stream.isTTY === true, - columns: stream.columns, - write: stream.write.bind(stream), - cursorTo: (x: number) => readline.cursorTo(stream, x), - clearLine: (dir: -1 | 0 | 1) => readline.clearLine(stream, dir), - moveCursor: (dx: number, dy: number) => readline.moveCursor(stream, dx, dy), - } as NodeJS.WriteStream; -} - -export function shouldEmitAgentProgress(params: { outputFormat: OutputFormat; printMode: boolean }): boolean { - const { outputFormat, printMode } = params; - if (printMode) { - return false; - } - if (outputFormat !== OutputFormat.Line) { - return false; - } - return process.stderr.isTTY === true; -} diff --git a/src/agent/prompt-builder.ts b/src/agent/prompt-builder.ts deleted file mode 100644 index 0d16870a..00000000 --- a/src/agent/prompt-builder.ts +++ /dev/null @@ -1,57 +0,0 @@ -export interface BuildAgentSystemPromptParams { - workspaceRoot: string; - fileRuleMatches: Array<{ file: string; ruleSource: string }>; - availableTools: Array<{ name: string; description: string }>; - userInstructions?: string; -} - -function formatBulletedList(values: string[]): string { - if (values.length === 0) { - return '- (none)'; - } - return values.map((value) => `- ${value}`).join('\n'); -} - -function formatFileRuleMatches( - matches: Array<{ file: string; ruleSource: string }> -): string { - if (matches.length === 0) { - return '- (none)'; - } - - const rulesByFile = new Map(); - for (const { file, ruleSource } of matches) { - const rules = rulesByFile.get(file) ?? []; - rules.push(ruleSource); - rulesByFile.set(file, rules); - } - - return Array.from(rulesByFile.entries()) - .map(([file, rules]) => `- ${file}\n${rules.map((rule) => ` - ${rule}`).join('\n')}`) - .join('\n'); -} - -export function buildAgentSystemPrompt(params: BuildAgentSystemPromptParams): string { - const date = new Date().toISOString().slice(0, 10); - const userInstructions = params.userInstructions?.trim(); - const fileRuleMatches = formatFileRuleMatches(params.fileRuleMatches); - - return `You are a senior technical writer. You review documentation files against source-backed rules to identify quality issues, inconsistencies, and violations. - -Your goal is to produce a thorough, complete review of every file against every matched rule. - -Workflow: -1. You are given matched file-rule pairs. Work through each file one at a time — complete every matched rule for a file before moving to the next. -2. For each file-rule pair, review the file against the rule. -3. After reviewing the file, read the rule. If the rule contains top-level review instructions — such as checking for documentation drift, verifying that certain files exist, or any other workspace-level check — carry them out and report any findings. -4. When every file has been reviewed against all of its matched rules, you MUST call the finalize_review tool. - -Available tools: -${formatBulletedList(params.availableTools.map((toolDef) => `${toolDef.name}: ${toolDef.description}`))} - -Review files and matched rules: -${fileRuleMatches}${userInstructions ? `\n\nUser Instructions (from VECTORLINT.md):\n${userInstructions}` : ''} - -Current date: ${date} -Workspace root: ${params.workspaceRoot}`; -} diff --git a/src/agent/review-session-store.ts b/src/agent/review-session-store.ts deleted file mode 100644 index d5fb7262..00000000 --- a/src/agent/review-session-store.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { randomUUID } from 'crypto'; -import { appendFile, mkdir, open, readFile } from 'fs/promises'; -import * as path from 'path'; -import { SESSION_EVENT_SCHEMA, SESSION_EVENT_TYPE, type SessionEvent } from './types'; - -type AppendableSessionEvent = Omit; - -export interface ReviewSessionStore { - sessionId: string; - sessionFilePath: string; - append: (event: AppendableSessionEvent) => Promise; - replay: () => Promise; - hasFinalizedEvent: () => Promise; -} - -function buildSessionFilePath(reviewsDir: string, sessionId: string): string { - return path.join(reviewsDir, `${sessionId}.jsonl`); -} - -async function createUniqueSessionFile(reviewsDir: string): Promise<{ sessionId: string; sessionFilePath: string }> { - while (true) { - const sessionId = randomUUID(); - const sessionFilePath = buildSessionFilePath(reviewsDir, sessionId); - - try { - const handle = await open(sessionFilePath, 'wx'); - await handle.close(); - return { sessionId, sessionFilePath }; - } catch (error) { - if (error instanceof Error && 'code' in error && (error as { code: string }).code === 'EEXIST') { - continue; - } - throw error; - } - } -} - -export async function createReviewSessionStore({ homeDir }: { homeDir: string }): Promise { - const reviewsDir = path.join(homeDir, '.vectorlint', 'reviews'); - await mkdir(reviewsDir, { recursive: true }); - - const { sessionId, sessionFilePath } = await createUniqueSessionFile(reviewsDir); - - async function append(event: AppendableSessionEvent): Promise { - const parsed = SESSION_EVENT_SCHEMA.parse({ - sessionId, - timestamp: new Date().toISOString(), - ...event, - }); - await appendFile(sessionFilePath, `${JSON.stringify(parsed)}\n`, 'utf8'); - } - - async function replay(): Promise { - let raw = ''; - try { - raw = await readFile(sessionFilePath, 'utf8'); - } catch (error) { - if (error instanceof Error && 'code' in error && (error as { code: string }).code === 'ENOENT') { - return []; - } - throw error; - } - - const events: SessionEvent[] = []; - const lines = raw.split('\n').filter((line) => line.trim().length > 0); - - for (const line of lines) { - try { - const candidate = JSON.parse(line) as unknown; - const parsed = SESSION_EVENT_SCHEMA.safeParse(candidate); - if (parsed.success) { - events.push(parsed.data); - } - } catch { - // Ignore malformed JSONL lines so replay can recover valid events. - } - } - - return events; - } - - async function hasFinalizedEvent(): Promise { - const events = await replay(); - return events.some((event) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized); - } - - return { - sessionId, - sessionFilePath, - append, - replay, - hasFinalizedEvent, - }; -} diff --git a/src/agent/rule-id.ts b/src/agent/rule-id.ts deleted file mode 100644 index 2ee68813..00000000 --- a/src/agent/rule-id.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { PromptFile } from '../prompts/prompt-loader'; - -export function buildRuleId(prompt: PromptFile): string { - const pack = prompt.pack || 'Default'; - const rule = String(prompt.meta.id || prompt.filename || 'Rule'); - return `${pack}.${rule}`; -} - -export function normalizeRuleSource(ruleSource: string): string { - return ruleSource.replace(/\\/g, '/').replace(/^\.\//, ''); -} diff --git a/src/agent/tools-registry.ts b/src/agent/tools-registry.ts deleted file mode 100644 index 4bc49dea..00000000 --- a/src/agent/tools-registry.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { AgentToolDefinition } from '../providers/llm-provider'; -import { - FINALIZE_REVIEW_INPUT_SCHEMA, - LINT_TOOL_INPUT_SCHEMA, - LIST_DIRECTORY_INPUT_SCHEMA, - READ_FILE_INPUT_SCHEMA, - SEARCH_CONTENT_INPUT_SCHEMA, - SEARCH_FILES_INPUT_SCHEMA, - TOP_LEVEL_REPORT_INPUT_SCHEMA, -} from './types'; - -export type AgentToolName = - | 'lint' - | 'report_finding' - | 'read_file' - | 'search_files' - | 'list_directory' - | 'search_content' - | 'finalize_review'; - -export type AgentToolHandler = (input: unknown) => Promise; -export type AgentToolHandlers = Record; - -export function createAgentTools(params: { - runTool: ( - toolName: AgentToolName, - input: unknown, - handler: AgentToolHandler - ) => Promise; - handlers: AgentToolHandlers; -}): Record { - const { runTool, handlers } = params; - - return { - lint: { - description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.', - inputSchema: LINT_TOOL_INPUT_SCHEMA, - execute: (input) => runTool('lint', input, handlers.lint), - }, - report_finding: { - description: 'Record a top-level finding for the report.', - inputSchema: TOP_LEVEL_REPORT_INPUT_SCHEMA, - execute: (input) => runTool('report_finding', input, handlers.report_finding), - }, - read_file: { - description: 'Read a file inside the workspace root.', - inputSchema: READ_FILE_INPUT_SCHEMA, - execute: (input) => runTool('read_file', input, handlers.read_file), - }, - search_files: { - description: 'Find files in the workspace by glob pattern.', - inputSchema: SEARCH_FILES_INPUT_SCHEMA, - execute: (input) => runTool('search_files', input, handlers.search_files), - }, - list_directory: { - description: 'List files and directories inside a path in the workspace.', - inputSchema: LIST_DIRECTORY_INPUT_SCHEMA, - execute: (input) => runTool('list_directory', input, handlers.list_directory), - }, - search_content: { - description: 'Search workspace text content by substring and optional glob.', - inputSchema: SEARCH_CONTENT_INPUT_SCHEMA, - execute: (input) => runTool('search_content', input, handlers.search_content), - }, - finalize_review: { - description: 'REQUIRED: Call this tool when all matched file-rule pairs have been reviewed.', - inputSchema: FINALIZE_REVIEW_INPUT_SCHEMA, - execute: (input) => runTool('finalize_review', input, handlers.finalize_review), - }, - }; -} - -export function listAvailableTools( - tools: Record -): Array<{ name: string; description: string }> { - return Object.entries(tools).map(([name, definition]) => ({ - name, - description: definition.description, - })); -} diff --git a/src/agent/types.ts b/src/agent/types.ts deleted file mode 100644 index e1f2407d..00000000 --- a/src/agent/types.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { z } from 'zod'; - -export const SESSION_EVENT_TYPE = { - SessionStarted: 'session_started', - ToolCallStarted: 'tool_call_started', - ToolCallFinished: 'tool_call_finished', - FindingRecordedInline: 'finding_recorded_inline', - FindingRecordedTopLevel: 'finding_recorded_top_level', - SessionFinalized: 'session_finalized', -} as const; - -export const LINT_TOOL_INPUT_SCHEMA = z.object({ - file: z.string().min(1), - ruleSource: z.string().min(1), - reviewInstruction: z.string().min(1).optional(), -}); - -export const TOP_LEVEL_REFERENCE_SCHEMA = z.object({ - file: z.string().min(1), - startLine: z.number().int().positive(), - endLine: z.number().int().positive(), -}); - -export const TOP_LEVEL_REPORT_INPUT_SCHEMA = z.object({ - kind: z.literal('top-level'), - ruleSource: z.string().min(1), - message: z.string().min(1), - suggestion: z.string().optional(), - references: z.array(TOP_LEVEL_REFERENCE_SCHEMA).optional(), -}); - -export const READ_FILE_INPUT_SCHEMA = z.object({ - path: z.string().min(1), -}); - -export const SEARCH_FILES_INPUT_SCHEMA = z.object({ - pattern: z.string().min(1), -}); - -export const LIST_DIRECTORY_INPUT_SCHEMA = z.object({ - path: z.string().min(1), -}); - -export const SEARCH_CONTENT_INPUT_SCHEMA = z.object({ - pattern: z.string().min(1), - path: z.string().optional(), - glob: z.string().optional(), -}); - -export const FINALIZE_REVIEW_INPUT_SCHEMA = z.object({ - summary: z.string().optional(), -}); - -const SESSION_EVENT_BASE_SCHEMA = z.object({ - sessionId: z.string().min(1), - timestamp: z.string().min(1), -}); - -const SESSION_STARTED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.SessionStarted), - payload: z.object({ - cwd: z.string().min(1), - targets: z.array(z.string().min(1)), - }), -}); - -const TOOL_CALL_STARTED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.ToolCallStarted), - payload: z.object({ - toolName: z.string().min(1), - input: z.unknown(), - }), -}); - -const TOOL_CALL_FINISHED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.ToolCallFinished), - payload: z.object({ - toolName: z.string().min(1), - ok: z.boolean(), - output: z.unknown().optional(), - error: z.string().optional(), - }), -}); - -const FINDING_RECORDED_INLINE_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.FindingRecordedInline), - payload: z.object({ - file: z.string().min(1), - line: z.number().int().positive(), - column: z.number().int().positive().optional(), - severity: z.enum(['error', 'warning']).optional(), - ruleId: z.string().min(1).optional(), - ruleSource: z.string().min(1), - message: z.string().min(1), - analysis: z.string().optional(), - suggestion: z.string().optional(), - fix: z.string().optional(), - match: z.string().optional(), - }), -}); - -const FINDING_RECORDED_TOP_LEVEL_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.FindingRecordedTopLevel), - payload: z.object({ - file: z.string().min(1).optional(), - line: z.number().int().positive().optional(), - column: z.number().int().positive().optional(), - severity: z.enum(['error', 'warning']).optional(), - ruleId: z.string().min(1).optional(), - ruleSource: z.string().min(1), - message: z.string().min(1), - suggestion: z.string().optional(), - references: z.array(TOP_LEVEL_REFERENCE_SCHEMA).optional(), - }), -}); - -const SESSION_FINALIZED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.SessionFinalized), - payload: z.object({ - totalFindings: z.number().int().nonnegative(), - summary: z.string().optional(), - }), -}); - -export const SESSION_EVENT_SCHEMA = z.discriminatedUnion('eventType', [ - SESSION_STARTED_EVENT_SCHEMA, - TOOL_CALL_STARTED_EVENT_SCHEMA, - TOOL_CALL_FINISHED_EVENT_SCHEMA, - FINDING_RECORDED_INLINE_EVENT_SCHEMA, - FINDING_RECORDED_TOP_LEVEL_EVENT_SCHEMA, - SESSION_FINALIZED_EVENT_SCHEMA, -]); - -export type SessionEvent = z.infer; diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 50dbffc8..b2e7f0b7 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -18,7 +18,7 @@ import { resolveTargets } from '../scan/file-resolver'; import { parseCliOptions, parseEnvironment } from '../boundaries/index'; import { handleUnknownError } from '../errors/index'; import { evaluateFiles } from './orchestrator'; -import { DEFAULT_REVIEW_MODE, OUTPUT_FORMATS, OutputFormat } from './types'; +import { DEFAULT_REVIEW_MODEL_CALL, OUTPUT_FORMATS, OutputFormat } from './types'; import { DEFAULT_CONFIG_FILENAME, USER_INSTRUCTION_FILENAME } from '../config/constants'; import { createWinstonLogger } from '../logging/winston-logger'; import { createNoopLogger } from '../logging/logger'; @@ -82,8 +82,8 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) - .option('-p, --print', 'Suppress interactive progress output in agent mode') + .option('--model-call ', 'Model call strategy: single (one structured call per rule), agent (bounded target-only paging), or auto (default)', DEFAULT_REVIEW_MODEL_CALL) + .option('--mode ', 'Removed: use --model-call instead') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') .action(async (paths: string[] = []) => { @@ -94,6 +94,15 @@ export function registerMainCommand(program: Command): void { return; } + // --mode is removed (audit Product Decision): the autonomous workspace-agent + // surface is gone. Fail clearly and point users at the replacement rather + // than silently mapping it onto --model-call. + const rawCliOpts = program.opts() as { mode?: unknown }; + if (rawCliOpts.mode !== undefined) { + console.error('Error: --mode is no longer supported. Use --model-call single|agent|auto instead.'); + process.exit(1); + } + // Parse and validate CLI options const cliOptions = await runOrExit('Parsing CLI options', () => parseCliOptions(program.opts())); @@ -192,6 +201,7 @@ export function registerMainCommand(program: Command): void { try { observability = await initializeObservability(env, runtimeLogger); + const requestBuilder = new DefaultRequestBuilder(directive, userInstructions.content || undefined); const provider = createProvider( env, { @@ -201,7 +211,7 @@ export function registerMainCommand(program: Command): void { logger: runtimeLogger, observability, }, - new DefaultRequestBuilder(directive, userInstructions.content || undefined) + requestBuilder ); // Create search provider if API key is available @@ -214,14 +224,14 @@ export function registerMainCommand(program: Command): void { prompts, rulesPath, provider, + requestBuilder, ...(searchProvider ? { searchProvider } : {}), concurrency: config.concurrency, verbose: cliOptions.verbose, logger: runtimeLogger, debugJson: cliOptions.debugJson, outputFormat: cliOptions.output, - mode: cliOptions.mode, - printMode: cliOptions.print, + modelCall: cliOptions.modelCall, scanPaths: config.scanPaths, pricing: { inputPricePerMillion: env.INPUT_PRICE_PER_MILLION, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 1e61a47d..205a2a80 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,41 +1,29 @@ import { readFileSync } from 'fs'; -import { readFile } from 'fs/promises'; import { randomUUID } from 'crypto'; import * as path from 'path'; -import * as os from 'os'; +import { pathToFileURL } from 'url'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { isJudgeResult } from '../prompts/schema'; -import { handleUnknownError, MissingDependencyError } from '../errors/index'; -import { processFindings } from '../findings'; -import { createEvaluator } from '../evaluators/index'; -import { Type, Severity } from '../evaluators/types'; +import { handleUnknownError } from '../errors/index'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; -import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from './types'; -import { runAgentExecutor, type AgentExecutorResult, type AgentFinding } from '../agent/executor'; -import { AgentProgressReporter, shouldEmitAgentProgress } from '../agent/progress'; +import { OutputFormat } from './types'; +import { executorFor } from '../executors'; +import { chooseModelCall } from '../review/executor'; +import { buildReviewRequest } from '../review/request-builder'; +import type { ReviewResult, ReviewSeverity, ReviewTarget } from '../review/types'; import type { - EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessPromptResultParams, - RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, - RunPromptEvaluationResultSuccess + EvaluationOptions, EvaluationResult, ReportIssueParams, EvaluateFileResult, } from './types'; import { calculateCost, TokenUsageStats } from '../providers/token-usage'; -import { calculateCheckScore } from '../scoring'; -import { countWords } from '../chunking/utils'; -import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; -import { - computeFilterDecision, - type FilterDecision, -} from "../evaluators/violation-filter"; -import { writeDebugRunArtifact } from "../debug/run-artifact"; +import { writeDebugRunArtifact } from '../debug/run-artifact'; +import { Severity } from '../evaluators/types'; function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } { const provider = process.env.LLM_PROVIDER; @@ -60,34 +48,6 @@ function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string return { ...(provider && { provider }), ...(name && { name }), ...(tag && { tag }) }; } - -/* - * Generic concurrency runner that executes workers in parallel up to a specified limit. - * Preserves result order matching input order. - */ -async function runWithConcurrency( - items: T[], - limit: number, - worker: (item: T, index: number) => Promise -): Promise { - const results: R[] = new Array(items.length); - let i = 0; - const workers = new Array(Math.min(limit, items.length)) - .fill(0) - .map(async () => { - while (true) { - const idx = i++; - if (idx >= items.length) break; - const item = items[idx]; - if (item !== undefined) { - results[idx] = await worker(item, idx); - } - } - }); - await Promise.all(workers); - return results; -} - /* * Reports an issue in either line or JSON format. */ @@ -154,259 +114,62 @@ function reportIssue(params: ReportIssueParams): void { } } -function getViolationFilterResults< - TViolation extends Parameters[0] ->( - violations: TViolation[] -): { - decisions: FilterDecision[]; - surfacedViolations: TViolation[]; -} { - const decisions = violations.map((v) => computeFilterDecision(v)); - const surfacedViolations = violations.filter( - (_v, i) => decisions[i]?.surface === true - ); - - return { decisions, surfacedViolations }; +function toSeverity(severity: ReviewSeverity): Severity { + return severity === 'error' ? Severity.ERROR : Severity.WARNING; } -/* - * Routes an evaluation result through the shared finding processor. - * Check results are verified, filtered, scored, and reported; judge/rubric - * results are rejected (Phase 3) because subjective scoring is not a - * future-facing review type. - */ -function routePromptResult( - params: ProcessPromptResultParams -): ErrorTrackingResult { - const { - promptFile, - result, - content, - relFile, - outputFormat, - jsonFormatter, - verbose, - debugJson, - } = params; - const meta = promptFile.meta; - const promptId = (meta.id || "").toString(); - - // Handle Check Result — routed through the shared finding processor - // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, - // filters, deduplicates, scores, and resolves severity; the orchestrator - // only feeds the returned ReviewResult to the existing formatter sinks. - if (!isJudgeResult(result)) { - const reviewResult = processFindings({ - pack: promptFile.pack, - ruleId: promptId, - ruleSource: promptFile.fullPath, - candidateFindings: result.violations, - wordCount: result.word_count, - promptMeta: { - ...(meta.severity !== undefined ? { severity: meta.severity } : {}), - ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - // Sanitize to the findings contract ({ id, name }): meta.criteria is - // PromptCriterionSpec[], which can carry legacy rubric weight/target. - ...(meta.criteria - ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } - : {}), - }, - targetContent: content, - }); - - // processFindings always returns exactly one score entry (processor contract). - const ruleScore = reviewResult.scores[0]!; - const severity = - ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - - // Report only verified findings through the existing line/json/rdjson/vale sink. - for (const finding of reviewResult.findings) { - reportIssue({ - file: relFile, - line: finding.line, - column: finding.column, - severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), - ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), - ...(finding.fix !== undefined ? { fix: finding.fix } : {}), - match: finding.match, - }); - } - - // Diagnostics (e.g. finding-evidence-not-locatable) surface in verbose mode, - // consistent with prior operational reporting. They are warn-level and do not - // flag the run as operationally failed (audit Finding #6). - if (verbose) { - for (const diagnostic of reviewResult.diagnostics) { - console.warn(`[vectorlint] ${diagnostic.message}`); - } - } - - // Verified finding count drives the counts (audit Finding #6). - const findingCount = reviewResult.findings.length; - const totalErrors = severity === Severity.ERROR ? findingCount : 0; - const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; - - const scoreEntry: EvaluationSummary = { - id: ruleScore.ruleId, - scoreText: ruleScore.scoreText, - score: ruleScore.score, - }; - - if (debugJson) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - const runId = randomUUID(); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - evaluation_type: "check", - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: decisions.map((d, i) => ({ - index: i, - surface: d.surface, - reasons: d.reasons, - })), - surfaced_violations: surfacedViolations, - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } - - return { - errors: totalErrors, - warnings: totalWarnings, - hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, - hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, - scoreEntries: [scoreEntry], - }; - } +function contentTypeFor(file: string): string { + return path.extname(file).toLowerCase() === '.md' ? 'text/markdown' : 'text/plain'; +} - // Judge/rubric reviews are not a future-facing review type (Phase 3). The - // prompt-meta boundary rejects `type: judge`, so a JudgeResult here means a - // caller bypassed that boundary. Refuse it explicitly rather than - // misprojecting it as a check result. No adapter is added for judge metadata. - if (verbose) { - console.warn( - '[vectorlint] Judge/rubric review results are no longer supported; skipping prompt.' - ); - } - return { - errors: 0, - warnings: 0, - hadOperationalErrors: true, - hadSeverityErrors: false, - scoreEntries: [], - }; +function emptyTokenUsage(): TokenUsageStats { + return { totalInputTokens: 0, totalOutputTokens: 0 }; } /* - * Runs a single prompt evaluation. - * BaseEvaluator auto-detects mode from criteria presence: - * - criteria defined → scored mode - * - no criteria → basic mode + * Writes a debug JSON artifact for a review run. The executor path reviews + * through the shared finding processor, so raw model output and per-candidate + * filter decisions are no longer reachable here; the artifact records the + * verified findings that surfaced and the review metadata instead. */ -async function runPromptEvaluation( - params: RunPromptEvaluationParams -): Promise { - const { promptFile, relFile, content, provider, searchProvider } = params; - +function writeReviewDebugArtifact(relFile: string, result: ReviewResult): void { + const runId = randomUUID(); + const model = getModelInfoFromEnv(); try { - const meta = promptFile.meta; - - const evaluatorType = String(meta.evaluator || Type.BASE); - const baseEvaluatorType = String(Type.BASE); - - // Specialized evaluators (e.g., technical-accuracy) require criteria - // BaseEvaluator handles both modes: scored (with criteria) and basic (without) - if (evaluatorType !== baseEvaluatorType) { - if ( - !meta || - !Array.isArray(meta.criteria) || - meta.criteria.length === 0 - ) { - throw new Error( - `Prompt ${promptFile.filename} has no criteria in frontmatter` - ); - } - } - const evaluator = createEvaluator( - evaluatorType, - provider, - promptFile, - searchProvider - ); - const result = await evaluator.evaluate(relFile, content); - - - const resultObj: RunPromptEvaluationResultSuccess = { ok: true, result }; - - return resultObj; - } catch (e: unknown) { - const err = handleUnknownError(e, `Running prompt ${promptFile.filename}`); - return { ok: false, error: err }; + const filePath = writeDebugRunArtifact(process.cwd(), runId, { + file: relFile, + ...(Object.keys(model).length > 0 ? { model } : {}), + ...(model.tag !== undefined ? { subdir: model.tag } : {}), + prompt: { + evaluation_type: 'check', + }, + raw_model_output: null, + filter_decisions: [], + surfaced_violations: result.findings, + }); + console.warn(`[vectorlint] Debug JSON written: ${filePath}`); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + console.warn(`[vectorlint] Debug JSON write failed: ${message}`); } } /* - * Evaluates a single file with all applicable prompts. + * Determines the source-backed prompts that apply to a file, honoring the + * configured scan paths. When no rules match but a VECTORLINT.md user + * instruction guide exists, a synthetic rule is added so the reviewer model + * evaluates against the user instructions in the system prompt. */ -async function evaluateFile( - params: EvaluateFileParams -): Promise { - const { file, options, jsonFormatter } = params; - const { - prompts, - provider, - searchProvider, - concurrency, - scanPaths, - outputFormat = OutputFormat.Line, - verbose, - debugJson, - } = options; - - let hadOperationalErrors = false; - let hadSeverityErrors = false; - let totalErrors = 0; - let totalWarnings = 0; - let requestFailures = 0; - let totalInputTokens = 0; - let totalOutputTokens = 0; - - const allScores = new Map(); - - const content = readFileSync(file, "utf-8"); - const relFile = path.relative(process.cwd(), file) || file; - - if (outputFormat === OutputFormat.Line) { - printFileHeader(relFile); - } - - // Determine applicable prompts for this file +function resolveApplicablePrompts( + relFile: string, + prompts: PromptFile[], + scanPaths: EvaluationOptions['scanPaths'], + userInstructionContent: string | undefined, +): PromptFile[] { const toRun: PromptFile[] = []; if (scanPaths && scanPaths.length > 0) { const resolver = new ScanPathResolver(); - // Extract available packs from loaded prompts const availablePacks = Array.from( new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p)) ); @@ -417,9 +180,7 @@ async function evaluateFile( availablePacks ); - // Filter prompts by active packs - only runs if explicitly in RunRules const activePrompts = prompts.filter((p) => { - // Prompts with empty pack (style guide only) always run if (p.pack === '') return true; if (!p.pack || !resolution.packs.includes(p.pack)) return false; if (!p.meta?.id) return true; @@ -433,13 +194,10 @@ async function evaluateFile( toRun.push(...activePrompts); } else { - // Fallback: When no scanPaths configured, run all prompts. toRun.push(...prompts); } - // If no rules matched but VECTORLINT.md exists, run an evaluation using it. - // The LLM will use the VECTORLINT.md content from the system prompt. - if (options.userInstructionContent) { + if (userInstructionContent) { toRun.push({ id: USER_INSTRUCTION_FILENAME, filename: USER_INSTRUCTION_FILENAME, @@ -454,346 +212,165 @@ async function evaluateFile( }); } - const results = await runWithConcurrency( - toRun, - concurrency, - async (prompt) => { - return runPromptEvaluation({ - promptFile: prompt, - relFile, - content, - provider, - ...(searchProvider !== undefined && { searchProvider }), - }); - } - ); + return toRun; +} - // Aggregate results from each prompt - for (let idx = 0; idx < toRun.length; idx++) { - const p = toRun[idx]; - const r = results[idx]; - if (!p || !r) continue; - - if (r.ok !== true) { - // Check if this is a missing dependency error - if so, skip gracefully - if (r.error instanceof MissingDependencyError) { - console.warn(`[vectorlint] Skipping ${p.filename}: ${r.error.message}`); - if (r.error.hint) { - console.warn(`[vectorlint] Hint: ${r.error.hint}`); - } - // Skip this evaluation entirely - don't count it as a failure - continue; - } +/* + * Evaluates a single file: builds a ReviewRequest from the applicable + * source-backed prompts, resolves the model-call strategy, dispatches through + * the selected ReviewExecutor, and routes the resulting ReviewResult to the + * existing line/json/vale output sinks. + */ +async function evaluateFile( + file: string, + options: EvaluationOptions, + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter, +): Promise { + const { + prompts, + scanPaths, + outputFormat = OutputFormat.Line, + verbose, + debugJson, + } = options; - // Other errors are actual failures - console.error(` Prompt failed: ${p.filename}`); - console.error(r.error); - hadOperationalErrors = true; - requestFailures += 1; - continue; - } + const allScores = new Map(); - // Accumulate token usage - if (r.result.usage) { - totalInputTokens += r.result.usage.inputTokens; - totalOutputTokens += r.result.usage.outputTokens; - } + const content = readFileSync(file, "utf-8"); + const relFile = path.relative(process.cwd(), file) || file; - const promptResult = routePromptResult({ - promptFile: p, - result: r.result, - content, - relFile, - outputFormat, - jsonFormatter, - verbose, - ...(debugJson !== undefined ? { debugJson } : {}), - }); - totalErrors += promptResult.errors; - totalWarnings += promptResult.warnings; - hadOperationalErrors = - hadOperationalErrors || promptResult.hadOperationalErrors; - hadSeverityErrors = hadSeverityErrors || promptResult.hadSeverityErrors; - - if (promptResult.scoreEntries && promptResult.scoreEntries.length > 0) { - const ruleName = (p.meta.id || p.filename).toString(); - allScores.set(ruleName, promptResult.scoreEntries); - } + if (outputFormat === OutputFormat.Line) { + printFileHeader(relFile); } - const tokenUsageStats: TokenUsageStats = { - totalInputTokens, - totalOutputTokens, - }; + const toRun = resolveApplicablePrompts( + relFile, + prompts, + scanPaths, + options.userInstructionContent, + ); - if (outputFormat === OutputFormat.Line) { - printEvaluationSummaries(allScores); - console.log(""); + // No applicable rules: nothing to review for this file. + if (toRun.length === 0) { + if (outputFormat === OutputFormat.Line) { + printEvaluationSummaries(allScores); + console.log(""); + } + return { + errors: 0, + warnings: 0, + requestFailures: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + tokenUsage: emptyTokenUsage(), + }; } - return { - errors: totalErrors, - warnings: totalWarnings, - requestFailures, - hadOperationalErrors, - hadSeverityErrors, - tokenUsage: tokenUsageStats + // Build the review request and dispatch through the executor selected by the + // resolved model-call strategy (audit Product Decision; Finding #2). + const target: ReviewTarget = { + uri: pathToFileURL(path.resolve(file)).href, + content, + contentType: contentTypeFor(file), + byteLength: Buffer.byteLength(content), }; -} - -function reportAgentFinding(params: { - finding: AgentFinding; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; -}): void { - const { finding, outputFormat, jsonFormatter } = params; - - reportIssue({ - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis ? { analysis: finding.analysis } : {}), - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(finding.fix ? { fix: finding.fix } : {}), - ...(finding.match ? { match: finding.match } : {}), + const request = buildReviewRequest({ + target, + prompts: toRun, + config: { modelCall: options.modelCall }, + }); + const resolvedModelCall = chooseModelCall(request.modelCall, { + targetBytes: request.target.byteLength ?? Buffer.byteLength(content), + rules: request.rules.length, + }); + const executor = executorFor(resolvedModelCall, { + structuredModelClient: options.provider, + toolCallingModelClient: options.provider, + builder: options.requestBuilder, }); -} - -type AgentRuleScore = { - ruleId: string; - score: number; - scoreText: string; -}; - -async function getAgentFileWordCount( - file: string, - workspaceRoot: string, - cache: Map -): Promise { - const workspaceRelative = path.relative(workspaceRoot, path.resolve(workspaceRoot, file)) || file; - if (cache.has(workspaceRelative)) { - return cache.get(workspaceRelative)!; - } - const absolutePath = path.resolve(workspaceRoot, workspaceRelative); + let result: ReviewResult; try { - const content = await readFile(absolutePath, 'utf-8'); - const words = Math.max(1, countWords(content) || 1); - cache.set(workspaceRelative, words); - return words; - } catch { - cache.set(workspaceRelative, 1); - return 1; + result = await executor.run(request); + } catch (e: unknown) { + const err = handleUnknownError(e, `Reviewing ${relFile}`); + console.error(`Error reviewing file ${relFile}: ${err.message}`); + return { + errors: 0, + warnings: 0, + requestFailures: 1, + hadOperationalErrors: true, + hadSeverityErrors: false, + tokenUsage: emptyTokenUsage(), + }; } -} -async function buildAgentRuleScores( - findings: AgentFinding[], - prompts: PromptFile[], - fileRuleMatches: Array<{ file: string; ruleSource: string }>, - workspaceRoot: string -): Promise { - const fileWordCountCache = new Map(); - const findingsByRule = new Map(); - const filesByRuleSource = new Map>(); - - for (const finding of findings) { - const existing = findingsByRule.get(finding.ruleId) ?? []; - existing.push(finding); - findingsByRule.set(finding.ruleId, existing); - } - for (const match of fileRuleMatches) { - const files = filesByRuleSource.get(match.ruleSource) ?? new Set(); - files.add(match.file); - filesByRuleSource.set(match.ruleSource, files); + // Route the ReviewResult's verified findings through the existing sinks. + for (const finding of result.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity: toSeverity(finding.severity), + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); } - const results: AgentRuleScore[] = []; - for (const prompt of prompts) { - const ruleId = buildRuleId(prompt); - const ruleFindings = findingsByRule.get(ruleId) ?? []; - const matchedFiles = filesByRuleSource.get(normalizeRuleSource(prompt.fullPath)) ?? new Set(); - - if (matchedFiles.size === 0) { - results.push({ - ruleId, - score: 10, - scoreText: '10.0/10', - }); - continue; - } + for (const score of result.scores) { + allScores.set(score.ruleId, [ + { id: score.ruleId, scoreText: score.scoreText, score: score.score }, + ]); + } - let totalWords = 0; - for (const file of matchedFiles) { - totalWords += await getAgentFileWordCount(file, workspaceRoot, fileWordCountCache); + if (verbose) { + for (const diagnostic of result.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } - - const syntheticViolations = Array.from({ length: ruleFindings.length }, (_, index) => ({ - line: index + 1, - description: 'Agent finding', - analysis: 'Agent finding', - })); - - const scored = calculateCheckScore( - syntheticViolations, - Math.max(1, totalWords), - { - strictness: prompt.meta.strictness, - promptSeverity: prompt.meta.severity, - } - ); - - results.push({ - ruleId, - score: scored.final_score, - scoreText: `${scored.final_score.toFixed(1)}/10`, - }); } - return results; -} -// Retained in quarantine: unreachable from the CLI after --mode agent was -// deprecated (now falls back to standard evaluation). Removed in Phase 4. -// eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine -async function evaluateFilesInAgentMode( - targets: string[], - options: EvaluationOptions, - outputFormat: OutputFormat, - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter -): Promise { - const workspaceRoot = inferAgentWorkspaceRoot(targets); - const progressReporter = new AgentProgressReporter( - shouldEmitAgentProgress({ - outputFormat, - printMode: options.printMode ?? false, - }) - ); + const totalErrors = result.findings.filter((f) => f.severity === 'error').length; + const totalWarnings = result.findings.filter((f) => f.severity === 'warning').length; - const agentResult: AgentExecutorResult = await runAgentExecutor({ - targets, - prompts: options.prompts, - provider: options.provider, - workspaceRoot, - scanPaths: options.scanPaths, - outputFormat, - printMode: options.printMode ?? false, - sessionHomeDir: os.homedir(), - progressReporter, - maxParallelToolCalls: 3, - maxRetries: options.agentMaxRetries ?? 10, - ...(options.userInstructionContent ? { userInstructions: options.userInstructionContent } : {}), - }); + const tokenUsage: TokenUsageStats = { + totalInputTokens: result.usage?.inputTokens ?? 0, + totalOutputTokens: result.usage?.outputTokens ?? 0, + }; - let totalErrors = 0; - let totalWarnings = 0; - const printedFileHeaders = new Set(); - for (const finding of agentResult.findings) { - if (outputFormat === OutputFormat.Line && !printedFileHeaders.has(finding.file)) { - printFileHeader(finding.file); - printedFileHeaders.add(finding.file); - } - reportAgentFinding({ finding, outputFormat, jsonFormatter }); - if (finding.severity === Severity.ERROR) { - totalErrors += 1; - } else { - totalWarnings += 1; - } + if (debugJson) { + writeReviewDebugArtifact(relFile, result); } if (outputFormat === OutputFormat.Line) { - const ruleScores = await buildAgentRuleScores( - agentResult.findings, - options.prompts, - agentResult.fileRuleMatches, - workspaceRoot - ); - const scoreSummary = new Map( - ruleScores.map((entry) => [ - entry.ruleId, - [{ id: 'overall', scoreText: entry.scoreText, score: entry.score }], - ]) - ); - printEvaluationSummaries(scoreSummary); - - if (agentResult.hadOperationalErrors) { - const message = agentResult.errorMessage ?? 'Agent run encountered an operational error.'; - console.error(`\n[agent] ${message}`); - } - } - - if ( - outputFormat === OutputFormat.Json || - outputFormat === OutputFormat.ValeJson || - outputFormat === OutputFormat.RdJson - ) { - console.log(jsonFormatter.toJson()); + printEvaluationSummaries(allScores); + console.log(""); } - const tokenUsage = { - totalInputTokens: agentResult.usage?.inputTokens ?? 0, - totalOutputTokens: agentResult.usage?.outputTokens ?? 0, - }; - return { - totalFiles: targets.length, - totalErrors, - totalWarnings, - requestFailures: agentResult.requestFailures, - hadOperationalErrors: agentResult.hadOperationalErrors, + errors: totalErrors, + warnings: totalWarnings, + requestFailures: 0, + hadOperationalErrors: result.hadOperationalErrors ?? false, hadSeverityErrors: totalErrors > 0, tokenUsage, }; } -function inferAgentWorkspaceRoot(targets: string[]): string { - if (targets.length === 0) { - return process.cwd(); - } - - const directories = targets.map((target) => path.dirname(path.resolve(target))); - let root = directories[0]!; - - for (const directory of directories.slice(1)) { - root = commonPathPrefix(root, directory); - } - - return root; -} - -function commonPathPrefix(left: string, right: string): string { - let candidate = path.resolve(left); - const target = path.resolve(right); - - while (true) { - const relative = path.relative(candidate, target); - const insideCandidate = !relative.startsWith('..') && !path.isAbsolute(relative); - if (insideCandidate) { - return candidate; - } - - const parent = path.dirname(candidate); - if (parent === candidate) { - return candidate; - } - candidate = parent; - } -} - /* - * Runs evaluations across all target files with configurable concurrency. - * Coordinates prompt-to-file mapping, evaluation execution, and result aggregation. - * Returns aggregated results for reporting. + * Runs reviews across all target files, dispatching each through the executor + * selected by the model-call strategy, and aggregates the results. */ export async function evaluateFiles( targets: string[], options: EvaluationOptions ): Promise { - const { outputFormat = OutputFormat.Line, mode = DEFAULT_REVIEW_MODE } = options; + const { outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -813,18 +390,10 @@ export async function evaluateFiles( jsonFormatter = new ValeJsonFormatter(); } - if (mode === AGENT_REVIEW_MODE) { - options.logger?.warn( - '--mode agent is deprecated and now falls back to standard mode. ' + - 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', - ); - // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. - } - for (const file of targets) { try { totalFiles += 1; - const fileResult = await evaluateFile({ file, options, jsonFormatter }); + const fileResult = await evaluateFile(file, options, jsonFormatter); totalErrors += fileResult.errors; totalWarnings += fileResult.warnings; requestFailures += fileResult.requestFailures; @@ -832,7 +401,6 @@ export async function evaluateFiles( hadOperationalErrors || fileResult.hadOperationalErrors; hadSeverityErrors = hadSeverityErrors || fileResult.hadSeverityErrors; - // Aggregate token usage if (fileResult.tokenUsage) { totalInputTokens += fileResult.tokenUsage.totalInputTokens; totalOutputTokens += fileResult.tokenUsage.totalOutputTokens; diff --git a/src/cli/types.ts b/src/cli/types.ts index a083f2e2..900b1037 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,15 +1,20 @@ import type { PromptFile } from '../prompts/prompt-loader'; -import type { LLMProvider } from '../providers/llm-provider'; +import type { StructuredModelClient } from '../providers/structured-model-client'; +import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; import type { SearchProvider } from '../providers/search-provider'; +import type { RequestBuilder } from '../providers/request-builder'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { PromptEvaluationResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; +import type { ReviewModelCall } from '../review/types'; + +export { REVIEW_MODEL_CALLS } from '../review/executor'; +export type { ReviewModelCall } from '../review/types'; export enum OutputFormat { Line = "line", @@ -27,25 +32,26 @@ export const OUTPUT_FORMATS = [ export const DEFAULT_OUTPUT_FORMAT = OUTPUT_FORMATS[0]; -export const REVIEW_MODES = ['standard', 'agent'] as const; -export const DEFAULT_REVIEW_MODE = REVIEW_MODES[0]; -export const AGENT_REVIEW_MODE = REVIEW_MODES[1]; - -export type ReviewMode = (typeof REVIEW_MODES)[number]; +/** + * How the reviewer model is invoked for a review (audit Product Decision; + * Finding #2). `single` is one structured call per rule/chunk; `agent` is a + * bounded target-only paging run; `auto` resolves via `chooseModelCall`. + */ +export const DEFAULT_REVIEW_MODEL_CALL: ReviewModelCall = 'auto'; export interface EvaluationOptions { prompts: PromptFile[]; rulesPath: string | undefined; - provider: LLMProvider; + provider: StructuredModelClient & ToolCallingModelClient; + /** Retained for the search-provider capability; the executor path reviews structured output. */ searchProvider?: SearchProvider; + requestBuilder: RequestBuilder; concurrency: number; verbose: boolean; debugJson?: boolean; scanPaths: FilePatternConfig[]; outputFormat?: OutputFormat; - mode?: ReviewMode; - printMode?: boolean; - agentMaxRetries?: number; + modelCall: ReviewModelCall; pricing?: PricingConfig; userInstructionContent?: string; logger?: Logger; @@ -69,15 +75,6 @@ export interface ErrorTrackingResult { scoreEntries?: EvaluationSummary[]; } -export interface EvaluationContext { - content: string; - relFile: string; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - verbose?: boolean; - debugJson?: boolean; -} - export interface ReportIssueParams { file: string; line: number; @@ -94,34 +91,6 @@ export interface ReportIssueParams { match?: string; } -export interface ProcessPromptResultParams extends EvaluationContext { - promptFile: PromptFile; - result: PromptEvaluationResult; -} - -export interface RunPromptEvaluationParams { - promptFile: PromptFile; - relFile: string; - content: string; - provider: LLMProvider; - searchProvider?: SearchProvider; -} - -export interface RunPromptEvaluationResultSuccess { - ok: true; - result: PromptEvaluationResult; -} - -export type RunPromptEvaluationResult = - | RunPromptEvaluationResultSuccess - | { ok: false; error: Error }; - -export interface EvaluateFileParams { - file: string; - options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; -} - export interface EvaluateFileResult extends ErrorTrackingResult { requestFailures: number; tokenUsage?: TokenUsageStats; diff --git a/src/executors/index.ts b/src/executors/index.ts new file mode 100644 index 00000000..2b565a29 --- /dev/null +++ b/src/executors/index.ts @@ -0,0 +1,38 @@ +import type { StructuredModelClient } from '../providers/structured-model-client'; +import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { RequestBuilder } from '../providers/request-builder'; +import type { ReviewExecutor } from '../review/executor'; +import { SingleModelCallExecutor } from './single-model-call-executor'; +import { AgentModelCallExecutor } from './agent-model-call-executor'; + +/** + * The model-client dependencies every executor composes. Both capabilities are + * supplied because the resolved {@link ModelCall} is not known until + * {@link chooseModelCall} runs at review time (audit Finding #2). + */ +export interface ExecutorDeps { + structuredModelClient: StructuredModelClient; + toolCallingModelClient: ToolCallingModelClient; + builder: RequestBuilder; +} + +/** A resolved reviewer model-call strategy (after `chooseModelCall`). */ +export type ModelCall = 'single' | 'agent'; + +/** + * Selects the {@link ReviewExecutor} for a resolved model call. `single` maps + * to {@link SingleModelCallExecutor}; `agent` maps to + * {@link AgentModelCallExecutor}. Callers resolve `auto` via + * {@link chooseModelCall} before calling this factory. + */ +export function executorFor(modelCall: ModelCall, deps: ExecutorDeps): ReviewExecutor { + if (modelCall === 'agent') { + return new AgentModelCallExecutor(deps.toolCallingModelClient, deps.builder); + } + return new SingleModelCallExecutor(deps.structuredModelClient); +} + +export { SingleModelCallExecutor } from './single-model-call-executor'; +export { AgentModelCallExecutor } from './agent-model-call-executor'; +export { TargetReadCapability } from './target-read-capability-adapter'; +export { REVIEW_BUDGET_EXCEEDED_CODE, splitRuleId } from './shared'; diff --git a/src/providers/llm-provider.ts b/src/providers/llm-provider.ts index 635d5ecd..50b0a21a 100644 --- a/src/providers/llm-provider.ts +++ b/src/providers/llm-provider.ts @@ -1,40 +1,20 @@ -import type { TokenUsage } from './token-usage'; import type { StructuredModelClient } from './structured-model-client'; /** - * `LLMResult` now lives on the permanent structured-output capability. It is + * `LLMResult` lives on the permanent structured-output capability. It is * re-exported here so existing deep imports (`from './llm-provider'`) keep - * compiling until the legacy provider surface is removed in a later task. + * compiling. */ export type { LLMResult } from './structured-model-client'; -export interface AgentToolDefinition { - description: string; - inputSchema: unknown; - execute: (input: unknown) => Promise; -} - -export interface AgentToolLoopParams { - systemPrompt: string; - prompt: string; - tools: Record; - maxSteps?: number; - maxRetries?: number; - maxParallelToolCalls?: number; -} - -export interface AgentToolLoopResult { - usage?: TokenUsage; -} - /** - * Temporary compile bridge, preserved only until later tasks remove the - * autonomous agent surface. It extends the permanent - * {@link StructuredModelClient} (single structured output) with the legacy - * `runAgentToolLoop` method still consumed by `src/agent/`. Do not expand this - * surface; `runAgentToolLoop` and the `Agent*` types above are deleted once the - * agent tree is removed (audit Finding #2; Product Decision). + * The provider capability historically named `LLMProvider`. The autonomous + * agent loop and the `Agent*` types that used to live here are removed (audit + * Finding #2; Product Decision): the provider surface is now the permanent + * {@link StructuredModelClient} structured-output capability, plus the bounded + * {@link ToolCallingModelClient} transport implemented by concrete providers. + * + * Kept as an alias so existing evaluator and factory imports continue to + * resolve the structured-output capability. */ -export interface LLMProvider extends StructuredModelClient { - runAgentToolLoop(params: AgentToolLoopParams): Promise; -} +export type LLMProvider = StructuredModelClient; diff --git a/src/providers/provider-factory.ts b/src/providers/provider-factory.ts index 3234087d..cbef40ef 100644 --- a/src/providers/provider-factory.ts +++ b/src/providers/provider-factory.ts @@ -4,7 +4,8 @@ import { createAnthropic } from '@ai-sdk/anthropic'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock'; import type { LanguageModel } from 'ai'; -import { LLMProvider } from './llm-provider'; +import type { StructuredModelClient } from './structured-model-client'; +import type { ToolCallingModelClient } from './tool-calling-model-client'; import { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; import { RequestBuilder } from './request-builder'; import type { EnvConfig } from '../schemas/env-schemas'; @@ -38,7 +39,7 @@ export function createProvider( envConfig: EnvConfig, options: ProviderOptions = {}, builder?: RequestBuilder -): LLMProvider { +): StructuredModelClient & ToolCallingModelClient { let model: LanguageModel; let modelName: string; let temperature = 0.2; diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index ed314001..dba87369 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -2,7 +2,8 @@ import { generateText, Output, NoObjectGeneratedError, stepCountIs, tool } from import type { LanguageModel } from 'ai'; import { z } from 'zod'; import pLimit from 'p-limit'; -import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from './llm-provider'; +import type { LLMProvider } from './llm-provider'; +import type { LLMResult } from './structured-model-client'; import type { ToolCallDefinition, ToolCallRunOptions, ToolCallingModelClient } from './tool-calling-model-client'; import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; @@ -252,70 +253,6 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { } } - runAgentToolLoop = async (params: AgentToolLoopParams): Promise => { - const maxParallel = params.maxParallelToolCalls ?? 1; - const limit = pLimit(maxParallel); - - const mappedTools = Object.fromEntries( - Object.entries(params.tools).map(([name, definition]) => [ - name, - tool({ - description: definition.description, - inputSchema: definition.inputSchema as z.ZodType, - execute: (input: unknown) => limit(() => definition.execute(input)), - }), - ]) - ); - - const result = await generateText({ - model: this.config.model, - system: params.systemPrompt, - prompt: params.prompt, - ...(params.maxRetries !== undefined ? { maxRetries: params.maxRetries } : {}), - ...this.getObservabilityOptions({ - operation: 'agent-tool-loop', - provider: this.config.providerName ?? 'unknown', - model: this.config.modelName ?? 'unknown', - }), - stopWhen: stepCountIs(params.maxSteps ?? 1000), - providerOptions: { - openai: { - parallelToolCalls: maxParallel > 1, - }, - }, - tools: mappedTools, - }); - - if (this.config.debug) { - for (const [i, step] of result.steps.entries()) { - const toolNames = step.toolCalls.map((c) => c.toolName).join(', ') || '(none)'; - this.logger.debug( - `[agent] step ${i + 1}: finishReason=${step.finishReason} tools=[${toolNames}]` - ); - if (step.text) { - this.logger.debug( - `[agent] step ${i + 1} text: ${step.text.slice(0, 500)}${step.text.length > 500 ? '...' : ''}` - ); - } - } - this.logger.debug(`[agent] final finishReason=${result.finishReason} steps=${result.steps.length}`); - if (result.text) { - this.logger.debug( - `[agent] final text: ${result.text.slice(0, 500)}${result.text.length > 500 ? '...' : ''}` - ); - } - } - - return result.usage - ? { - usage: { - inputTokens: result.usage.inputTokens ?? 0, - outputTokens: result.usage.outputTokens ?? 0, - }, - } - : {}; - } - private getObservabilityOptions(context: AIExecutionContext): Record { if (!this.observability) { return {}; diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 654f41df..468a0aca 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { DEFAULT_OUTPUT_FORMAT, DEFAULT_REVIEW_MODE, OutputFormat, REVIEW_MODES } from '../cli/types'; +import { DEFAULT_OUTPUT_FORMAT, DEFAULT_REVIEW_MODEL_CALL, OutputFormat, REVIEW_MODEL_CALLS } from '../cli/types'; // CLI options schema for command line argument validation export const CLI_OPTIONS_SCHEMA = z.object({ @@ -8,8 +8,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.nativeEnum(OutputFormat).default(DEFAULT_OUTPUT_FORMAT), - mode: z.enum(REVIEW_MODES).default(DEFAULT_REVIEW_MODE), - print: z.boolean().default(false), + modelCall: z.enum(REVIEW_MODEL_CALLS).default(DEFAULT_REVIEW_MODEL_CALL), prompts: z.string().optional(), config: z.string().optional(), }); diff --git a/tests/agent/agent-executor.test.ts b/tests/agent/agent-executor.test.ts deleted file mode 100644 index 2f3d1ec5..00000000 --- a/tests/agent/agent-executor.test.ts +++ /dev/null @@ -1,841 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { afterEach, describe, expect, it } from 'vitest'; -import type { PromptFile } from '../../src/prompts/prompt-loader'; -import type { LLMProvider } from '../../src/providers/llm-provider'; -import { Severity } from '../../src/evaluators/types'; -import { OutputFormat } from '../../src/cli/types'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; -import { AgentToolError } from '../../src/errors'; - -function makePrompt(): PromptFile { - return { - id: 'consistency', - filename: 'consistency.md', - fullPath: 'packs/default/consistency.md', - pack: 'Default', - body: 'Find inconsistent wording.', - meta: { - id: 'Consistency', - name: 'Consistency', - type: 'check', - severity: Severity.WARNING, - }, - }; -} - -function makeProvider( - script: (params: Record) => Promise<{ usage?: { inputTokens: number; outputTokens: number } }> -): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Bad phrase used', - analysis: 'This wording is inconsistent.', - message: 'Use consistent wording', - suggestion: 'Replace bad phrase', - fix: 'better phrase', - rule_quote: 'Avoid vague wording', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: script as unknown as never, - } as unknown as LLMProvider; -} - -describe('agent executor', () => { - const tempDirs: string[] = []; - - function createTempRepo(): string { - const repo = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-agent-')); - tempDirs.push(repo); - return repo; - } - - afterEach(() => { - for (const dir of tempDirs.splice(0, tempDirs.length)) { - rmSync(dir, { recursive: true, force: true }); - } - }); - - it('exposes only non-mutating analysis tools plus finalize_review', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = (params.tools ?? {}) as Record; - const names = Object.keys(tools).sort(); - expect(names).toEqual([ - 'finalize_review', - 'lint', - 'list_directory', - 'read_file', - 'report_finding', - 'search_content', - 'search_files', - ]); - - const finalize = tools.finalize_review as { execute: (input: unknown) => Promise }; - await finalize.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.fileRuleMatches).toEqual([ - { file: 'doc.md', ruleSource: 'packs/default/consistency.md' }, - ]); - }); - - it('returns explicit tool error for unknown ruleSource with valid-source hints', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - let errorMessage = ''; - try { - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/does-not-exist.md', - }); - } catch (error) { - expect(error).toBeInstanceOf(AgentToolError); - errorMessage = error instanceof Error ? error.message : String(error); - } - expect(errorMessage).toContain('Unknown ruleSource'); - expect(errorMessage).toContain('Valid sources'); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - }); - - it('returns explicit tool error for unknown ruleSource in report_finding', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/does-not-exist.md', - message: 'Unknown source', - }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('Unknown ruleSource'); - expect(result.errorMessage).toContain('Valid sources'); - }); - - it('returns findings reconstructed from persisted inline and top-level session events', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({ summary: 'done' }); - return { usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - const inlineEventCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.FindingRecordedInline - ).length; - const topLevelEventCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.FindingRecordedTopLevel - ).length; - const replayableFindingEvents = inlineEventCount + topLevelEventCount; - - expect(replayableFindingEvents).toBeGreaterThanOrEqual(2); - expect(result.findings.length).toBe(replayableFindingEvents); - expect(result.findings.some((finding: { line: number }) => finding.line > 1)).toBe(false); - expect( - result.findings.some( - (finding: { ruleId: string; ruleSource: string }) => - finding.ruleId === 'Default.Consistency' && - finding.ruleSource === 'packs/default/consistency.md' - ) - ).toBe(true); - expect(result.events.some((event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized)).toBe(true); - }); - - it('aggregates nested lint usage with agent loop usage', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [], - }, - usage: { - inputTokens: 7, - outputTokens: 3, - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 11, outputTokens: 5 } }; - }, - }; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.usage).toEqual({ - inputTokens: 18, - outputTokens: 8, - }); - }); - - it('falls back to matching all prompts when scanPaths is empty', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.fileRuleMatches).toEqual([ - { file: 'doc.md', ruleSource: 'packs/default/consistency.md' }, - ]); - }); - - it('records the required session event stream and preserves lifecycle ordering', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue', - }); - await tools.finalize_review.execute({ summary: 'done' }); - - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const eventTypes = result.events.map((event: { eventType: string }) => event.eventType); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.SessionStarted); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.ToolCallStarted); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.ToolCallFinished); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.FindingRecordedInline); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.FindingRecordedTopLevel); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.SessionFinalized); - expect(eventTypes.indexOf(SESSION_EVENT_TYPE.SessionStarted)).toBeLessThan( - eventTypes.indexOf(SESSION_EVENT_TYPE.SessionFinalized) - ); - expect(eventTypes.indexOf(SESSION_EVENT_TYPE.FindingRecordedInline)).toBeLessThan( - eventTypes.indexOf(SESSION_EVENT_TYPE.SessionFinalized) - ); - }); - - it('returns an operational error when finalize_review is called more than once', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - await tools.finalize_review.execute({ summary: 'first' }); - await tools.finalize_review.execute({ summary: 'second' }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('finalize_review'); - const finalizedCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized - ).length; - expect(finalizedCount).toBe(1); - }); - - it.each([ - { - toolName: 'read_file', - input: { path: '../outside.md' }, - }, - { - toolName: 'search_files', - input: { pattern: '../*.md' }, - }, - { - toolName: 'list_directory', - input: { path: '../' }, - }, - { - toolName: 'search_content', - input: { pattern: 'bad phrase', path: '../', glob: '**/*.md' }, - }, - ])( - 'returns explicit tool errors when $toolName is asked to access outside repository bounds', - async ({ toolName, input }) => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - let toolError = ''; - let toolErrorValue: unknown; - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - try { - await tools[toolName]!.execute(input); - } catch (error) { - toolErrorValue = error; - toolError = error instanceof Error ? error.message : String(error); - } - - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(toolErrorValue).toBeInstanceOf(AgentToolError); - expect(toolError).toBeTruthy(); - expect(toolError).toMatch(/outside|workspace|root|bounds/i); - } - ); - - it('continues producing findings after a recoverable tool error and reports the tool diagnostic', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - let toolError = ''; - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - try { - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/does-not-exist.md', - }); - } catch (error) { - toolError = error instanceof Error ? error.message : String(error); - } - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({ summary: 'done' }); - - return { usage: { inputTokens: 2, outputTokens: 2 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(toolError).toContain('Unknown ruleSource'); - expect(result.hadOperationalErrors).toBe(false); - expect(result.findings.length).toBeGreaterThan(0); - expect( - result.events.some( - (event: { eventType: string; payload?: { ok?: boolean } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallFinished && event.payload?.ok === false - ) - ).toBe(true); - }); - - it('allows read-only search_content tool usage without requiring mutation capabilities', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\nanother line\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - const output = await tools.search_content.execute({ - pattern: 'bad phrase', - path: '.', - glob: '**/*.md', - }); - - expect(output).toBeTruthy(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - }); - - it('marks the run as operationally failed but preserves findings when finalize_review is missing', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('finalize_review'); - expect(result.findings.length).toBeGreaterThan(0); - }); - - it('uses reviewInstruction to override the prompt body for that lint invocation', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const promptBodies: string[] = []; - const provider: LLMProvider = { - runPromptStructured(_content, promptText: string) { - promptBodies.push(promptText); - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - reviewInstruction: 'Review this file for wording consistency using the evidence you gathered.', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(promptBodies.length).toBeGreaterThan(0); - expect(promptBodies[0]).toBe( - 'Review this file for wording consistency using the evidence you gathered.' - ); - }); - - it('keeps lint prompt body unchanged when reviewInstruction is not provided', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const promptBodies: string[] = []; - const provider: LLMProvider = { - runPromptStructured(_content, promptText: string) { - promptBodies.push(promptText); - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(promptBodies.length).toBeGreaterThan(0); - expect(promptBodies[0]).toBe('Find inconsistent wording.'); - }); - - it('records judge-style violations as inline findings in agent mode', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const basePrompt = makePrompt(); - const judgePrompt: PromptFile = { - ...basePrompt, - body: 'Judge the document for clarity.', - meta: { - ...basePrompt.meta, - type: 'judge', - criteria: [{ id: 'Clarity', name: 'Clarity', weight: 1 }], - }, - }; - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - criteria: [ - { - name: 'Clarity', - score: 2, - summary: 'Needs work', - reasoning: 'The wording is unclear.', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Unclear wording', - analysis: 'This phrase is vague.', - message: 'Use clearer wording', - suggestion: 'Replace the vague phrase', - fix: 'better phrase', - rule_quote: 'Prefer precise language', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [judgePrompt], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.findings).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - file: 'doc.md', - ruleId: 'Default.Consistency', - message: 'Use clearer wording', - }), - ]) - ); - }); - - it('redacts raw read_file content from persisted tool_call_finished events', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'secret text\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.read_file.execute({ path: 'doc.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const readFileEvent = result.events.find( - (event: { - eventType: string; - payload?: { toolName?: string; output?: { path?: string; contentLength?: number } }; - }) => event.eventType === SESSION_EVENT_TYPE.ToolCallFinished && event.payload?.toolName === 'read_file' - ); - - expect(readFileEvent?.payload?.output).toEqual({ - path: 'doc.md', - contentLength: 'secret text\n'.length, - }); - }); - - it('records started and failed events for visible-tool path errors', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: '../outside.md' })).rejects.toThrow(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const started = result.events.find( - (event: { eventType: string; payload?: { toolName?: string } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallStarted && event.payload?.toolName === 'read_file' - ); - const failed = result.events.find( - (event: { eventType: string; payload?: { toolName?: string; ok?: boolean; error?: string } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallFinished - && event.payload?.toolName === 'read_file' - && event.payload?.ok === false - ); - - expect(started).toBeDefined(); - expect(failed?.payload?.error).toContain('outside workspace root'); - }); -}); diff --git a/tests/agent/progress.test.ts b/tests/agent/progress.test.ts deleted file mode 100644 index df213ab4..00000000 --- a/tests/agent/progress.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { AgentProgressReporter } from '../../src/agent/progress'; - -describe('agent progress reporter', () => { - afterEach(() => { - vi.useRealTimers(); - vi.restoreAllMocks(); - }); - - it('animates spinner frames independently of content changes while active', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - reporter.showVisibleToolStart({ - toolName: 'lint', - path: 'README.md', - ruleName: 'Repetition', - ruleText: '# Repetition Flag any instance where the same wording repeats', - }); - - const initialWrites = writeSpy.mock.calls.length; - vi.advanceTimersByTime(250); - - expect(writeSpy.mock.calls.length).toBeGreaterThan(initialWrites); - - reporter.finishRun(); - }); - - it('is a no-op when disabled', () => { - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(false); - reporter.startFile('README.md', 'Repetition'); - reporter.finishRun(); - - expect(writeSpy).not.toHaveBeenCalled(); - }); - - it.each([ - { elapsedMs: 0, expected: 'Completed review in 0s.' }, - { elapsedMs: 60_000, expected: 'Completed review in 1m 0s.' }, - { elapsedMs: 3_600_000, expected: 'Completed review in 1h 0m 0s.' }, - ])('formats elapsed time boundaries for $expected', ({ elapsedMs, expected }) => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(elapsedMs); - reporter.finishRun(); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain(expected); - }); - - it('formats the completion footer with elapsed time', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(85_000); - reporter.finishRun(); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain('Completed review in 1m 25s.'); - }); - - it('formats a failed footer when the run ends with errors', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(85_000); - reporter.finishRun('failed'); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain('Review failed after 1m 25s.'); - }); -}); diff --git a/tests/agent/prompt-builder.test.ts b/tests/agent/prompt-builder.test.ts deleted file mode 100644 index fe31074c..00000000 --- a/tests/agent/prompt-builder.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('agent prompt builder', () => { - it('builds a non-empty system prompt for valid inputs', async () => { - const { buildAgentSystemPrompt } = await import('../../src/agent/prompt-builder'); - - const prompt = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [ - { file: 'README.md', ruleSource: 'packs/default/ai-pattern.md' }, - { file: 'README.md', ruleSource: 'packs/default/consistency.md' }, - ], - availableTools: [ - { name: 'read_file', description: 'Read a file inside the workspace root.' }, - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - { name: 'finalize_review', description: 'Finalize review output and close the session.' }, - ], - }); - - expect(typeof prompt).toBe('string'); - expect(prompt.length).toBeGreaterThan(0); - expect(prompt).toContain('Workspace root: /workspace'); - expect(prompt).toContain('matched rules'); - }); - - it('supports optional user instructions without breaking prompt generation', async () => { - const { buildAgentSystemPrompt } = await import('../../src/agent/prompt-builder'); - - const withoutUserInstructions = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [{ file: 'README.md', ruleSource: 'packs/default/consistency.md' }], - availableTools: [ - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - ], - }); - - const prompt = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [{ file: 'README.md', ruleSource: 'packs/default/consistency.md' }], - availableTools: [ - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - ], - userInstructions: 'Always enforce concise phrasing.', - }); - - expect(typeof withoutUserInstructions).toBe('string'); - expect(typeof prompt).toBe('string'); - expect(withoutUserInstructions.length).toBeGreaterThan(0); - expect(prompt.length).toBeGreaterThan(withoutUserInstructions.length); - }); -}); diff --git a/tests/agent/review-session-store.test.ts b/tests/agent/review-session-store.test.ts deleted file mode 100644 index 0258b561..00000000 --- a/tests/agent/review-session-store.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { appendFileSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; - -beforeEach(() => { - vi.resetModules(); -}); - -afterEach(() => { - vi.doUnmock('crypto'); - vi.resetModules(); -}); - -describe('review session store', () => { - it('creates session files in the VectorLint reviews directory', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - expect(store.sessionFilePath).toContain( - `${path.sep}.vectorlint${path.sep}reviews${path.sep}` - ); - expect(path.basename(store.sessionFilePath)).toMatch(/\.jsonl$/); - expect(readFileSync(store.sessionFilePath, 'utf8')).toBe(''); - }); - - it('persists each appended event as a valid JSONL record', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0, summary: 'done' }, - }); - - const raw = readFileSync(store.sessionFilePath, 'utf8'); - const lines = raw.trim().split('\n'); - expect(lines).toHaveLength(2); - - const parsed = lines.map((line) => JSON.parse(line) as { eventType?: string }); - expect(parsed.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - }); - - it('reads persisted session events in order and reports completion state', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - expect(await store.hasFinalizedEvent()).toBe(false); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0, summary: 'done' }, - }); - - const events = await store.replay(); - expect(events.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - expect(await store.hasFinalizedEvent()).toBe(true); - }); - - it('recovers valid events when an existing session file contains malformed JSONL lines', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - - appendFileSync(store.sessionFilePath, 'not-json\n', 'utf8'); - appendFileSync(store.sessionFilePath, '{"eventType":"unknown"}\n', 'utf8'); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0 }, - }); - - const events = await store.replay(); - expect(events.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - }); - - it('creates a unique session file when an initial generated session id collides', async () => { - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const reviewsDir = path.join(home, '.vectorlint', 'reviews'); - mkdirSync(reviewsDir, { recursive: true }); - writeFileSync(path.join(reviewsDir, 'collision-id.jsonl'), '', 'utf8'); - - const randomUUIDMock = vi - .fn() - .mockReturnValueOnce('collision-id') - .mockReturnValueOnce('fresh-id'); - - vi.doMock('crypto', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - randomUUID: randomUUIDMock, - }; - }); - - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const store = await createReviewSessionStore({ homeDir: home }); - expect(path.basename(store.sessionFilePath)).toBe('fresh-id.jsonl'); - expect(readFileSync(store.sessionFilePath, 'utf8')).toBe(''); - }); -}); diff --git a/tests/agent/types-contract.test.ts b/tests/agent/types-contract.test.ts deleted file mode 100644 index 6363587a..00000000 --- a/tests/agent/types-contract.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; - -describe('agent contracts', () => { - it('accepts ruleSource-based inputs for lint and top-level findings', async () => { - const contracts = await import('../../src/agent/types'); - - const lintInput = contracts.LINT_TOOL_INPUT_SCHEMA.parse({ - file: 'docs/guide.md', - ruleSource: 'packs/default/consistency.md', - reviewInstruction: 'Review this file for consistency.', - }); - expect(lintInput.ruleSource).toBe('packs/default/consistency.md'); - - const topLevel = contracts.TOP_LEVEL_REPORT_INPUT_SCHEMA.parse({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file inconsistency', - references: [{ file: 'docs/guide.md', startLine: 2, endLine: 4 }], - }); - expect(topLevel.kind).toBe('top-level'); - expect(topLevel.ruleSource).toBe('packs/default/consistency.md'); - }); - - it('accepts finalized session events with completion metadata', async () => { - const contracts = await import('../../src/agent/types'); - - const event = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:00.000Z', - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 1, summary: 'done' }, - }); - - expect(event.eventType).toBe(SESSION_EVENT_TYPE.SessionFinalized); - expect(event.payload.totalFindings).toBe(1); - }); - - it('accepts required tool and finding session event variants for deterministic replay', async () => { - const contracts = await import('../../src/agent/types'); - - const started = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:00.000Z', - eventType: SESSION_EVENT_TYPE.ToolCallStarted, - payload: { toolName: 'lint', input: { file: 'docs/guide.md' } }, - }); - - const finished = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:01.000Z', - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { toolName: 'lint', ok: true }, - }); - - const inlineFinding = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:02.000Z', - eventType: SESSION_EVENT_TYPE.FindingRecordedInline, - payload: { - file: 'docs/guide.md', - line: 2, - message: 'Inconsistent term', - ruleSource: 'packs/default/consistency.md', - }, - }); - - const topLevelFinding = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:03.000Z', - eventType: SESSION_EVENT_TYPE.FindingRecordedTopLevel, - payload: { - message: 'Cross-file mismatch', - ruleSource: 'packs/default/consistency.md', - }, - }); - - expect(started.eventType).toBe(SESSION_EVENT_TYPE.ToolCallStarted); - expect(finished.eventType).toBe(SESSION_EVENT_TYPE.ToolCallFinished); - expect(inlineFinding.eventType).toBe(SESSION_EVENT_TYPE.FindingRecordedInline); - expect(topLevelFinding.eventType).toBe(SESSION_EVENT_TYPE.FindingRecordedTopLevel); - }); -}); diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index 359da161..7d652c46 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -97,8 +97,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - mode: 'standard', - print: false, + modelCall: 'auto', config: undefined, }); MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); @@ -142,8 +141,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - mode: 'standard', - print: false, + modelCall: 'auto', config: undefined, }); const observability = { diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts deleted file mode 100644 index 0b61ea63..00000000 --- a/tests/orchestrator-agent-output.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; -import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { evaluateFiles } from '../src/cli/orchestrator'; -import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; -import type { PromptFile } from '../src/prompts/prompt-loader'; -import type { LLMProvider } from '../src/providers/llm-provider'; -import type { Logger } from '../src/logging/logger'; -import { Severity } from '../src/evaluators/types'; - -function makePrompt(): PromptFile { - const id = 'consistency'; - const name = 'Consistency'; - return { - id, - filename: `${id}.md`, - fullPath: 'packs/default/consistency.md', - pack: 'Default', - body: 'Find inconsistent wording', - meta: { - id: name, - name, - type: 'check', - severity: Severity.WARNING, - }, - }; -} - -type LoggerSpy = Logger & { warn: ReturnType }; - -function makeLogger(): LoggerSpy { - return { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - } as unknown as LoggerSpy; -} - -interface StandardProviderSpies { - provider: LLMProvider; - runPromptStructured: ReturnType; - runAgentToolLoop: ReturnType; -} - -function makeStandardProvider(): StandardProviderSpies { - const runPromptStructured = vi.fn().mockResolvedValue({ - data: { reasoning: 'ok', violations: [] }, - }); - const runAgentToolLoop = vi.fn().mockResolvedValue({ - usage: { inputTokens: 0, outputTokens: 0 }, - }); - const provider = { runPromptStructured, runAgentToolLoop } as unknown as LLMProvider; - return { provider, runPromptStructured, runAgentToolLoop }; -} - -// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no -// longer reaches the autonomous agent executor: it warns through the injected -// logger and falls back to standard evaluation. The retained agent executor -// code is covered directly by tests/agent/* and removed in Phase 4. -describe('agent mode deprecation', () => { - const tempRepos: string[] = []; - - function createTempRepo(): string { - const repo = mkdtempSync(path.join(process.cwd(), 'tmp-agent-orch-')); - tempRepos.push(repo); - return repo; - } - - beforeEach(() => { - vi.spyOn(console, 'log').mockImplementation(() => undefined); - vi.spyOn(console, 'warn').mockImplementation(() => undefined); - vi.spyOn(console, 'error').mockImplementation(() => undefined); - }); - - afterEach(() => { - vi.restoreAllMocks(); - for (const repo of tempRepos.splice(0, tempRepos.length)) { - rmSync(repo, { recursive: true, force: true }); - } - }); - - it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); - const logger = makeLogger(); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - logger, - }); - - expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('deprecated'), - ); - expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), - ); - // Standard evaluation ran; the autonomous executor did not. - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(runPromptStructured).toHaveBeenCalled(); - expect(result.totalFiles).toBe(1); - }); - - it('does not invoke the agent executor in line output mode', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); - const logger = makeLogger(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - logger, - }); - - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(runPromptStructured).toHaveBeenCalled(); - }); - - it('stays silent in standard mode without a logger', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runAgentToolLoop } = makeStandardProvider(); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: DEFAULT_REVIEW_MODE, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - }); - - // No deprecation warning and no executor invocation in standard mode. - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(result.totalFiles).toBe(1); - }); -}); diff --git a/tests/orchestrator-check-processor.test.ts b/tests/orchestrator-check-processor.test.ts deleted file mode 100644 index eaaa1529..00000000 --- a/tests/orchestrator-check-processor.test.ts +++ /dev/null @@ -1,356 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { Result } from "../src/output/json-formatter"; -import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { RawCheckResult } from "../src/prompts/schema"; - -const { EVALUATE_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), -})); - -type CheckViolation = RawCheckResult["violations"][number]; - -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -} as const; - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-check-proc-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -/** - * Builds a check violation that passes the confidence gate by default. Real - * model output always carries `message`; the helper sets it explicitly so the - * test reflects production behavior rather than the analysis fallback. - */ -function makeCheckViolation( - overrides: Partial = {} -): CheckViolation { - return { - line: 1, - analysis: "Issue 1", - message: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - quoted_text: "Alpha text", - context_before: "", - context_after: "", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - confidence: 0.9, - ...overrides, - }; -} - -function makeCheckResult(params: { - violations: CheckViolation[]; - wordCount?: number; -}): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: params.violations, - word_count: params.wordCount ?? 100, - }; -} - -describe("standard check evaluation via the shared finding processor", () => { - const originalThreshold = process.env.CONFIDENCE_THRESHOLD; - - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - delete process.env.CONFIDENCE_THRESHOLD; - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - if (originalThreshold === undefined) { - delete process.env.CONFIDENCE_THRESHOLD; - } else { - process.env.CONFIDENCE_THRESHOLD = originalThreshold; - } - vi.restoreAllMocks(); - }); - - it("reports fully locatable findings, score, and counts unchanged", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckPrompt", - name: "Check Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ message: "First issue", analysis: "First issue" }), - makeCheckViolation({ - line: 2, - quoted_text: "Beta text", - message: "Second issue", - analysis: "Second issue", - suggestion: "Suggestion 2", - fix: "Fix 2", - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Both quotes anchor: 2 verified findings over 100 words -> 8.0/10. - expect(run.totalWarnings).toBe(2); - expect(run.totalErrors).toBe(0); - expect(run.hadOperationalErrors).toBe(false); - - const scoreLine = logCalls.find((l) => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("8.0/10"); - // The line summary prints the last segment of the Pack.Rule score id. - expect(scoreLine).toContain("CheckPrompt"); - }); - - it("counts only verified findings and excludes unanchored quotes from the score", async () => { - // One anchored + one unanchored quote. Both pass the confidence gate, so - // only the verifier distinguishes them. Verified count = 1 -> 9.0/10. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CountingFixPrompt", - name: "Counting Fix Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: "Alpha text" }), - makeCheckViolation({ - line: 9, - quoted_text: "this quote is not anywhere in the content", - message: "Ghost issue", - analysis: "Ghost issue", - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Intentional Phase 3 fix: the unanchored quote is a diagnostic, not a - // finding; it contributes neither to the count nor the score. - expect(run.totalWarnings).toBe(1); - expect(run.hadOperationalErrors).toBe(false); - - const scoreLine = logCalls.find((l) => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("9.0/10"); - }); - - it("does not flag severity errors when no finding can be verified", async () => { - // severity=error, but the only quote cannot anchor -> 0 verified findings -> - // perfect 10.0/10 -> severity resolves to warning -> no error exit signal. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "ErrorNoVerifiedPrompt", - name: "Error No Verified Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - line: 9, - quoted_text: "nowhere to be found", - message: "Ghost", - analysis: "Ghost", - }), - ], - wordCount: 100, - }) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(0); - expect(run.hadSeverityErrors).toBe(false); - expect(run.hadOperationalErrors).toBe(false); - }); - - it("flags severity errors when verified error-severity findings exist", async () => { - // 1 verified finding over 10 words -> density 10 -> score 0.0 < 10 -> - // prompt severity error applies. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "ErrorVerifiedPrompt", - name: "Error Verified Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [makeCheckViolation({ quoted_text: "Alpha text" })], - wordCount: 10, - }) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(1); - expect(run.hadSeverityErrors).toBe(true); - }); - - it("emits verified findings with anchored location through the JSON sink", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckJsonPrompt", - name: "Check JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ message: "First", analysis: "First" }), - makeCheckViolation({ - line: 2, - quoted_text: "Beta text", - message: "Second", - analysis: "Second", - }), - ], - wordCount: 100, - }) - ); - - await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.Json, - }); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Result; - const issues = Object.values(parsed.files).flatMap((file) => file.issues); - - expect(issues).toHaveLength(2); - expect(issues[0]).toMatchObject({ - line: 1, - column: 1, - severity: Severity.WARNING, - message: "First", - rule: "TestPack.CheckJsonPrompt", - match: "Alpha text", - }); - expect(issues[1]).toMatchObject({ - line: 2, - message: "Second", - match: "Beta text", - }); - }); - - it("omits unanchored quotes from the Vale JSON output", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckValePrompt", - name: "Check Vale Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: "Alpha text" }), - makeCheckViolation({ - line: 9, - quoted_text: "missing quote", - message: "Ghost", - analysis: "Ghost", - }), - ], - wordCount: 100, - }) - ); - - await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.ValeJson, - }); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as ValeOutput; - const issues = Object.values(parsed).flat(); - - expect(issues).toHaveLength(1); - expect(issues[0]).toMatchObject({ - Check: "TestPack.CheckValePrompt", - Line: 1, - Match: "Alpha text", - Severity: "warning", - }); - }); -}); diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts deleted file mode 100644 index b4d2cf89..00000000 --- a/tests/orchestrator-filtering.test.ts +++ /dev/null @@ -1,412 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { Result } from "../src/output/json-formatter"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { JudgeResult, RawCheckResult } from "../src/prompts/schema"; - -const { EVALUATE_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), -})); - -type CheckViolation = RawCheckResult["violations"][number]; -type JudgeViolation = JudgeResult["criteria"][number]["violations"][number]; - -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-filtering-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -} as const; - -const EMPTY_CHECK_NOTES: JudgeViolation["check_notes"] = { - rule_supports_claim: "", - evidence_exact: "", - context_supports_violation: "", - plausible_non_violation: "", - fix_is_drop_in: "", - fix_preserves_meaning: "", -}; - -function makeCheckViolation( - overrides: Partial = {} -): CheckViolation { - return { - line: 1, - analysis: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - quoted_text: "Alpha text", - context_before: "", - context_after: "", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - confidence: 0.9, - ...overrides, - }; -} - -function makeJudgeViolation( - overrides: Partial = {} -): JudgeViolation { - const { check_notes: checkNotesOverrides, ...rest } = overrides; - return { - line: 1, - quoted_text: "Alpha text", - context_before: "", - context_after: "", - description: "Issue 1", - analysis: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - check_notes: { - ...EMPTY_CHECK_NOTES, - ...(checkNotesOverrides ?? {}), - }, - confidence: 0.9, - ...rest, - }; -} - -function makeCheckResult(params: { - violations: CheckViolation[]; - wordCount?: number; -}): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: params.violations, - word_count: params.wordCount ?? 100, - }; -} - -function makeJudgeResult(violations: JudgeViolation[]): JudgeResult { - return { - type: EvaluationType.JUDGE, - final_score: 5, - criteria: [ - { - name: "Clarity", - weight: 1, - score: 2, - normalized_score: 5, - weighted_points: 1, - summary: "Needs work", - reasoning: "Reasoning", - violations, - }, - ], - }; -} - -describe("CLI violation filtering", () => { - const originalThreshold = process.env.CONFIDENCE_THRESHOLD; - - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - delete process.env.CONFIDENCE_THRESHOLD; - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - if (originalThreshold === undefined) { - delete process.env.CONFIDENCE_THRESHOLD; - } else { - process.env.CONFIDENCE_THRESHOLD = originalThreshold; - } - vi.restoreAllMocks(); - }); - - it("filters low-confidence check violations from CLI counts by default", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckPrompt", - name: "Check Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation(), - makeCheckViolation({ - line: 2, - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - quoted_text: "Beta text", - confidence: 0.2, - }), - ], - }) - ); - - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalWarnings).toBe(1); - - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation(), - makeCheckViolation({ - line: 2, - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - quoted_text: "Beta text", - confidence: 0.2, - }), - ], - }) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalWarnings).toBe(2); - }); - - it("does not mark severity error when no check violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckErrorPrompt", - name: "Check Error Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalErrors).toBe(0); - expect(defaultRun.hadSeverityErrors).toBe(false); - - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalErrors).toBe(1); - expect(zeroThresholdRun.hadSeverityErrors).toBe(true); - }); - - it("score reflects only surfaced violations, not filtered-out ones", async () => { - // 100-word file: 2 violations from model, 1 fails confidence gate - // With default threshold, only 1 violation surfaces - // Density: 1/100 * 100 * 10 = 10 penalty → score = 9.0 - // If bug were present (scoring all 2): 2/100 * 100 * 10 = 20 penalty → score = 8.0 - - const content = new Array(100).fill("word").join(" ") + "\n"; - const targetFile = createTempFile(content); - - const prompt = createPrompt({ - id: "ScorePrompt", - name: "Score Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: content.split(" ")[0] ?? "word" }), - makeCheckViolation({ - quoted_text: content.split(" ")[1] ?? "word", - confidence: 0.2, // fails confidence gate — should NOT affect score - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Score should reflect 1 surfaced violation, not 2 - const scoreLine = logCalls.find(l => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("9.0/10"); - expect(scoreLine).not.toContain("8.0/10"); - }); - - it("rejects judge/rubric results instead of projecting them as check findings", async () => { - // Judge/rubric reviews are not a future-facing review type (Phase 3). The - // orchestrator refuses a JudgeResult rather than running the old criterion - // extraction/reporting path: no findings are projected and the run is - // flagged with an operational error. - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "JudgePrompt", - name: "Judge Prompt", - type: "judge", - criteria: [{ id: "Clarity", name: "Clarity", weight: 1 }], - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeJudgeResult([ - makeJudgeViolation(), - makeJudgeViolation({ - line: 2, - quoted_text: "Beta text", - confidence: 0.2, - }), - ]) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(0); - expect(run.totalWarnings).toBe(0); - expect(run.hadOperationalErrors).toBe(true); - expect(run.hadSeverityErrors).toBe(false); - }); - - it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckJsonPrompt", - name: "Check JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const run = await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.Json, - }); - - expect(run.totalWarnings).toBe(0); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Result; - const allIssues = Object.values(parsed.files).flatMap((file) => file.issues); - - expect(allIssues).toHaveLength(0); - expect(JSON.stringify(parsed)).not.toContain("No issues found"); - }); - - it("does not emit dummy issues in Vale JSON output when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckValeJsonPrompt", - name: "Check Vale JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const run = await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.ValeJson, - }); - - expect(run.totalWarnings).toBe(0); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as ValeOutput; - const allIssues = Object.values(parsed).flat(); - - expect(allIssues).toHaveLength(0); - expect(JSON.stringify(parsed)).not.toContain("No issues found"); - }); -}); diff --git a/tests/orchestrator-finding-criteria.test.ts b/tests/orchestrator-finding-criteria.test.ts deleted file mode 100644 index dc0b1ad8..00000000 --- a/tests/orchestrator-finding-criteria.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { RawCheckResult } from "../src/prompts/schema"; -import type { ReviewResult } from "../src/review/types"; - -const { EVALUATE_MOCK, PROCESS_FINDINGS_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), - PROCESS_FINDINGS_MOCK: vi.fn<(input: unknown) => ReviewResult>(), -})); - -// createEvaluator returns a controllable evaluator so evaluateFiles reaches -// routePromptResult -> processFindings without a real model call. -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -// Replace only processFindings (the findings boundary); keep every other -// findings export real so unrelated imports stay intact. -vi.mock("../src/findings", async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, processFindings: PROCESS_FINDINGS_MOCK }; -}); - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-criteria-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -function makeCheckResult(): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: [], - word_count: 100, - }; -} - -describe("standard check orchestration sanitizes criteria at the findings boundary", () => { - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - PROCESS_FINDINGS_MOCK.mockReset(); - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it("drops rubric weight/target from criteria before reaching processFindings", async () => { - const targetFile = createTempFile("Alpha text\n"); - // PromptCriterionSpec admits legacy rubric fields (weight, target). The - // orchestrator must strip them so only { id, name } crosses the boundary. - const prompt = createPrompt({ - id: "SanitizePrompt", - name: "Sanitize Prompt", - type: "check", - severity: Severity.WARNING, - criteria: [ - { id: "Hedging", name: "Hedge words", weight: 3, target: { regex: "x" } }, - ], - }); - - EVALUATE_MOCK.mockResolvedValue(makeCheckResult()); - PROCESS_FINDINGS_MOCK.mockReturnValue({ - findings: [], - scores: [ - { - ruleId: "TestPack.SanitizePrompt", - score: 10, - scoreText: "10.0/10", - severity: "warning", - }, - ], - diagnostics: [], - }); - - await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(PROCESS_FINDINGS_MOCK).toHaveBeenCalledTimes(1); - const input = PROCESS_FINDINGS_MOCK.mock.calls[0]![0] as { - promptMeta: { criteria?: Array> }; - }; - - // toEqual fails if any extra (weight/target) key survives the mapping. - expect(input.promptMeta.criteria).toEqual([{ id: "Hedging", name: "Hedge words" }]); - for (const criterion of input.promptMeta.criteria ?? []) { - expect(criterion).not.toHaveProperty("weight"); - expect(criterion).not.toHaveProperty("target"); - } - }); -}); diff --git a/tests/providers/llm-provider-contract.test.ts b/tests/providers/llm-provider-contract.test.ts deleted file mode 100644 index 249854ca..00000000 --- a/tests/providers/llm-provider-contract.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { LanguageModel } from 'ai'; -import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; - -describe('LLMProvider agent contract', () => { - it('supports agent-mode execution via a provider loop interface', () => { - const provider = new VercelAIProvider({ - model: {} as unknown as LanguageModel, - }); - - const runAgentToolLoop = (provider as { runAgentToolLoop?: unknown }).runAgentToolLoop; - expect(typeof runAgentToolLoop).toBe('function'); - }); -}); diff --git a/tests/providers/structured-model-client.test.ts b/tests/providers/structured-model-client.test.ts index 7fa6b61d..45db05cc 100644 --- a/tests/providers/structured-model-client.test.ts +++ b/tests/providers/structured-model-client.test.ts @@ -45,13 +45,11 @@ describe('StructuredModelClient contract', () => { const onlyRunPromptStructured: StructuredKeys extends 'runPromptStructured' ? true : false = true; expect(onlyRunPromptStructured).toBe(true); - // Runtime guard: a minimal implementation exposes only runPromptStructured. + // Runtime guard: a minimal implementation exposes exactly one capability. const client: StructuredModelClient = { runPromptStructured: () => Promise.resolve({ data: null }), }; - expect( - (client as unknown as Record).runAgentToolLoop, - ).toBeUndefined(); + expect(Object.keys(client)).toEqual(['runPromptStructured']); }); it('is satisfied by VercelAIProvider', () => { diff --git a/tests/providers/tool-calling-model-client.test.ts b/tests/providers/tool-calling-model-client.test.ts index ccf84f68..a2239552 100644 --- a/tests/providers/tool-calling-model-client.test.ts +++ b/tests/providers/tool-calling-model-client.test.ts @@ -66,8 +66,6 @@ describe('ToolCallingModelClient contract', () => { 'read_file', 'search_content', 'report_finding', - 'reviewInstruction', - 'runAgentToolLoop', ]; for (const concept of forbidden) { expect(keys).not.toContain(concept); diff --git a/tests/providers/vercel-ai-provider-agent-loop.test.ts b/tests/providers/vercel-ai-provider-agent-loop.test.ts deleted file mode 100644 index b1bca660..00000000 --- a/tests/providers/vercel-ai-provider-agent-loop.test.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { z } from 'zod'; -import type { LanguageModel } from 'ai'; -import { createMockLogger } from '../utils'; - -const MOCK_GENERATE_TEXT = vi.hoisted(() => vi.fn()); -const MOCK_STEP_COUNT_IS = vi.hoisted(() => vi.fn(() => ({ type: 'stepCount' }))); -const MOCK_TOOL = vi.hoisted(() => - vi.fn((definition: Record) => definition) -); - -const ERROR_CLASSES = vi.hoisted(() => { - class NoObjectGeneratedError extends Error { - text: string; - - constructor(message: string, text: string) { - super(message); - this.name = 'NoObjectGeneratedError'; - this.text = text; - } - - static isInstance(error: unknown): error is NoObjectGeneratedError { - return error instanceof NoObjectGeneratedError; - } - } - - return { NoObjectGeneratedError }; -}); - -vi.mock('ai', () => { - const { NoObjectGeneratedError } = ERROR_CLASSES; - return { - generateText: MOCK_GENERATE_TEXT, - stepCountIs: MOCK_STEP_COUNT_IS, - tool: MOCK_TOOL, - Output: { - object: vi.fn((schema: unknown) => ({ - _outputType: 'object', - schema, - })), - }, - NoObjectGeneratedError, - }; -}); - -import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; - -describe('VercelAIProvider agent loop', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('applies configured retry and tool-concurrency limits during agent execution', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - const runAgentToolLoop = ( - provider as unknown as { - runAgentToolLoop?: (params: Record) => Promise; - } - ).runAgentToolLoop; - - expect(typeof runAgentToolLoop).toBe('function'); - if (typeof runAgentToolLoop !== 'function') { - throw new Error('runAgentToolLoop is not implemented'); - } - - await runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - maxRetries: 4, - maxSteps: 42, - maxParallelToolCalls: 1, - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as { - maxRetries?: number; - providerOptions?: { openai?: { parallelToolCalls?: boolean } }; - }; - - expect(call.maxRetries).toBe(4); - expect(call.providerOptions).toEqual({ - openai: { parallelToolCalls: false }, - }); - }); - - it('adds observability options to agent-loop generateText calls', async () => { - const observability = { - init: vi.fn(), - decorateCall: vi.fn(() => ({ - experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.agent-tool-loop' }, - })), - shutdown: vi.fn(), - }; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - steps: [], - finishReason: 'stop', - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, - providerName: 'openai', - modelName: 'gpt-4o-mini', - observability, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - expect(observability.decorateCall).toHaveBeenCalledWith({ - operation: 'agent-tool-loop', - provider: 'openai', - model: 'gpt-4o-mini', - }); - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; - expect(call).toHaveProperty('experimental_telemetry'); - }); - - it('continues agent-loop AI calls when observability decoration fails', async () => { - const logger = createMockLogger(); - const observability = { - init: vi.fn(), - decorateCall: vi.fn(() => { - throw new Error('telemetry failed'); - }), - shutdown: vi.fn(), - }; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - steps: [], - finishReason: 'stop', - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, - providerName: 'openai', - modelName: 'gpt-4o-mini', - logger, - observability, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; - expect(call).not.toHaveProperty('experimental_telemetry'); - expect(logger.warn).toHaveBeenCalledWith( - '[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', - expect.objectContaining({ error: 'telemetry failed', operation: 'agent-tool-loop' }) - ); - }); - - it('limits concurrent tool executes to maxParallelToolCalls', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - MOCK_GENERATE_TEXT.mockImplementation(async (args: Record) => { - const tools = args.tools as Record Promise }>; - await Promise.all(Object.values(tools).map((t) => t.execute({}))); - return { text: 'done', usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const trackingExecute = async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return {}; - }; - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - maxParallelToolCalls: 2, - tools: { - tool_a: { description: 'a', inputSchema: z.object({}), execute: trackingExecute }, - tool_b: { description: 'b', inputSchema: z.object({}), execute: trackingExecute }, - tool_c: { description: 'c', inputSchema: z.object({}), execute: trackingExecute }, - tool_d: { description: 'd', inputSchema: z.object({}), execute: trackingExecute }, - }, - }); - - expect(maxConcurrent).toBeLessThanOrEqual(2); - expect(maxConcurrent).toBeGreaterThan(1); - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as { - providerOptions?: { openai?: { parallelToolCalls?: boolean } }; - }; - expect(call.providerOptions).toEqual({ - openai: { parallelToolCalls: true }, - }); - }); - - it('defaults tool concurrency to 1 when maxParallelToolCalls is not supplied', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - MOCK_GENERATE_TEXT.mockImplementation(async (args: Record) => { - const tools = args.tools as Record Promise }>; - await Promise.all(Object.values(tools).map((t) => t.execute({}))); - return { text: 'done', usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const trackingExecute = async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return {}; - }; - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - tool_a: { description: 'a', inputSchema: z.object({}), execute: trackingExecute }, - tool_b: { description: 'b', inputSchema: z.object({}), execute: trackingExecute }, - tool_c: { description: 'c', inputSchema: z.object({}), execute: trackingExecute }, - }, - }); - - expect(maxConcurrent).toBe(1); - }); - - it('emits agent-loop debug output through the injected logger', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'final summary', - finishReason: 'stop', - steps: [ - { - finishReason: 'tool-calls', - text: 'step text', - toolCalls: [{ toolName: 'lint' }], - }, - ], - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const logger = createMockLogger(); - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - debug: true, - logger, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - expect(logger.debug).toHaveBeenCalled(); - expect( - logger.debug.mock.calls.some(([message]) => - String(message).includes('[agent] step 1: finishReason=tool-calls tools=[lint]') - ) - ).toBe(true); - expect( - logger.debug.mock.calls.some(([message]) => - String(message).includes('[agent] final finishReason=stop steps=1') - ) - ).toBe(true); - }); -}); From 6e2a04feda87a83ec9feb4c2c8259d12470c9fc8 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:28:56 +0100 Subject: [PATCH 23/26] refactor(cli): reject legacy mode flag - Reject the removed --mode option without silently remapping it.\n- Cover single, agent, and auto executor dispatch plus ReviewResult routing.\n- Preserve the breaking CLI migration to --model-call single|agent|auto. --- src/cli/commands.ts | 3 +- tests/cli-mode-rejection.test.ts | 37 +++ tests/orchestrator-executor-dispatch.test.ts | 229 +++++++++++++++++++ 3 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 tests/cli-mode-rejection.test.ts create mode 100644 tests/orchestrator-executor-dispatch.test.ts diff --git a/src/cli/commands.ts b/src/cli/commands.ts index b2e7f0b7..9c0e362a 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -97,8 +97,7 @@ export function registerMainCommand(program: Command): void { // --mode is removed (audit Product Decision): the autonomous workspace-agent // surface is gone. Fail clearly and point users at the replacement rather // than silently mapping it onto --model-call. - const rawCliOpts = program.opts() as { mode?: unknown }; - if (rawCliOpts.mode !== undefined) { + if (program.opts().mode !== undefined) { console.error('Error: --mode is no longer supported. Use --model-call single|agent|auto instead.'); process.exit(1); } diff --git a/tests/cli-mode-rejection.test.ts b/tests/cli-mode-rejection.test.ts new file mode 100644 index 00000000..38c0f3db --- /dev/null +++ b/tests/cli-mode-rejection.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Command } from 'commander'; + +describe('CLI --mode rejection', () => { + let exitSpy: ReturnType; + + beforeEach(() => { + vi.spyOn(console, 'error').mockImplementation(() => undefined); + exitSpy = vi.spyOn(process, 'exit').mockImplementation( + ((code?: string | number | null) => { + throw new Error(`exit:${code ?? ''}`); + }) as never, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('rejects --mode and points users at --model-call instead of mapping it', async () => { + const { registerMainCommand } = await import('../src/cli/commands'); + const program = new Command(); + registerMainCommand(program); + + await expect( + program.parseAsync(['node', 'test', 'README.md', '--mode', 'agent']), + ).rejects.toThrow('exit:1'); + + expect(vi.mocked(console.error)).toHaveBeenCalledWith( + expect.stringContaining('--mode is no longer supported'), + ); + expect(vi.mocked(console.error)).toHaveBeenCalledWith( + expect.stringContaining('--model-call'), + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); +}); diff --git a/tests/orchestrator-executor-dispatch.test.ts b/tests/orchestrator-executor-dispatch.test.ts new file mode 100644 index 00000000..b4a1d093 --- /dev/null +++ b/tests/orchestrator-executor-dispatch.test.ts @@ -0,0 +1,229 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import * as path from 'path'; +import { evaluateFiles } from '../src/cli/orchestrator'; +import { OutputFormat, type EvaluationOptions } from '../src/cli/types'; +import { Severity } from '../src/evaluators/types'; +import type { PromptFile } from '../src/prompts/prompt-loader'; +import type { ReviewRequest, ReviewResult } from '../src/review/types'; + +const { EXECUTOR_FOR_MOCK, FAKE_EXECUTOR_RUN } = vi.hoisted(() => ({ + EXECUTOR_FOR_MOCK: vi.fn(), + FAKE_EXECUTOR_RUN: vi.fn(), +})); + +// Replace executorFor with a spy that returns a controllable executor so the +// orchestrator's wiring (ReviewRequest build -> chooseModelCall -> dispatch -> +// ReviewResult routing) is exercised without a real model call. +vi.mock('../src/executors', () => ({ + executorFor: EXECUTOR_FOR_MOCK, +})); + +function makePrompt(id: string): PromptFile { + return { + id, + filename: `${id}.md`, + fullPath: path.join(process.cwd(), 'prompts', `${id}.md`), + pack: 'TestPack', + body: `Rule body for ${id}`, + meta: { id, name: id, type: 'check', severity: Severity.WARNING }, + }; +} + +function makeOptions(prompts: PromptFile[], overrides: Partial = {}): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + requestBuilder: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Json, + modelCall: 'auto', + ...overrides, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), 'vectorlint-dispatch-')); + const filePath = path.join(dir, 'input.md'); + writeFileSync(filePath, content); + return filePath; +} + +function makeReviewResult(overrides: Partial = {}): ReviewResult { + return { + findings: [ + { + ruleId: 'TestPack.CheckPrompt', + ruleSource: path.join(process.cwd(), 'prompts', 'CheckPrompt.md'), + severity: 'warning', + message: 'Vague advice found', + line: 1, + column: 1, + match: 'vague text', + suggestion: 'Be specific.', + }, + ], + scores: [ + { + ruleId: 'TestPack.CheckPrompt', + score: 8, + scoreText: '8.0/10', + severity: 'warning', + findingCount: 1, + }, + ], + diagnostics: [], + hadOperationalErrors: false, + usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5 }, + ...overrides, + }; +} + +describe('orchestrator executor dispatch', () => { + beforeEach(() => { + EXECUTOR_FOR_MOCK.mockReset(); + FAKE_EXECUTOR_RUN.mockReset(); + EXECUTOR_FOR_MOCK.mockReturnValue({ run: FAKE_EXECUTOR_RUN }); + FAKE_EXECUTOR_RUN.mockResolvedValue(makeReviewResult()); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('resolves auto to single for a normal-sized target and routes findings to JSON output', async () => { + const file = createTempFile('vague text here\n'); + const run = await evaluateFiles([file], makeOptions([makePrompt('CheckPrompt')])); + + // auto + small target + one rule resolves to the single executor. + expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); + + // The verified finding reaches the JSON sink. + const parsed = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) as { + files: Record> }>; + }; + const issues = Object.values(parsed.files).flatMap((f) => f.issues); + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + line: 1, + severity: Severity.WARNING, + message: 'Vague advice found', + rule: 'TestPack.CheckPrompt', + match: 'vague text', + }); + + expect(run.totalWarnings).toBe(1); + expect(run.totalErrors).toBe(0); + expect(run.hadSeverityErrors).toBe(false); + expect(run.hadOperationalErrors).toBe(false); + }); + + it('resolves auto to agent for a large target', async () => { + const file = createTempFile(`${'x'.repeat(650_000)}\n`); + await evaluateFiles([file], makeOptions([makePrompt('BigPrompt')])); + + expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); + }); + + it('honors an explicit single modelCall even for a large target', async () => { + const file = createTempFile(`${'x'.repeat(650_000)}\n`); + await evaluateFiles([file], makeOptions([makePrompt('ForcedSingle')], { modelCall: 'single' })); + + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); + }); + + it('honors an explicit agent modelCall even for a small target', async () => { + const file = createTempFile('small\n'); + await evaluateFiles([file], makeOptions([makePrompt('ForcedAgent')], { modelCall: 'agent' })); + + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); + }); + + it('forwards the built ReviewRequest (target + rules + modelCall) to the executor', async () => { + const file = createTempFile('target content line one\n'); + await evaluateFiles([file], makeOptions([makePrompt('FwdPrompt')], { modelCall: 'agent' })); + + expect(FAKE_EXECUTOR_RUN).toHaveBeenCalledTimes(1); + const request = FAKE_EXECUTOR_RUN.mock.calls[0]![0] as ReviewRequest; + expect(request.target.content).toBe('target content line one\n'); + expect(request.rules).toHaveLength(1); + expect(request.rules[0]?.body).toBe('Rule body for FwdPrompt'); + expect(request.modelCall).toBe('agent'); + }); + + it('aggregates error-severity findings and flags hadSeverityErrors', async () => { + FAKE_EXECUTOR_RUN.mockResolvedValue( + makeReviewResult({ + findings: [ + { + ruleId: 'TestPack.CheckPrompt', + ruleSource: 'src', + severity: 'error', + message: 'Severe issue', + line: 2, + column: 1, + match: 'vague text', + }, + ], + scores: [ + { ruleId: 'TestPack.CheckPrompt', score: 0, scoreText: '0.0/10', severity: 'error', findingCount: 1 }, + ], + }), + ); + const file = createTempFile('vague text\n'); + + const run = await evaluateFiles([file], makeOptions([makePrompt('ErrorPrompt')])); + + expect(run.totalErrors).toBe(1); + expect(run.hadSeverityErrors).toBe(true); + }); + + it('routes ReviewResult diagnostics to verbose console output', async () => { + FAKE_EXECUTOR_RUN.mockResolvedValue( + makeReviewResult({ + findings: [], + diagnostics: [{ level: 'warn', code: 'finding-evidence-not-locatable', message: 'could not anchor quote' }], + }), + ); + const file = createTempFile('vague text\n'); + + await evaluateFiles([file], makeOptions([makePrompt('DiagPrompt')], { verbose: true })); + + expect(vi.mocked(console.warn)).toHaveBeenCalledWith( + expect.stringContaining('could not anchor quote'), + ); + }); + + it('aggregates token usage from the ReviewResult', async () => { + const file = createTempFile('vague text\n'); + const run = await evaluateFiles([file], makeOptions([makePrompt('UsagePrompt')])); + + expect(run.tokenUsage?.totalInputTokens).toBe(10); + expect(run.tokenUsage?.totalOutputTokens).toBe(5); + }); + + it('skips the executor when no prompts apply', async () => { + const file = createTempFile('content\n'); + // A scan-path config that runs a pack none of the prompts belong to. + const run = await evaluateFiles( + [file], + makeOptions([makePrompt('OrphanPrompt')], { + scanPaths: [{ pattern: '**/*.md', runRules: ['OtherPack'], overrides: {} }], + }), + ); + + expect(EXECUTOR_FOR_MOCK).not.toHaveBeenCalled(); + expect(run.totalFiles).toBe(1); + expect(run.totalWarnings).toBe(0); + }); +}); From 64e434d88350204c20c2eb671d9fa4181059d7d6 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:39:52 +0100 Subject: [PATCH 24/26] refactor(cli): harden config and payload telemetry - create global config resources with restrictive permissions\n- propagate opt-in payload telemetry through both executors\n- cover CLI, package entrypoints, observability, and provider logging --- src/config/global-config.ts | 4 ++-- src/executors/agent-model-call-executor.ts | 6 +++++- src/executors/single-model-call-executor.ts | 5 ++++- src/observability/ai-observability.ts | 1 + src/observability/langfuse-observability.ts | 4 ++-- src/providers/request-builder.ts | 1 + src/providers/tool-calling-model-client.ts | 2 ++ src/providers/vercel-ai-provider.ts | 6 ++++++ tests/global-config.test.ts | 17 ++++++++++++++- tests/main-command-observability.test.ts | 4 ++-- .../langfuse-observability.test.ts | 21 ++++++++++++++++++- tests/package-entrypoint.test.ts | 21 +++++++++++++++++++ tests/vercel-ai-provider.test.ts | 1 + 13 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 tests/package-entrypoint.test.ts diff --git a/src/config/global-config.ts b/src/config/global-config.ts index 98b1a7fc..6a694122 100644 --- a/src/config/global-config.ts +++ b/src/config/global-config.ts @@ -74,11 +74,11 @@ export function ensureGlobalConfig(): string { const configDir = path.dirname(configPath); if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); + mkdirSync(configDir, { recursive: true, mode: 0o700 }); } if (!existsSync(configPath)) { - writeFileSync(configPath, DEFAULT_GLOBAL_CONFIG_TEMPLATE, 'utf-8'); + writeFileSync(configPath, DEFAULT_GLOBAL_CONFIG_TEMPLATE, { encoding: 'utf-8', mode: 0o600 }); } return configPath; diff --git a/src/executors/agent-model-call-executor.ts b/src/executors/agent-model-call-executor.ts index 1b09f388..1679fecb 100644 --- a/src/executors/agent-model-call-executor.ts +++ b/src/executors/agent-model-call-executor.ts @@ -50,7 +50,10 @@ export class AgentModelCallExecutor implements ReviewExecutor { const capability = new TargetReadCapability(request.target.content); // Exactly one executor-owned tool is exposed: target-section paging. const tools = buildReadTargetSectionTool(capability); - const context = buildEvalContext(request.target.uri); + const context = { + ...buildEvalContext(request.target.uri), + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }; const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; @@ -136,6 +139,7 @@ export class AgentModelCallExecutor implements ReviewExecutor { // emitting its structured finding set. maxSteps: request.budget.maxChunksPerRule, maxParallelToolCalls: 1, + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, }, }); diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts index 3809cbb3..2bb87ec4 100644 --- a/src/executors/single-model-call-executor.ts +++ b/src/executors/single-model-call-executor.ts @@ -50,7 +50,10 @@ export class SingleModelCallExecutor implements ReviewExecutor { async run(request: ReviewRequest): Promise { const schema = buildCheckLLMSchema(); - const context = buildEvalContext(request.target.uri); + const context = { + ...buildEvalContext(request.target.uri), + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }; const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index e16a3c9d..88a76f26 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -4,6 +4,7 @@ export interface AIExecutionContext { model: string; evaluator?: string; rule?: string; + recordPayloadTelemetry?: boolean; } export interface AIObservability { diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index 25f4d32c..bae2d555 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -63,8 +63,8 @@ export class LangfuseObservability implements AIObservability { ...(context.evaluator ? { evaluator: context.evaluator } : {}), ...(context.rule ? { rule: context.rule } : {}), }, - recordInputs: true, - recordOutputs: true, + recordInputs: context.recordPayloadTelemetry === true, + recordOutputs: context.recordPayloadTelemetry === true, }, }; } diff --git a/src/providers/request-builder.ts b/src/providers/request-builder.ts index 806c53e8..9e0b239c 100644 --- a/src/providers/request-builder.ts +++ b/src/providers/request-builder.ts @@ -2,6 +2,7 @@ export interface EvalContext { fileType?: string; + recordPayloadTelemetry?: boolean; } export interface RequestBuilder { diff --git a/src/providers/tool-calling-model-client.ts b/src/providers/tool-calling-model-client.ts index 2139c8f2..a038e7e5 100644 --- a/src/providers/tool-calling-model-client.ts +++ b/src/providers/tool-calling-model-client.ts @@ -21,6 +21,8 @@ export interface ToolCallDefinition { * transport honors them. It does not invent its own product-level loop. */ export interface ToolCallRunOptions { + /** Opt in to recording prompt and generated payloads in telemetry. */ + recordPayloadTelemetry?: boolean; /** Maximum number of model steps (tool-call rounds) before the run stops. */ maxSteps?: number; /** Provider retry budget for transient failures. */ diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index dba87369..634e08f8 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -91,6 +91,9 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { model: this.config.modelName ?? 'unknown', ...(evaluator ? { evaluator } : {}), ...(rule ? { rule } : {}), + ...(context?.recordPayloadTelemetry !== undefined + ? { recordPayloadTelemetry: context.recordPayloadTelemetry } + : {}), }); const result = await generateText({ @@ -197,6 +200,9 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { operation: 'tool-calling', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', + ...(options.recordPayloadTelemetry !== undefined + ? { recordPayloadTelemetry: options.recordPayloadTelemetry } + : {}), }), stopWhen: stepCountIs(options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS), providerOptions: { diff --git a/tests/global-config.test.ts b/tests/global-config.test.ts index 2a8d8c7b..acee276c 100644 --- a/tests/global-config.test.ts +++ b/tests/global-config.test.ts @@ -3,7 +3,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; -import { loadGlobalConfig } from '../src/config/global-config'; +import { ensureGlobalConfig, loadGlobalConfig } from '../src/config/global-config'; import { GLOBAL_CONFIG_DIR, GLOBAL_CONFIG_FILE } from '../src/config/constants'; // Mock fs and os @@ -49,6 +49,21 @@ BOOL_KEY = true expect(fs.readFileSync).toHaveBeenCalledWith(mockConfigPath, 'utf-8'); }); + it('creates the global config directory and file with restrictive modes', () => { + vi.mocked(fs.existsSync).mockReturnValue(false); + + expect(ensureGlobalConfig()).toBe(mockConfigPath); + expect(fs.mkdirSync).toHaveBeenCalledWith( + path.dirname(mockConfigPath), + { recursive: true, mode: 0o700 }, + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + mockConfigPath, + expect.any(String), + { encoding: 'utf-8', mode: 0o600 }, + ); + }); + it('should NOT overwrite existing env vars', () => { const mockToml = ` [env] diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index 7d652c46..d1619ad9 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -97,7 +97,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - modelCall: 'auto', + modelCall: 'single', config: undefined, }); MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); @@ -141,7 +141,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - modelCall: 'auto', + modelCall: 'single', config: undefined, }); const observability = { diff --git a/tests/observability/langfuse-observability.test.ts b/tests/observability/langfuse-observability.test.ts index 56448489..ca0f73c4 100644 --- a/tests/observability/langfuse-observability.test.ts +++ b/tests/observability/langfuse-observability.test.ts @@ -26,7 +26,7 @@ describe('LangfuseObservability', () => { SHUTDOWN_MOCK.mockResolvedValue(undefined); }); - it('returns AI SDK telemetry options with full payload recording enabled', () => { + it('does not record payloads by default', () => { const subject = new LangfuseObservability({ publicKey: 'pk-lf-test', secretKey: 'sk-lf-test', @@ -49,6 +49,25 @@ describe('LangfuseObservability', () => { evaluator: 'clarity', rule: 'no-fluff', }, + recordInputs: false, + recordOutputs: false, + }, + }); + }); + + it('records payloads only when the review output policy opts in', () => { + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + }); + + expect(subject.decorateCall({ + operation: 'structured-eval', + provider: 'openai', + model: 'gpt-4o', + recordPayloadTelemetry: true, + })).toMatchObject({ + experimental_telemetry: { recordInputs: true, recordOutputs: true, }, diff --git a/tests/package-entrypoint.test.ts b/tests/package-entrypoint.test.ts new file mode 100644 index 00000000..ff99ac77 --- /dev/null +++ b/tests/package-entrypoint.test.ts @@ -0,0 +1,21 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const PACKAGE_PATH = resolve(dirname(fileURLToPath(import.meta.url)), '../package.json'); + +describe('package entrypoints', () => { + it('keeps both published bin aliases pointed at the built CLI', () => { + const packageJson = JSON.parse(readFileSync(PACKAGE_PATH, 'utf8')) as { + main?: string; + bin?: Record; + }; + + expect(packageJson.main).toBe('dist/index.js'); + expect(packageJson.bin).toEqual({ + vectorlint: './dist/index.js', + veclint: './dist/index.js', + }); + }); +}); diff --git a/tests/vercel-ai-provider.test.ts b/tests/vercel-ai-provider.test.ts index ab258dd4..80bbaf5f 100644 --- a/tests/vercel-ai-provider.test.ts +++ b/tests/vercel-ai-provider.test.ts @@ -391,6 +391,7 @@ describe('VercelAIProvider', () => { }) as Record, }) ); + expect(logger.debug).not.toHaveBeenCalledWith('Test content'); }); it('does not log when debug is disabled', async () => { From 0c18c2dfbb58cb03854bfad19d40cc3f69a79c26 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 13:01:26 +0100 Subject: [PATCH 25/26] refactor(observability): remove legacy agent telemetry operation - Keep observability operation types aligned with the executor-only review surface --- src/observability/ai-observability.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index 88a76f26..7ed80e03 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -1,5 +1,5 @@ export interface AIExecutionContext { - operation: 'structured-eval' | 'tool-calling' | 'agent-tool-loop'; + operation: 'structured-eval' | 'tool-calling'; provider: string; model: string; evaluator?: string; From 9c122f81ff2f60bbae4ad02dd861c705b0b82e17 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 13:34:45 +0100 Subject: [PATCH 26/26] refactor(errors): drop orphaned AgentToolError class - AgentToolError's only consumers were src/agent/executor.ts, src/agent/path-utils.ts, and tests/agent/agent-executor.test.ts, all deleted with the autonomous workspace-agent surface - Remove the now-unused class so no agent-tied symbol lingers in the error hierarchy Refs: docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md (Product Decision; Finding #2) --- src/errors/index.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/errors/index.ts b/src/errors/index.ts index 86e257ba..ca196d95 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -30,14 +30,6 @@ export class ProcessingError extends VectorlintError { } } -// Agent tool/runtime error for agent-mode operational failures -export class AgentToolError extends VectorlintError { - constructor(message: string, code = 'AGENT_TOOL_ERROR') { - super(message, code); - this.name = 'AgentToolError'; - } -} - // Missing dependency error for when required dependencies are not available export class MissingDependencyError extends VectorlintError { constructor(