From 43985c703fe556cad238d6dfe61311f89966857b Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 18 Feb 2026 21:18:14 -0500 Subject: [PATCH 1/2] feat: add `quorum doctor` command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validates the user's Quorum setup in one command: - Config file exists and parses correctly (YAML validation) - Node.js version compatibility (requires ≥20) - Quorum version (current vs latest on npm) - Each configured provider: lightweight API probe to verify auth Output uses ✅/❌/⚠️ icons with a summary line. Exit code 1 if any errors, 0 otherwise. --- src/cli/doctor.test.ts | 139 +++++++++++++++++++++++++++++ src/cli/doctor.ts | 192 +++++++++++++++++++++++++++++++++++++++++ src/cli/index.ts | 2 + 3 files changed, 333 insertions(+) create mode 100644 src/cli/doctor.test.ts create mode 100644 src/cli/doctor.ts diff --git a/src/cli/doctor.test.ts b/src/cli/doctor.test.ts new file mode 100644 index 0000000..a2fd542 --- /dev/null +++ b/src/cli/doctor.test.ts @@ -0,0 +1,139 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Mock modules before importing the module under test +vi.mock('../config.js', () => ({ + loadConfig: vi.fn(), + CONFIG_PATH: '/home/test/.quorum/config.yaml', +})); + +vi.mock('../providers/base.js', () => ({ + createProvider: vi.fn(), +})); + +import { runDoctor } from './doctor.js'; +import { loadConfig } from '../config.js'; +import { createProvider } from '../providers/base.js'; +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; + +// Mock existsSync for config file check +vi.mock('node:fs', async () => { + const actual = await vi.importActual('node:fs'); + return { ...actual, existsSync: vi.fn(actual.existsSync), readFileSync: actual.readFileSync }; +}); + +vi.mock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises'); + return { ...actual, readFile: vi.fn(actual.readFile) }; +}); + +// Mock fetch for npm version check +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +beforeEach(() => { + vi.clearAllMocks(); + // Default: config file exists + (existsSync as any).mockImplementation((path: string) => { + if (path === '/home/test/.quorum/config.yaml') return true; + const { existsSync: real } = vi.importActual('node:fs') as any; + return real(path); + }); + // Mock readFile for config path to return valid YAML + (readFile as any).mockImplementation(async (path: string, enc?: string) => { + if (path === '/home/test/.quorum/config.yaml') { + return 'providers:\n - name: test\n provider: openai\n model: gpt-4o\n'; + } + const actual = await vi.importActual('node:fs/promises'); + return actual.readFile(path, enc as any); + }); + // Default: npm returns current version + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ version: '0.0.0' }), // will differ from actual + }); +}); + +describe('doctor', () => { + it('returns 0 when all checks pass', async () => { + (loadConfig as any).mockResolvedValue({ + providers: [{ name: 'test-provider', provider: 'openai', model: 'gpt-4o' }], + }); + (createProvider as any).mockResolvedValue({ + generate: vi.fn().mockResolvedValue('ok'), + }); + + const spy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const code = await runDoctor(); + spy.mockRestore(); + + // May be 0 or non-zero depending on version mismatch (warn, not error) + // Provider check should pass + expect(code).toBe(0); + }); + + it('returns 1 when config file is missing', async () => { + (existsSync as any).mockImplementation((path: string) => { + if (path === '/home/test/.quorum/config.yaml') return false; + const { existsSync: real } = vi.importActual('node:fs') as any; + return real(path); + }); + + const spy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const code = await runDoctor(); + spy.mockRestore(); + + expect(code).toBe(1); + }); + + it('returns 1 when a provider fails auth', async () => { + (loadConfig as any).mockResolvedValue({ + providers: [{ name: 'bad-provider', provider: 'openai', model: 'gpt-4o' }], + }); + (createProvider as any).mockRejectedValue( + Object.assign(new Error('Unauthorized'), { status: 401 }), + ); + + const spy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const code = await runDoctor(); + spy.mockRestore(); + + expect(code).toBe(1); + }); + + it('returns 1 when provider connection is refused', async () => { + (loadConfig as any).mockResolvedValue({ + providers: [{ name: 'ollama', provider: 'ollama', model: 'llama3' }], + }); + (createProvider as any).mockRejectedValue( + Object.assign(new Error('fetch failed: ECONNREFUSED'), { code: 'ECONNREFUSED' }), + ); + + const spy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const code = await runDoctor(); + spy.mockRestore(); + + expect(code).toBe(1); + }); + + it('handles multiple providers with mixed results', async () => { + (loadConfig as any).mockResolvedValue({ + providers: [ + { name: 'good', provider: 'openai', model: 'gpt-4o' }, + { name: 'bad', provider: 'deepseek', model: 'deepseek-chat' }, + ], + }); + (createProvider as any).mockImplementation(async (config: any) => { + if (config.name === 'good') { + return { generate: vi.fn().mockResolvedValue('ok') }; + } + throw Object.assign(new Error('402 Payment Required'), { status: 402 }); + }); + + const spy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const code = await runDoctor(); + spy.mockRestore(); + + expect(code).toBe(1); + }); +}); diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts new file mode 100644 index 0000000..7f0001f --- /dev/null +++ b/src/cli/doctor.ts @@ -0,0 +1,192 @@ +import type { Command } from 'commander'; +import pc from 'picocolors'; +import { existsSync, readFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { parse } from 'yaml'; +import { loadConfig, CONFIG_PATH } from '../config.js'; +import { createProvider } from '../providers/base.js'; +import type { ProviderConfig } from '../types.js'; + +// ── Types ────────────────────────────────────────────────────────────────── + +type Status = 'ok' | 'warn' | 'error'; + +interface CheckResult { + status: Status; + label: string; + detail: string; +} + +// ── Symbols ──────────────────────────────────────────────────────────────── + +function icon(s: Status): string { + switch (s) { + case 'ok': + return pc.green('✅'); + case 'warn': + return pc.yellow('⚠️'); + case 'error': + return pc.red('❌'); + } +} + +// ── Individual checks ────────────────────────────────────────────────────── + +async function checkConfig(): Promise { + const label = 'Config'; + const path = CONFIG_PATH; + + if (!existsSync(path)) { + return { status: 'error', label, detail: `${path} not found — run \`quorum init\`` }; + } + + try { + const raw = await readFile(path, 'utf-8'); + const parsed = parse(raw); + if (!parsed || !Array.isArray(parsed.providers)) { + return { status: 'error', label, detail: `${path} missing 'providers' array` }; + } + return { status: 'ok', label, detail: `${tildefy(path)} found and valid` }; + } catch (e: any) { + return { status: 'error', label, detail: `${tildefy(path)} parse error: ${e.message}` }; + } +} + +function checkNodeVersion(): CheckResult { + const label = 'Node.js'; + const major = parseInt(process.versions.node.split('.')[0], 10); + const version = `v${process.versions.node}`; + if (major >= 20) { + return { status: 'ok', label, detail: `${version} (requires ≥20)` }; + } + return { status: 'error', label, detail: `${version} — requires ≥20, please upgrade` }; +} + +async function checkQuorumVersion(): Promise { + const label = 'Quorum'; + const pkgPath = new URL('../../package.json', import.meta.url); + const currentVersion = JSON.parse(readFileSync(pkgPath, 'utf-8')).version as string; + + try { + const res = await fetch('https://registry.npmjs.org/quorum-ai/latest', { + signal: AbortSignal.timeout(5000), + }); + if (!res.ok) { + return { status: 'warn', label, detail: `v${currentVersion} (couldn't check latest)` }; + } + const data = (await res.json()) as { version: string }; + const latest = data.version; + if (currentVersion === latest) { + return { status: 'ok', label, detail: `v${currentVersion} (latest)` }; + } + return { + status: 'warn', + label, + detail: `v${currentVersion} — update available: v${latest}`, + }; + } catch { + return { status: 'warn', label, detail: `v${currentVersion} (couldn't check latest)` }; + } +} + +async function checkProvider(config: ProviderConfig): Promise { + const label = config.name; + const start = Date.now(); + + try { + const adapter = await createProvider(config); + const result = await adapter.generate('Say "ok".', 'Respond with only the word ok.'); + const elapsed = Date.now() - start; + if (result && result.length > 0) { + return { status: 'ok', label, detail: `${config.model} — authenticated, ${elapsed}ms` }; + } + return { status: 'warn', label, detail: `${config.model} — empty response, ${elapsed}ms` }; + } catch (e: any) { + const detail = diagnoseError(e, config); + return { status: 'error', label, detail: `${config.model} — ${detail}` }; + } +} + +function diagnoseError(e: any, config: ProviderConfig): string { + const msg = e.message || String(e); + const status = e.status || e.statusCode; + + if (status === 401 || msg.includes('401')) return '401 Unauthorized — check API key'; + if (status === 402 || msg.includes('402')) return '402 Insufficient Balance'; + if (status === 403 || msg.includes('403')) return '403 Forbidden — check permissions'; + if (status === 429 || msg.includes('429')) return '429 Rate Limited — try again later'; + if (msg.includes('ECONNREFUSED')) return `connection refused (is ${config.provider} running?)`; + if (msg.includes('ENOTFOUND')) return 'DNS resolution failed — check network'; + if (msg.includes('ETIMEDOUT') || msg.includes('timed out')) return 'request timed out'; + if (msg.includes('fetch failed')) return 'network error — check connectivity'; + + // Truncate long messages + return msg.length > 100 ? msg.slice(0, 100) + '…' : msg; +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +function tildefy(path: string): string { + const home = homedir(); + return path.startsWith(home) ? '~' + path.slice(home.length) : path; +} + +function pad(s: string, len: number): string { + return s.length >= len ? s : s + ' '.repeat(len - s.length); +} + +// ── Main ─────────────────────────────────────────────────────────────────── + +export async function runDoctor(): Promise { + const results: CheckResult[] = []; + + // System checks + const [configResult, versionResult] = await Promise.all([checkConfig(), checkQuorumVersion()]); + const nodeResult = checkNodeVersion(); + + results.push(configResult, nodeResult, versionResult); + + // Provider checks (only if config is valid) + if (configResult.status !== 'error') { + const config = await loadConfig(); + if (config.providers.length > 0) { + const providerResults = await Promise.all(config.providers.map(checkProvider)); + results.push(...providerResults); + } + } + + // Print results + const maxLabel = Math.max(...results.map((r) => r.label.length)); + console.log(''); + for (const r of results) { + console.log(`${icon(r.status)} ${pad(r.label, maxLabel + 2)}${r.detail}`); + } + + // Summary + const ok = results.filter((r) => r.status === 'ok').length; + const warns = results.filter((r) => r.status === 'warn').length; + const errors = results.filter((r) => r.status === 'error').length; + + console.log(''); + const parts: string[] = []; + if (ok > 0) parts.push(pc.green(`${ok} healthy`)); + if (errors > 0) parts.push(pc.red(`${errors} error${errors > 1 ? 's' : ''}`)); + if (warns > 0) parts.push(pc.yellow(`${warns} warning${warns > 1 ? 's' : ''}`)); + console.log(parts.join(', ')); + console.log(''); + + return errors > 0 ? 1 : 0; +} + +// ── CLI registration ─────────────────────────────────────────────────────── + +export function registerDoctorCommand(program: Command): void { + program + .command('doctor') + .description('Check your Quorum setup — config, providers, connectivity') + .action(async () => { + const exitCode = await runDoctor(); + process.exit(exitCode); + }); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index e692981..358af0d 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -37,6 +37,7 @@ import { registerAuthCommand } from './auth.js'; import { registerSessionCommands } from './session.js'; import { registerAnalysisCommands } from './analysis.js'; import { registerGovernanceCommands } from './governance.js'; +import { registerDoctorCommand } from './doctor.js'; const program = new Command(); @@ -52,6 +53,7 @@ registerAuthCommand(program); registerSessionCommands(program); registerAnalysisCommands(program); registerGovernanceCommands(program); +registerDoctorCommand(program); // Ensure clean exit after any command (prevents event-loop hangs from dangling handles) program.hook('postAction', (_thisCommand, actionCommand) => { From 395441eff31acfd99fd068771f6da4647b52e9c2 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 30 Mar 2026 18:40:33 -0400 Subject: [PATCH 2/2] feat: track token usage per provider response Co-Authored-By: Paperclip --- eslint.config.js | 2 +- src/cli/analysis.ts | 4 +- src/cli/doctor.ts | 3 +- src/cli/providers.ts | 3 +- src/cli/session.ts | 188 +++++++++++++++++++----- src/council-v2.ts | 264 ++++++++++++++++++++++++++-------- src/providers/base.ts | 60 ++++++-- src/session.ts | 2 + src/types.ts | 31 +++- src/usage.ts | 150 +++++++++++++++++++ tests/cli-integration.test.ts | 94 +++++++++++- tsconfig.eslint.json | 9 ++ 12 files changed, 704 insertions(+), 106 deletions(-) create mode 100644 src/usage.ts create mode 100644 tsconfig.eslint.json diff --git a/eslint.config.js b/eslint.config.js index 3bbc3b5..dc6bb4b 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -10,7 +10,7 @@ export default tseslint.config( { languageOptions: { parserOptions: { - projectService: true, + project: './tsconfig.eslint.json', tsconfigRootDir: import.meta.dirname, }, }, diff --git a/src/cli/analysis.ts b/src/cli/analysis.ts index 35a3d17..baf89e2 100644 --- a/src/cli/analysis.ts +++ b/src/cli/analysis.ts @@ -255,10 +255,12 @@ ${pc.dim(' $ quorum diff session1 session2 --json')} console.log(pc.dim(`Analyzing with ${providerConfig.name}...`)); } try { - analysisNarrative = await adapter.generate( + const analysisResponse = await adapter.generate( prompt, 'You are an expert analyst comparing two AI deliberation outcomes. Be concise and insightful.', ); + analysisNarrative = + typeof analysisResponse === 'string' ? analysisResponse : analysisResponse.content; } catch (err) { if (!opts.json) { console.error( diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 7f0001f..2b41885 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -97,8 +97,9 @@ async function checkProvider(config: ProviderConfig): Promise { try { const adapter = await createProvider(config); const result = await adapter.generate('Say "ok".', 'Respond with only the word ok.'); + const text = typeof result === 'string' ? result : result.content; const elapsed = Date.now() - start; - if (result && result.length > 0) { + if (text && text.length > 0) { return { status: 'ok', label, detail: `${config.model} — authenticated, ${elapsed}ms` }; } return { status: 'warn', label, detail: `${config.model} — empty response, ${elapsed}ms` }; diff --git a/src/cli/providers.ts b/src/cli/providers.ts index 3ca86a4..59c1785 100644 --- a/src/cli/providers.ts +++ b/src/cli/providers.ts @@ -201,7 +201,8 @@ ${pc.dim(' $ quorum init --non-interactive')} 'Say "OK" in one word.', 'You are a helpful assistant. Reply concisely.', ); - console.log(pc.green(`✅ "${response.slice(0, 50)}"`)); + const text = typeof response === 'string' ? response : response.content; + console.log(pc.green(`✅ "${text.slice(0, 50)}"`)); } catch (err) { console.log(pc.red(`❌ ${err instanceof Error ? err.message.slice(0, 80) : 'failed'}`)); } diff --git a/src/cli/session.ts b/src/cli/session.ts index 976e970..e9de13f 100644 --- a/src/cli/session.ts +++ b/src/cli/session.ts @@ -8,6 +8,122 @@ import { loadConfig, loadAgentProfile } from '../config.js'; import { CouncilV2 } from '../council-v2.js'; import { createProvider } from '../providers/base.js'; import { CLIError, readStdin, resolveLastSession } from './helpers.js'; +import { + collectUsageRecordsFromPhase, + formatUsd, + summarizeUsageRecords, + type SessionUsageRecord, + type SessionUsageSummary, +} from '../usage.js'; + +const SESSION_PHASE_FILES = [ + { file: '01-gather', name: 'GATHER' }, + { file: '02-plan', name: 'PLAN' }, + { file: '03-formulate', name: 'FORMULATE' }, + { file: '04-debate', name: 'DEBATE' }, + { file: '05-adjust', name: 'ADJUST' }, + { file: '06-rebuttal', name: 'REBUTTAL' }, + { file: '07-vote', name: 'VOTE' }, +]; + +async function resolveSessionArg(sessionPath: string): Promise { + if (sessionPath !== 'last') return sessionPath; + + const sessionsDir = pathJoin(homedir(), '.quorum', 'sessions'); + const indexPath = pathJoin(sessionsDir, 'index.json'); + if (existsSync(indexPath)) { + try { + const entries = JSON.parse(await readFile(indexPath, 'utf-8')) as Array<{ + sessionId: string; + }>; + if (entries.length > 0) { + return pathJoin(sessionsDir, entries[entries.length - 1].sessionId); + } + } catch { + // fall through + } + } + return resolveLastSession(sessionsDir); +} + +async function loadSessionUsageSummary(sessionPath: string): Promise { + const synthPath = pathJoin(sessionPath, 'synthesis.json'); + if (existsSync(synthPath)) { + try { + const synth = JSON.parse(await readFile(synthPath, 'utf-8')); + if (synth.usageSummary) return synth.usageSummary as SessionUsageSummary; + } catch { + // fall through + } + } + + const records: SessionUsageRecord[] = []; + for (const { file } of SESSION_PHASE_FILES) { + const phasePath = pathJoin(sessionPath, `${file}.json`); + if (!existsSync(phasePath)) continue; + const phase = JSON.parse(await readFile(phasePath, 'utf-8')); + records.push(...collectUsageRecordsFromPhase(phase)); + } + + if (existsSync(synthPath)) { + try { + const synth = JSON.parse(await readFile(synthPath, 'utf-8')); + if (Array.isArray(synth.usageRecords)) records.push(...synth.usageRecords); + } catch { + // ignore malformed synthesis extras + } + } + + return summarizeUsageRecords(records); +} + +function printUsageSummary(summary: SessionUsageSummary): void { + if (summary.providers.length === 0) { + console.log(pc.dim('No usage data recorded for this session.')); + return; + } + + const widths = { + name: Math.max(8, ...summary.providers.map((row) => row.name.length)), + provider: Math.max(7, ...summary.providers.map((row) => (row.provider ?? '-').length)), + model: Math.max(5, ...summary.providers.map((row) => (row.model ?? '-').length)), + input: Math.max(5, ...summary.providers.map((row) => String(row.input).length)), + output: Math.max(6, ...summary.providers.map((row) => String(row.output).length)), + total: Math.max(5, ...summary.providers.map((row) => String(row.totalTokens).length)), + cost: Math.max(8, ...summary.providers.map((row) => formatUsd(row.totalCost).length)), + }; + const pad = (value: string | number, width: number) => String(value).padEnd(width); + + console.log(pc.bold('Usage Summary')); + console.log( + [ + pad('Name', widths.name), + pad('API', widths.provider), + pad('Model', widths.model), + pad('Input', widths.input), + pad('Output', widths.output), + pad('Total', widths.total), + pad('Cost', widths.cost), + ].join(' '), + ); + for (const row of summary.providers) { + console.log( + [ + pad(row.name, widths.name), + pad(row.provider ?? '-', widths.provider), + pad(row.model ?? '-', widths.model), + pad(row.input, widths.input), + pad(row.output, widths.output), + pad(row.totalTokens, widths.total), + pad(formatUsd(row.totalCost), widths.cost), + ].join(' '), + ); + } + console.log(''); + console.log( + `${pc.bold('Totals')}: ${summary.totals.totalTokens} tokens across ${summary.totals.calls} calls, ${formatUsd(summary.totals.totalCost)}`, + ); +} export function registerSessionCommands(program: Command): void { // --- quorum session --- @@ -28,27 +144,7 @@ ${pc.dim(' $ quorum session ~/.quorum/sessions/abc123 --phase synthesis')} `, ) .action(async (sessionPath: string, opts) => { - // Resolve "last" to most recent session - if (sessionPath === 'last') { - const sessionsDir = pathJoin(homedir(), '.quorum', 'sessions'); - const indexPath = pathJoin(sessionsDir, 'index.json'); - if (existsSync(indexPath)) { - try { - const entries = JSON.parse(await readFile(indexPath, 'utf-8')) as Array<{ - sessionId: string; - }>; - if (entries.length > 0) { - sessionPath = pathJoin(sessionsDir, entries[entries.length - 1].sessionId); - } else { - sessionPath = await resolveLastSession(sessionsDir); - } - } catch { - sessionPath = await resolveLastSession(pathJoin(homedir(), '.quorum', 'sessions')); - } - } else { - sessionPath = await resolveLastSession(pathJoin(homedir(), '.quorum', 'sessions')); - } - } + sessionPath = await resolveSessionArg(sessionPath); const phaseName = opts.phase as string | undefined; @@ -120,17 +216,7 @@ ${pc.dim(' $ quorum session ~/.quorum/sessions/abc123 --phase synthesis')} ); } - const phases = [ - { file: '01-gather', name: 'GATHER' }, - { file: '02-plan', name: 'PLAN' }, - { file: '03-formulate', name: 'FORMULATE' }, - { file: '04-debate', name: 'DEBATE' }, - { file: '05-adjust', name: 'ADJUST' }, - { file: '06-rebuttal', name: 'REBUTTAL' }, - { file: '07-vote', name: 'VOTE' }, - ]; - - for (const { file, name } of phases) { + for (const { file, name } of SESSION_PHASE_FILES) { const phasePath = `${sessionPath}/${file}.json`; if (!existsSync(phasePath)) continue; const phase = JSON.parse(await readFile(phasePath, 'utf-8')); @@ -179,6 +265,44 @@ ${pc.dim(' $ quorum session ~/.quorum/sessions/abc123 --phase synthesis')} /* no uncertainty data */ } + const usageSummary = await loadSessionUsageSummary(sessionPath); + if (usageSummary.providers.length > 0) { + console.log(''); + printUsageSummary(usageSummary); + } + + console.log(''); + }); + + // --- quorum usage --- + program + .command('usage ') + .description('Show token and cost usage for a saved session. Use "last" for most recent.') + .addHelpText( + 'after', + ` +${pc.dim('Examples:')} +${pc.dim(' $ quorum usage last')} +${pc.dim(' $ quorum usage ~/.quorum/sessions/abc123')} +`, + ) + .action(async (sessionPath: string) => { + sessionPath = await resolveSessionArg(sessionPath); + + const metaPath = pathJoin(sessionPath, 'meta.json'); + if (!existsSync(metaPath)) { + throw new CLIError(pc.red(`Session not found: ${sessionPath}`)); + } + + const meta = JSON.parse(await readFile(metaPath, 'utf-8')); + const usageSummary = await loadSessionUsageSummary(sessionPath); + + console.log(''); + console.log(pc.bold(pc.cyan('═══ Usage ═══'))); + console.log(pc.dim(`Question: ${String(meta.input ?? '').slice(0, 200)}`)); + console.log(pc.dim(`Session: ${sessionPath}`)); + console.log(''); + printUsageSummary(usageSummary); console.log(''); }); diff --git a/src/council-v2.ts b/src/council-v2.ts index 3440675..a9054b9 100644 --- a/src/council-v2.ts +++ b/src/council-v2.ts @@ -9,11 +9,26 @@ import { SessionStore, type PhaseOutput } from './session.js'; import { availableInput, fitToBudget } from './context.js'; -import type { AgentProfile, ProviderAdapter, ProviderConfig, Synthesis } from './types.js'; +import type { + AgentProfile, + ProviderAdapter, + ProviderConfig, + ProviderGenerateResult, + ProviderResponse, + Synthesis, +} from './types.js'; import { tallyWithMethod, type Ballot } from './voting.js'; import { generateHeatmap } from './heatmap.js'; import { runHook, type HookEnv } from './hooks.js'; import { executeTools } from './tools.js'; +import { + buildProviderMetadata, + collectUsageRecordsFromPhase, + normalizeProviderResponse, + summarizeUsageRecords, + type PhaseRunResult, + type SessionUsageRecord, +} from './usage.js'; import { generateEvidenceReport, EVIDENCE_INSTRUCTION, @@ -146,6 +161,7 @@ export class CouncilV2 { duration: number; responses: Record; }> = []; + private sessionUsageRecords: SessionUsageRecord[] = []; constructor( adapters: ProviderAdapter[], @@ -347,7 +363,7 @@ export class CouncilV2 { `\nAvailable tools: ${toolList.join(', ')}. Max 3 tool uses per response.`; } const sys = this.prompt('gather', gatherSys, adapter.name); - let response = await this.adapterGenerate(adapter, input, sys); + let response = (await this.adapterGenerate(adapter, input, sys)).content; // Tool execution in gather phase if (this.profile.tools) { @@ -368,7 +384,7 @@ export class CouncilV2 { .map((tr) => `[${tr.tool}] Input: ${tr.input}\nOutput: ${tr.output}`) .join('\n\n'); const followUp = `Your previous response invoked tools. Here are the results:\n\n${toolSummary}\n\nPlease incorporate these findings into a revised, comprehensive response.`; - response = await this.adapterGenerate(adapter, followUp, sys); + response = (await this.adapterGenerate(adapter, followUp, sys)).content; } } @@ -854,9 +870,9 @@ export class CouncilV2 { this.emit('response', { provider: `${rtAdapter.name} (red-team)`, phase: 'red_team' }); // Parse and score - const attacks = parseRedTeamResponse(rtResponse, currentPositions); + const attacks = parseRedTeamResponse(rtResponse.content, currentPositions); this.redTeamResult = scoreResilience(attacks, currentPositions); - this.redTeamResult.rawResponse = rtResponse; + this.redTeamResult.rawResponse = rtResponse.content; this.redTeamResult.attackPacks = packNames; this.emit('redTeam', { result: this.redTeamResult }); @@ -1115,17 +1131,21 @@ export class CouncilV2 { ].join('\n'); this.currentPhase = 'SYNTHESIZE'; - let synthContent: string; + let synthResponse: ProviderResponse; if (this.streaming) { try { - synthContent = await this.generateStreaming(synthAdapter, synthPrompt, synthSys); + synthResponse = await this.generateStreaming(synthAdapter, synthPrompt, synthSys); } catch { - synthContent = await this.generateWithRetry(synthAdapter, synthPrompt, synthSys); + synthResponse = await this.generateWithRetry(synthAdapter, synthPrompt, synthSys); } } else { - synthContent = await this.generateWithRetry(synthAdapter, synthPrompt, synthSys); + synthResponse = await this.generateWithRetry(synthAdapter, synthPrompt, synthSys); } + const synthContent = synthResponse.content; this.emit('response', { provider: synthAdapter.name, phase: 'synthesize' }); + const synthesisUsageRecords: SessionUsageRecord[] = [ + this.buildUsageRecord('SYNTHESIZE', synthAdapter.name, synthResponse), + ].filter((record): record is SessionUsageRecord => record !== null); // Parse scores — flexible regex const consensusMatch = synthContent.match(/Consensus[:\s]*\*?\*?([\d.]+)/i); @@ -1156,7 +1176,14 @@ export class CouncilV2 { ].join('\n'); const wwcmSys = `You are a critical thinker examining a council's conclusion for potential weaknesses and conditions under which it should be revised.`; try { - whatWouldChange = await this.generateWithRetry(synthAdapter, wwcmPrompt, wwcmSys); + const wwcmResponse = await this.generateWithRetry(synthAdapter, wwcmPrompt, wwcmSys); + whatWouldChange = wwcmResponse.content; + const wwcmUsageRecord = this.buildUsageRecord( + 'WHAT_WOULD_CHANGE', + synthAdapter.name, + wwcmResponse, + ); + if (wwcmUsageRecord) synthesisUsageRecords.push(wwcmUsageRecord); this.emit('response', { provider: synthAdapter.name, phase: 'what_would_change' }); } catch (err) { this.emit('warn', { @@ -1180,7 +1207,15 @@ export class CouncilV2 { }; try { - await this.store.writeSynthesis({ ...synthesis, votes }); + await this.store.writeSynthesis({ + ...synthesis, + votes, + usageRecords: synthesisUsageRecords, + usageSummary: summarizeUsageRecords([ + ...this.sessionUsageRecords, + ...synthesisUsageRecords, + ]), + }); } catch (err) { this.emit('warn', { message: `Failed to write synthesis: ${err instanceof Error ? err.message : err}`, @@ -1455,10 +1490,36 @@ export class CouncilV2 { } }), ); - return Object.fromEntries(results); + const responses = Object.fromEntries( + results.map(([providerName, response]) => [ + providerName, + typeof response === 'string' ? response : response.content, + ]), + ); + const responseMetadata: PhaseRunResult['responseMetadata'] = {}; + for (const [providerName, response] of results) { + const metadata = + typeof response === 'string' + ? buildProviderMetadata( + this.normalizeAdapterResult( + this.adapters.find((a) => a.name === providerName)!, + response, + ), + ) + : buildProviderMetadata(response); + if (metadata) responseMetadata[providerName] = metadata; + } + return { + responses, + ...(Object.keys(responseMetadata).length > 0 ? { responseMetadata } : {}), + }; } else { // Sequential execution const results: Record = {}; + const responseMetadata: Record< + string, + NonNullable> + > = {}; for (const providerName of phase.participants) { const adapter = this.adapters.find((a) => a.name === providerName); if (!adapter) { @@ -1492,7 +1553,9 @@ export class CouncilV2 { try { const response = await this.adapterGenerate(adapter, prompt, sys); this.emit('response', { provider: providerName, phase: phase.name }); - results[providerName] = response; + results[providerName] = response.content; + const metadata = buildProviderMetadata(response); + if (metadata) responseMetadata[providerName] = metadata; } catch (err) { this.emit('warn', { message: `${providerName} failed in ${phase.name}: ${err instanceof Error ? err.message : String(err)}`, @@ -1500,7 +1563,10 @@ export class CouncilV2 { results[providerName] = `[${providerName} failed]`; } } - return results; + return { + responses: results, + ...(Object.keys(responseMetadata).length > 0 ? { responseMetadata } : {}), + }; } }, ); @@ -1556,16 +1622,20 @@ export class CouncilV2 { const synthPrompt = `Original question: ${input}\n\n## Responses:\n${allPositionText}\n\nProduce:\n## Synthesis\n[Best answer]\n\n## Minority Report\n[Dissenting views]\n\n## Scores\nConsensus: [0.0-1.0]\nConfidence: [0.0-1.0]`; this.currentPhase = 'SYNTHESIZE'; - let synthContent: string; + let synthResponse: ProviderResponse; if (this.streaming) { try { - synthContent = await this.generateStreaming(synthProvider, synthPrompt, synthSys); + synthResponse = await this.generateStreaming(synthProvider, synthPrompt, synthSys); } catch { - synthContent = await this.generateWithRetry(synthProvider, synthPrompt, synthSys); + synthResponse = await this.generateWithRetry(synthProvider, synthPrompt, synthSys); } } else { - synthContent = await this.generateWithRetry(synthProvider, synthPrompt, synthSys); + synthResponse = await this.generateWithRetry(synthProvider, synthPrompt, synthSys); } + const synthContent = synthResponse.content; + const synthesisUsageRecords: SessionUsageRecord[] = [ + this.buildUsageRecord('SYNTHESIZE', synthProvider.name, synthResponse), + ].filter((record): record is SessionUsageRecord => record !== null); const consensusMatch = synthContent.match(/Consensus[:\s]*\*?\*?([\d.]+)/i); const confidenceMatch = synthContent.match(/Confidence[:\s]*\*?\*?([\d.]+)/i); @@ -1584,7 +1654,15 @@ export class CouncilV2 { }; try { - await this.store.writeSynthesis({ ...synthesis, votes }); + await this.store.writeSynthesis({ + ...synthesis, + votes, + usageRecords: synthesisUsageRecords, + usageSummary: summarizeUsageRecords([ + ...this.sessionUsageRecords, + ...synthesisUsageRecords, + ]), + }); } catch { /* non-fatal */ } @@ -1818,17 +1896,21 @@ export class CouncilV2 { ].join('\n'); this.currentPhase = 'SYNTHESIZE'; - let synthContent: string; + let synthResponse: ProviderResponse; if (this.streaming) { try { - synthContent = await this.generateStreaming(synthAdapter, synthPrompt, synthSys); + synthResponse = await this.generateStreaming(synthAdapter, synthPrompt, synthSys); } catch { - synthContent = await this.generateWithRetry(synthAdapter, synthPrompt, synthSys); + synthResponse = await this.generateWithRetry(synthAdapter, synthPrompt, synthSys); } } else { - synthContent = await this.generateWithRetry(synthAdapter, synthPrompt, synthSys); + synthResponse = await this.generateWithRetry(synthAdapter, synthPrompt, synthSys); } + const synthContent = synthResponse.content; this.emit('response', { provider: synthAdapter.name, phase: 'synthesize' }); + const synthesisUsageRecords: SessionUsageRecord[] = [ + this.buildUsageRecord('SYNTHESIZE', synthAdapter.name, synthResponse), + ].filter((record): record is SessionUsageRecord => record !== null); const consensusMatch = synthContent.match(/Consensus[:\s]*\*?\*?([\d.]+)/i); const confidenceMatch = synthContent.match(/Confidence[:\s]*\*?\*?([\d.]+)/i); @@ -1857,7 +1939,15 @@ export class CouncilV2 { }; try { - await this.store.writeSynthesis({ ...synthesis, votes }); + await this.store.writeSynthesis({ + ...synthesis, + votes, + usageRecords: synthesisUsageRecords, + usageSummary: summarizeUsageRecords([ + ...this.sessionUsageRecords, + ...synthesisUsageRecords, + ]), + }); } catch (err) { this.emit('warn', { message: `Failed to write synthesis: ${err instanceof Error ? err.message : err}`, @@ -1992,26 +2082,33 @@ export class CouncilV2 { prompt: string, system: string, fallback?: string, - ): Promise { + ): Promise { for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { try { - let result: string; + let result: ProviderResponse; if (this.streaming && adapter.generateStream) { const start = Date.now(); this.emit('stream:start', { provider: adapter.name, phase: this.currentPhase }); - result = await adapter.generateStream(prompt, system, (delta) => { - this.onStreamDelta(adapter.name, this.currentPhase, delta); - this.emit('stream:delta', { provider: adapter.name, phase: this.currentPhase, delta }); - }); + result = this.normalizeAdapterResult( + adapter, + await adapter.generateStream(prompt, system, (delta) => { + this.onStreamDelta(adapter.name, this.currentPhase, delta); + this.emit('stream:delta', { + provider: adapter.name, + phase: this.currentPhase, + delta, + }); + }), + ); this.emit('stream:end', { provider: adapter.name, phase: this.currentPhase, duration: Date.now() - start, }); } else { - result = await adapter.generate(prompt, system); + result = this.normalizeAdapterResult(adapter, await adapter.generate(prompt, system)); } - if (result && result.trim().length > 0) return result; + if (result.content && result.content.trim().length > 0) return result; this.emit('warn', { message: `${adapter.name} returned empty response (attempt ${attempt + 1}/${MAX_RETRIES + 1})`, @@ -2039,7 +2136,38 @@ export class CouncilV2 { message: `${adapter.name} exhausted retries, using fallback`, phase: 'generate', }); - return fallbackText; + return this.normalizeAdapterResult(adapter, fallbackText); + } + + private normalizeAdapterResult( + adapter: ProviderAdapter, + result: ProviderGenerateResult, + ): ProviderResponse { + return normalizeProviderResponse(result, { + provider: adapter.config?.provider, + model: adapter.config?.model, + }); + } + + private buildUsageRecord( + phase: string, + name: string, + response: ProviderResponse, + ): SessionUsageRecord | null { + if (!response.provider && !response.model && !response.usage) return null; + return { + phase, + name, + provider: response.provider, + model: response.model, + usage: response.usage, + }; + } + + private isPhaseRunResult( + value: PhaseRunResult | Record, + ): value is PhaseRunResult { + return typeof value === 'object' && value !== null && 'responses' in value; } /** @@ -2087,7 +2215,7 @@ export class CouncilV2 { private async runPhase( fileKey: string, phaseName: string, - fn: () => Promise>, + fn: () => Promise>, ): Promise { this.currentPhase = phaseName; @@ -2097,12 +2225,16 @@ export class CouncilV2 { this.emit('phase', { phase: phaseName }); const start = Date.now(); - const responses = await fn(); + const phaseResult = await fn(); + const normalized = this.isPhaseRunResult(phaseResult) + ? phaseResult + : { responses: phaseResult }; const output: PhaseOutput = { phase: phaseName, timestamp: start, duration: Date.now() - start, - responses, + responses: normalized.responses, + ...(normalized.responseMetadata ? { responseMetadata: normalized.responseMetadata } : {}), }; try { await this.store.writePhase(fileKey, output); @@ -2119,6 +2251,7 @@ export class CouncilV2 { duration: output.duration, responses: output.responses, }); + this.sessionUsageRecords.push(...collectUsageRecordsFromPhase(output)); // Post-hook — write phase output to temp file for QUORUM_PHASE_OUTPUT const { writeFileSync, unlinkSync } = await import('node:fs'); @@ -2126,7 +2259,7 @@ export class CouncilV2 { const { join } = await import('node:path'); const tmpFile = join(tmpdir(), `quorum-phase-${phaseKey}-${Date.now()}.json`); try { - writeFileSync(tmpFile, JSON.stringify(responses, null, 2), 'utf-8'); + writeFileSync(tmpFile, JSON.stringify(output.responses, null, 2), 'utf-8'); await this.executeHook(`post-${phaseKey}`, { QUORUM_PHASE_OUTPUT: tmpFile }); } finally { try { @@ -2143,15 +2276,15 @@ export class CouncilV2 { * Run fn for all adapters in parallel, with retry/fallback per adapter. */ private async parallel( - fn: (adapter: ProviderAdapter, index: number) => Promise, + fn: (adapter: ProviderAdapter, index: number) => Promise, fallbacks?: Record, - ): Promise> { + ): Promise { const results = await Promise.all( this.adapters.map(async (adapter, i) => { for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { try { - const result = await fn(adapter, i); - if (result && result.trim().length > 0) { + const result = this.normalizeAdapterResult(adapter, await fn(adapter, i)); + if (result.content && result.content.trim().length > 0) { this.emit('response', { provider: adapter.name }); return [adapter.name, result] as const; } @@ -2181,10 +2314,19 @@ export class CouncilV2 { message: `${adapter.name} exhausted retries, using fallback`, }); this.emit('response', { provider: adapter.name, fallback: true }); - return [adapter.name, fallback] as const; + return [adapter.name, this.normalizeAdapterResult(adapter, fallback)] as const; }), ); - return Object.fromEntries(results); + const responses = Object.fromEntries(results.map(([name, result]) => [name, result.content])); + const responseMetadata: PhaseRunResult['responseMetadata'] = {}; + for (const [name, result] of results) { + const metadata = buildProviderMetadata(result); + if (metadata) responseMetadata[name] = metadata; + } + return { + responses, + ...(Object.keys(responseMetadata).length > 0 ? { responseMetadata } : {}), + }; } /** @@ -2195,14 +2337,17 @@ export class CouncilV2 { adapter: ProviderAdapter, prompt: string, system: string, - ): Promise { + ): Promise { if (this.streaming && adapter.generateStream) { const start = Date.now(); this.emit('stream:start', { provider: adapter.name, phase: this.currentPhase }); - const result = await adapter.generateStream(prompt, system, (delta) => { - this.onStreamDelta(adapter.name, this.currentPhase, delta); - this.emit('stream:delta', { provider: adapter.name, phase: this.currentPhase, delta }); - }); + const result = this.normalizeAdapterResult( + adapter, + await adapter.generateStream(prompt, system, (delta) => { + this.onStreamDelta(adapter.name, this.currentPhase, delta); + this.emit('stream:delta', { provider: adapter.name, phase: this.currentPhase, delta }); + }), + ); this.emit('stream:end', { provider: adapter.name, phase: this.currentPhase, @@ -2210,21 +2355,28 @@ export class CouncilV2 { }); return result; } - return adapter.generate(prompt, system); + return this.normalizeAdapterResult(adapter, await adapter.generate(prompt, system)); } /** * Generate for a given adapter — uses streaming when available, otherwise direct. * Intended for use inside parallel() callbacks instead of adapter.generate(). */ - async adapterGenerate(adapter: ProviderAdapter, prompt: string, system: string): Promise { + async adapterGenerate( + adapter: ProviderAdapter, + prompt: string, + system: string, + ): Promise { if (this.streaming && adapter.generateStream) { const start = Date.now(); this.emit('stream:start', { provider: adapter.name, phase: this.currentPhase }); - const result = await adapter.generateStream(prompt, system, (delta) => { - this.onStreamDelta(adapter.name, this.currentPhase, delta); - this.emit('stream:delta', { provider: adapter.name, phase: this.currentPhase, delta }); - }); + const result = this.normalizeAdapterResult( + adapter, + await adapter.generateStream(prompt, system, (delta) => { + this.onStreamDelta(adapter.name, this.currentPhase, delta); + this.emit('stream:delta', { provider: adapter.name, phase: this.currentPhase, delta }); + }), + ); this.emit('stream:end', { provider: adapter.name, phase: this.currentPhase, @@ -2232,7 +2384,7 @@ export class CouncilV2 { }); return result; } - return adapter.generate(prompt, system); + return this.normalizeAdapterResult(adapter, await adapter.generate(prompt, system)); } private budgetFor(adapterIndex: number): number { @@ -2730,7 +2882,7 @@ export class CouncilV2 { emit('response', { provider: judgeAdapter.name }); emit('phase:done', { phase: 'JUDGMENT', duration: 0 }); - return comparison; + return typeof comparison === 'string' ? comparison : comparison.content; } /** diff --git a/src/providers/base.ts b/src/providers/base.ts index b3776c2..ea553e8 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,4 +1,4 @@ -import type { ProviderAdapter, ProviderConfig } from '../types.js'; +import type { ProviderAdapter, ProviderConfig, ProviderResponse, ProviderUsage } from '../types.js'; import { resolveCredential } from '../auth.js'; import { completeSimple, streamSimple, getModels } from '@mariozechner/pi-ai'; import type { @@ -7,6 +7,8 @@ import type { SimpleStreamOptions, KnownProvider, AssistantMessageEvent, + AssistantMessage, + Usage, } from '@mariozechner/pi-ai'; // ============================================================================ @@ -275,6 +277,31 @@ export async function createProvider(config: ProviderConfig): Promise { + if (!usage) return undefined; + return { + input: usage.input, + output: usage.output, + cacheRead: usage.cacheRead, + cacheWrite: usage.cacheWrite, + totalTokens: usage.totalTokens, + cost: { + input: usage.cost.input, + output: usage.cost.output, + cacheRead: usage.cost.cacheRead, + cacheWrite: usage.cost.cacheWrite, + total: usage.cost.total, + }, + }; + }; + + const buildResponse = (result: AssistantMessage, fallbackText?: string): ProviderResponse => ({ + content: extractText(result) || fallbackText || '', + provider: model.provider, + model: model.id, + usage: normalizeUsage(result.usage), + }); + const withTimeout = (promise: Promise, label: string): Promise => { return Promise.race([ promise, @@ -295,7 +322,7 @@ export async function createProvider(config: ProviderConfig): Promise ac.abort(), timeoutMs); const stream = streamSimple(model, buildContext(prompt, systemPrompt), buildOpts()); let text = ''; + let latestUsage: ProviderUsage | undefined; try { const abortPromise = new Promise((_, reject) => { ac.signal.addEventListener( @@ -338,6 +366,12 @@ export async function createProvider(config: ProviderConfig): Promise) { if (ac.signal.aborted) break; + if ('partial' in event && event.partial?.usage) { + latestUsage = normalizeUsage(event.partial.usage); + } + if (event.type === 'done') { + latestUsage = normalizeUsage(event.message.usage); + } if (event.type === 'text_delta') { lastChunkTime = Date.now(); gotFirstChunk = true; @@ -355,17 +389,19 @@ export async function createProvider(config: ProviderConfig): Promise if (config.model) args.push('-m', config.model); const { spawn } = await import('node:child_process'); - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { let settled = false; const proc = spawn('gemini', args, { env: { ...process.env }, @@ -417,7 +453,11 @@ async function createGeminiCli(config: ProviderConfig): Promise if (settled) return; settled = true; if (code === 0) { - resolve(stdout.trim()); + resolve({ + content: stdout.trim(), + provider: config.provider, + model: config.model, + }); } else { reject(new Error(`gemini-cli exited ${code}: ${stderr}`)); } diff --git a/src/session.ts b/src/session.ts index 930a1ca..5734cbd 100644 --- a/src/session.ts +++ b/src/session.ts @@ -8,12 +8,14 @@ import { existsSync } from 'node:fs'; import { join } from 'node:path'; import { homedir } from 'node:os'; import { computePhaseHash, type HashChainEntry } from './integrity.js'; +import type { ProviderResponseMetadata } from './types.js'; export interface PhaseOutput { phase: string; timestamp: number; duration: number; responses: Record; + responseMetadata?: Record; } export class SessionStore { diff --git a/src/types.ts b/src/types.ts index cb2492e..17e75fe 100644 --- a/src/types.ts +++ b/src/types.ts @@ -56,15 +56,42 @@ export interface AuthStore { export interface ProviderAdapter { name: string; config?: ProviderConfig; - generate(prompt: string, systemPrompt?: string): Promise; + generate(prompt: string, systemPrompt?: string): Promise; /** Streaming generate — calls onDelta with text chunks, returns full text */ generateStream?( prompt: string, systemPrompt: string | undefined, onDelta: (delta: string) => void, - ): Promise; + ): Promise; } +export interface ProviderUsage { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + totalTokens: number; + cost: { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + total: number; + }; +} + +export interface ProviderResponseMetadata { + provider?: string; + model?: string; + usage?: ProviderUsage; +} + +export interface ProviderResponse extends ProviderResponseMetadata { + content: string; +} + +export type ProviderGenerateResult = string | ProviderResponse; + // --- Agent File (Deliberation Profile) --- export interface PhasePrompts { diff --git a/src/usage.ts b/src/usage.ts new file mode 100644 index 0000000..a451f6e --- /dev/null +++ b/src/usage.ts @@ -0,0 +1,150 @@ +import type { PhaseOutput } from './session.js'; +import type { + ProviderGenerateResult, + ProviderResponse, + ProviderResponseMetadata, + ProviderUsage, +} from './types.js'; + +export interface PhaseRunResult { + responses: Record; + responseMetadata?: Record; +} + +export interface SessionUsageRecord { + phase: string; + name: string; + provider?: string; + model?: string; + usage?: ProviderUsage; +} + +export interface SessionUsageSummaryRow { + name: string; + provider?: string; + model?: string; + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + totalTokens: number; + totalCost: number; + calls: number; + phases: string[]; +} + +export interface SessionUsageSummary { + providers: SessionUsageSummaryRow[]; + totals: Omit; +} + +export function normalizeProviderResponse( + result: ProviderGenerateResult, + defaults?: Omit, +): ProviderResponse { + if (typeof result === 'string') { + return { + content: result, + provider: defaults?.provider, + model: defaults?.model, + usage: defaults?.usage, + }; + } + return { + content: result.content, + provider: result.provider ?? defaults?.provider, + model: result.model ?? defaults?.model, + usage: result.usage ?? defaults?.usage, + }; +} + +export function buildProviderMetadata( + response: ProviderResponse, +): ProviderResponseMetadata | undefined { + if (!response.provider && !response.model && !response.usage) return undefined; + return { + provider: response.provider, + model: response.model, + usage: response.usage, + }; +} + +export function collectUsageRecordsFromPhase(phase: PhaseOutput): SessionUsageRecord[] { + const metadata = phase.responseMetadata ?? {}; + return Object.entries(metadata).map(([name, entry]) => ({ + phase: phase.phase, + name, + provider: entry.provider, + model: entry.model, + usage: entry.usage, + })); +} + +export function summarizeUsageRecords(records: SessionUsageRecord[]): SessionUsageSummary { + const rows = new Map(); + + for (const record of records) { + if (!record.usage) continue; + + const key = `${record.name}::${record.provider ?? ''}::${record.model ?? ''}`; + const existing = rows.get(key) ?? { + name: record.name, + provider: record.provider, + model: record.model, + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + totalCost: 0, + calls: 0, + phases: [], + }; + + existing.input += record.usage.input; + existing.output += record.usage.output; + existing.cacheRead += record.usage.cacheRead; + existing.cacheWrite += record.usage.cacheWrite; + existing.totalTokens += record.usage.totalTokens; + existing.totalCost += record.usage.cost.total; + existing.calls += 1; + if (!existing.phases.includes(record.phase)) existing.phases.push(record.phase); + + rows.set(key, existing); + } + + const providers = [...rows.values()].sort((a, b) => { + if (b.totalCost !== a.totalCost) return b.totalCost - a.totalCost; + if (b.totalTokens !== a.totalTokens) return b.totalTokens - a.totalTokens; + return a.name.localeCompare(b.name); + }); + + return { + providers, + totals: providers.reduce( + (acc, row) => { + acc.input += row.input; + acc.output += row.output; + acc.cacheRead += row.cacheRead; + acc.cacheWrite += row.cacheWrite; + acc.totalTokens += row.totalTokens; + acc.totalCost += row.totalCost; + acc.calls += row.calls; + return acc; + }, + { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + totalCost: 0, + calls: 0, + }, + ), + }; +} + +export function formatUsd(value: number): string { + return `$${value.toFixed(6)}`; +} diff --git a/tests/cli-integration.test.ts b/tests/cli-integration.test.ts index 07e9a29..b8f8823 100644 --- a/tests/cli-integration.test.ts +++ b/tests/cli-integration.test.ts @@ -3,7 +3,7 @@ import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { join } from 'node:path'; import { readFileSync } from 'node:fs'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; const exec = promisify(execFile); @@ -64,7 +64,7 @@ describe('Basic CLI', () => { it('--help lists expected top-level commands', async () => { const { stdout, exitCode } = await run('--help'); expect(exitCode).toBe(0); - for (const cmd of ['ask', 'review', 'providers', 'auth', 'session', 'history']) { + for (const cmd of ['ask', 'review', 'providers', 'auth', 'session', 'usage', 'history']) { expect(stdout).toContain(cmd); } }); @@ -131,6 +131,96 @@ describe('Error paths', () => { const { exitCode } = await runIsolated('history'); expect(exitCode).toBe(0); }); + + it('usage "last" with no sessions dir exits non-zero', async () => { + const { stderr, exitCode } = await runIsolated('usage', 'last'); + expect(exitCode).not.toBe(0); + expect(stderr).toMatch(/no sessions|not found|error/i); + }); +}); + +describe('Usage command', () => { + it('usage last prints token and cost summary from session metadata', async () => { + const tmpHome = await mkdtemp(join(tmpdir(), 'quorum-usage-test-')); + try { + const sessionId = 'session-123'; + const sessionsDir = join(tmpHome, '.quorum', 'sessions'); + const sessionDir = join(sessionsDir, sessionId); + await mkdir(sessionDir, { recursive: true }); + + await writeFile( + join(sessionsDir, 'index.json'), + JSON.stringify([{ sessionId }], null, 2), + 'utf-8', + ); + await writeFile( + join(sessionDir, 'meta.json'), + JSON.stringify( + { + input: 'How expensive was this run?', + profile: 'test', + providers: [{ name: 'claude', provider: 'anthropic', model: 'claude-sonnet' }], + startedAt: Date.now(), + }, + null, + 2, + ), + 'utf-8', + ); + await writeFile( + join(sessionDir, '01-gather.json'), + JSON.stringify( + { + phase: 'GATHER', + timestamp: Date.now(), + duration: 100, + responses: { claude: 'Answer' }, + responseMetadata: { + claude: { + provider: 'anthropic', + model: 'claude-sonnet', + usage: { + input: 120, + output: 80, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 200, + cost: { + input: 0.00036, + output: 0.0012, + cacheRead: 0, + cacheWrite: 0, + total: 0.00156, + }, + }, + }, + }, + }, + null, + 2, + ), + 'utf-8', + ); + + const { stdout } = await exec('node', [CLI, 'usage', 'last'], { + timeout: 10_000, + env: { + ...process.env, + HOME: tmpHome, + XDG_CONFIG_HOME: join(tmpHome, '.config'), + NO_COLOR: '1', + FORCE_COLOR: '0', + }, + }); + + expect(stdout).toContain('Usage Summary'); + expect(stdout).toContain('claude'); + expect(stdout).toContain('200'); + expect(stdout).toContain('$0.001560'); + } finally { + await rm(tmpHome, { recursive: true, force: true }); + } + }); }); // ─── Subcommand Help ──────────────────────────────────────────────────────── diff --git a/tsconfig.eslint.json b/tsconfig.eslint.json new file mode 100644 index 0000000..679575a --- /dev/null +++ b/tsconfig.eslint.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "noEmit": true, + "allowJs": true + }, + "include": ["src/**/*", "tests/**/*", "eslint.config.js"] +}