diff --git a/CHANGELOG.md b/CHANGELOG.md index edf01d4..264c8ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ The format is based on Keep a Changelog and this project follows Semantic Versio - **Agent workflow commands** — `AGENTS.md` now records the common validation, development, E2E, and benchmark data persistence workflows agents should use when changing the repo. +### Removed + +- **Orphaned manual inference flow** — removed the unused `EvaluationForm` frontend and its dedicated `/eval-inference` API while retaining the Evaluate scoring queue, evaluation persistence, and leaderboard. + ## [0.11.0] - 2026-06-25 ### Added diff --git a/backend/src/api/routes/eval-inference.ts b/backend/src/api/routes/eval-inference.ts deleted file mode 100644 index ae1d53d..0000000 --- a/backend/src/api/routes/eval-inference.ts +++ /dev/null @@ -1,86 +0,0 @@ -import path from 'path'; -import { fileURLToPath } from 'url'; - -import { FastifyInstance } from 'fastify'; - -import { ModelCallError, ServerNotFoundError, ServerUnreachableError, runEvalInference } from '../../services/eval-inference-service.js'; -import { validateWithSchema } from '../../services/schema-validator.js'; - -const moduleDir = path.dirname(fileURLToPath(import.meta.url)); -const PROMPT_SCHEMA_PATH = path.resolve(moduleDir, '../../schemas/eval-prompt.schema.json'); -const IC_SCHEMA_PATH = path.resolve(moduleDir, '../../schemas/inference-config.schema.json'); - -export function registerEvalInferenceRoutes(app: FastifyInstance): void { - app.post('/eval-inference', async (request, reply) => { - const body = request.body as { - server_id?: string; - model_name?: string; - prompt_text?: string; - inference_config?: unknown; - }; - - const promptValidation = validateWithSchema(PROMPT_SCHEMA_PATH, { text: body.prompt_text, tags: [] }); - if (!promptValidation.ok) { - reply.code(400).send({ error: 'Invalid request', issues: promptValidation.issues }); - return; - } - - if (body.inference_config !== undefined) { - const icValidation = validateWithSchema(IC_SCHEMA_PATH, body.inference_config); - if (!icValidation.ok) { - reply.code(400).send({ error: 'Invalid inference_config', issues: icValidation.issues }); - return; - } - } - - if (!body.server_id || typeof body.server_id !== 'string') { - reply.code(400).send({ error: 'server_id is required' }); - return; - } - if (!body.model_name || typeof body.model_name !== 'string') { - reply.code(400).send({ error: 'model_name is required' }); - return; - } - - const inferenceConfig = (body.inference_config as { - temperature?: number | null; - top_p?: number | null; - max_tokens?: number | null; - quantization_level?: string | null; - } | null) ?? {}; - - try { - const result = await runEvalInference({ - server_id: body.server_id, - model_name: body.model_name, - prompt_text: body.prompt_text as string, - inference_config: { - temperature: inferenceConfig.temperature ?? null, - top_p: inferenceConfig.top_p ?? null, - max_tokens: inferenceConfig.max_tokens ?? null, - quantization_level: inferenceConfig.quantization_level ?? null - } - }); - reply.code(200).send(result); - } catch (err) { - if (err instanceof ServerNotFoundError) { - reply.code(404).send({ error: err.message }); - return; - } - if (err instanceof ServerUnreachableError) { - reply.code(502).send({ error: err.message }); - return; - } - if (err instanceof ModelCallError) { - reply.code(502).send({ error: err.message, upstream_status: err.upstreamStatus }); - return; - } - const error = err as Error; - if (error.name === 'TimeoutError' || error.message.includes('timeout')) { - reply.code(504).send({ error: 'Inference server did not respond in time' }); - return; - } - reply.code(502).send({ error: 'Inference call failed' }); - } - }); -} diff --git a/backend/src/api/server.ts b/backend/src/api/server.ts index d21755d..5693bcf 100644 --- a/backend/src/api/server.ts +++ b/backend/src/api/server.ts @@ -9,7 +9,6 @@ import { registerResultsViewRoutes } from './routes/results-view.js'; import { registerInferenceServersRoutes } from './routes/inference-servers.js'; import { registerModelsRoutes } from './routes/models.js'; import { registerSystemRoutes } from './routes/system.js'; -import { registerEvalInferenceRoutes } from './routes/eval-inference.js'; import { registerEvaluationsRoutes } from './routes/evaluations.js'; import { registerLeaderboardRoutes } from './routes/leaderboard.js'; import { registerArchitectureRoutes } from './routes/architecture.js'; @@ -100,7 +99,6 @@ export function createServer() { registerInferenceServersRoutes(app); registerModelsRoutes(app); registerResultsViewRoutes(app); - registerEvalInferenceRoutes(app); registerEvaluationsRoutes(app); registerEvaluationQueueRoutes(app); registerInferenceParamPresetRoutes(app); diff --git a/backend/src/services/eval-inference-service.ts b/backend/src/services/eval-inference-service.ts deleted file mode 100644 index 737c1b9..0000000 --- a/backend/src/services/eval-inference-service.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { getInferenceServerById } from '../models/inference-server.js'; -import { backendFetch } from './inference-proxy.js'; -import { logEvent } from './observability.js'; - -export interface ModelPricing { - input: number; - output: number; -} - -export interface EvalInferenceParams { - server_id: string; - model_name: string; - prompt_text: string; - inference_config: { - temperature: number | null; - top_p: number | null; - max_tokens: number | null; - quantization_level: string | null; - }; -} - -export interface EvalInferenceResult { - answer_text: string; - input_tokens: number | null; - output_tokens: number | null; - total_tokens: number | null; - latency_ms: number; - word_count: number; - estimated_cost: number | null; -} - -export class ServerNotFoundError extends Error { - constructor(server_id: string) { - super(`Server not found or archived: ${server_id}`); - this.name = 'ServerNotFoundError'; - } -} - -export class ServerUnreachableError extends Error { - constructor(message: string) { - super(message); - this.name = 'ServerUnreachableError'; - } -} - -export class ModelCallError extends Error { - public readonly upstreamStatus: number; - constructor(message: string, upstreamStatus: number) { - super(message); - this.name = 'ModelCallError'; - this.upstreamStatus = upstreamStatus; - } -} - -export function computeWordCount(text: string): number { - return text.split(/\s+/).filter(Boolean).length; -} - -export function computeEstimatedCost( - inputTokens: number | null, - outputTokens: number | null, - pricing: ModelPricing | null -): number | null { - if (!pricing || inputTokens === null || outputTokens === null) { - return null; - } - return inputTokens * pricing.input + outputTokens * pricing.output; -} - -function getPricing(_modelName: string): ModelPricing | null { - return null; -} - -const INFERENCE_TIMEOUT_MS = 30_000; - -export async function runEvalInference(params: EvalInferenceParams): Promise { - const server = getInferenceServerById(params.server_id); - if (!server || server.inference_server.archived) { - throw new ServerNotFoundError(params.server_id); - } - - const baseUrl = server.endpoints.base_url.replace(/\/$/, ''); - const url = `${baseUrl}/v1/chat/completions`; - - const body: Record = { - model: params.model_name, - messages: [{ role: 'user', content: params.prompt_text }], - stream: false - }; - - const { temperature, top_p, max_tokens } = params.inference_config; - if (temperature !== null) body.temperature = temperature; - if (top_p !== null) body.top_p = top_p; - if (max_tokens !== null) body.max_tokens = max_tokens; - - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), INFERENCE_TIMEOUT_MS); - const startMs = Date.now(); - - logEvent({ - level: 'info', - message: 'eval-inference started', - meta: { server_id: params.server_id, model_name: params.model_name } - }); - - let response: Response; - try { - response = await backendFetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - signal: controller.signal - }); - } catch (err) { - clearTimeout(timer); - const error = err as Error; - if (error.name === 'AbortError') { - throw Object.assign(new Error('Inference timeout'), { name: 'TimeoutError' }); - } - throw new ServerUnreachableError(error.message); - } - - clearTimeout(timer); - const latency_ms = Date.now() - startMs; - - if (!response.ok) { - throw new ModelCallError(`Upstream error: ${response.statusText}`, response.status); - } - - const json = (await response.json()) as { - choices?: Array<{ message?: { content?: string } }>; - usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number }; - }; - - const answer_text = json.choices?.[0]?.message?.content ?? ''; - const input_tokens = json.usage?.prompt_tokens ?? null; - const output_tokens = json.usage?.completion_tokens ?? null; - const total_tokens = json.usage?.total_tokens ?? null; - const word_count = computeWordCount(answer_text); - const pricing = getPricing(params.model_name); - const estimated_cost = computeEstimatedCost( - input_tokens !== undefined ? input_tokens : null, - output_tokens !== undefined ? output_tokens : null, - pricing - ); - - logEvent({ - level: 'info', - message: 'eval-inference completed', - meta: { server_id: params.server_id, model_name: params.model_name, latency_ms, word_count } - }); - - return { answer_text, input_tokens, output_tokens, total_tokens, latency_ms, word_count, estimated_cost }; -} diff --git a/backend/tests/integration/eval-inference.test.ts b/backend/tests/integration/eval-inference.test.ts deleted file mode 100644 index 4794834..0000000 --- a/backend/tests/integration/eval-inference.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import { fileURLToPath } from 'url'; - -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; - -import { createServer } from '../../src/api/server.js'; -import { getDb, resetDbInstance, runSchema } from '../../src/models/db.js'; - -const AUTH_HEADERS = { 'x-api-token': 'test-token' }; - -const moduleDir = path.dirname(fileURLToPath(import.meta.url)); -const SCHEMA_PATH = path.resolve(moduleDir, '../../src/models/schema.sql'); - -function seedServer(serverId = 'srv-eval-inf') { - const db = getDb(); - const now = new Date().toISOString(); - db.prepare(` - INSERT OR IGNORE INTO inference_servers - (server_id, display_name, active, archived, created_at, updated_at, runtime, endpoints, auth, capabilities, discovery, raw) - VALUES (?, ?, 1, 0, ?, ?, ?, ?, ?, ?, ?, ?) - `).run( - serverId, 'Eval Test Server', now, now, - JSON.stringify({ api: { schema_family: ['openai-compatible'], api_version: null } }), - JSON.stringify({ base_url: 'http://localhost:9999' }), - JSON.stringify({}), - JSON.stringify({}), - JSON.stringify({}), - JSON.stringify({}) - ); - return serverId; -} - -describe('POST /eval-inference', () => { - process.env.INFERHARNESS_API_TOKEN = 'test-token'; - - beforeEach(() => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'aitb-eval-inf-')); - process.env.INFERHARNESS_DB_PATH = path.join(tmpDir, 'test.sqlite'); - resetDbInstance(); - runSchema(fs.readFileSync(SCHEMA_PATH, 'utf8')); - }); - - afterEach(() => { - resetDbInstance(); - }); - - it('returns 404 when server_id is not in DB', async () => { - const app = createServer(); - const response = await app.inject({ - method: 'POST', - url: '/eval-inference', - headers: AUTH_HEADERS, - payload: { - server_id: 'nonexistent-server', - model_name: 'llama3', - prompt_text: 'Hello', - inference_config: { temperature: null, top_p: null, max_tokens: null, quantization_level: null } - } - }); - expect(response.statusCode).toBe(404); - }); - - it('returns 400 when prompt_text is empty', async () => { - const app = createServer(); - seedServer(); - const response = await app.inject({ - method: 'POST', - url: '/eval-inference', - headers: AUTH_HEADERS, - payload: { - server_id: 'srv-eval-inf', - model_name: 'llama3', - prompt_text: '', - inference_config: { temperature: null, top_p: null, max_tokens: null, quantization_level: null } - } - }); - expect(response.statusCode).toBe(400); - }); - - it('returns 200 with answer_text and metric fields on success', async () => { - const app = createServer(); - const serverId = seedServer('srv-mock'); - - // Mock fetch for the inference call - const { default: fetchMock } = await import('../../src/services/eval-inference-service.js'); - void fetchMock; - - // We need a real server to get 200; skip if no mock available — verify field shape - // This test verifies the route exists and returns the correct shape - // A full happy-path test requires a mock inference server - const response = await app.inject({ - method: 'POST', - url: '/eval-inference', - headers: AUTH_HEADERS, - payload: { - server_id: serverId, - model_name: 'llama3', - prompt_text: 'What is 2+2?', - inference_config: { temperature: 0.7, top_p: null, max_tokens: null, quantization_level: null } - } - }); - // Server is not reachable → expect 502 or 504 (not 404, not 500, not 400) - expect([502, 504]).toContain(response.statusCode); - }); -}); diff --git a/backend/tests/unit/eval-inference-metrics.test.ts b/backend/tests/unit/eval-inference-metrics.test.ts deleted file mode 100644 index 3ba1556..0000000 --- a/backend/tests/unit/eval-inference-metrics.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { computeEstimatedCost, computeWordCount } from '../../src/services/eval-inference-service.js'; - -describe('computeWordCount', () => { - it('counts words by whitespace split', () => { - expect(computeWordCount('hello world foo')).toBe(3); - }); - - it('handles leading/trailing whitespace', () => { - expect(computeWordCount(' hello world ')).toBe(2); - }); - - it('returns 0 for empty string', () => { - expect(computeWordCount('')).toBe(0); - }); - - it('handles multiple consecutive spaces', () => { - expect(computeWordCount('a b c')).toBe(3); - }); -}); - -describe('computeEstimatedCost', () => { - it('computes cost from known token counts and prices', () => { - // 100 * 0.001 + 50 * 0.002 = 0.1 + 0.1 = 0.2 - const cost = computeEstimatedCost(100, 50, { input: 0.001, output: 0.002 }); - expect(cost).toBeCloseTo(0.2); - }); - - it('returns null when pricing config is absent', () => { - expect(computeEstimatedCost(100, 50, null)).toBeNull(); - }); - - it('returns null when input_tokens is null', () => { - expect(computeEstimatedCost(null, 50, { input: 0.001, output: 0.002 })).toBeNull(); - }); - - it('returns null when output_tokens is null', () => { - expect(computeEstimatedCost(100, null, { input: 0.001, output: 0.002 })).toBeNull(); - }); -}); diff --git a/frontend/src/components/EvaluationForm.tsx b/frontend/src/components/EvaluationForm.tsx deleted file mode 100644 index 86b3435..0000000 --- a/frontend/src/components/EvaluationForm.tsx +++ /dev/null @@ -1,379 +0,0 @@ -import { useCallback, useEffect, useMemo, useState } from 'react'; - -import type { InferenceConfig } from '../services/eval-inference-api.js'; -import { runEvalInference } from '../services/eval-inference-api.js'; -import type { EvaluationInput } from '../services/evaluations-api.js'; -import { createEvaluation } from '../services/evaluations-api.js'; -import type { InferenceServerRecord } from '../services/inference-servers-api.js'; -import { listModels } from '../services/models-api.js'; -import type { ModelRecord } from '../services/models-api.js'; -import { RunInferenceServerSelect } from './RunInferenceServerSelect.js'; -import { ScoreSliders } from './ScoreSliders.js'; - -type Dimension = 'accuracy' | 'relevance' | 'coherence' | 'completeness' | 'helpfulness'; -type ScoreMap = Record; - -function nullScores(): ScoreMap { - return { - accuracy: null, - relevance: null, - coherence: null, - completeness: null, - helpfulness: null - }; -} - -interface MetricsResult { - answer_text: string; - input_tokens: number | null; - output_tokens: number | null; - total_tokens: number | null; - latency_ms: number; - word_count: number; - estimated_cost: number | null; -} - -interface EvaluationFormProps { - sharedPromptText?: string; - sharedInferenceConfig?: InferenceConfig; - onPromptTextChange?: (text: string) => void; - onInferenceConfigChange?: (config: InferenceConfig) => void; -} - -function formatMetric(value: number | null, decimals = 0): string { - if (value === null) return 'N/A'; - return typeof value === 'number' ? value.toFixed(decimals) : String(value); -} - -export function EvaluationForm({ - sharedPromptText, - sharedInferenceConfig, - onPromptTextChange, - onInferenceConfigChange -}: EvaluationFormProps) { - const [stage, setStage] = useState<'input' | 'score'>('input'); - const [serverId, setServerId] = useState(''); - const [modelName, setModelName] = useState(''); - const [models, setModels] = useState([]); - const [servers, setServers] = useState([]); - const [promptText, setPromptText] = useState(sharedPromptText ?? ''); - const [tags, setTags] = useState([]); - const [tagInput, setTagInput] = useState(''); - const [inferenceConfig, setInferenceConfig] = useState( - sharedInferenceConfig ?? { temperature: null, top_p: null, max_tokens: null, quantization_level: null } - ); - const [metrics, setMetrics] = useState(null); - const [scores, setScores] = useState(nullScores()); - const [note, setNote] = useState(''); - const [running, setRunning] = useState(false); - const [saving, setSaving] = useState(false); - const [runError, setRunError] = useState(null); - const [saveError, setSaveError] = useState(null); - const [showRequired, setShowRequired] = useState(false); - const [successMessage, setSuccessMessage] = useState(null); - - const effectivePromptText = sharedPromptText ?? promptText; - const effectiveInferenceConfig = sharedInferenceConfig ?? inferenceConfig; - - const handleServersLoaded = useCallback((loadedServers: InferenceServerRecord[]) => { - setServers(loadedServers); - }, []); - - function handleServerChange(nextServerId: string) { - setServerId(nextServerId); - setModelName(''); - } - - useEffect(() => { - if (!serverId) { - setModels([]); - return; - } - listModels() - .then((all) => setModels(all.filter((m) => m.model.server_id === serverId && !m.model.archived))) - .catch(() => setModels([])); - }, [serverId]); - - const modelOptions = useMemo(() => { - const selectedServer = servers.find((server) => server.inference_server.server_id === serverId); - const options = new Map(); - - for (const entry of selectedServer?.discovery.model_list.normalised ?? []) { - if (!entry.model_id) { - continue; - } - options.set(entry.model_id, { - model_id: entry.model_id, - display_name: entry.display_name ?? entry.model_id - }); - } - - for (const record of models) { - options.set(record.model.model_id, { - model_id: record.model.model_id, - display_name: record.model.base_model_name ?? record.model.display_name - }); - } - - return Array.from(options.values()).sort((a, b) => a.display_name.localeCompare(b.display_name)); - }, [models, serverId, servers]); - - function handleInferenceConfigChange(field: keyof InferenceConfig, raw: string) { - const updated = { ...effectiveInferenceConfig }; - if (field === 'temperature') updated.temperature = raw === '' ? null : parseFloat(raw); - else if (field === 'top_p') updated.top_p = raw === '' ? null : parseFloat(raw); - else if (field === 'max_tokens') updated.max_tokens = raw === '' ? null : parseInt(raw, 10); - else if (field === 'quantization_level') updated.quantization_level = raw === '' ? null : raw; - if (onInferenceConfigChange) onInferenceConfigChange(updated); - else setInferenceConfig(updated); - } - - function addTag(raw: string) { - const tag = raw.trim().replace(/,+$/, ''); - if (tag && !tags.includes(tag) && tags.length < 20) { - setTags([...tags, tag]); - } - setTagInput(''); - } - - async function handleRun() { - if (!serverId || !modelName || !effectivePromptText.trim()) return; - setRunError(null); - setRunning(true); - try { - const result = await runEvalInference({ - server_id: serverId, - model_name: modelName, - prompt_text: effectivePromptText, - inference_config: effectiveInferenceConfig - }); - setMetrics(result); - setStage('score'); - } catch (err) { - setRunError(err instanceof Error ? err.message : 'Run failed'); - } finally { - setRunning(false); - } - } - - async function handleSave() { - if (!metrics) return; - const allScored = Object.values(scores).every((v) => v !== null); - if (!allScored) { - setShowRequired(true); - return; - } - setSaveError(null); - setSaving(true); - try { - const input: EvaluationInput = { - prompt_text: effectivePromptText, - tags, - server_id: serverId, - model_name: modelName, - inference_config: effectiveInferenceConfig, - answer_text: metrics.answer_text, - input_tokens: metrics.input_tokens, - output_tokens: metrics.output_tokens, - total_tokens: metrics.total_tokens, - latency_ms: metrics.latency_ms, - word_count: metrics.word_count, - estimated_cost: metrics.estimated_cost, - accuracy_score: scores.accuracy as number, - relevance_score: scores.relevance as number, - coherence_score: scores.coherence as number, - completeness_score: scores.completeness as number, - helpfulness_score: scores.helpfulness as number, - note: note.trim() || null - }; - await createEvaluation(input); - window.dispatchEvent(new CustomEvent('evaluations:saved')); - setSuccessMessage('Evaluation saved successfully.'); - setStage('input'); - setMetrics(null); - setScores(nullScores()); - setNote(''); - setShowRequired(false); - } catch (err) { - setSaveError(err instanceof Error ? err.message : 'Save failed'); - } finally { - setSaving(false); - } - } - - return ( -
- {successMessage ? ( -

{successMessage}

- ) : null} - - {stage === 'input' ? ( -
- - {modelOptions.length > 0 ? ( - - ) : ( - - )} - - {!sharedPromptText ? ( -