diff --git a/package-lock.json b/package-lock.json index 89ef280..44e4450 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,8 @@ "js-yaml": "^4.1.1" }, "bin": { - "preflight-dev": "bin/cli.js" + "preflight-dev": "bin/cli.js", + "preflight-dev-serve": "bin/serve.js" }, "devDependencies": { "@eslint/js": "^10.0.1", @@ -29,7 +30,7 @@ "vitest": "^4.0.18" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/@esbuild/aix-ppc64": { diff --git a/src/cli/init.ts b/src/cli/init.ts index dfaaa25..4a7b5ed 100644 --- a/src/cli/init.ts +++ b/src/cli/init.ts @@ -92,7 +92,7 @@ async function main(): Promise { if (profile === "full") { console.log("\nFull profile uses embeddings for vector search."); - const provider = await ask("Embedding provider [local/openai] (default: local): "); + const provider = await ask("Embedding provider [local/openai/voyage] (default: local): "); if (provider.trim().toLowerCase() === "openai") { const key = await ask("OpenAI API key (or set OPENAI_API_KEY later): "); if (key.trim()) { diff --git a/src/lib/embeddings.ts b/src/lib/embeddings.ts index 69b5883..5a8fe5a 100644 --- a/src/lib/embeddings.ts +++ b/src/lib/embeddings.ts @@ -102,11 +102,67 @@ class OpenAIEmbeddingProvider implements EmbeddingProvider { } } +// --- Voyage AI Provider --- + +class VoyageEmbeddingProvider implements EmbeddingProvider { + dimensions = 1024; + private apiKey: string; + private model: string; + + constructor(apiKey: string, model = "voyage-3") { + this.apiKey = apiKey; + this.model = model; + // voyage-3 outputs 1024 dims, voyage-3-lite outputs 512 + if (model === "voyage-3-lite") this.dimensions = 512; + } + + async embed(text: string): Promise { + const [result] = await this.embedBatch([text]); + return result; + } + + async embedBatch(texts: string[]): Promise { + const results: number[][] = []; + const processed = texts.map(preprocessText); + + // Voyage supports up to 128 texts per batch + for (let i = 0; i < processed.length; i += 128) { + const batch = processed.slice(i, i + 128); + const resp = await fetch("https://api.voyageai.com/v1/embeddings", { + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: this.model, + input: batch, + input_type: "document", + }), + }); + + if (!resp.ok) { + const err = await resp.text(); + throw new Error(`Voyage AI embeddings API error ${resp.status}: ${err}`); + } + + const data = await resp.json(); + const sorted = data.data.sort((a: any, b: any) => a.index - b.index); + for (const item of sorted) { + results.push(item.embedding); + } + } + + return results; + } +} + // --- Factory --- export interface EmbeddingConfig { - provider: "local" | "openai"; + provider: "local" | "openai" | "voyage"; apiKey?: string; + model?: string; } export function createEmbeddingProvider(config: EmbeddingConfig): EmbeddingProvider { @@ -114,5 +170,9 @@ export function createEmbeddingProvider(config: EmbeddingConfig): EmbeddingProvi if (!config.apiKey) throw new Error("OpenAI API key required for openai embedding provider"); return new OpenAIEmbeddingProvider(config.apiKey); } + if (config.provider === "voyage") { + if (!config.apiKey) throw new Error("Voyage AI API key required for voyage embedding provider"); + return new VoyageEmbeddingProvider(config.apiKey, config.model); + } return new LocalEmbeddingProvider(); } diff --git a/src/lib/timeline-db.ts b/src/lib/timeline-db.ts index 49b4f78..987b04f 100644 --- a/src/lib/timeline-db.ts +++ b/src/lib/timeline-db.ts @@ -55,7 +55,7 @@ export interface ProjectInfo { } export interface TimelineConfig { - embedding_provider: "local" | "openai"; + embedding_provider: "local" | "openai" | "voyage"; embedding_model: string; openai_api_key?: string; indexed_projects: Record { }); expect(provider.dimensions).toBe(1536); }); + + it("throws when voyage provider has no API key", () => { + expect(() => + createEmbeddingProvider({ provider: "voyage" }), + ).toThrow("API key required"); + }); + + it("returns voyage provider with 1024 dimensions (voyage-3)", () => { + const provider = createEmbeddingProvider({ + provider: "voyage", + apiKey: "pa-test-key", + }); + expect(provider.dimensions).toBe(1024); + }); + + it("returns voyage provider with 512 dimensions for voyage-3-lite", () => { + const provider = createEmbeddingProvider({ + provider: "voyage", + apiKey: "pa-test-key", + model: "voyage-3-lite", + }); + expect(provider.dimensions).toBe(512); + }); });