feat: scheduled ingest with deduplication

.env.example

@@ -11,12 +11,18 @@ GOOGLE_API_KEY=your_key_here
 # DATABASE_PATH=data/proxy.db  # SQLite file path (default: data/proxy.db relative to gateway cwd)
 
 # Service ports
+PORT=3001
 # OPENROUTER_PORT=4010 # OpenRouter integration (memory API served here)
 # DASHBOARD_PORT=3000  # Dashboard UI
 
 # Memory is embedded in the OpenRouter process (no separate service).
 # MEMORY_DB_PATH=./memory.db   # SQLite path for memory store (used by OpenRouter)
 
+# Scheduled ingest (deduplicated memory)
+# CONVERSATION_LOG_PATH=./data/conversation-log.jsonl
+# INGEST_CHECKPOINT_PATH=./data/ingest-checkpoint.json
+# MEMORY_INGEST_URL=http://localhost:4010
+# INGEST_RATE_LIMIT_MS=1000
 # Optional x402 passthrough for OpenRouter access
 X402_BASE_URL=x402_supported_provider_url
 PRIVATE_KEY=

gateway/src/costs/ollama.yaml

@@ -0,0 +1,59 @@
+provider: "ollama"
+currency: "USD"
+unit: "MTok"
+models:
+  # Ollama runs models locally — all costs are zero.
+  # Users may add custom model entries here if needed.
+  llama3.3:
+    input: 0.00
+    output: 0.00
+  llama3.2:
+    input: 0.00
+    output: 0.00
+  llama3.1:
+    input: 0.00
+    output: 0.00
+  llama3:
+    input: 0.00
+    output: 0.00
+  gemma3:
+    input: 0.00
+    output: 0.00
+  gemma2:
+    input: 0.00
+    output: 0.00
+  qwen3:
+    input: 0.00
+    output: 0.00
+  qwen2.5-coder:
+    input: 0.00
+    output: 0.00
+  deepseek-r1:
+    input: 0.00
+    output: 0.00
+  deepseek-coder-v2:
+    input: 0.00
+    output: 0.00
+  phi4:
+    input: 0.00
+    output: 0.00
+  phi3:
+    input: 0.00
+    output: 0.00
+  mistral:
+    input: 0.00
+    output: 0.00
+  mixtral:
+    input: 0.00
+    output: 0.00
+  codellama:
+    input: 0.00
+    output: 0.00
+  starcoder2:
+    input: 0.00
+    output: 0.00
+metadata:
+  last_updated: "2026-02-03"
+  source: "https://ollama.com"
+  notes: "Ollama runs models locally. All API costs are zero — hardware costs are borne by the user."
+  version: "1.0"

gateway/src/domain/services/provider-registry.ts

@@ -0,0 +1,92 @@
+import { AIProvider } from '../types/provider.js';
+import { AnthropicProvider } from '../providers/anthropic-provider.js';
+import { OpenAIProvider } from '../providers/openai-provider.js';
+import { OpenRouterProvider } from '../providers/openrouter-provider.js';
+import { XAIProvider } from '../providers/xai-provider.js';
+import { ZAIProvider } from '../providers/zai-provider.js';
+import { GoogleProvider } from '../providers/google-provider.js';
+import { OllamaProvider } from '../providers/ollama-provider.js';
+
+export enum Provider {
+  ANTHROPIC = 'anthropic',
+  OPENAI = 'openai',
+  OPENROUTER = 'openrouter',
+  XAI = 'xAI',
+  ZAI = 'zai',
+  GOOGLE = 'google',
+  OLLAMA = 'ollama'
+}
+
+export interface ProviderSelectionRule {
+  match: (modelName: string) => boolean;
+}
+
+export interface ProviderPlugin {
+  id: Provider;
+  create: () => AIProvider;
+  selectionRules?: ProviderSelectionRule[];
+}
+
+/**
+  * Central registry for provider creation and selection hints.
+  * Keeps wiring in one place and reduces per-provider boilerplate.
+  */
+export class ProviderRegistry {
+  private readonly instances = new Map<Provider, AIProvider>();
+
+  constructor(private readonly plugins: ProviderPlugin[]) {}
+
+  listProviders(): Provider[] {
+    return this.plugins.map(p => p.id);
+  }
+
+  getOrCreateProvider(id: Provider): AIProvider {
+    if (!this.instances.has(id)) {
+      const plugin = this.plugins.find(p => p.id === id);
+      if (!plugin) {
+        throw new Error(`Unknown provider: ${id}`);
+      }
+      this.instances.set(id, plugin.create());
+    }
+
+    const provider = this.instances.get(id);
+    if (!provider) {
+      throw new Error(`Failed to create provider: ${id}`);
+    }
+    return provider;
+  }
+
+  getAvailableProviders(): Provider[] {
+    return this.listProviders().filter(id => {
+      const provider = this.getOrCreateProvider(id);
+      return provider.isConfigured();
+    });
+  }
+
+  /**
+   * Return the first preferred provider whose rule matches the model name and is available.
+   */
+  findPreferredProvider(modelName: string, available: Provider[]): Provider | undefined {
+    for (const plugin of this.plugins) {
+      if (!plugin.selectionRules || !available.includes(plugin.id)) continue;
+      if (plugin.selectionRules.some(rule => rule.match(modelName))) {
+        return plugin.id;
+      }
+    }
+    return undefined;
+  }
+}
+
+export function createDefaultProviderRegistry(): ProviderRegistry {
+  const plugins: ProviderPlugin[] = [
+    { id: Provider.ANTHROPIC, create: () => new AnthropicProvider() },
+    { id: Provider.OPENAI, create: () => new OpenAIProvider() },
+    { id: Provider.OPENROUTER, create: () => new OpenRouterProvider() },
+    { id: Provider.XAI, create: () => new XAIProvider(), selectionRules: [{ match: model => model.includes('grok-') || model.includes('grok_beta') }] },
+    { id: Provider.ZAI, create: () => new ZAIProvider() },
+    { id: Provider.GOOGLE, create: () => new GoogleProvider(), selectionRules: [{ match: model => model.toLowerCase().includes('gemini') }] },
+    { id: Provider.OLLAMA, create: () => new OllamaProvider(), selectionRules: [{ match: model => model.startsWith('ollama/') }] },
+  ];
+
+  return new ProviderRegistry(plugins);
+}

gateway/src/infrastructure/config/app-config.ts

@@ -0,0 +1,167 @@
+/**
+ * Centralized application configuration
+ * All environment variables are validated and accessed through this class
+ */
+
+export class AppConfig {
+  // Server configuration
+  readonly server = {
+    port: this.getNumber('PORT', 3001),
+    environment: this.getString('NODE_ENV', 'development'),
+    isDevelopment: this.getString('NODE_ENV', 'development') === 'development',
+    isProduction: this.getString('NODE_ENV', 'development') === 'production',
+    version: this.getOptionalString('npm_package_version') || 'dev',
+  };
+
+  // x402 Payment configuration
+  readonly x402 = {
+    enabled: this.has('PRIVATE_KEY'),
+    privateKey: this.getOptionalString('PRIVATE_KEY'),
+    baseUrl: this.getString('X402_BASE_URL', 'https://x402.ekailabs.xyz'),
+
+    // Helper methods
+    get chatCompletionsUrl() {
+      return `${this.baseUrl}/v1/chat/completions`;
+    },
+    get messagesUrl() {
+      return `${this.baseUrl}/v1/messages`;
+    },
+  };
+
+  // Provider API Keys
+  readonly providers = {
+    anthropic: {
+      apiKey: this.getOptionalString('ANTHROPIC_API_KEY'),
+      enabled: this.has('ANTHROPIC_API_KEY'),
+    },
+    openai: {
+      apiKey: this.getOptionalString('OPENAI_API_KEY'),
+      enabled: this.has('OPENAI_API_KEY'),
+    },
+    openrouter: {
+      apiKey: this.getOptionalString('OPENROUTER_API_KEY'),
+      enabled: this.has('OPENROUTER_API_KEY'),
+    },
+    xai: {
+      apiKey: this.getOptionalString('XAI_API_KEY'),
+      enabled: this.has('XAI_API_KEY'),
+    },
+    zai: {
+      apiKey: this.getOptionalString('ZAI_API_KEY'),
+      enabled: this.has('ZAI_API_KEY'),
+    },
+    google: {
+      apiKey: this.getOptionalString('GOOGLE_API_KEY'),
+      enabled: this.has('GOOGLE_API_KEY'),
+    },
+    ollama: {
+      baseUrl: this.getString('OLLAMA_BASE_URL', 'http://localhost:11434/v1'),
+      apiKey: this.getOptionalString('OLLAMA_API_KEY'),
+      enabled: this.has('OLLAMA_BASE_URL'),
+    },
+  };
+
+  // Telemetry configuration
+  readonly telemetry = {
+    enabled: this.getBoolean('ENABLE_TELEMETRY', true),
+    endpoint: this.getOptionalString('TELEMETRY_ENDPOINT'),
+  };
+
+  // OpenRouter-specific configuration
+  readonly openrouter = {
+    skipPricingRefresh: this.getBoolean('SKIP_OPENROUTER_PRICING_REFRESH', false),
+    pricingTimeoutMs: this.getNumber('OPENROUTER_PRICING_TIMEOUT_MS', 4000),
+    pricingRetries: this.getNumber('OPENROUTER_PRICING_RETRIES', 2),
+  };
+
+  // Feature flags
+  readonly features = {
+    usageTracking: this.getBoolean('ENABLE_USAGE_TRACKING', true),
+  };
+
+  // Memory service configuration (FIFO file backend by default)
+  readonly memory = {
+    backend: this.getString('MEMORY_BACKEND', 'file'),
+    maxItems: this.getNumber('MEMORY_MAX_ITEMS', 20),
+  } as const;
+
+  // Helper methods
+  private has(key: string): boolean {
+    return !!process.env[key];
+  }
+
+  private getString(key: string, defaultValue: string): string;
+  private getString(key: string): string;
+  private getString(key: string, defaultValue?: string): string {
+    const value = process.env[key] || defaultValue;
+    if (value === undefined) {
+      throw new Error(`Missing required environment variable: ${key}`);
+    }
+    return value;
+  }
+
+  private getOptionalString(key: string): string | undefined {
+    return process.env[key];
+  }
+
+  private getNumber(key: string, defaultValue: number): number {
+    const value = process.env[key];
+    if (!value) return defaultValue;
+    const num = parseInt(value, 10);
+    if (isNaN(num)) {
+      throw new Error(`Invalid number for environment variable ${key}: ${value}`);
+    }
+    return num;
+  }
+
+  private getBoolean(key: string, defaultValue: boolean): boolean {
+    const value = process.env[key];
+    if (!value) return defaultValue;
+    return value.toLowerCase() === 'true' || value === '1';
+  }
+
+  /**
+   * Validate that at least one authentication method is configured
+   */
+  validate(): void {
+    const hasApiKeys = Object.values(this.providers).some(p => p.enabled);
+    const hasX402 = this.x402.enabled;
+
+    if (!hasApiKeys && !hasX402) {
+      throw new Error(
+        'No authentication configured. Set either:\n' +
+        '  1. At least one provider API key (ANTHROPIC_API_KEY, OPENAI_API_KEY, etc.)\n' +
+        '  2. PRIVATE_KEY for x402 payment mode'
+      );
+    }
+  }
+
+  /**
+   * Get human-readable mode description
+   */
+  getMode(): 'x402-only' | 'hybrid' | 'byok' {
+    const hasApiKeys = Object.values(this.providers).some(p => p.enabled);
+    const hasX402 = this.x402.enabled;
+
+    if (!hasApiKeys && hasX402) return 'x402-only';
+    if (hasApiKeys && hasX402) return 'hybrid';
+    return 'byok';
+  }
+}
+
+// Singleton instance
+let configInstance: AppConfig | null = null;
+
+export function getConfig(): AppConfig {
+  if (!configInstance) {
+    configInstance = new AppConfig();
+    configInstance.validate();
+  }
+  return configInstance;
+}
+
+// For testing - reset config
+export function resetConfig(): void {
+  configInstance = null;
+}
+