diff --git a/.claude/settings.local.json b/.claude/settings.local.json index e518c4f..69052df 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -77,7 +77,18 @@ "Bash(kill:*)", "Bash(grep:*)", "WebFetch(domain:opencode.ai)", - "Bash(find:*)" + "Bash(find:*)", + "WebFetch(domain:www.databricks.com)", + "WebFetch(domain:docs.databricks.com)", + "Bash(env:*)", + "Bash(DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test:*)", + "Bash(DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node:*)", + "Bash(gh pr list:*)", + "Bash(gh pr diff:*)", + "Bash(PREFER_OLLAMA=true node:*)", + "Bash(DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com MODEL_PROVIDER=azure-openai AZURE_OPENAI_ENDPOINT=https://test.openai.azure.com AZURE_OPENAI_API_KEY=test-key node:*)", + "Bash(git stash:*)", + "WebFetch(domain:docs.ollama.com)" ], "deny": [], "ask": [] diff --git a/.env.example b/.env.example index 644358f..7c37a1f 100644 --- a/.env.example +++ b/.env.example @@ -1,90 +1,62 @@ -# Lynkr Configuration +# ============================================================================== +# LYNKR CONFIGURATION - All Environment Variables # Copy this file to .env and fill in your values +# ============================================================================== # ============================================================================== # Model Provider Configuration # ============================================================================== -# Primary model provider to use -# Options: databricks, azure-anthropic, azure-openai, openrouter, openai, ollama, llamacpp, lmstudio, bedrock, zai, vertex -# Default: databricks +# Primary model provider — controls routing when TIER_* vars are NOT configured (static mode). +# When all 4 TIER_* vars are set, tier routing overrides this for request routing. +# Even with tier routing active, MODEL_PROVIDER is still used for: +# - Startup checks (e.g. waiting for Ollama to be reachable) +# - Provider discovery API (/v1/providers) +# - Default provider when a TIER_* value has no "provider:" prefix +# Options: databricks, azure-anthropic, azure-openai, openrouter, openai, ollama, llamacpp, lmstudio, bedrock, zai, vertex, moonshot +# Note: PREFER_OLLAMA is deprecated and has no effect. Use TIER_SIMPLE=ollama: instead. MODEL_PROVIDER=ollama # ============================================================================== -# Ollama Configuration (Hybrid Routing) +# Databricks Configuration # ============================================================================== -# Enable Ollama preference for simple requests -PREFER_OLLAMA=false +# DATABRICKS_API_BASE=https://your-workspace.cloud.databricks.com +# DATABRICKS_API_KEY=dapi1234567890abcdef +# DATABRICKS_ENDPOINT_PATH=/serving-endpoints/your-endpoint/invocations -# Ollama model to use (must be compatible with tool calling) -# Options: qwen2.5-coder:latest, llama3.1, mistral-nemo, nemotron-3-nano:30b-cloud, etc. -OLLAMA_MODEL=qwen2.5-coder:latest +# ============================================================================== +# Ollama Configuration (Local Models) +# ============================================================================== -# Ollama endpoint (default: http://localhost:11434) +# Ollama endpoint OLLAMA_ENDPOINT=http://localhost:11434 -# Ollama embeddings configuration (for Cursor @Codebase semantic search) -# Embedding models for local, privacy-first semantic search -# Popular models: -# - nomic-embed-text (768 dim, 137M params, best all-around) - RECOMMENDED -# - mxbai-embed-large (1024 dim, 335M params, higher quality) -# - all-minilm (384 dim, 23M params, fastest/smallest) -# -# Pull model: ollama pull nomic-embed-text -# OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text -# OLLAMA_EMBEDDINGS_ENDPOINT=http://localhost:11434/api/embeddings - -# Fallback provider when primary provider fails or for complex requests -# Options: databricks, azure-anthropic, azure-openai, openrouter, openai, bedrock -# Note: Local providers (ollama, llamacpp, lmstudio) cannot be used as fallback -FALLBACK_PROVIDER=databricks +# Ollama timeout in milliseconds +OLLAMA_TIMEOUT_MS=120000 -# Enable automatic fallback (true = transparent fallback, false = fail on provider error) -FALLBACK_ENABLED=false +# Ollama model to use (must be compatible with tool calling) +# Options: qwen2.5-coder:latest, llama3.1, mistral-nemo, etc. +OLLAMA_MODEL=qwen2.5-coder:latest # Max tools for routing to Ollama (requests with more tools go to cloud) OLLAMA_MAX_TOOLS_FOR_ROUTING=3 -# ============================================================================== -# Databricks Configuration -# ============================================================================== - -# DATABRICKS_API_BASE=https://your-workspace.cloud.databricks.com -# DATABRICKS_API_KEY=dapi1234567890abcdef - -# ============================================================================== -# AWS Bedrock Configuration -# ============================================================================== - -# Bedrock API Key (Bearer token) - REQUIRED -# Generate from AWS Console → Bedrock → API Keys -# See: https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys-generate.html -# AWS_BEDROCK_API_KEY=your-bedrock-api-key-here - -# AWS region (default: us-east-1) -# Available regions: us-east-1, us-west-2, us-east-2, ap-southeast-1, ap-northeast-1, eu-central-1 -# AWS_BEDROCK_REGION=us-east-2 - -# Bedrock model ID to use -# Claude models (recommended): -# - anthropic.claude-3-5-sonnet-20241022-v2:0 (best for tool calling) -# - anthropic.claude-3-opus-20240229-v1:0 (most capable) -# - anthropic.claude-3-haiku-20240307-v1:0 (fast, cheap) -# Other models: -# - us.deepseek.r1-v1:0 (DeepSeek R1 - reasoning model) -# - qwen.qwen3-coder-480b-a35b-v1:0 (coding specialist) -# - minimax.minimax-m2 (MiniMax M2) -# - amazon.titan-text-express-v1 -# - meta.llama3-1-70b-instruct-v1:0 -# AWS_BEDROCK_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0 +# Ollama embeddings configuration (for Cursor @Codebase semantic search) +# Pull model: ollama pull nomic-embed-text +OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text +OLLAMA_EMBEDDINGS_ENDPOINT=http://localhost:11434/api/embeddings # ============================================================================== -# Azure Anthropic Configuration +# OpenRouter Configuration (100+ Models via Single API) # ============================================================================== -# AZURE_ANTHROPIC_ENDPOINT=https://your-anthropic.openai.azure.com -# AZURE_ANTHROPIC_API_KEY=your-azure-key +# Get your API key from: https://openrouter.ai/keys +# OPENROUTER_API_KEY=sk-or-v1-your-key-here +OPENROUTER_MODEL=openai/gpt-4o-mini +OPENROUTER_EMBEDDINGS_MODEL=openai/text-embedding-ada-002 +OPENROUTER_ENDPOINT=https://openrouter.ai/api/v1/chat/completions +OPENROUTER_MAX_TOOLS_FOR_ROUTING=15 # ============================================================================== # Azure OpenAI Configuration @@ -92,15 +64,22 @@ OLLAMA_MAX_TOOLS_FOR_ROUTING=3 # Azure OpenAI endpoint (supports both standard and AI Foundry formats) # Standard: https://.openai.azure.com -# AI Foundry: https://.services.ai.azure.com/models/chat/completions?api-version=2024-05-01-preview +# AI Foundry: https://.services.ai.azure.com/models/chat/completions?api-version=... # AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com - # AZURE_OPENAI_API_KEY=your-azure-openai-key # AZURE_OPENAI_DEPLOYMENT=gpt-4o -# AZURE_OPENAI_API_VERSION=2024-05-01-preview +# AZURE_OPENAI_API_VERSION=2024-08-01-preview # ============================================================================== -# OpenAI Configuration (Direct OpenAI API) +# Azure Anthropic Configuration +# ============================================================================== + +# AZURE_ANTHROPIC_ENDPOINT=https://your-anthropic.openai.azure.com +# AZURE_ANTHROPIC_API_KEY=your-azure-key +# AZURE_ANTHROPIC_VERSION=2023-06-01 + +# ============================================================================== +# OpenAI Configuration (Direct) # ============================================================================== # OPENAI_API_KEY=sk-your-openai-api-key @@ -109,88 +88,76 @@ OLLAMA_MAX_TOOLS_FOR_ROUTING=3 # OPENAI_ORGANIZATION=org-your-org-id # ============================================================================== -# OpenRouter Configuration (100+ Models via Single API) +# AWS Bedrock Configuration # ============================================================================== -# Get your API key from: https://openrouter.ai/keys -# OPENROUTER_API_KEY=sk-or-v1-your-key-here - -# Model to use (default: openai/gpt-4o-mini) -# Popular options: -# - nvidia/nemotron-3-nano-30b-a3b:free (FREE) -# - anthropic/claude-3.5-sonnet ($3/$15 per 1M) -# - openai/gpt-4o-mini ($0.15/$0.60 per 1M) -# OPENROUTER_MODEL=openai/gpt-4o-mini +# Generate from AWS Console → Bedrock → API Keys +# AWS_BEDROCK_API_KEY=your-bedrock-bearer-token +# AWS_BEDROCK_REGION=us-east-1 +# AWS_BEDROCK_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0 # ============================================================================== # llama.cpp Configuration (Local GGUF Models) # ============================================================================== -# LLAMACPP_ENDPOINT=http://localhost:8080 -# LLAMACPP_MODEL=default -# LLAMACPP_TIMEOUT_MS=120000 +LLAMACPP_ENDPOINT=http://localhost:8080 +LLAMACPP_MODEL=default +LLAMACPP_TIMEOUT_MS=120000 # LLAMACPP_API_KEY=your-optional-api-key - -# llama.cpp embeddings configuration -# LLAMACPP_EMBEDDINGS_ENDPOINT=http://localhost:8080/embeddings +LLAMACPP_EMBEDDINGS_ENDPOINT=http://localhost:8080/embeddings # ============================================================================== -# LM Studio Configuration (Local Models with GUI) +# LM Studio Configuration # ============================================================================== -# LMSTUDIO_ENDPOINT=http://localhost:1234 -# LMSTUDIO_MODEL=default -# LMSTUDIO_TIMEOUT_MS=120000 +LMSTUDIO_ENDPOINT=http://localhost:1234 +LMSTUDIO_MODEL=default +LMSTUDIO_TIMEOUT_MS=120000 # LMSTUDIO_API_KEY=your-optional-api-key # ============================================================================== # Z.AI (Zhipu AI) Configuration - ~1/7 cost of Anthropic # ============================================================================== -# Z.AI provides GLM models through an Anthropic-compatible API -# Get your API key from: https://z.ai/ or https://open.bigmodel.cn/ # ZAI_API_KEY=your-zai-api-key - -# Z.AI endpoint (default: https://api.z.ai/api/anthropic/v1/messages) # ZAI_ENDPOINT=https://api.z.ai/api/anthropic/v1/messages - -# Model to use (GLM-4.7 is equivalent to Claude Sonnet, GLM-4.5-Air is like Haiku) -# Options: GLM-4.7, GLM-4.5-Air, GLM-4-Plus # ZAI_MODEL=GLM-4.7 +# ============================================================================== +# Moonshot AI (Kimi) Configuration +# ============================================================================== + +# MOONSHOT_API_KEY=your-moonshot-api-key +# MOONSHOT_ENDPOINT=https://api.moonshot.ai/v1/chat/completions +# MOONSHOT_MODEL=kimi-k2-turbo-preview + # ============================================================================== # Google Vertex AI Configuration (Gemini Models) # ============================================================================== -# Google AI API Key (required) # Get your API key from: https://aistudio.google.com/app/apikey # VERTEX_API_KEY=your-google-api-key -# or use: GOOGLE_API_KEY=your-google-api-key - -# Gemini model to use (default: gemini-2.0-flash) -# Options: -# - gemini-2.0-flash (fast, good for most tasks) - DEFAULT -# - gemini-2.0-flash-lite (fastest, cheapest) -# - gemini-2.5-pro (most capable, best quality) -# - gemini-1.5-pro (previous generation) -# - gemini-1.5-flash (previous generation, fast) +# GOOGLE_API_KEY=your-google-api-key # VERTEX_MODEL=gemini-2.0-flash -# Model mapping from Claude names: -# claude-sonnet-* → gemini-2.0-flash -# claude-haiku-* → gemini-2.0-flash-lite -# claude-opus-* → gemini-2.5-pro +# ============================================================================== +# Fallback Configuration +# ============================================================================== + +# Enable automatic fallback when tier provider fails +FALLBACK_ENABLED=false + +# Fallback provider (cannot be local: ollama, llamacpp, lmstudio) +# Options: databricks, azure-anthropic, azure-openai, openrouter, openai, bedrock +FALLBACK_PROVIDER=databricks # ============================================================================== # Embeddings Provider Override # ============================================================================== -# By default, embeddings use same provider as MODEL_PROVIDER (if supported) -# To force a specific provider, set: -# EMBEDDINGS_PROVIDER=ollama # Use Ollama embeddings -# EMBEDDINGS_PROVIDER=llamacpp # Use llama.cpp embeddings -# EMBEDDINGS_PROVIDER=openrouter # Use OpenRouter embeddings -# EMBEDDINGS_PROVIDER=openai # Use OpenAI embeddings +# Force a specific embeddings provider (default: same as MODEL_PROVIDER) +# Options: ollama, llamacpp, openrouter, openai +# EMBEDDINGS_PROVIDER=ollama # ============================================================================== # Server Configuration @@ -198,83 +165,165 @@ OLLAMA_MAX_TOOLS_FOR_ROUTING=3 PORT=8081 LOG_LEVEL=info -WEB_SEARCH_ENDPOINT=http://localhost:8888/search +# NODE_ENV=development -# Policy Configuration -POLICY_MAX_STEPS=20 -POLICY_MAX_TOOL_CALLS=12 +# File logging (persistent logs with automatic rotation via pino-roll) +# LOG_FILE_ENABLED=true +# LOG_FILE_PATH=./logs/lynkr.log +# LOG_FILE_LEVEL=debug +# LOG_FILE_FREQUENCY=daily +# LOG_FILE_MAX_FILES=14 -# Tool loop guard - max tool results in conversation before force-terminating -# Prevents infinite tool loops. Set higher for complex multi-step tasks. -POLICY_TOOL_LOOP_THRESHOLD=10 +# Maximum JSON request body size +REQUEST_JSON_LIMIT=1gb + +# Session database path +SESSION_DB_PATH=./data/sessions.db -# Workspace for embeddings/indexing +# Workspace root directory WORKSPACE_ROOT=/path/to/your/workspace -WORKSPACE_INDEX_ENABLED=true -# Tool execution mode: where to execute tools (Write, Read, Bash, etc.) -# - server: Execute tools on the server (default, for standalone proxy use) +# ============================================================================== +# Tool Execution Mode +# ============================================================================== + +# Where to execute tools +# - server: Execute tools on the proxy server (default) # - client/passthrough: Return tool calls to CLI for local execution TOOL_EXECUTION_MODE=server -# Suggestion mode model override -# Controls which model handles suggestion mode (predicting next user input). -# Values: -# default - Use the same model as MODEL_PROVIDER (no change) -# none - Skip suggestion mode LLM calls entirely (saves GPU time) -# - Use a specific model (e.g. "llama3.1" for a lighter model) -SUGGESTION_MODE_MODEL=default - # Enable/disable automatic tool injection for local models INJECT_TOOLS_LLAMACPP=true INJECT_TOOLS_OLLAMA=true +# Suggestion mode model override +# Values: default (same as MODEL_PROVIDER), none (skip), or name +SUGGESTION_MODE_MODEL=default + +# ============================================================================== +# Rate Limiting +# ============================================================================== + +RATE_LIMIT_ENABLED=true +RATE_LIMIT_WINDOW_MS=60000 +RATE_LIMIT_MAX=100 +RATE_LIMIT_KEY_BY=session + +# ============================================================================== +# Web Search Configuration +# ============================================================================== + +WEB_SEARCH_ENDPOINT=http://localhost:8888/search +# WEB_SEARCH_API_KEY= +WEB_SEARCH_ALLOW_ALL=true +# WEB_SEARCH_ALLOWED_HOSTS=localhost,127.0.0.1 +WEB_SEARCH_TIMEOUT_MS=10000 +WEB_FETCH_BODY_PREVIEW_MAX=10000 +WEB_SEARCH_RETRY_ENABLED=true +WEB_SEARCH_MAX_RETRIES=2 + +# ============================================================================== +# Policy Configuration +# ============================================================================== + +POLICY_MAX_STEPS=20 +POLICY_MAX_TOOL_CALLS=12 +# POLICY_DISALLOWED_TOOLS=dangerous_tool1,dangerous_tool2 + +# Tool loop guard threshold +POLICY_TOOL_LOOP_THRESHOLD=10 + +# Git policy +POLICY_GIT_ALLOW_PUSH=false +POLICY_GIT_ALLOW_PULL=true +POLICY_GIT_ALLOW_COMMIT=true +# POLICY_GIT_TEST_COMMAND=npm test +POLICY_GIT_REQUIRE_TESTS=false +# POLICY_GIT_COMMIT_REGEX=^(feat|fix|docs|style|refactor|test|chore): +POLICY_GIT_AUTOSTASH=false + +# File access policy +# POLICY_FILE_ALLOWED_PATHS=/path1,/path2 +POLICY_FILE_BLOCKED_PATHS=/.env,.env,/etc/passwd,/etc/shadow + +# Safe commands +POLICY_SAFE_COMMANDS_ENABLED=true +# POLICY_SAFE_COMMANDS_CONFIG={"allowed":["ls","cat","grep"]} + +# ============================================================================== +# Agents Configuration +# ============================================================================== + +AGENTS_ENABLED=true +AGENTS_MAX_CONCURRENT=10 +AGENTS_DEFAULT_MODEL=haiku +AGENTS_MAX_STEPS=15 +AGENTS_TIMEOUT=300000 + +# ============================================================================== +# MCP Sandbox Configuration +# ============================================================================== + +MCP_SANDBOX_ENABLED=true +# MCP_SANDBOX_IMAGE=node:20-alpine +MCP_SANDBOX_RUNTIME=docker +MCP_SANDBOX_CONTAINER_WORKSPACE=/workspace +MCP_SANDBOX_MOUNT_WORKSPACE=true +MCP_SANDBOX_ALLOW_NETWORKING=false +MCP_SANDBOX_NETWORK_MODE=none +MCP_SANDBOX_PASSTHROUGH_ENV=PATH,LANG,LC_ALL,TERM,HOME +# MCP_SANDBOX_EXTRA_MOUNTS=/host/path:/container/path:ro +MCP_SANDBOX_TIMEOUT_MS=20000 +# MCP_SANDBOX_USER=node +# MCP_SANDBOX_ENTRYPOINT=/bin/sh +MCP_SANDBOX_REUSE_SESSION=true +MCP_SANDBOX_READ_ONLY_ROOT=false +MCP_SANDBOX_NO_NEW_PRIVILEGES=true +MCP_SANDBOX_DROP_CAPABILITIES=ALL +# MCP_SANDBOX_ADD_CAPABILITIES=NET_BIND_SERVICE +MCP_SANDBOX_MEMORY_LIMIT=512m +MCP_SANDBOX_CPU_LIMIT=1.0 +MCP_SANDBOX_PIDS_LIMIT=100 + +# MCP permissions +MCP_SANDBOX_PERMISSION_MODE=auto +# MCP_SANDBOX_PERMISSION_ALLOW=tool1,tool2 +# MCP_SANDBOX_PERMISSION_DENY=tool3,tool4 + +# MCP server manifest +# MCP_SERVER_MANIFEST=~/.claude/mcp/servers.json +MCP_MANIFEST_DIRS=~/.claude/mcp + +# ============================================================================== +# Prompt Cache Configuration +# ============================================================================== + +PROMPT_CACHE_ENABLED=true +PROMPT_CACHE_MAX_ENTRIES=1000 +PROMPT_CACHE_TTL_MS=300000 + # ============================================================================== # Semantic Response Cache # ============================================================================== -# Enable semantic caching (requires embeddings provider) -# Caches LLM responses and returns them for semantically similar prompts -# Requires: Ollama with nomic-embed-text, or another embeddings provider +# Requires an embeddings provider SEMANTIC_CACHE_ENABLED=false - -# Similarity threshold for cache hits (0.0-1.0, higher = stricter matching) -# 0.95 = very similar prompts only, 0.90 = more lenient SEMANTIC_CACHE_THRESHOLD=0.95 # ============================================================================== # Long-Term Memory System (Titans-Inspired) # ============================================================================== -# Enable/disable the entire memory system MEMORY_ENABLED=true - -# Maximum number of memories to inject into each request MEMORY_RETRIEVAL_LIMIT=5 - -# Minimum surprise score (0.0-1.0) required to store a memory MEMORY_SURPRISE_THRESHOLD=0.3 - -# Auto-delete memories older than this many days MEMORY_MAX_AGE_DAYS=90 - -# Maximum total memories to keep MEMORY_MAX_COUNT=10000 - -# Enable importance decay over time -MEMORY_DECAY_ENABLED=true - -# Days for importance to decay by 50% -MEMORY_DECAY_HALF_LIFE=30 - -# Include global memories in all sessions MEMORY_INCLUDE_GLOBAL=true - -# Where to inject memories (system or assistant_preamble) MEMORY_INJECTION_FORMAT=system - -# Enable automatic extraction MEMORY_EXTRACTION_ENABLED=true +MEMORY_DECAY_ENABLED=true +MEMORY_DECAY_HALF_LIFE=30 # ============================================================================== # Token Optimization Settings (60-80% Cost Reduction) @@ -301,107 +350,49 @@ TOON_FAIL_OPEN=true TOON_LOG_STATS=true # ============================================================================== -# Smart Tool Selection (Advanced Token Optimization) +# Smart Tool Selection # ============================================================================== # Selection strategy: heuristic, aggressive, or conservative SMART_TOOL_SELECTION_MODE=heuristic - -# Maximum token budget for tools per request SMART_TOOL_SELECTION_TOKEN_BUDGET=2500 # ============================================================================== -# Performance & Security +# Test Configuration # ============================================================================== -# API retry configuration -API_RETRY_MAX_RETRIES=3 -API_RETRY_INITIAL_DELAY=1000 -API_RETRY_MAX_DELAY=30000 - -# Load shedding thresholds -LOAD_SHEDDING_HEAP_THRESHOLD=90 -LOAD_SHEDDING_EVENT_LOOP_DELAY=100 +# WORKSPACE_TEST_COMMAND=npm test +# WORKSPACE_TEST_ARGS=--coverage +WORKSPACE_TEST_TIMEOUT_MS=600000 +WORKSPACE_TEST_SANDBOX=auto +WORKSPACE_TEST_COVERAGE_FILES=coverage/coverage-summary.json +# WORKSPACE_TEST_PROFILES=[{"name":"unit","command":"npm test"}] # ============================================================================== # Hot Reload Configuration # ============================================================================== -# Enable hot reload of configuration (default: true) -# When enabled, changes to .env are applied without restart HOT_RELOAD_ENABLED=true - -# Debounce delay in ms (prevents rapid reloads) HOT_RELOAD_DEBOUNCE_MS=1000 -# ============================================================================== -# Quick Start Examples -# ============================================================================== - -# 100% Local (FREE) - Ollama: -# MODEL_PROVIDER=ollama -# OLLAMA_MODEL=qwen2.5-coder:latest -# npm start - -# AWS Bedrock: -# MODEL_PROVIDER=bedrock -# AWS_BEDROCK_API_KEY=your-key -# AWS_BEDROCK_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0 -# npm start - -# OpenRouter (simplest cloud): -# MODEL_PROVIDER=openrouter -# OPENROUTER_API_KEY=sk-or-v1-your-key -# npm start - -# Azure OpenAI: -# MODEL_PROVIDER=azure-openai -# AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com -# AZURE_OPENAI_API_KEY=your-key -# AZURE_OPENAI_DEPLOYMENT=gpt-4o -# npm start - -# Z.AI (Zhipu - ~1/7 cost of Anthropic): -# MODEL_PROVIDER=zai -# ZAI_API_KEY=your-zai-api-key -# ZAI_MODEL=GLM-4.7 -# npm start - -# Google Gemini (via Vertex AI): -# MODEL_PROVIDER=vertex -# VERTEX_API_KEY=your-google-api-key -# VERTEX_MODEL=gemini-2.0-flash -# npm start - # ============================================================================== # Headroom Context Compression (Sidecar) # ============================================================================== -# Headroom provides 47-92% token reduction through intelligent context compression. -# It runs as a Python sidecar container managed automatically by Lynkr via Docker. -# -# Features: -# - Smart Crusher: Statistical JSON compression for tool outputs -# - Cache Aligner: Stabilizes dynamic content for provider cache hits -# - CCR (Compress-Cache-Retrieve): Reversible compression with on-demand retrieval -# - Rolling Window: Token budget enforcement with turn-based windowing -# - LLMLingua (optional): ML-based 20x compression with GPU acceleration - -# Enable/disable Headroom compression (default: false) + +# Enable Headroom compression (47-92% token reduction) HEADROOM_ENABLED=false -# Sidecar endpoint (auto-configured when Docker is enabled) +# Sidecar endpoint HEADROOM_ENDPOINT=http://localhost:8787 -# Request timeout in milliseconds +# Request timeout and minimum tokens HEADROOM_TIMEOUT_MS=5000 - -# Minimum tokens to trigger compression (skip small requests) HEADROOM_MIN_TOKENS=500 -# Operating mode: "audit" (observe only) or "optimize" (apply transforms) +# Operating mode: audit (observe) or optimize (apply) HEADROOM_MODE=optimize -# Provider for cache optimization hints: anthropic, openai, google +# Provider for cache hints: anthropic, openai, google HEADROOM_PROVIDER=anthropic # Log level: debug, info, warning, error @@ -410,50 +401,27 @@ HEADROOM_LOG_LEVEL=info # ============================================================================== # Headroom Docker Configuration # ============================================================================== -# When enabled, Lynkr automatically manages the Headroom container lifecycle -# Enable Docker container management (default: true when HEADROOM_ENABLED=true) HEADROOM_DOCKER_ENABLED=true - -# Docker image to use HEADROOM_DOCKER_IMAGE=lynkr/headroom-sidecar:latest - -# Container name HEADROOM_DOCKER_CONTAINER_NAME=lynkr-headroom - -# Port mapping HEADROOM_DOCKER_PORT=8787 - -# Resource limits HEADROOM_DOCKER_MEMORY_LIMIT=512m HEADROOM_DOCKER_CPU_LIMIT=1.0 - -# Restart policy: no, always, unless-stopped, on-failure HEADROOM_DOCKER_RESTART_POLICY=unless-stopped - -# Docker network (optional, leave empty for default bridge) # HEADROOM_DOCKER_NETWORK=lynkr-network - -# Build from local source instead of pulling image -# HEADROOM_DOCKER_AUTO_BUILD=true # HEADROOM_DOCKER_BUILD_CONTEXT=./headroom-sidecar +# HEADROOM_DOCKER_AUTO_BUILD=true # ============================================================================== # Headroom Transform Settings # ============================================================================== -# Smart Crusher (statistical JSON compression) HEADROOM_SMART_CRUSHER=true HEADROOM_SMART_CRUSHER_MIN_TOKENS=200 HEADROOM_SMART_CRUSHER_MAX_ITEMS=15 - -# Tool Crusher (fixed-rules compression for tool outputs) HEADROOM_TOOL_CRUSHER=true - -# Cache Aligner (stabilize dynamic content like UUIDs, timestamps) HEADROOM_CACHE_ALIGNER=true - -# Rolling Window (context overflow management) HEADROOM_ROLLING_WINDOW=true HEADROOM_KEEP_TURNS=3 @@ -461,29 +429,32 @@ HEADROOM_KEEP_TURNS=3 # Headroom CCR (Compress-Cache-Retrieve) # ============================================================================== -# Enable CCR for reversible compression with on-demand retrieval HEADROOM_CCR=true - -# TTL for cached content in seconds (default: 5 minutes) HEADROOM_CCR_TTL=300 # ============================================================================== -# Headroom LLMLingua (Optional ML Compression) +# Headroom LLMLingua (ML Compression - Requires GPU) # ============================================================================== -# LLMLingua-2 provides ML-based 20x compression using BERT token classification. -# Requires GPU for reasonable performance, or use CPU with longer timeouts. -# Enable LLMLingua (default: false, requires GPU recommended) HEADROOM_LLMLINGUA=false - -# Device: cuda, cpu, auto HEADROOM_LLMLINGUA_DEVICE=auto # ============================================================================== -# Prompt Cache Configuration +# Tiered Model Routing (Recommended for Cost Optimization) # ============================================================================== - -# Enable prompt caching (caches exact prompts) -PROMPT_CACHE_ENABLED=true -PROMPT_CACHE_MAX_ENTRIES=1000 -PROMPT_CACHE_TTL_MS=300000 +# When all 4 TIER_* vars are set, they OVERRIDE MODEL_PROVIDER for routing. +# Each request is scored for complexity (0-100) and routed to the matching tier: +# SIMPLE (0-25) → cheap/local models COMPLEX (51-75) → capable cloud models +# MEDIUM (26-50) → mid-range models REASONING (76-100) → best available +# +# Format: TIER_=provider:model +# All 4 tiers must be configured to enable tiered routing. +# If any are missing, tiered routing is disabled and MODEL_PROVIDER is used directly. +# +# Supported providers: ollama, openai, azure-openai, openrouter, +# databricks, bedrock, vertex, zai, moonshot, llamacpp, lmstudio +# +TIER_SIMPLE=ollama:llama3.2 +TIER_MEDIUM=openrouter:openai/gpt-4o-mini +TIER_COMPLEX=azure-openai:gpt-4o +TIER_REASONING=azure-openai:gpt-4o diff --git a/Dockerfile b/Dockerfile index 59c57a1..691a885 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,7 +56,7 @@ COPY --from=build --chown=node:node /app/index.js /app/package.json ./ COPY --from=build --chown=node:node /app/node_modules ./node_modules COPY --from=build --chown=node:node /app/src ./src -VOLUME ["/app/data"] +VOLUME ["/app/data", "/app/logs"] EXPOSE 8081 @@ -75,17 +75,27 @@ ENV MODEL_PROVIDER="databricks" \ LOG_LEVEL="info" \ WORKSPACE_ROOT="/workspace" \ WEB_SEARCH_ENDPOINT="http://searxng:8888/search" \ - NODE_ENV="production" + NODE_ENV="production" \ + REQUEST_JSON_LIMIT="1gb" \ + SESSION_DB_PATH="/app/data/sessions.db" + +# File Logging (persistent logs with pino-roll rotation) +ENV LOG_FILE_ENABLED="false" \ + LOG_FILE_PATH="/app/logs/lynkr.log" \ + LOG_FILE_LEVEL="debug" \ + LOG_FILE_FREQUENCY="daily" \ + LOG_FILE_MAX_FILES="14" # Databricks Configuration (default provider) ENV DATABRICKS_API_BASE="https://example.cloud.databricks.com" \ DATABRICKS_API_KEY="replace-with-databricks-pat" -# Ollama Configuration (for hybrid routing) +# Ollama Configuration (for tier-based routing) # Recommended models: llama3.1:8b, llama3.2, qwen2.5:14b, mistral:7b-instruct -ENV PREFER_OLLAMA="false" \ - OLLAMA_ENDPOINT="http://localhost:11434" \ +# Configure via TIER_* env vars: TIER_SIMPLE=ollama:llama3.2 +ENV OLLAMA_ENDPOINT="http://localhost:11434" \ OLLAMA_MODEL="llama3.1:8b" \ + OLLAMA_TIMEOUT_MS="120000" \ OLLAMA_MAX_TOOLS_FOR_ROUTING="3" \ OLLAMA_EMBEDDINGS_MODEL="nomic-embed-text" \ OLLAMA_EMBEDDINGS_ENDPOINT="http://localhost:11434/api/embeddings" @@ -99,45 +109,99 @@ ENV OPENROUTER_API_KEY="" \ OPENROUTER_MAX_TOOLS_FOR_ROUTING="15" # Azure OpenAI Configuration (optional) -# IMPORTANT: Set full endpoint URL including deployment path -# Example: https://your-resource.openai.azure.com/openai/deployments/YOUR-DEPLOYMENT/chat/completions?api-version=2025-01-01-preview -# Deployment options: gpt-4o, gpt-4o-mini, gpt-5-chat, o1-preview, o3-mini ENV AZURE_OPENAI_ENDPOINT="" \ AZURE_OPENAI_API_KEY="" \ - AZURE_OPENAI_DEPLOYMENT="gpt-4o" + AZURE_OPENAI_DEPLOYMENT="gpt-4o" \ + AZURE_OPENAI_API_VERSION="2024-08-01-preview" # Hybrid Routing & Fallback Configuration -# Options: databricks, azure-openai, azure-anthropic, openrouter, bedrock, openai -# Note: Local providers (ollama, llamacpp, lmstudio) cannot be used as fallback ENV FALLBACK_ENABLED="true" \ FALLBACK_PROVIDER="databricks" # Azure Anthropic Configuration (optional) ENV AZURE_ANTHROPIC_ENDPOINT="" \ - AZURE_ANTHROPIC_API_KEY="" + AZURE_ANTHROPIC_API_KEY="" \ + AZURE_ANTHROPIC_VERSION="2023-06-01" # AWS Bedrock Configuration (optional) -# Supports Claude, Titan, Llama, Jurassic, Cohere, Mistral models ENV AWS_BEDROCK_API_KEY="" \ AWS_BEDROCK_REGION="us-east-1" \ AWS_BEDROCK_MODEL_ID="anthropic.claude-3-5-sonnet-20241022-v2:0" -# llama.cpp Configuration (optional - for local GGUF models) +# llama.cpp Configuration (optional) ENV LLAMACPP_ENDPOINT="http://localhost:8080" \ LLAMACPP_MODEL="default" \ LLAMACPP_EMBEDDINGS_ENDPOINT="http://localhost:8080/embeddings" \ LLAMACPP_TIMEOUT_MS="120000" +# LM Studio Configuration (optional) +ENV LMSTUDIO_ENDPOINT="http://localhost:1234" \ + LMSTUDIO_MODEL="default" \ + LMSTUDIO_TIMEOUT_MS="120000" + # OpenAI Configuration (optional) ENV OPENAI_API_KEY="" \ OPENAI_MODEL="gpt-4o" \ OPENAI_ENDPOINT="https://api.openai.com/v1/chat/completions" +# Z.AI Configuration (optional) +ENV ZAI_API_KEY="" \ + ZAI_ENDPOINT="https://api.z.ai/api/anthropic/v1/messages" \ + ZAI_MODEL="GLM-4.7" + +# Google Vertex AI Configuration (optional) +ENV VERTEX_API_KEY="" \ + VERTEX_MODEL="gemini-2.0-flash" + # Embeddings Provider Override (optional) -# Options: ollama, llamacpp, openrouter, openai -# By default, uses same provider as MODEL_PROVIDER ENV EMBEDDINGS_PROVIDER="" +# Tool Injection & Suggestion Mode +ENV INJECT_TOOLS_LLAMACPP="true" \ + INJECT_TOOLS_OLLAMA="true" \ + SUGGESTION_MODE_MODEL="default" + +# Rate Limiting +ENV RATE_LIMIT_ENABLED="true" \ + RATE_LIMIT_WINDOW_MS="60000" \ + RATE_LIMIT_MAX="100" \ + RATE_LIMIT_KEY_BY="session" + +# Web Search Configuration +ENV WEB_SEARCH_ALLOW_ALL="true" \ + WEB_SEARCH_TIMEOUT_MS="10000" \ + WEB_FETCH_BODY_PREVIEW_MAX="10000" \ + WEB_SEARCH_RETRY_ENABLED="true" \ + WEB_SEARCH_MAX_RETRIES="2" + +# Policy Configuration +ENV POLICY_MAX_STEPS="20" \ + POLICY_MAX_TOOL_CALLS="12" \ + POLICY_TOOL_LOOP_THRESHOLD="10" \ + POLICY_GIT_ALLOW_PUSH="false" \ + POLICY_GIT_ALLOW_PULL="true" \ + POLICY_GIT_ALLOW_COMMIT="true" \ + POLICY_GIT_REQUIRE_TESTS="false" \ + POLICY_GIT_AUTOSTASH="false" \ + POLICY_FILE_BLOCKED_PATHS="/.env,.env,/etc/passwd,/etc/shadow" \ + POLICY_SAFE_COMMANDS_ENABLED="true" + +# Agents Configuration +ENV AGENTS_ENABLED="true" \ + AGENTS_MAX_CONCURRENT="10" \ + AGENTS_DEFAULT_MODEL="haiku" \ + AGENTS_MAX_STEPS="15" \ + AGENTS_TIMEOUT="300000" + +# Prompt Cache Configuration +ENV PROMPT_CACHE_ENABLED="true" \ + PROMPT_CACHE_MAX_ENTRIES="1000" \ + PROMPT_CACHE_TTL_MS="300000" + +# Semantic Response Cache +ENV SEMANTIC_CACHE_ENABLED="false" \ + SEMANTIC_CACHE_THRESHOLD="0.95" + # Production Hardening Defaults ENV CIRCUIT_BREAKER_FAILURE_THRESHOLD="5" \ CIRCUIT_BREAKER_SUCCESS_THRESHOLD="2" \ @@ -160,6 +224,34 @@ ENV MEMORY_ENABLED="true" \ MEMORY_DEDUP_ENABLED="true" \ MEMORY_DEDUP_LOOKBACK="5" +# Token Optimization +ENV TOKEN_TRACKING_ENABLED="true" \ + TOOL_TRUNCATION_ENABLED="true" \ + SYSTEM_PROMPT_MODE="dynamic" \ + TOOL_DESCRIPTIONS="minimal" \ + HISTORY_COMPRESSION_ENABLED="true" \ + HISTORY_KEEP_RECENT_TURNS="10" \ + HISTORY_SUMMARIZE_OLDER="true" \ + TOKEN_BUDGET_WARNING="100000" \ + TOKEN_BUDGET_MAX="180000" \ + TOKEN_BUDGET_ENFORCEMENT="true" + +# Smart Tool Selection +ENV SMART_TOOL_SELECTION_MODE="heuristic" \ + SMART_TOOL_SELECTION_TOKEN_BUDGET="2500" + +# Hot Reload +ENV HOT_RELOAD_ENABLED="true" \ + HOT_RELOAD_DEBOUNCE_MS="1000" + +# Tiered Model Routing (optional) +# Format: TIER_=provider:model +# All 4 tiers must be set to enable tiered routing +# ENV TIER_SIMPLE="ollama:llama3.2" \ +# TIER_MEDIUM="openrouter:openai/gpt-4o-mini" \ +# TIER_COMPLEX="azure-openai:gpt-4o" \ +# TIER_REASONING="azure-openai:gpt-4o" + # Switch to non-root user USER node diff --git a/README.md b/README.md index 69d017a..89dd677 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ Lynkr supports [ClawdBot](https://github.com/openclaw/openclaw) via its OpenAI-c ### Getting Started - 📦 **[Installation Guide](documentation/installation.md)** - Detailed installation for all methods -- ⚙️ **[Provider Configuration](documentation/providers.md)** - Complete setup for all 9+ providers +- ⚙️ **[Provider Configuration](documentation/providers.md)** - Complete setup for all 12+ providers - 🎯 **[Quick Start Examples](documentation/installation.md#quick-start-examples)** - Copy-paste configs ### IDE & CLI Integration @@ -277,7 +277,7 @@ Lynkr supports [ClawdBot](https://github.com/openclaw/openclaw) via its OpenAI-c ## Key Features Highlights -- ✅ **Multi-Provider Support** - 9+ providers including local (Ollama, llama.cpp) and cloud (Bedrock, Databricks, OpenRouter) +- ✅ **Multi-Provider Support** - 12+ providers including local (Ollama, llama.cpp) and cloud (Bedrock, Databricks, OpenRouter, Moonshot AI) - ✅ **60-80% Cost Reduction** - Token optimization with smart tool selection, prompt caching, memory deduplication - ✅ **100% Local Option** - Run completely offline with Ollama/llama.cpp (zero cloud dependencies) - ✅ **OpenAI Compatible** - Works with Cursor IDE, Continue.dev, and any OpenAI-compatible client diff --git a/config/model-tiers.json b/config/model-tiers.json new file mode 100644 index 0000000..94c3812 --- /dev/null +++ b/config/model-tiers.json @@ -0,0 +1,89 @@ +{ + "tiers": { + "SIMPLE": { + "description": "Greetings, simple Q&A, confirmations, basic lookups", + "range": [0, 25], + "priority": 1, + "preferred": { + "ollama": ["llama3.2", "gemma2", "phi3", "qwen2.5:7b", "mistral"], + "llamacpp": ["default"], + "lmstudio": ["default"], + "openai": ["gpt-4o-mini", "gpt-3.5-turbo"], + "azure-openai": ["gpt-4o-mini", "gpt-35-turbo"], + "anthropic": ["claude-3-haiku-20240307", "claude-3-5-haiku-20241022"], + "bedrock": ["anthropic.claude-3-haiku-20240307-v1:0", "amazon.nova-lite-v1:0"], + "databricks": ["databricks-claude-haiku-4-5", "databricks-gpt-5-nano"], + "google": ["gemini-2.0-flash", "gemini-1.5-flash"], + "openrouter": ["google/gemini-flash-1.5", "deepseek/deepseek-chat"], + "zai": ["GLM-4-Flash"], + "moonshot": ["kimi-k2-turbo-preview"] + } + }, + "MEDIUM": { + "description": "Code reading, simple edits, research, documentation", + "range": [26, 50], + "priority": 2, + "preferred": { + "ollama": ["qwen2.5:32b", "deepseek-coder:33b", "codellama:34b"], + "llamacpp": ["default"], + "lmstudio": ["default"], + "openai": ["gpt-4o", "gpt-4-turbo"], + "azure-openai": ["gpt-4o", "gpt-4"], + "anthropic": ["claude-sonnet-4-20250514", "claude-3-5-sonnet-20241022"], + "bedrock": ["anthropic.claude-3-5-sonnet-20241022-v2:0", "amazon.nova-pro-v1:0"], + "databricks": ["databricks-claude-sonnet-4-5", "databricks-gpt-5-1"], + "google": ["gemini-1.5-pro", "gemini-2.0-pro"], + "openrouter": ["anthropic/claude-3.5-sonnet", "openai/gpt-4o"], + "zai": ["GLM-4.7"], + "moonshot": ["kimi-k2-turbo-preview"] + } + }, + "COMPLEX": { + "description": "Multi-file changes, debugging, architecture, refactoring", + "range": [51, 75], + "priority": 3, + "preferred": { + "ollama": ["qwen2.5:72b", "llama3.1:70b", "deepseek-coder-v2:236b"], + "openai": ["o1-mini", "o3-mini", "gpt-4o"], + "azure-openai": ["o1-mini", "gpt-4o"], + "anthropic": ["claude-sonnet-4-20250514", "claude-3-5-sonnet-20241022"], + "bedrock": ["anthropic.claude-3-5-sonnet-20241022-v2:0"], + "databricks": ["databricks-claude-sonnet-4-5", "databricks-gpt-5-1-codex-max"], + "google": ["gemini-2.5-pro", "gemini-1.5-pro"], + "openrouter": ["anthropic/claude-3.5-sonnet", "meta-llama/llama-3.1-405b"], + "zai": ["GLM-4.7"], + "moonshot": ["kimi-k2-turbo-preview"] + } + }, + "REASONING": { + "description": "Complex analysis, security audits, novel problems, deep thinking", + "range": [76, 100], + "priority": 4, + "preferred": { + "openai": ["o1", "o1-pro", "o3"], + "azure-openai": ["o1", "o1-pro"], + "anthropic": ["claude-opus-4-20250514", "claude-3-opus-20240229"], + "bedrock": ["anthropic.claude-3-opus-20240229-v1:0"], + "databricks": ["databricks-claude-opus-4-6", "databricks-claude-opus-4-5", "databricks-gpt-5-2"], + "google": ["gemini-2.5-pro"], + "openrouter": ["anthropic/claude-3-opus", "deepseek/deepseek-reasoner", "openai/o1"], + "deepseek": ["deepseek-reasoner", "deepseek-r1"], + "moonshot": ["kimi-k2-thinking", "kimi-k2-turbo-preview"] + } + } + }, + "localProviders": { + "ollama": { "free": true, "defaultTier": "SIMPLE" }, + "llamacpp": { "free": true, "defaultTier": "SIMPLE" }, + "lmstudio": { "free": true, "defaultTier": "SIMPLE" } + }, + "providerAliases": { + "azure": "azure-openai", + "aws": "bedrock", + "amazon": "bedrock", + "claude": "anthropic", + "gemini": "google", + "vertex": "google", + "kimi": "moonshot" + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 8b7c3d0..8d18466 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,11 @@ services: # - llama3.2 (latest) # - qwen2.5:14b (strong reasoning, 7b struggles with tools) # - mistral:7b-instruct (fast and capable) - PREFER_OLLAMA: ${PREFER_OLLAMA:-true} + # Tier-based routing (set all 4 to enable) + TIER_SIMPLE: ${TIER_SIMPLE:-} + TIER_MEDIUM: ${TIER_MEDIUM:-} + TIER_COMPLEX: ${TIER_COMPLEX:-} + TIER_REASONING: ${TIER_REASONING:-} # OLLAMA_ENDPOINT: http://ollama:11434 OLLAMA_ENDPOINT: http://host.docker.internal:11434 OLLAMA_MODEL: ${OLLAMA_MODEL:-llama3.1:8b} @@ -64,23 +68,15 @@ services: # ============================================================ # AZURE OPENAI CONFIGURATION # ============================================================ - # Required when MODEL_PROVIDER=azure-openai - # IMPORTANT: Use FULL endpoint URL including deployment path and API version - # Format: https://YOUR-RESOURCE.openai.azure.com/openai/deployments/YOUR-DEPLOYMENT/chat/completions?api-version=2025-01-01-preview - # Get credentials from: https://portal.azure.com → Azure OpenAI → Keys and Endpoint - # Deployment options: gpt-4o, gpt-4o-mini, gpt-5-chat, o1-preview, o3-mini AZURE_OPENAI_ENDPOINT: ${AZURE_OPENAI_ENDPOINT:-} AZURE_OPENAI_API_KEY: ${AZURE_OPENAI_API_KEY:-} AZURE_OPENAI_DEPLOYMENT: ${AZURE_OPENAI_DEPLOYMENT:-gpt-4o} + AZURE_OPENAI_API_VERSION: ${AZURE_OPENAI_API_VERSION:-2024-08-01-preview} # ============================================================ # HYBRID ROUTING & FALLBACK # ============================================================ - # Enable/disable fallback to cloud providers FALLBACK_ENABLED: ${FALLBACK_ENABLED:-true} - # Fallback provider when Ollama can't handle request - # Options: databricks, azure-openai, azure-anthropic, openrouter, bedrock, openai - # Note: Local providers (ollama, llamacpp, lmstudio) cannot be used as fallback FALLBACK_PROVIDER: ${FALLBACK_PROVIDER:-databricks} # ============================================================ @@ -94,12 +90,11 @@ services: # ============================================================ AZURE_ANTHROPIC_ENDPOINT: ${AZURE_ANTHROPIC_ENDPOINT:-} AZURE_ANTHROPIC_API_KEY: ${AZURE_ANTHROPIC_API_KEY:-} + AZURE_ANTHROPIC_VERSION: ${AZURE_ANTHROPIC_VERSION:-2023-06-01} # ============================================================ # AWS BEDROCK CONFIGURATION (OPTIONAL) # ============================================================ - # Supports Claude, Titan, Llama, Jurassic, Cohere, Mistral models - # Get API key from AWS Console → Bedrock → API Keys AWS_BEDROCK_API_KEY: ${AWS_BEDROCK_API_KEY:-} AWS_BEDROCK_REGION: ${AWS_BEDROCK_REGION:-us-east-1} AWS_BEDROCK_MODEL_ID: ${AWS_BEDROCK_MODEL_ID:-anthropic.claude-3-5-sonnet-20241022-v2:0} @@ -107,12 +102,18 @@ services: # ============================================================ # LLAMA.CPP CONFIGURATION (OPTIONAL) # ============================================================ - # For local GGUF models LLAMACPP_ENDPOINT: ${LLAMACPP_ENDPOINT:-http://localhost:8080} LLAMACPP_MODEL: ${LLAMACPP_MODEL:-default} LLAMACPP_EMBEDDINGS_ENDPOINT: ${LLAMACPP_EMBEDDINGS_ENDPOINT:-http://localhost:8080/embeddings} LLAMACPP_TIMEOUT_MS: ${LLAMACPP_TIMEOUT_MS:-120000} + # ============================================================ + # LM STUDIO CONFIGURATION (OPTIONAL) + # ============================================================ + LMSTUDIO_ENDPOINT: ${LMSTUDIO_ENDPOINT:-http://localhost:1234} + LMSTUDIO_MODEL: ${LMSTUDIO_MODEL:-default} + LMSTUDIO_TIMEOUT_MS: ${LMSTUDIO_TIMEOUT_MS:-120000} + # ============================================================ # OPENAI CONFIGURATION (OPTIONAL) # ============================================================ @@ -120,11 +121,22 @@ services: OPENAI_MODEL: ${OPENAI_MODEL:-gpt-4o} OPENAI_ENDPOINT: ${OPENAI_ENDPOINT:-https://api.openai.com/v1/chat/completions} + # ============================================================ + # Z.AI CONFIGURATION (OPTIONAL) + # ============================================================ + ZAI_API_KEY: ${ZAI_API_KEY:-} + ZAI_ENDPOINT: ${ZAI_ENDPOINT:-https://api.z.ai/api/anthropic/v1/messages} + ZAI_MODEL: ${ZAI_MODEL:-GLM-4.7} + + # ============================================================ + # GOOGLE VERTEX AI CONFIGURATION (OPTIONAL) + # ============================================================ + VERTEX_API_KEY: ${VERTEX_API_KEY:-} + VERTEX_MODEL: ${VERTEX_MODEL:-gemini-2.0-flash} + # ============================================================ # EMBEDDINGS PROVIDER OVERRIDE (OPTIONAL) # ============================================================ - # Options: ollama, llamacpp, openrouter, openai - # By default, uses same provider as MODEL_PROVIDER EMBEDDINGS_PROVIDER: ${EMBEDDINGS_PROVIDER:-} # ============================================================ @@ -132,16 +144,127 @@ services: # ============================================================ PORT: ${PORT:-8081} LOG_LEVEL: ${LOG_LEVEL:-info} + NODE_ENV: ${NODE_ENV:-production} + REQUEST_JSON_LIMIT: ${REQUEST_JSON_LIMIT:-1gb} + SESSION_DB_PATH: /app/data/sessions.db WEB_SEARCH_ENDPOINT: ${WEB_SEARCH_ENDPOINT:-http://searxng:8888/search} WORKSPACE_ROOT: /workspace # ============================================================ - # PRODUCTION HARDENING (OPTIONAL) + # FILE LOGGING (pino-roll rotation) + # ============================================================ + LOG_FILE_ENABLED: ${LOG_FILE_ENABLED:-false} + LOG_FILE_PATH: /app/logs/lynkr.log + LOG_FILE_LEVEL: ${LOG_FILE_LEVEL:-debug} + LOG_FILE_FREQUENCY: ${LOG_FILE_FREQUENCY:-daily} + LOG_FILE_MAX_FILES: ${LOG_FILE_MAX_FILES:-14} + + # ============================================================ + # TOOL INJECTION & SUGGESTION MODE + # ============================================================ + INJECT_TOOLS_LLAMACPP: ${INJECT_TOOLS_LLAMACPP:-true} + INJECT_TOOLS_OLLAMA: ${INJECT_TOOLS_OLLAMA:-true} + SUGGESTION_MODE_MODEL: ${SUGGESTION_MODE_MODEL:-default} + + # ============================================================ + # RATE LIMITING + # ============================================================ + RATE_LIMIT_ENABLED: ${RATE_LIMIT_ENABLED:-true} + RATE_LIMIT_WINDOW_MS: ${RATE_LIMIT_WINDOW_MS:-60000} + RATE_LIMIT_MAX: ${RATE_LIMIT_MAX:-100} + RATE_LIMIT_KEY_BY: ${RATE_LIMIT_KEY_BY:-session} + + # ============================================================ + # WEB SEARCH + # ============================================================ + WEB_SEARCH_ALLOW_ALL: ${WEB_SEARCH_ALLOW_ALL:-true} + WEB_SEARCH_TIMEOUT_MS: ${WEB_SEARCH_TIMEOUT_MS:-10000} + WEB_FETCH_BODY_PREVIEW_MAX: ${WEB_FETCH_BODY_PREVIEW_MAX:-10000} + WEB_SEARCH_RETRY_ENABLED: ${WEB_SEARCH_RETRY_ENABLED:-true} + WEB_SEARCH_MAX_RETRIES: ${WEB_SEARCH_MAX_RETRIES:-2} + + # ============================================================ + # POLICY CONFIGURATION + # ============================================================ + POLICY_MAX_STEPS: ${POLICY_MAX_STEPS:-20} + POLICY_MAX_TOOL_CALLS: ${POLICY_MAX_TOOL_CALLS:-12} + POLICY_TOOL_LOOP_THRESHOLD: ${POLICY_TOOL_LOOP_THRESHOLD:-10} + POLICY_GIT_ALLOW_PUSH: ${POLICY_GIT_ALLOW_PUSH:-false} + POLICY_GIT_ALLOW_PULL: ${POLICY_GIT_ALLOW_PULL:-true} + POLICY_GIT_ALLOW_COMMIT: ${POLICY_GIT_ALLOW_COMMIT:-true} + POLICY_GIT_REQUIRE_TESTS: ${POLICY_GIT_REQUIRE_TESTS:-false} + POLICY_GIT_AUTOSTASH: ${POLICY_GIT_AUTOSTASH:-false} + POLICY_FILE_BLOCKED_PATHS: ${POLICY_FILE_BLOCKED_PATHS:-/.env,.env,/etc/passwd,/etc/shadow} + POLICY_SAFE_COMMANDS_ENABLED: ${POLICY_SAFE_COMMANDS_ENABLED:-true} + + # ============================================================ + # AGENTS CONFIGURATION + # ============================================================ + AGENTS_ENABLED: ${AGENTS_ENABLED:-true} + AGENTS_MAX_CONCURRENT: ${AGENTS_MAX_CONCURRENT:-10} + AGENTS_DEFAULT_MODEL: ${AGENTS_DEFAULT_MODEL:-haiku} + AGENTS_MAX_STEPS: ${AGENTS_MAX_STEPS:-15} + AGENTS_TIMEOUT: ${AGENTS_TIMEOUT:-300000} + + # ============================================================ + # PROMPT & SEMANTIC CACHE + # ============================================================ + PROMPT_CACHE_ENABLED: ${PROMPT_CACHE_ENABLED:-true} + PROMPT_CACHE_MAX_ENTRIES: ${PROMPT_CACHE_MAX_ENTRIES:-1000} + PROMPT_CACHE_TTL_MS: ${PROMPT_CACHE_TTL_MS:-300000} + SEMANTIC_CACHE_ENABLED: ${SEMANTIC_CACHE_ENABLED:-false} + SEMANTIC_CACHE_THRESHOLD: ${SEMANTIC_CACHE_THRESHOLD:-0.95} + + # ============================================================ + # PRODUCTION HARDENING # ============================================================ CIRCUIT_BREAKER_FAILURE_THRESHOLD: ${CIRCUIT_BREAKER_FAILURE_THRESHOLD:-5} CIRCUIT_BREAKER_TIMEOUT: ${CIRCUIT_BREAKER_TIMEOUT:-60000} LOAD_SHEDDING_MEMORY_THRESHOLD: ${LOAD_SHEDDING_MEMORY_THRESHOLD:-0.85} + # ============================================================ + # LONG-TERM MEMORY (Titans-inspired) + # ============================================================ + MEMORY_ENABLED: ${MEMORY_ENABLED:-true} + MEMORY_RETRIEVAL_LIMIT: ${MEMORY_RETRIEVAL_LIMIT:-5} + MEMORY_SURPRISE_THRESHOLD: ${MEMORY_SURPRISE_THRESHOLD:-0.3} + MEMORY_MAX_AGE_DAYS: ${MEMORY_MAX_AGE_DAYS:-90} + MEMORY_MAX_COUNT: ${MEMORY_MAX_COUNT:-10000} + MEMORY_INCLUDE_GLOBAL: ${MEMORY_INCLUDE_GLOBAL:-true} + MEMORY_INJECTION_FORMAT: ${MEMORY_INJECTION_FORMAT:-system} + MEMORY_EXTRACTION_ENABLED: ${MEMORY_EXTRACTION_ENABLED:-true} + MEMORY_DECAY_ENABLED: ${MEMORY_DECAY_ENABLED:-true} + MEMORY_DECAY_HALF_LIFE: ${MEMORY_DECAY_HALF_LIFE:-30} + + # ============================================================ + # TOKEN OPTIMIZATION (60-80% cost reduction) + # ============================================================ + TOKEN_TRACKING_ENABLED: ${TOKEN_TRACKING_ENABLED:-true} + TOOL_TRUNCATION_ENABLED: ${TOOL_TRUNCATION_ENABLED:-true} + MEMORY_FORMAT: ${MEMORY_FORMAT:-compact} + MEMORY_DEDUP_ENABLED: ${MEMORY_DEDUP_ENABLED:-true} + MEMORY_DEDUP_LOOKBACK: ${MEMORY_DEDUP_LOOKBACK:-5} + SYSTEM_PROMPT_MODE: ${SYSTEM_PROMPT_MODE:-dynamic} + TOOL_DESCRIPTIONS: ${TOOL_DESCRIPTIONS:-minimal} + HISTORY_COMPRESSION_ENABLED: ${HISTORY_COMPRESSION_ENABLED:-true} + HISTORY_KEEP_RECENT_TURNS: ${HISTORY_KEEP_RECENT_TURNS:-10} + HISTORY_SUMMARIZE_OLDER: ${HISTORY_SUMMARIZE_OLDER:-true} + TOKEN_BUDGET_WARNING: ${TOKEN_BUDGET_WARNING:-100000} + TOKEN_BUDGET_MAX: ${TOKEN_BUDGET_MAX:-180000} + TOKEN_BUDGET_ENFORCEMENT: ${TOKEN_BUDGET_ENFORCEMENT:-true} + + # ============================================================ + # SMART TOOL SELECTION + # ============================================================ + SMART_TOOL_SELECTION_MODE: ${SMART_TOOL_SELECTION_MODE:-heuristic} + SMART_TOOL_SELECTION_TOKEN_BUDGET: ${SMART_TOOL_SELECTION_TOKEN_BUDGET:-2500} + + # ============================================================ + # HOT RELOAD + # ============================================================ + HOT_RELOAD_ENABLED: ${HOT_RELOAD_ENABLED:-true} + HOT_RELOAD_DEBOUNCE_MS: ${HOT_RELOAD_DEBOUNCE_MS:-1000} + # ============================================================ # HEADROOM CONTEXT COMPRESSION (OPTIONAL) # ============================================================ @@ -163,9 +286,20 @@ services: HEADROOM_CCR_TTL: ${HEADROOM_CCR_TTL:-300} HEADROOM_LLMLINGUA: ${HEADROOM_LLMLINGUA:-false} + # ============================================================ + # TIERED MODEL ROUTING (OPTIONAL) + # ============================================================ + # Format: TIER_=provider:model + # All 4 must be set to enable tiered routing + # TIER_SIMPLE: ${TIER_SIMPLE:-} + # TIER_MEDIUM: ${TIER_MEDIUM:-} + # TIER_COMPLEX: ${TIER_COMPLEX:-} + # TIER_REASONING: ${TIER_REASONING:-} + volumes: - - ./data:/app/data # Persist SQLite databases - - .:/workspace # Mount workspace + - ./data:/app/data # Persist SQLite databases + - ./logs:/app/logs # Persist log files + - .:/workspace # Mount workspace restart: unless-stopped networks: - lynkr-network diff --git a/docs/docs.html b/docs/docs.html index 71d5d12..22fe761 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -51,6 +51,7 @@
Features
  • Core Features
    • +
  • Routing & Model Tiering
  • Token Optimization
  • Memory System
  • Headroom Compression
    • diff --git a/docs/index.md b/docs/index.md index b907f6c..157824e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -311,6 +311,13 @@ +
    + 🌙 +
    Moonshot AI
    +
    Cloud
    + +
    +
    Z
    z.ai
    diff --git a/documentation/README.md b/documentation/README.md index d872796..06adaba 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -9,7 +9,7 @@ Welcome to the comprehensive documentation for Lynkr, the self-hosted Claude Cod New to Lynkr? Start here: - **[Installation Guide](installation.md)** - Complete installation instructions for all methods (npm, git clone, homebrew, Docker) -- **[Provider Configuration](providers.md)** - Detailed setup for all 9+ supported providers (Databricks, Bedrock, OpenRouter, Ollama, llama.cpp, Azure OpenAI, Azure Anthropic, OpenAI, LM Studio) +- **[Provider Configuration](providers.md)** - Detailed setup for all 12+ supported providers (Databricks, Bedrock, OpenRouter, Ollama, llama.cpp, Azure OpenAI, Azure Anthropic, OpenAI, LM Studio, Moonshot AI, Z.AI, Vertex AI) - **[Quick Start Examples](installation.md#quick-start-examples)** - Copy-paste configurations to get running fast --- @@ -30,6 +30,7 @@ Connect Lynkr to your development tools: Understand Lynkr's capabilities: - **[Architecture & Features](features.md)** - System architecture, request flow, format conversion, and core capabilities +- **[Routing & Model Tiering](routing.md)** - 4-tier model system, 15-dimension complexity scoring, agentic workflow detection, and cost optimization - **[Memory System](memory-system.md)** - Titans-inspired long-term memory with surprise-based filtering and decay - **[Token Optimization](token-optimization.md)** - Achieve 60-80% cost reduction through smart tool selection, prompt caching, and memory deduplication - **[Headroom Compression](headroom.md)** - 47-92% token reduction through intelligent context compression (Smart Crusher, CCR, LLMLingua) @@ -73,7 +74,7 @@ Get help and contribute: - [Installation](installation.md) | [Providers](providers.md) | [Claude Code](claude-code-cli.md) | [Codex CLI](codex-cli.md) | [Cursor](cursor-integration.md) | [Embeddings](embeddings.md) ### Features & Optimization -- [Features](features.md) | [Memory System](memory-system.md) | [Token Optimization](token-optimization.md) | [Headroom](headroom.md) | [Tools](tools.md) +- [Features](features.md) | [Routing](routing.md) | [Memory System](memory-system.md) | [Token Optimization](token-optimization.md) | [Headroom](headroom.md) | [Tools](tools.md) ### Deployment & Production - [Docker](docker.md) | [Production](production.md) | [API Reference](api.md) diff --git a/documentation/claude-code-cli.md b/documentation/claude-code-cli.md index 81035c5..26dc005 100644 --- a/documentation/claude-code-cli.md +++ b/documentation/claude-code-cli.md @@ -11,7 +11,7 @@ Lynkr acts as a drop-in replacement for Anthropic's backend, enabling Claude Cod ### Why Use Lynkr with Claude Code CLI? - 💰 **60-80% cost savings** through token optimization -- 🔓 **Provider choice** - Use any of 9+ supported providers +- 🔓 **Provider choice** - Use any of 12+ supported providers - 🏠 **Self-hosted** - Full control over your AI infrastructure - 🔒 **Local option** - Run 100% offline with Ollama or llama.cpp - ✅ **Zero code changes** - Drop-in replacement for Anthropic backend @@ -74,7 +74,7 @@ export DATABRICKS_API_BASE=https://your-workspace.databricks.com export DATABRICKS_API_KEY=dapi1234567890abcdef ``` -See [Provider Configuration Guide](providers.md) for all 9+ providers. +See [Provider Configuration Guide](providers.md) for all 12+ providers. --- @@ -341,15 +341,16 @@ export MODEL_PROVIDER=databricks --- -## Hybrid Routing (Cost Optimization) +## Tier-Based Routing (Cost Optimization) -Use local Ollama for simple tasks, fallback to cloud for complex ones: +Use local Ollama for simple tasks, cloud for complex ones: ```bash -# Configure hybrid routing -export MODEL_PROVIDER=ollama -export OLLAMA_MODEL=llama3.1:8b -export PREFER_OLLAMA=true +# Configure tier-based routing (set all 4 to enable) +export TIER_SIMPLE=ollama:llama3.2 +export TIER_MEDIUM=openrouter:openai/gpt-4o-mini +export TIER_COMPLEX=databricks:databricks-claude-sonnet-4-5 +export TIER_REASONING=databricks:databricks-claude-sonnet-4-5 export FALLBACK_ENABLED=true export FALLBACK_PROVIDER=databricks export DATABRICKS_API_BASE=https://your-workspace.databricks.com @@ -360,13 +361,15 @@ lynkr start ``` **How it works:** -- **0-2 tools**: Ollama (free, local, fast) -- **3-15 tools**: OpenRouter (if configured) or fallback -- **16+ tools**: Databricks/Azure (most capable) -- **Ollama failures**: Automatic transparent fallback to cloud +- Each request is scored for complexity (0-100) and mapped to a tier +- **SIMPLE (0-25)**: Ollama (free, local, fast) +- **MEDIUM (26-50)**: OpenRouter (affordable cloud) +- **COMPLEX (51-75)**: Databricks (most capable) +- **REASONING (76-100)**: Databricks (best available) +- **Provider failures**: Automatic transparent fallback to cloud **Cost savings:** -- **65-100%** for requests that stay on Ollama +- **65-100%** for requests routed to local models - **40-87%** faster for simple requests --- @@ -534,9 +537,13 @@ claude "What files are in the current directory?" - Local (Ollama): Should be 100-500ms - Cloud: Should be 500ms-2s -2. **Enable hybrid routing:** +2. **Enable tier-based routing:** ```bash - export PREFER_OLLAMA=true + # Set all 4 TIER_* env vars to enable tier-based routing + export TIER_SIMPLE=ollama:llama3.2 + export TIER_MEDIUM=openrouter:openai/gpt-4o-mini + export TIER_COMPLEX=azure-openai:gpt-4o + export TIER_REASONING=azure-openai:gpt-4o export FALLBACK_ENABLED=true ``` @@ -655,7 +662,7 @@ Claude Code CLI (displays result) ## Next Steps -- **[Provider Configuration](providers.md)** - Configure all 9+ providers +- **[Provider Configuration](providers.md)** - Configure all 12+ providers - **[Installation Guide](installation.md)** - Detailed installation - **[Features Guide](features.md)** - Learn about advanced features - **[Token Optimization](token-optimization.md)** - Maximize cost savings diff --git a/documentation/cursor-integration.md b/documentation/cursor-integration.md index 52e50c2..d8f0ff8 100644 --- a/documentation/cursor-integration.md +++ b/documentation/cursor-integration.md @@ -534,11 +534,14 @@ AWS_BEDROCK_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0 - **Cloud** (OpenRouter/Databricks): Should be 500ms-2s - **Distant regions**: Can be 2-5s -2. **Enable hybrid routing** for speed: +2. **Enable tier-based routing** for speed: ```env - # Use Ollama for simple requests (fast) - # Cloud for complex requests - PREFER_OLLAMA=true + # Use Ollama for simple requests (fast), cloud for complex requests + # Set all 4 TIER_* env vars to enable tier-based routing + TIER_SIMPLE=ollama:llama3.2 + TIER_MEDIUM=openrouter:openai/gpt-4o-mini + TIER_COMPLEX=azure-openai:gpt-4o + TIER_REASONING=azure-openai:gpt-4o FALLBACK_ENABLED=true ``` @@ -675,12 +678,12 @@ OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text ### Setup 3: Hybrid (Best of Both Worlds) ```bash -# Chat: Ollama for simple requests, Databricks for complex -PREFER_OLLAMA=true +# Chat: Tier-based routing (set all 4 to enable) +TIER_SIMPLE=ollama:llama3.2 +TIER_MEDIUM=openrouter:openai/gpt-4o-mini +TIER_COMPLEX=databricks:databricks-claude-sonnet-4-5 +TIER_REASONING=databricks:databricks-claude-sonnet-4-5 FALLBACK_ENABLED=true -OLLAMA_MODEL=llama3.1:8b - -# Fallback to Databricks for complex requests FALLBACK_PROVIDER=databricks DATABRICKS_API_BASE=https://your-workspace.databricks.com DATABRICKS_API_KEY=your-key @@ -688,15 +691,15 @@ DATABRICKS_API_KEY=your-key # Embeddings: Ollama (local, private) OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text -# Cost: Mostly FREE (Ollama handles 70-80% of requests) -# Only complex tool-heavy requests go to Databricks +# Cost: Mostly FREE (Ollama handles 70-80% of simple requests) +# Only complex/reasoning requests go to Databricks ``` **Benefits:** -- ✅ Mostly FREE (70-80% of requests on Ollama) +- ✅ Mostly FREE (70-80% of requests on Ollama via TIER_SIMPLE) - ✅ Private embeddings (local search) - ✅ Cloud quality for complex tasks -- ✅ Automatic intelligent routing +- ✅ Automatic intelligent tier-based routing --- @@ -704,7 +707,7 @@ OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text | Aspect | Cursor Native | Lynkr + Cursor | |--------|---------------|----------------| -| **Providers** | OpenAI only | 9+ providers (Bedrock, Databricks, OpenRouter, Ollama, llama.cpp, etc.) | +| **Providers** | OpenAI only | 12+ providers (Bedrock, Databricks, OpenRouter, Ollama, llama.cpp, Moonshot, etc.) | | **Costs** | OpenAI pricing | 60-80% cheaper (or 100% FREE with Ollama) | | **Privacy** | Cloud-only | Can run 100% locally (Ollama + local embeddings) | | **Embeddings** | Built-in (cloud) | 4 options: Ollama (local), llama.cpp (local), OpenRouter (cloud), OpenAI (cloud) | diff --git a/documentation/docker.md b/documentation/docker.md index 8cca33e..6d06d63 100644 --- a/documentation/docker.md +++ b/documentation/docker.md @@ -73,10 +73,14 @@ services: ports: - "8081:8081" environment: - # Hybrid routing: Ollama first, fallback to cloud + # Tier-based routing: local for simple, cloud for complex - MODEL_PROVIDER=ollama - OLLAMA_API_BASE=http://ollama:11434 - - PREFER_OLLAMA=true + # Set all 4 TIER_* vars to enable tier-based routing + - TIER_SIMPLE=ollama:llama3.2 + - TIER_MEDIUM=ollama:llama3.2 + - TIER_COMPLEX=databricks:databricks-claude-sonnet-4-5 + - TIER_REASONING=databricks:databricks-claude-sonnet-4-5 - FALLBACK_ENABLED=true - FALLBACK_PROVIDER=databricks - DATABRICKS_API_BASE=${DATABRICKS_API_BASE} @@ -452,8 +456,11 @@ environment: - DATABRICKS_API_BASE=https://your-workspace.databricks.com - DATABRICKS_API_KEY=${DATABRICKS_API_KEY} - # Hybrid routing - - PREFER_OLLAMA=true + # Tier-based routing (set all 4 to enable) + - TIER_SIMPLE=ollama:llama3.2 + - TIER_MEDIUM=openrouter:openai/gpt-4o-mini + - TIER_COMPLEX=databricks:databricks-claude-sonnet-4-5 + - TIER_REASONING=databricks:databricks-claude-sonnet-4-5 - FALLBACK_ENABLED=true - FALLBACK_PROVIDER=databricks diff --git a/documentation/embeddings.md b/documentation/embeddings.md index 42c7960..65831cc 100644 --- a/documentation/embeddings.md +++ b/documentation/embeddings.md @@ -532,10 +532,12 @@ OPENROUTER_EMBEDDINGS_MODEL=openai/text-embedding-3-small **Best for:** Privacy + Quality + Cost Optimization ```env -# Chat: Ollama + Cloud fallback -PREFER_OLLAMA=true +# Chat: Tier-based routing (set all 4 to enable) +TIER_SIMPLE=ollama:llama3.2 +TIER_MEDIUM=openrouter:openai/gpt-4o-mini +TIER_COMPLEX=databricks:databricks-claude-sonnet-4-5 +TIER_REASONING=databricks:databricks-claude-sonnet-4-5 FALLBACK_ENABLED=true -OLLAMA_MODEL=llama3.1:8b FALLBACK_PROVIDER=databricks DATABRICKS_API_BASE=https://your-workspace.databricks.com DATABRICKS_API_KEY=your-key @@ -547,10 +549,10 @@ OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text ``` **Benefits:** -- ✅ 70-80% of chat requests FREE (Ollama) +- ✅ 70-80% of chat requests FREE (Ollama via TIER_SIMPLE) - ✅ 100% private embeddings (local) - ✅ Cloud quality for complex tasks -- ✅ Intelligent automatic routing +- ✅ Intelligent automatic tier-based routing --- diff --git a/documentation/faq.md b/documentation/faq.md index 22c4aff..b91bc5c 100644 --- a/documentation/faq.md +++ b/documentation/faq.md @@ -8,11 +8,11 @@ Common questions about Lynkr, installation, configuration, and usage. ### What is Lynkr? -Lynkr is a self-hosted proxy server that enables Claude Code CLI and Cursor IDE to work with multiple LLM providers (Databricks, AWS Bedrock, OpenRouter, Ollama, etc.) instead of being locked to Anthropic's API. +Lynkr is a self-hosted proxy server that enables Claude Code CLI and Cursor IDE to work with multiple LLM providers (Databricks, AWS Bedrock, OpenRouter, Ollama, Moonshot AI, etc.) instead of being locked to Anthropic's API. **Key benefits:** - 💰 **60-80% cost savings** through token optimization -- 🔓 **Provider flexibility** - Choose from 9+ providers +- 🔓 **Provider flexibility** - Choose from 12+ providers - 🔒 **Privacy** - Run 100% locally with Ollama or llama.cpp - ✅ **Zero code changes** - Drop-in replacement for Anthropic backend @@ -67,7 +67,7 @@ Lynkr itself is **100% FREE** and open source (Apache 2.0 license). | Feature | Native Claude Code | Lynkr | |---------|-------------------|-------| -| **Providers** | Anthropic only | 9+ providers | +| **Providers** | Anthropic only | 12+ providers | | **Cost** | Full Anthropic pricing | 60-80% cheaper | | **Local models** | ❌ Cloud-only | ✅ Ollama, llama.cpp | | **Privacy** | ☁️ Cloud | 🔒 Can run 100% locally | @@ -126,6 +126,11 @@ See [Installation Guide](installation.md) for all methods. - **Setup:** 5 minutes - **Cost:** ~$10-20/month +**For Affordable Cloud + Reasoning:** +- ✅ **Moonshot AI** - Kimi K2, thinking models +- **Setup:** 2 minutes +- **Cost:** ~$5-10/month + **For Enterprise:** - ✅ **Databricks** - Claude 4.5, enterprise SLA - **Setup:** 10 minutes @@ -137,23 +142,71 @@ See [Provider Configuration Guide](providers.md) for detailed comparison. ### Can I use multiple providers? -**Yes!** Lynkr supports hybrid routing: +**Yes!** Lynkr supports tier-based routing: ```bash -# Use Ollama for simple requests, Databricks for complex ones -export PREFER_OLLAMA=true -export OLLAMA_MODEL=llama3.1:8b +# Set all 4 TIER_* env vars to enable tier-based routing +export TIER_SIMPLE=ollama:llama3.2 +export TIER_MEDIUM=openrouter:openai/gpt-4o-mini +export TIER_COMPLEX=azure-openai:gpt-4o +export TIER_REASONING=azure-openai:gpt-4o export FALLBACK_ENABLED=true export FALLBACK_PROVIDER=databricks ``` **How it works:** -- **0-2 tools**: Ollama (free, local, fast) -- **3-15 tools**: OpenRouter (if configured) or fallback -- **16+ tools**: Databricks/Azure (most capable) -- **Ollama failures**: Automatic transparent fallback +- Each request is scored for complexity (0-100) and mapped to a tier +- **SIMPLE (0-25)**: Ollama (free, local, fast) or Moonshot (affordable cloud) +- **MEDIUM (26-50)**: OpenRouter or mid-range cloud model +- **COMPLEX (51-75)**: Capable cloud models +- **REASONING (76-100)**: Best available models +- **Provider failures**: Automatic transparent fallback + +**Cost savings:** 65-100% for requests routed to local/cheap models. + +--- + +### What is MODEL_PROVIDER and do I still need it? + +`MODEL_PROVIDER` sets a single static provider for all requests. When you set `MODEL_PROVIDER=ollama`, every request goes to Ollama regardless of complexity. + +**With TIER_\* vars configured:** `MODEL_PROVIDER` is not used for routing — the tier system picks the provider per-request. However, `MODEL_PROVIDER` is still read for startup checks (e.g. waiting for Ollama) and as a fallback default in edge cases. Keep it set to your most-used provider. + +**Without TIER_\* vars:** `MODEL_PROVIDER` is the only thing that controls where requests go. + +--- + +### How do MODEL_PROVIDER and TIER_\* work together? + +They are two separate routing modes: + +| Scenario | What happens | +|----------|-------------| +| `MODEL_PROVIDER` only | Static routing — all requests go to that provider | +| All 4 `TIER_*` set | Tier routing — TIER_\* **overrides** MODEL_PROVIDER for routing | +| Only 1-3 `TIER_*` set | Tier routing disabled — falls back to `MODEL_PROVIDER` | +| Both set | TIER_\* takes priority for routing; MODEL_PROVIDER is kept as a config default | + +**Example:** If you have `MODEL_PROVIDER=ollama` and `TIER_COMPLEX=databricks:claude-sonnet`, complex requests go to Databricks even though MODEL_PROVIDER says ollama. + +--- + +### What happens if I only set some TIER_\* vars? + +All 4 must be set (`TIER_SIMPLE`, `TIER_MEDIUM`, `TIER_COMPLEX`, `TIER_REASONING`) for tier routing to activate. If any are missing, tier routing is disabled entirely and `MODEL_PROVIDER` is used for all requests. + +This is intentional — partial tier config could lead to unexpected gaps where some complexity levels have no provider assigned. + +--- + +### What is FALLBACK_PROVIDER? + +The fallback provider is a safety net for when the tier-selected provider fails (timeout, connection refused, rate limit). If `FALLBACK_ENABLED=true` and the primary provider for a request fails, Lynkr retries the request against `FALLBACK_PROVIDER` transparently. -**Cost savings:** 65-100% for requests that stay on Ollama. +- Only triggers when tier routing is active +- Cannot be a local provider (ollama, llamacpp, lmstudio) — use cloud providers +- Defaults to `databricks` +- If you don't have cloud credentials, set `FALLBACK_ENABLED=false` --- @@ -227,6 +280,7 @@ See [Embeddings Guide](embeddings.md) for details. | **OpenRouter** | 500ms-2s | $-$$ | Excellent | Flexibility, 100+ models | | **Databricks/Azure** | 500ms-2s | $$$ | Excellent | Enterprise, Claude 4.5 | | **AWS Bedrock** | 500ms-2s | $-$$$ | Excellent* | AWS, 100+ models | +| **Moonshot AI** | 500ms-2s | $ | Good | Affordable, thinking models | | **OpenAI** | 500ms-2s | $$ | Excellent | GPT-4o, o1, o3 | _* Tool calling only supported by Claude models on Bedrock_ diff --git a/documentation/features.md b/documentation/features.md index ada49b9..2257d57 100644 --- a/documentation/features.md +++ b/documentation/features.md @@ -26,6 +26,7 @@ Complete guide to Lynkr's architecture, request flow, and core capabilities. ├──→ Databricks (Claude 4.5) ├──→ AWS Bedrock (100+ models) ├──→ OpenRouter (100+ models) + ├──→ Moonshot AI (Kimi K2) ├──→ Ollama (local, free) ├──→ llama.cpp (local, free) ├──→ Azure OpenAI (GPT-4o, o1) @@ -52,17 +53,19 @@ Complete guide to Lynkr's architecture, request flow, and core capabilities. ### 2. Provider Routing -**Smart Routing Logic:** +**4-Tier Intelligent Routing:** -```javascript -if (PREFER_OLLAMA && toolCount <= OLLAMA_MAX_TOOLS_FOR_ROUTING) { - provider = "ollama"; // Local, fast, free -} else if (toolCount <= OPENROUTER_MAX_TOOLS_FOR_ROUTING) { - provider = "openrouter"; // Cloud, moderate complexity -} else { - provider = fallbackProvider; // Databricks/Azure, complex -} -``` +Lynkr uses a multi-phase complexity analysis to route each request to the optimal model tier: + +| Tier | Score | Routes To | +|------|-------|-----------| +| SIMPLE (0-25) | Greetings, simple Q&A | Cheap/local models (Ollama, llama.cpp) | +| MEDIUM (26-50) | Code reading, simple edits | Mid-range models (GPT-4o, Claude Sonnet) | +| COMPLEX (51-75) | Multi-file changes, debugging | Capable models (o1-mini, Claude Sonnet) | +| REASONING (76-100) | Security audits, architecture | Best models (o1, Claude Opus) | + +Includes agentic workflow detection, 15-dimension weighted scoring, and cost optimization. +See **[Routing & Model Tiering](routing.md)** for full details. **Automatic Fallback:** - If primary provider fails → Use FALLBACK_PROVIDER @@ -171,6 +174,8 @@ data: {} - `invokeOllama()` - Ollama local - `invokeLlamaCpp()` - llama.cpp - `invokeBedrock()` - AWS Bedrock +- `invokeMoonshot()` - Moonshot AI (Kimi) +- `invokeZai()` - Z.AI (Zhipu AI) **Format converters:** - `openrouter-utils.js` - OpenAI format conversion @@ -271,14 +276,15 @@ data: {} ### 1. Multi-Provider Support -**9+ Providers:** -- Cloud: Databricks, Bedrock, OpenRouter, Azure, OpenAI +**12+ Providers:** +- Cloud: Databricks, Bedrock, OpenRouter, Azure, OpenAI, Moonshot AI, Z.AI, Vertex AI - Local: Ollama, llama.cpp, LM Studio **Hybrid Routing:** -- Automatic provider selection -- Transparent failover -- Cost optimization +- [4-tier intelligent routing](routing.md) with complexity scoring +- Automatic provider selection and transparent failover +- Agentic workflow detection with tier upgrades +- Cost optimization with multi-source pricing ### 2. Token Optimization @@ -383,6 +389,7 @@ PROMPT_CACHE_MAX_ENTRIES=256 ## Next Steps +- **[Routing & Model Tiering](routing.md)** - Intelligent routing and scoring algorithm - **[Memory System](memory-system.md)** - Long-term memory details - **[Token Optimization](token-optimization.md)** - Cost reduction strategies - **[Production Guide](production.md)** - Deploy to production diff --git a/documentation/installation.md b/documentation/installation.md index 477b137..f600747 100644 --- a/documentation/installation.md +++ b/documentation/installation.md @@ -16,6 +16,7 @@ Before installing Lynkr, ensure you have: - **OpenRouter API key** (get from [openrouter.ai/keys](https://openrouter.ai/keys)) - **Azure OpenAI** or **Azure Anthropic** subscription - **OpenAI API key** (get from [platform.openai.com/api-keys](https://platform.openai.com/api-keys)) + - **Moonshot AI API key** (get from [platform.moonshot.ai](https://platform.moonshot.ai)) - **Ollama** installed locally (for free local models) - Optional: **Docker** for containerized deployment or MCP sandboxing - Optional: **Claude Code CLI** (latest release) for CLI usage @@ -236,6 +237,25 @@ MEMORY_RETRIEVAL_LIMIT=5 --- +## Understanding Provider Selection + +Lynkr has two modes for selecting which AI provider handles your requests: + +| Mode | Config | How it works | Best for | +|------|--------|-------------|----------| +| **Static** | `MODEL_PROVIDER=ollama` | All requests go to one provider | Simple setups, single provider | +| **Tier-based** | All 4 `TIER_*` vars set | Requests route by complexity score | Cost optimization, multi-provider | + +**Static mode** — Set `MODEL_PROVIDER` to your provider. Every request goes there. Simple and predictable. + +**Tier-based mode** — Set all 4 `TIER_*` env vars (`TIER_SIMPLE`, `TIER_MEDIUM`, `TIER_COMPLEX`, `TIER_REASONING`). Each request is scored for complexity and routed to the appropriate tier's provider. When all 4 are set, they **override** `MODEL_PROVIDER` for routing decisions. + +> **Note:** If only some `TIER_*` vars are set (not all 4), tier routing is disabled and `MODEL_PROVIDER` is used instead. `MODEL_PROVIDER` is always required as a fallback default even when tiers are configured. + +See [Tier-Based Routing](#tier-based-routing-cost-optimization) below for full setup, or pick a single provider from the Quick Start examples to get running immediately. + +--- + ## Quick Start Examples Choose your provider and follow the setup steps: @@ -501,7 +521,36 @@ lynkr start --- -### 9. LM Studio (Local with GUI) +### 9. Moonshot AI / Kimi (Affordable Cloud) + +**Best for:** Affordable cloud models, thinking/reasoning models + +```bash +# Install +npm install -g lynkr + +# Configure +export MODEL_PROVIDER=moonshot +export MOONSHOT_API_KEY=sk-your-moonshot-api-key +export MOONSHOT_MODEL=kimi-k2-turbo-preview + +# Start +lynkr start +``` + +**Get Moonshot API key:** +1. Visit [platform.moonshot.ai](https://platform.moonshot.ai) +2. Sign up or log in +3. Create a new API key +4. Add credits to your account + +**Available models:** +- `kimi-k2-turbo-preview` - Fast, efficient, tool calling support +- `kimi-k2-thinking` - Chain-of-thought reasoning model + +--- + +### 10. LM Studio (Local with GUI) **Best for:** Local models with graphical interface @@ -525,19 +574,20 @@ lynkr start --- -## Hybrid Routing (Cost Optimization) +## Tier-Based Routing (Cost Optimization) -**Use local Ollama for simple tasks, fallback to cloud for complex ones:** +**Use local Ollama for simple tasks, cloud for complex ones:** ```bash # Start Ollama ollama serve -ollama pull llama3.1:8b +ollama pull llama3.2 -# Configure hybrid routing -export MODEL_PROVIDER=ollama -export OLLAMA_MODEL=llama3.1:8b -export PREFER_OLLAMA=true +# Configure tier-based routing (set all 4 to enable) +export TIER_SIMPLE=ollama:llama3.2 +export TIER_MEDIUM=openrouter:openai/gpt-4o-mini +export TIER_COMPLEX=databricks:databricks-claude-sonnet-4-5 +export TIER_REASONING=databricks:databricks-claude-sonnet-4-5 export FALLBACK_ENABLED=true export FALLBACK_PROVIDER=databricks export DATABRICKS_API_BASE=https://your-workspace.databricks.com @@ -548,13 +598,15 @@ lynkr start ``` **How it works:** -- **0-2 tools**: Ollama (free, local, fast) -- **3-15 tools**: OpenRouter (if configured) or fallback to Databricks -- **16+ tools**: Databricks/Azure (most capable) -- **Ollama failures**: Automatic transparent fallback to cloud +- Each request is scored for complexity (0-100) and mapped to a tier +- **SIMPLE (0-25)**: Ollama (free, local, fast) +- **MEDIUM (26-50)**: OpenRouter (affordable cloud) +- **COMPLEX (51-75)**: Databricks (most capable) +- **REASONING (76-100)**: Databricks (best available) +- **Provider failures**: Automatic transparent fallback to cloud **Cost savings:** -- **65-100%** for requests that stay on Ollama +- **65-100%** for requests routed to local models - **40-87%** faster for simple requests - **Privacy**: Simple queries never leave your machine @@ -614,7 +666,7 @@ See [Provider Configuration Guide](providers.md) for complete environment variab | Variable | Description | Default | |----------|-------------|---------| -| `MODEL_PROVIDER` | Provider to use (`databricks`, `bedrock`, `openrouter`, `ollama`, `llamacpp`, `azure-openai`, `azure-anthropic`, `openai`, `lmstudio`) | `databricks` | +| `MODEL_PROVIDER` | Provider to use (`databricks`, `bedrock`, `openrouter`, `ollama`, `llamacpp`, `azure-openai`, `azure-anthropic`, `openai`, `lmstudio`, `moonshot`, `zai`, `vertex`) | `databricks` | | `PORT` | HTTP port for proxy server | `8081` | | `WORKSPACE_ROOT` | Workspace directory path | `process.cwd()` | | `LOG_LEVEL` | Logging level (`error`, `warn`, `info`, `debug`) | `info` | diff --git a/documentation/production.md b/documentation/production.md index 50b4e11..8f74031 100644 --- a/documentation/production.md +++ b/documentation/production.md @@ -190,15 +190,35 @@ METRICS_ENABLED=true # default: true ### 6. Structured Logging -JSON logs with request ID correlation. +JSON logs with request ID correlation via [Pino](https://github.com/pinojs/pino). + +**Log Level Philosophy:** +- **`info`** — Meaningful milestones: request received (minimal), request completed (duration + tokens), errors, retries, fallbacks +- **`debug`** — Operational details: request body previews, tool injection, streaming chunks, intermediate conversions, tool mapping + +**Console Configuration:** +```bash +LOG_LEVEL=info # options: error, warn, info, debug (default: info) +REQUEST_LOGGING_ENABLED=true # default: true +``` + +In development mode (`NODE_ENV=development`), logs are pretty-printed via `pino-pretty`. + +**File Logging (optional):** + +Persistent log files with automatic daily rotation via [pino-roll](https://github.com/pinojs/pino-roll). Enable by setting `LOG_FILE_ENABLED=true`. -**Configuration:** ```bash -LOG_LEVEL=info # options: error, warn, info, debug -REQUEST_LOGGING_ENABLED=true # default: true +LOG_FILE_ENABLED=true # default: false +LOG_FILE_PATH=./logs/lynkr.log # default: /logs/lynkr.log +LOG_FILE_LEVEL=debug # default: debug (captures all levels) +LOG_FILE_FREQUENCY=daily # options: daily, hourly, custom (default: daily) +LOG_FILE_MAX_FILES=14 # rotated files to keep (default: 14) ``` -**Log format:** +Rotated files are named with timestamps (e.g., `lynkr.log.2025-07-12`). The log directory is created automatically. + +**Log format (JSON):** ```json { "level": "info", @@ -216,10 +236,25 @@ REQUEST_LOGGING_ENABLED=true # default: true } ``` +**Querying log files:** +```bash +# Tail live logs +tail -f ./logs/lynkr.log | npx pino-pretty + +# Find errors in the last 24 hours +cat ./logs/lynkr.log | jq 'select(.level >= 50)' + +# Filter by provider +cat ./logs/lynkr.log | jq 'select(.provider == "databricks")' + +# Search for slow requests (>2s) +cat ./logs/lynkr.log | jq 'select(.duration > 2000)' +``` + **Log aggregation:** -- Stdout (captured by Docker/K8s) -- Parse with structured log tools -- Send to Elasticsearch, Splunk, etc. +- **Stdout** — Captured by Docker/K8s log drivers +- **File rotation** — For standalone deployments or local debugging +- **External** — Forward JSON logs to Elasticsearch, Splunk, Grafana Loki, etc. ### 7. Health Checks diff --git a/documentation/providers.md b/documentation/providers.md index 892e293..6ef7450 100644 --- a/documentation/providers.md +++ b/documentation/providers.md @@ -1,6 +1,6 @@ # Provider Configuration Guide -Complete configuration reference for all 9+ supported LLM providers. Each provider section includes setup instructions, model options, pricing, and example configurations. +Complete configuration reference for all 12+ supported LLM providers. Each provider section includes setup instructions, model options, pricing, and example configurations. --- @@ -18,6 +18,7 @@ Lynkr supports multiple AI model providers, giving you flexibility in choosing t | **Azure OpenAI** | Cloud | GPT-4o, GPT-5, o1, o3 | $$$ | Cloud | Medium | | **Azure Anthropic** | Cloud | Claude models | $$$ | Cloud | Medium | | **OpenAI** | Cloud | GPT-4o, o1, o3 | $$$ | Cloud | Easy | +| **Moonshot AI (Kimi)** | Cloud | Kimi K2 (thinking + turbo) | $ | Cloud | Easy | | **LM Studio** | Local | Local models with GUI | **FREE** | 🔒 100% Local | Easy | | **MLX OpenAI Server** | Local | Apple Silicon optimized | **FREE** | 🔒 100% Local | Easy | @@ -25,7 +26,11 @@ Lynkr supports multiple AI model providers, giving you flexibility in choosing t ## Configuration Methods -### Environment Variables (Quick Start) +There are two routing modes. Choose based on your needs: + +### Static Routing (Single Provider) + +Set `MODEL_PROVIDER` to send all requests to one provider. All requests go to this provider regardless of complexity: ```bash export MODEL_PROVIDER=databricks @@ -34,6 +39,23 @@ export DATABRICKS_API_KEY=your-key lynkr start ``` +### Tier-Based Routing (Recommended for Cost Optimization) + +Set **all 4** `TIER_*` vars to route requests by complexity. Each request is scored 0-100 and routed to the `provider:model` matching its complexity tier. When all four are configured, they **override** `MODEL_PROVIDER` for routing decisions: + +```bash +export MODEL_PROVIDER=ollama # Still needed for startup checks +export TIER_SIMPLE=ollama:llama3.2 # Score 0-25 → local (free) +export TIER_MEDIUM=openrouter:openai/gpt-4o-mini # Score 26-50 → affordable cloud +export TIER_COMPLEX=databricks:claude-sonnet # Score 51-75 → capable cloud +export TIER_REASONING=databricks:claude-sonnet # Score 76-100 → best available +lynkr start +``` + +> **Important:** All 4 `TIER_*` vars must be set to enable tier routing. If any are missing, tier routing is disabled and `MODEL_PROVIDER` is used for all requests. `MODEL_PROVIDER` should always be set — even with tier routing active, it is used for startup checks, provider discovery, and as the default provider when a `TIER_*` value has no `provider:` prefix. +> +> **`PREFER_OLLAMA` is deprecated** and has no effect. Use `TIER_SIMPLE=ollama:` to route simple requests to Ollama. See [Routing Precedence](routing.md#routing-precedence) for full details. + ### .env File (Recommended for Production) ```bash @@ -46,11 +68,17 @@ nano .env Example `.env`: ```env -MODEL_PROVIDER=databricks +MODEL_PROVIDER=ollama DATABRICKS_API_BASE=https://your-workspace.databricks.com DATABRICKS_API_KEY=dapi1234567890abcdef PORT=8081 LOG_LEVEL=info + +# Tier routing (optional — set all 4 to enable) +TIER_SIMPLE=ollama:llama3.2 +TIER_MEDIUM=openrouter:openai/gpt-4o-mini +TIER_COMPLEX=databricks:claude-sonnet +TIER_REASONING=databricks:claude-sonnet ``` --- @@ -685,7 +713,82 @@ LMSTUDIO_API_KEY=your-optional-api-key --- -### 10. MLX OpenAI Server (Apple Silicon) +### 10. Moonshot AI / Kimi (OpenAI-Compatible) + +**Best for:** Affordable cloud models, thinking/reasoning models, OpenAI-compatible API + +#### Configuration + +```env +MODEL_PROVIDER=moonshot +MOONSHOT_API_KEY=sk-your-moonshot-api-key +MOONSHOT_ENDPOINT=https://api.moonshot.ai/v1/chat/completions +MOONSHOT_MODEL=kimi-k2-turbo-preview +``` + +#### Getting Moonshot API Key + +1. Visit [platform.moonshot.ai](https://platform.moonshot.ai) +2. Sign up or log in +3. Navigate to API Keys section +4. Create a new API key +5. Add credits to your account + +#### Available Models + +```env +MOONSHOT_MODEL=kimi-k2-turbo-preview # Fast, efficient (recommended) +MOONSHOT_MODEL=kimi-k2-thinking # Chain-of-thought reasoning model +``` + +**Model Details:** + +| Model | Type | Best For | +|-------|------|----------| +| `kimi-k2-turbo-preview` | Standard | Fast responses, tool calling, general tasks | +| `kimi-k2-thinking` | Thinking/Reasoning | Complex analysis, multi-step reasoning | + +#### How It Works + +Moonshot uses an **OpenAI-compatible** chat completions API. Lynkr handles all format conversion automatically: + +1. Claude Code CLI sends Anthropic-format request to Lynkr +2. Lynkr converts Anthropic messages → OpenAI chat completions format +3. Request is sent to Moonshot's `/v1/chat/completions` endpoint +4. Moonshot response is converted back to Anthropic format +5. Claude Code CLI receives a standard Anthropic response + +#### Thinking Model Support + +When using `kimi-k2-thinking`, the model returns both `reasoning_content` (chain-of-thought) and `content` (final answer). Lynkr automatically extracts only the final answer for clean CLI output. The reasoning content is used as a fallback only when the final answer is empty. + +#### Important Notes + +- **Streaming:** Streaming is disabled for Moonshot (responses arrive as complete JSON). This ensures clean terminal rendering since OpenAI SSE → Anthropic SSE conversion is not yet implemented. +- **Rate Limits:** Moonshot has a max concurrency of ~3 requests. Lynkr retries with backoff on 429 errors. +- **Tool Calling:** Full tool calling support via OpenAI function calling format (automatically converted from Anthropic format). +- **System Messages:** Moonshot natively supports the `system` role, so system prompts are passed directly. + +#### Benefits + +- ✅ **Affordable** — Competitive pricing for capable models +- ✅ **Thinking models** — Chain-of-thought reasoning with `kimi-k2-thinking` +- ✅ **Full tool calling** — Native function calling support +- ✅ **OpenAI-compatible** — Standard chat completions API +- ✅ **System role support** — Native system message handling + +#### Test Connection + +```bash +curl -X POST https://api.moonshot.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $MOONSHOT_API_KEY" \ + -d '{"model":"kimi-k2-turbo-preview","messages":[{"role":"user","content":"Hello"}]}' +``` + +--- + +### 11. MLX OpenAI Server (Apple Silicon) **Best for:** Maximum performance on Apple Silicon Macs (M1/M2/M3/M4) @@ -776,56 +879,53 @@ curl -X POST http://localhost:8000/v1/chat/completions -H "Content-Type: applica --- -## Hybrid Routing & Fallback +## Tier-Based Routing & Fallback -### Intelligent 3-Tier Routing +### Intelligent 4-Tier Routing Optimize costs by routing requests based on complexity: ```env -# Enable hybrid routing -PREFER_OLLAMA=true -FALLBACK_ENABLED=true +# Tier-based routing (set all 4 to enable) +TIER_SIMPLE=ollama:llama3.2 +TIER_MEDIUM=openrouter:openai/gpt-4o-mini +TIER_COMPLEX=azure-openai:gpt-4o +TIER_REASONING=azure-openai:gpt-4o -# Configure providers for each tier -MODEL_PROVIDER=ollama -OLLAMA_MODEL=llama3.1:8b -OLLAMA_MAX_TOOLS_FOR_ROUTING=3 +FALLBACK_ENABLED=true -# Mid-tier (moderate complexity) +# Provider credentials +OLLAMA_ENDPOINT=http://localhost:11434 OPENROUTER_API_KEY=your-key -OPENROUTER_MODEL=openai/gpt-4o-mini -OPENROUTER_MAX_TOOLS_FOR_ROUTING=15 - -# Heavy workload (complex requests) -FALLBACK_PROVIDER=databricks -DATABRICKS_API_BASE=your-base -DATABRICKS_API_KEY=your-key +AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/... +AZURE_OPENAI_API_KEY=your-key ``` ### How It Works **Routing Logic:** -1. **0-2 tools**: Try Ollama first (free, local, fast) -2. **3-15 tools**: Route to OpenRouter (affordable cloud) -3. **16+ tools**: Route directly to Databricks/Azure (most capable) +1. Each request is scored for complexity (0-100) +2. Score maps to a tier: SIMPLE (0-25), MEDIUM (26-50), COMPLEX (51-75), REASONING (76-100) +3. The request is routed to the provider:model configured for that tier **Automatic Fallback:** -- ❌ If Ollama fails → Fallback to OpenRouter or Databricks -- ❌ If OpenRouter fails → Fallback to Databricks -- ✅ Transparent to the user +- If the selected provider fails, Lynkr falls back to `FALLBACK_PROVIDER` +- Transparent to the user ### Cost Savings -- **65-100%** for requests that stay on Ollama +- **65-100%** for requests routed to local/cheap models - **40-87%** faster for simple requests -- **Privacy**: Simple queries never leave your machine +- **Privacy**: Simple queries can stay on your machine when using a local TIER_SIMPLE model ### Configuration Options | Variable | Description | Default | |----------|-------------|---------| -| `PREFER_OLLAMA` | Enable Ollama preference for simple requests | `false` | +| `TIER_SIMPLE` | Model for simple tier (`provider:model`) | *required for tier routing* | +| `TIER_MEDIUM` | Model for medium tier (`provider:model`) | *required for tier routing* | +| `TIER_COMPLEX` | Model for complex tier (`provider:model`) | *required for tier routing* | +| `TIER_REASONING` | Model for reasoning tier (`provider:model`) | *required for tier routing* | | `FALLBACK_ENABLED` | Enable automatic fallback | `true` | | `FALLBACK_PROVIDER` | Provider to use when primary fails | `databricks` | | `OLLAMA_MAX_TOOLS_FOR_ROUTING` | Max tools to route to Ollama | `3` | @@ -841,7 +941,7 @@ DATABRICKS_API_KEY=your-key | Variable | Description | Default | |----------|-------------|---------| -| `MODEL_PROVIDER` | Primary provider (`databricks`, `bedrock`, `openrouter`, `ollama`, `llamacpp`, `azure-openai`, `azure-anthropic`, `openai`, `lmstudio`) | `databricks` | +| `MODEL_PROVIDER` | Primary provider (`databricks`, `bedrock`, `openrouter`, `ollama`, `llamacpp`, `azure-openai`, `azure-anthropic`, `openai`, `lmstudio`, `zai`, `moonshot`, `vertex`) | `databricks` | | `PORT` | HTTP port for proxy server | `8081` | | `WORKSPACE_ROOT` | Workspace directory path | `process.cwd()` | | `LOG_LEVEL` | Logging level (`error`, `warn`, `info`, `debug`) | `info` | @@ -858,17 +958,19 @@ See individual provider sections above for complete variable lists. ### Feature Comparison -| Feature | Databricks | Bedrock | OpenAI | Azure OpenAI | Azure Anthropic | OpenRouter | Ollama | llama.cpp | LM Studio | -|---------|-----------|---------|--------|--------------|-----------------|------------|--------|-----------|-----------| -| **Setup Complexity** | Medium | Easy | Easy | Medium | Medium | Easy | Easy | Medium | Easy | -| **Cost** | $$$ | $-$$$ | $$ | $$ | $$$ | $-$$ | **Free** | **Free** | **Free** | -| **Latency** | Low | Low | Low | Low | Low | Medium | **Very Low** | **Very Low** | **Very Low** | -| **Model Variety** | 2 | **100+** | 10+ | 10+ | 2 | **100+** | 50+ | Unlimited | 50+ | -| **Tool Calling** | Excellent | Excellent* | Excellent | Excellent | Excellent | Good | Fair | Good | Fair | -| **Context Length** | 200K | Up to 300K | 128K | 128K | 200K | Varies | 32K-128K | Model-dependent | 32K-128K | -| **Streaming** | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | -| **Privacy** | Enterprise | Enterprise | Third-party | Enterprise | Enterprise | Third-party | **Local** | **Local** | **Local** | -| **Offline** | No | No | No | No | No | No | **Yes** | **Yes** | **Yes** | +| Feature | Databricks | Bedrock | OpenAI | Azure OpenAI | Azure Anthropic | OpenRouter | Moonshot | Ollama | llama.cpp | LM Studio | +|---------|-----------|---------|--------|--------------|-----------------|------------|----------|--------|-----------|-----------| +| **Setup Complexity** | Medium | Easy | Easy | Medium | Medium | Easy | Easy | Easy | Medium | Easy | +| **Cost** | $$$ | $-$$$ | $$ | $$ | $$$ | $-$$ | $ | **Free** | **Free** | **Free** | +| **Latency** | Low | Low | Low | Low | Low | Medium | Low | **Very Low** | **Very Low** | **Very Low** | +| **Model Variety** | 2 | **100+** | 10+ | 10+ | 2 | **100+** | 2+ | 50+ | Unlimited | 50+ | +| **Tool Calling** | Excellent | Excellent* | Excellent | Excellent | Excellent | Good | Good | Fair | Good | Fair | +| **Context Length** | 200K | Up to 300K | 128K | 128K | 200K | Varies | 128K | 32K-128K | Model-dependent | 32K-128K | +| **Streaming** | Yes | Yes | Yes | Yes | Yes | Yes | Non-streaming** | Yes | Yes | Yes | +| **Privacy** | Enterprise | Enterprise | Third-party | Enterprise | Enterprise | Third-party | Third-party | **Local** | **Local** | **Local** | +| **Offline** | No | No | No | No | No | No | No | **Yes** | **Yes** | **Yes** | + +_** Moonshot uses non-streaming mode (responses arrive as complete JSON) for clean terminal rendering_ _* Tool calling only supported by Claude models on Bedrock_ @@ -882,6 +984,7 @@ _* Tool calling only supported by Claude models on Bedrock_ | **OpenRouter** | GPT-4o mini | $0.15 | $0.60 | | **OpenAI** | GPT-4o | $2.50 | $10.00 | | **Azure OpenAI** | GPT-4o | $2.50 | $10.00 | +| **Moonshot** | Kimi K2 Turbo | See moonshot.ai | See moonshot.ai | | **Ollama** | Any model | **FREE** | **FREE** | | **llama.cpp** | Any model | **FREE** | **FREE** | | **LM Studio** | Any model | **FREE** | **FREE** | diff --git a/documentation/routing.md b/documentation/routing.md new file mode 100644 index 0000000..c937f9a --- /dev/null +++ b/documentation/routing.md @@ -0,0 +1,476 @@ +# Intelligent Routing & Model Tiering + +Lynkr's intelligent routing system automatically selects the optimal model and provider for each request based on complexity analysis, agentic workflow detection, and cost optimization. + +--- + +## Overview + +``` +Request → Force Patterns → Tool Thresholds → Complexity Analysis → Agentic Detection → Tier Selection → Cost Optimization → Provider +``` + +The routing pipeline evaluates every incoming request through multiple stages to determine which model tier and provider should handle it. Simple requests go to cheap/local models, complex ones go to powerful cloud models. + +**Key benefits:** +- 60-80% cost reduction by routing simple tasks to cheaper models +- Better quality on complex tasks by using capable models when needed +- Automatic agentic workflow detection with tier upgrades +- Multi-source pricing for optimal cost decisions + +--- + +## 4-Tier Model System + +Every request is mapped to one of four complexity tiers: + +| Tier | Score Range | Description | Example Tasks | +|------|-----------|-------------|---------------| +| **SIMPLE** | 0-25 | Greetings, simple Q&A, confirmations | "Hello", "What is a variable?", "Yes" | +| **MEDIUM** | 26-50 | Code reading, simple edits, research | "Read this file", "Fix this typo", "Search for X" | +| **COMPLEX** | 51-75 | Multi-file changes, debugging, architecture | "Refactor auth module", "Debug this race condition" | +| **REASONING** | 76-100 | Complex analysis, security audits, novel problems | "Security audit", "Design microservices architecture" | + +### Configuration + +Tiers are configured via mandatory environment variables in `provider:model` format: + +```bash +# Required - one per tier +TIER_SIMPLE=ollama:llama3.2 +TIER_MEDIUM=openai:gpt-4o +TIER_COMPLEX=openai:o1-mini +TIER_REASONING=openai:o1 + +# Examples with other providers +TIER_SIMPLE=ollama:qwen2.5-coder +TIER_MEDIUM=databricks:databricks-claude-sonnet-4-5 +TIER_COMPLEX=azure-openai:gpt-5.2-chat +TIER_REASONING=databricks:databricks-claude-opus-4-6 +``` + +If a model name is given without a provider prefix, the default provider (`MODEL_PROVIDER`) is used. + +### Routing Precedence + +There are three routing-related settings. Here is exactly how they interact: + +#### 1. `TIER_*` Environment Variables (Highest Priority) + +When **all four** `TIER_*` vars are set (`TIER_SIMPLE`, `TIER_MEDIUM`, `TIER_COMPLEX`, `TIER_REASONING`), tiered routing is **active**. Every incoming request is scored for complexity (0-100), mapped to a tier, and routed to the `provider:model` specified in the matching `TIER_*` var. + +In this mode, `MODEL_PROVIDER` is **not consulted** for routing decisions. The provider comes directly from the `TIER_*` value (e.g., `ollama:llama3.2` routes to Ollama, `openai:gpt-4o` routes to OpenAI). + +If any of the four `TIER_*` vars are missing, tiered routing is **completely disabled** and the system falls back to `MODEL_PROVIDER`. + +#### 2. `MODEL_PROVIDER` (Default / Fallback) + +`MODEL_PROVIDER` controls routing in two scenarios: + +- **When tiered routing is disabled** (any `TIER_*` var missing) — all requests go to the provider set in `MODEL_PROVIDER`, regardless of complexity. This is static routing. +- **When a `TIER_*` value has no provider prefix** (e.g., `TIER_SIMPLE=llama3.2` instead of `TIER_SIMPLE=ollama:llama3.2`) — `MODEL_PROVIDER` is used as the default provider for that tier. + +Even when tiered routing is active and overrides it for request routing, `MODEL_PROVIDER` is still used for: +- **Startup checks** — e.g., if `MODEL_PROVIDER=ollama`, the server waits for Ollama to be reachable before accepting requests +- **Provider discovery API** (`/v1/providers`) — marks which provider is "primary" in the response +- **Embeddings routing** — the OpenAI-compatible router checks `MODEL_PROVIDER` for embedding provider selection + +**Always set `MODEL_PROVIDER`** even when using tier routing. + +#### 3. `PREFER_OLLAMA` (Removed) + +`PREFER_OLLAMA` is **deprecated and has no effect**. If set, a warning is logged at startup: + +``` +[DEPRECATION] PREFER_OLLAMA is removed. Use TIER_* env vars for routing. +``` + +To route simple requests to Ollama, use `TIER_SIMPLE=ollama:` instead. + +#### Summary Table + +| Configuration | Routing Behavior | +|---|---| +| All 4 `TIER_*` set | Tier routing active. Each request scored and routed to its tier's `provider:model`. `MODEL_PROVIDER` ignored for routing. | +| 1-3 `TIER_*` set | Tier routing **disabled**. All requests go to `MODEL_PROVIDER` (static). | +| No `TIER_*` set | Static routing. All requests go to `MODEL_PROVIDER`. | +| `TIER_*` value without provider prefix | `MODEL_PROVIDER` used as the default provider for that tier. | +| `PREFER_OLLAMA` set | No effect. Deprecation warning logged. | + +#### Example: Mixed Local + Cloud Setup + +```bash +MODEL_PROVIDER=ollama # Startup checks + default provider +TIER_SIMPLE=ollama:llama3.2 # Score 0-25 → Ollama (free, local) +TIER_MEDIUM=openai:gpt-4o # Score 26-50 → OpenAI +TIER_COMPLEX=databricks:claude-sonnet-4-5 # Score 51-75 → Databricks +TIER_REASONING=databricks:claude-opus-4-6 # Score 76-100 → Databricks +``` + +In this setup, a "Hello" message (score ~5) routes to Ollama. A "Refactor the auth module" message (score ~65) routes to Databricks. `MODEL_PROVIDER=ollama` ensures the server waits for Ollama at startup but does not affect where complex requests go. + +### Tier Config File + +Additional tier preferences (fallback models per provider) can be defined in `config/model-tiers.json`: + +```json +{ + "tiers": { + "SIMPLE": { "preferred": { "ollama": ["llama3.2"], "openai": ["gpt-4o-mini"] } }, + "MEDIUM": { "preferred": { "openai": ["gpt-4o"], "anthropic": ["claude-sonnet-4-20250514"] } }, + "COMPLEX": { "preferred": { "openai": ["o1-mini"], "anthropic": ["claude-sonnet-4-20250514"] } }, + "REASONING": { "preferred": { "openai": ["o1"], "anthropic": ["claude-opus-4-20250514"] } } + }, + "localProviders": { + "ollama": { "free": true, "defaultTier": "SIMPLE" }, + "llamacpp": { "free": true, "defaultTier": "SIMPLE" }, + "lmstudio": { "free": true, "defaultTier": "SIMPLE" } + } +} +``` + +--- + +## Complexity Scoring Algorithm + +The complexity analyzer implements 4 phases to produce a score from 0-100. + +### Phase 1: Basic Scoring + +Three components scored independently: + +**Token Count (0-20 points):** + +| Tokens | Score | +|--------|-------| +| < 500 | 0 | +| 500-999 | 4 | +| 1,000-1,999 | 8 | +| 2,000-3,999 | 12 | +| 4,000-7,999 | 16 | +| 8,000+ | 20 | + +**Tool Count (0-20 points):** + +| Tools | Score | +|-------|-------| +| 0 | 0 | +| 1-3 | 4 | +| 4-6 | 8 | +| 7-10 | 12 | +| 11-15 | 16 | +| 16+ | 20 | + +**Task Type (0-25 points):** +- Greetings / yes-no: 0-2 +- Simple questions: 3 +- General non-technical: 5 +- Technical content: 10 +- Refactoring: 16 +- New implementation: 18 +- From scratch: 20 +- Entire codebase scope: 22 +- Force cloud patterns (security audit, architecture review): 25 + +### Phase 2: Advanced Classification + +Additional scoring on top of Phase 1: + +**Code Complexity (0-20 points):** + +| Pattern | Points | +|---------|--------| +| Multi-file operations | +5 | +| Architecture concerns | +5 | +| Security | +4 | +| Concurrency | +3 | +| Performance | +3 | +| Database operations | +3 | +| Testing | +2 | + +**Reasoning Requirements (0-15 points):** + +| Pattern | Points | +|---------|--------| +| Step-by-step reasoning | +4 | +| Trade-off analysis | +4 | +| General analysis | +3 | +| Planning | +3 | +| Edge cases | +2 | + +**Conversation Bonus:** +- 6-10 messages: +2 +- 11+ messages: +5 + +The standard score is the sum of all components, capped at 100. + +### Weighted Scoring Mode (15 Dimensions) + +When `ROUTING_WEIGHTED_SCORING=true`, the analyzer uses a 15-dimension weighted scoring system instead of the standard additive scoring: + +``` +Score = Sum of (dimension_value * weight) for all 15 dimensions +``` + +#### Dimension Weights + +**Content Analysis (35% total):** + +| Dimension | Weight | Measures | +|-----------|--------|----------| +| tokenCount | 0.08 | Request size (token estimate) | +| promptComplexity | 0.10 | Sentence structure, average length | +| technicalDepth | 0.10 | Technical keyword density | +| domainSpecificity | 0.07 | Number of specialized domains (security, ML, distributed, database, frontend, devops) | + +**Tool Analysis (25% total):** + +| Dimension | Weight | Measures | +|-----------|--------|----------| +| toolCount | 0.08 | Number of tools in request | +| toolComplexity | 0.10 | Weighted average of tool complexity (Bash=0.9, Write=0.8, Edit=0.7, Read=0.3, Glob/Grep=0.2) | +| toolChainPotential | 0.07 | Sequential operation indicators ("then", "after", "step 1") | + +**Reasoning Requirements (25% total):** + +| Dimension | Weight | Measures | +|-----------|--------|----------| +| multiStepReasoning | 0.10 | Step-by-step / planning patterns | +| codeGeneration | 0.08 | Code creation requests | +| analysisDepth | 0.07 | Trade-off / analysis patterns | + +**Context Factors (15% total):** + +| Dimension | Weight | Measures | +|-----------|--------|----------| +| conversationDepth | 0.05 | Message count in conversation | +| priorToolUsage | 0.05 | Tool results already in conversation | +| ambiguity | 0.05 | Inverse of request specificity | + +Each dimension is scored 0-100 independently, then multiplied by its weight. The final score is the rounded sum. + +### Phase 3: Metrics Tracking + +Every routing decision is recorded in-memory (last 1,000 decisions) for analytics: +- Total decisions, local vs. cloud split +- Average complexity score +- Per-provider and per-tier distribution + +Metrics are exposed via the `/metrics` endpoint and `X-Lynkr-*` response headers. + +### Phase 4: Embeddings-Based Similarity (Optional) + +When an embeddings model is configured (`OLLAMA_EMBEDDINGS_MODEL`), the analyzer can compare request content against reference embeddings for complex and simple tasks using cosine similarity. This produces a score adjustment of -10 to +10 points. + +--- + +## Agentic Workflow Detection + +The agentic detector identifies multi-step tool chains and autonomous agent patterns, boosting the complexity tier accordingly. + +### Agent Types + +| Type | Score Boost | Min Tier | Description | +|------|------------|----------|-------------| +| **SINGLE_SHOT** | +0 | SIMPLE | Simple request-response, no tool chains | +| **TOOL_CHAIN** | +15 | MEDIUM | Sequential tool usage (read -> edit -> test) | +| **ITERATIVE** | +25 | COMPLEX | Retry loops, debugging cycles, iterative refinement | +| **AUTONOMOUS** | +35 | REASONING | Open-ended tasks, full autonomy, complex decision making | + +### Detection Signals + +The detector evaluates 6 signal categories: + +**1. Tool Count** +- 4-5 tools: +8 +- 6-10 tools: +15 +- 11+ tools: +25 + +**2. Agentic Tools Present** (Bash, Write, Edit, Task, Git, Test) +- 1 agentic tool: +8 +- 2-3 agentic tools: +15 +- 4+ agentic tools: +25 + +**3. Prior Tool Results** (already in an agentic loop) +- 1-2 tool results: +10 +- 3-5 tool results: +20 +- 6+ tool results: +30 + +**4. Content Pattern Matching** +- Autonomous patterns ("figure out", "solve", "make it work"): +25 +- Iterative patterns ("keep trying", "debug", "retry"): +20 +- Tool chain patterns ("then use", "next step", "step 1"): +15 +- Multi-file work: +15 +- Planning required: +10 +- Implementation + testing: +15 + +**5. Conversation Depth** +- 5-8 messages: +6 +- 9-15 messages: +12 +- 16+ messages: +20 + +**6. Content Length** +- 2,000+ characters: +10 + +### Classification Thresholds + +| Agent Type | Score Threshold | Additional Conditions | +|------------|----------------|----------------------| +| AUTONOMOUS | >= 60 | or autonomous pattern + score >= 40 | +| ITERATIVE | >= 40 | or deep tool loop + score >= 30 | +| TOOL_CHAIN | >= 20 | or many agentic tools present | +| SINGLE_SHOT | < 20 | Default | + +When an agentic workflow is detected (`score >= 25`), the complexity score is boosted by the agent type's `scoreBoost` value, and the tier is upgraded to at least the agent type's `minTier`. + +--- + +## Force Patterns + +Certain requests bypass the scoring algorithm entirely: + +### Force Local (always local model) +- Greetings: "hi", "hello", "thanks", "bye" +- Time queries: "what time is it" +- Confirmations: "yes", "no", "ok", "sure" +- Help requests: "help", "commands" + +### Force Cloud (always cloud model) +- Security audits/reviews +- Architecture design/review +- Complete codebase refactoring +- Code/PR reviews +- Complex debugging +- Production incidents + +--- + +## Cost Optimization + +When `ROUTING_COST_OPTIMIZATION=true`, the router checks if a cheaper model can handle the determined tier. + +### Model Registry + +Pricing data is fetched from three sources (in priority order): + +1. **LiteLLM** (highest priority) - Community-maintained pricing from [BerriAI/litellm](https://github.com/BerriAI/litellm) +2. **models.dev** - API pricing aggregator +3. **Databricks Fallback** - Hardcoded pricing for common models (Claude, Llama, GPT, Gemini, DBRX) + +Pricing data is cached locally in `data/model-prices-cache.json` with a 24-hour TTL. Background refresh happens automatically when the cache is stale. + +### Cost Tracking + +The optimizer tracks costs at both session and global levels: +- Per-request cost recording (input + output tokens) +- Per-model, per-provider, per-tier breakdowns +- Savings calculation when routing to cheaper alternatives + +### Pricing Lookup + +The registry supports flexible model name lookup: +- Direct match: `gpt-4o` +- Provider prefix stripping: `databricks-claude-sonnet-4-5` -> `claude-sonnet-4-5` +- Fuzzy matching for partial names + +--- + +## Routing Headers + +Every response includes routing metadata in `X-Lynkr-*` headers: + +| Header | Description | Example | +|--------|-------------|---------| +| `X-Lynkr-Routing-Method` | How the decision was made | `tier_config`, `force`, `tool_threshold`, `agentic`, `cost_optimized` | +| `X-Lynkr-Provider` | Selected provider | `databricks`, `ollama`, `openrouter` | +| `X-Lynkr-Complexity-Score` | Complexity score (0-100) | `42` | +| `X-Lynkr-Complexity-Threshold` | Score threshold for cloud routing | `40` | +| `X-Lynkr-Routing-Reason` | Human-readable reason | `force_local_pattern`, `autonomous_workflow` | +| `X-Lynkr-Tier` | Selected model tier | `SIMPLE`, `MEDIUM`, `COMPLEX`, `REASONING` | +| `X-Lynkr-Model` | Selected model | `llama3.2`, `gpt-4o`, `claude-opus-4-6` | +| `X-Lynkr-Agentic` | Agentic workflow type (if detected) | `TOOL_CHAIN`, `ITERATIVE`, `AUTONOMOUS` | +| `X-Lynkr-Cost-Optimized` | Whether cost optimization was applied | `true` | + +--- + +## Configuration Reference + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `TIER_SIMPLE` | *required* | Model for simple tier (`provider:model`) | +| `TIER_MEDIUM` | *required* | Model for medium tier (`provider:model`) | +| `TIER_COMPLEX` | *required* | Model for complex tier (`provider:model`) | +| `TIER_REASONING` | *required* | Model for reasoning tier (`provider:model`) | +| `SMART_TOOL_SELECTION_MODE` | `heuristic` | Scoring mode: `aggressive` (threshold=60), `heuristic` (threshold=40), `conservative` (threshold=25) | +| `ROUTING_WEIGHTED_SCORING` | `false` | Enable 15-dimension weighted scoring | +| `ROUTING_AGENTIC_DETECTION` | `true` | Enable agentic workflow detection | +| `ROUTING_COST_OPTIMIZATION` | `false` | Enable cost-based model selection | +| `OLLAMA_MAX_TOOLS_FOR_ROUTING` | `3` | Max tools before routing away from Ollama | +| `OPENROUTER_MAX_TOOLS_FOR_ROUTING` | `15` | Max tools before routing away from OpenRouter | +| `OLLAMA_EMBEDDINGS_MODEL` | *(none)* | Embeddings model for Phase 4 similarity | + +### Smart Tool Selection Modes + +| Mode | Threshold | Behavior | +|------|-----------|----------| +| `aggressive` | 60 | More requests go to local (saves cost) | +| `heuristic` | 40 | Balanced local/cloud split | +| `conservative` | 25 | More requests go to cloud (better quality) | + +--- + +## Routing Decision Flow + +``` +1. Are all 4 TIER_* env vars configured? + └─ No → Return static provider (MODEL_PROVIDER), skip all routing + +2. Does content match FORCE_LOCAL patterns? + └─ Yes → Route to local provider + +3. Does content match FORCE_CLOUD patterns? + └─ Yes → Route to best cloud provider (requires FALLBACK_ENABLED) + +4. Analyze complexity: + └─ Calculate score 0-100 (standard or weighted mode) + +5. Optional: Embeddings adjustment: + └─ Adjust score by -10 to +10 based on semantic similarity + +6. Agentic detection: + └─ If agentic → Boost score, enforce minimum tier + └─ If AUTONOMOUS → Force cloud provider + +7. Map score to tier (SIMPLE/MEDIUM/COMPLEX/REASONING) + +8. Select provider:model from matching TIER_* env var + +9. Optional: Cost optimization + └─ Check for cheaper model that can handle the tier + +10. Return { provider, model, tier, score, method } +``` + +--- + +## Source Files + +| File | Description | +|------|-------------| +| `src/routing/index.js` | Main routing orchestrator (`determineProviderSmart()`) | +| `src/routing/complexity-analyzer.js` | 4-phase complexity analysis, 15-dimension weighted scoring | +| `src/routing/agentic-detector.js` | Agentic workflow detection and classification | +| `src/routing/model-tiers.js` | Tier definitions, model selection from `TIER_*` env vars | +| `src/routing/model-registry.js` | Multi-source pricing (LiteLLM, models.dev, Databricks fallback) | +| `src/routing/cost-optimizer.js` | Cost tracking, cheapest model finder, savings calculation | + +--- + +## Next Steps + +- **[Features Overview](features.md)** - Architecture and request flow +- **[Token Optimization](token-optimization.md)** - Cost reduction strategies +- **[Provider Configuration](providers.md)** - Setting up providers +- **[Production Guide](production.md)** - Deploy with routing enabled diff --git a/documentation/token-optimization.md b/documentation/token-optimization.md index 0e00437..7138f0d 100644 --- a/documentation/token-optimization.md +++ b/documentation/token-optimization.md @@ -252,14 +252,16 @@ LOG_LEVEL=info # No configuration needed ``` -### 2. Use Hybrid Routing +### 2. Use Tier-Based Routing ```bash -# Route simple requests to free Ollama -PREFER_OLLAMA=true +# Route simple requests to free Ollama, complex to cloud +# Set all 4 TIER_* env vars to enable tier-based routing +TIER_SIMPLE=ollama:llama3.2 +TIER_MEDIUM=openrouter:openai/gpt-4o-mini +TIER_COMPLEX=azure-openai:gpt-4o +TIER_REASONING=azure-openai:gpt-4o FALLBACK_ENABLED=true - -# Complex requests automatically go to cloud FALLBACK_PROVIDER=databricks ``` diff --git a/documentation/troubleshooting.md b/documentation/troubleshooting.md index ce511c9..3a5206e 100644 --- a/documentation/troubleshooting.md +++ b/documentation/troubleshooting.md @@ -400,6 +400,75 @@ Error: Cannot find module 'xxx' --- +### Moonshot AI (Kimi) + +**Issue:** Rate limited (429) + +**Symptoms:** +- `429 Too Many Requests` +- `Rate limit exceeded` +- Responses failing intermittently + +**Solutions:** + +1. **Reduce concurrency:** + Moonshot has a max concurrency of ~3 requests. Lynkr retries automatically with backoff, but sustained high concurrency will trigger 429s. + +2. **Use turbo model:** + ```bash + # Turbo has higher rate limits than thinking model + export MOONSHOT_MODEL=kimi-k2-turbo-preview + ``` + +3. **Enable fallback:** + ```bash + export FALLBACK_ENABLED=true + export FALLBACK_PROVIDER=openrouter + ``` + +**Issue:** Authentication failed + +**Symptoms:** +- `401 Unauthorized` +- `Invalid API key` + +**Solutions:** + +1. **Check API key format:** + ```bash + echo $MOONSHOT_API_KEY + # Should start with: sk- + ``` + +2. **Regenerate API key:** + - Visit [platform.moonshot.ai](https://platform.moonshot.ai) + - Generate a new key + - Update environment variable + +3. **Check endpoint:** + ```bash + echo $MOONSHOT_ENDPOINT + # Should be: https://api.moonshot.ai/v1/chat/completions + ``` + +**Issue:** Reasoning content displayed in output + +**Symptoms:** +- Response includes chain-of-thought text before the actual answer +- Long preambles like "The user is asking me to..." + +**Solutions:** + +This happens when using `kimi-k2-thinking` model. Lynkr should automatically strip reasoning content and only show the final answer. If you see reasoning in the output: + +1. **Update Lynkr** to the latest version +2. **Switch to turbo model** if reasoning output is not needed: + ```bash + export MOONSHOT_MODEL=kimi-k2-turbo-preview + ``` + +--- + ### llama.cpp **Issue:** Server not responding @@ -556,9 +625,13 @@ Error: Cannot find module 'xxx' export OLLAMA_MODEL=llama3.1:8b ``` -3. **Enable hybrid routing:** +3. **Enable tier-based routing:** ```bash - export PREFER_OLLAMA=true + # Set all 4 TIER_* env vars to enable tier-based routing + export TIER_SIMPLE=ollama:llama3.2 + export TIER_MEDIUM=openrouter:openai/gpt-4o-mini + export TIER_COMPLEX=azure-openai:gpt-4o + export TIER_REASONING=azure-openai:gpt-4o export FALLBACK_ENABLED=true ``` @@ -746,10 +819,13 @@ Restart Lynkr after configuration. export LOAD_SHEDDING_ACTIVE_REQUESTS_THRESHOLD=100 ``` -2. **Use local provider for simple requests:** +2. **Use tier-based routing to send simple requests to local models:** ```bash - export PREFER_OLLAMA=true - export OLLAMA_MODEL=llama3.1:8b + # Set all 4 TIER_* env vars to enable tier-based routing + export TIER_SIMPLE=ollama:llama3.2 + export TIER_MEDIUM=openrouter:openai/gpt-4o-mini + export TIER_COMPLEX=azure-openai:gpt-4o + export TIER_REASONING=azure-openai:gpt-4o ``` 3. **Enable circuit breaker:** diff --git a/install.sh b/install.sh index 50d8b1d..00389d3 100755 --- a/install.sh +++ b/install.sh @@ -134,10 +134,15 @@ MODEL_PROVIDER=ollama PORT=8080 # Ollama Configuration (default for local development) -PREFER_OLLAMA=true OLLAMA_MODEL=qwen2.5-coder:7b OLLAMA_ENDPOINT=http://localhost:11434 +# Tier-based routing (uncomment and configure to enable) +# TIER_SIMPLE=ollama:qwen2.5-coder:7b +# TIER_MEDIUM=ollama:qwen2.5-coder:7b +# TIER_COMPLEX=ollama:qwen2.5-coder:7b +# TIER_REASONING=ollama:qwen2.5-coder:7b + # Long-Term Memory System (Titans-Inspired) - Enabled by default MEMORY_ENABLED=true MEMORY_RETRIEVAL_LIMIT=5 diff --git a/package-lock.json b/package-lock.json index 2befc91..699ee41 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "openai": "^6.14.0", "pino": "^8.17.2", "pino-http": "^8.6.0", + "pino-roll": "^4.0.0", "undici": "^6.22.0" }, "bin": { @@ -1298,6 +1299,16 @@ "node": ">= 8" } }, + "node_modules/date-fns": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz", + "integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" + } + }, "node_modules/dateformat": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", @@ -3326,6 +3337,25 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/pino-roll": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pino-roll/-/pino-roll-4.0.0.tgz", + "integrity": "sha512-axI1aQaIxXdw1F4OFFli1EDxIrdYNGLowkw/ZoZogX8oCSLHUghzwVVXUS8U+xD/Savwa5IXpiXmsSGKFX/7Sg==", + "license": "MIT", + "dependencies": { + "date-fns": "^4.1.0", + "sonic-boom": "^4.0.1" + } + }, + "node_modules/pino-roll/node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/pino-std-serializers": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz", diff --git a/package.json b/package.json index 0590bab..0bb8dea 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "openai": "^6.14.0", "pino": "^8.17.2", "pino-http": "^8.6.0", + "pino-roll": "^4.0.0", "undici": "^6.22.0" }, "optionalDependencies": { diff --git a/scripts/setup.js b/scripts/setup.js index 3f0f53d..4a7840b 100755 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -251,7 +251,6 @@ async function createEnvFile() { if (ollamaOnly) { let envContent = fs.readFileSync(envPath, "utf-8"); envContent = envContent.replace(/^# MODEL_PROVIDER=databricks/m, "MODEL_PROVIDER=ollama"); - envContent = envContent.replace(/^PREFER_OLLAMA=true/m, "# PREFER_OLLAMA=true # Not needed when MODEL_PROVIDER=ollama"); envContent = envContent.replace(/^FALLBACK_ENABLED=true/m, "FALLBACK_ENABLED=false"); fs.writeFileSync(envPath, envContent); } diff --git a/src/api/middleware/session.js b/src/api/middleware/session.js index f19c58a..15381da 100644 --- a/src/api/middleware/session.js +++ b/src/api/middleware/session.js @@ -45,8 +45,21 @@ function sessionMiddleware(req, res, next) { // Add sessionId to logger context for this request req.log = logger.child({ sessionId }); - const session = getOrCreateSession(sessionId); - req.session = session; + // Skip DB persistence for auto-generated (ephemeral) session IDs. + // These are created when the client doesn't send a session header, + // so storing them just bloats the DB with throwaway records. + if (req.generatedSessionId) { + req.session = { + id: sessionId, + createdAt: Date.now(), + updatedAt: Date.now(), + metadata: {}, + history: [], + _ephemeral: true, + }; + } else { + req.session = getOrCreateSession(sessionId); + } return next(); } catch (err) { return next(err); diff --git a/src/api/openai-router.js b/src/api/openai-router.js index ab283e7..0153619 100644 --- a/src/api/openai-router.js +++ b/src/api/openai-router.js @@ -21,9 +21,9 @@ const orchestrator = require("../orchestrator"); const { getSession } = require("../sessions"); const { convertOpenAIToAnthropic, - convertAnthropicToOpenAI, - convertAnthropicStreamChunkToOpenAI + convertAnthropicToOpenAI } = require("../clients/openai-format"); +const { IDE_SAFE_TOOLS } = require("../clients/standard-tools"); const router = express.Router(); @@ -60,13 +60,14 @@ function detectClient(headers) { */ const CLIENT_TOOL_MAPPINGS = { // ============== CODEX CLI ============== - // Tools: shell_command, read_file, write_file, apply_patch, glob_file_search, rg, list_dir + // Confirmed tools: shell, apply_patch, read_file, write_file, list_dir, glob_file_search, + // rg, web_search, update_plan, view_image, memory + // NOT supported: spawn_agent/spawn_thread (Task has no Codex equivalent) codex: { "Bash": { - name: "shell_command", + name: "shell", mapArgs: (a) => ({ - command: a.command || "", - workdir: a.cwd || a.working_directory + command: ["bash", "-c", a.command || ""] }) }, "Read": { @@ -113,6 +114,18 @@ const CLIENT_TOOL_MAPPINGS = { mapArgs: (a) => ({ path: a.path || a.directory }) + }, + "TodoWrite": { + name: "update_plan", + mapArgs: (a) => ({ + todos: a.todos || [] + }) + }, + "WebSearch": { + name: "web_search", + mapArgs: (a) => ({ + query: a.query || "" + }) } }, @@ -321,14 +334,7 @@ function mapToolForClient(toolName, argsJson, clientType) { }; } -/** - * Check if client is a known AI coding tool that needs tool mapping - * @param {Object} headers - Request headers - * @returns {boolean} - */ -function isKnownClient(headers) { - return detectClient(headers) !== "unknown"; -} + /** * POST /v1/chat/completions @@ -374,7 +380,7 @@ router.post("/chat/completions", async (req, res) => { : JSON.stringify(m.content).substring(0, 200) })); - logger.info({ + logger.debug({ endpoint: "/v1/chat/completions", model: req.body.model, messageCount: req.body.messages?.length, @@ -393,6 +399,30 @@ router.post("/chat/completions", async (req, res) => { // Convert OpenAI request to Anthropic format const anthropicRequest = convertOpenAIToAnthropic(req.body); + // Inject tools if client didn't send any. + // Two-layer filtering: + // 1. IDE_SAFE_TOOLS = STANDARD_TOOLS minus AskUserQuestion (can't work through proxy) + // 2. For known clients (codex, cline, etc.), further filter to only tools + // that have a mapping in CLIENT_TOOL_MAPPINGS — this ensures clients like + // Codex don't see tools they can't handle (Task, WebFetch, NotebookEdit) + // while Claude Code (unknown client) gets the full IDE_SAFE_TOOLS set. + const clientType = detectClient(req.headers); + if (!anthropicRequest.tools || anthropicRequest.tools.length === 0) { + const clientMappings = CLIENT_TOOL_MAPPINGS[clientType]; + const clientTools = clientMappings + ? IDE_SAFE_TOOLS.filter(t => clientMappings[t.name]) + : IDE_SAFE_TOOLS; + anthropicRequest.tools = clientTools; + logger.debug({ + clientType, + injectedToolCount: clientTools.length, + injectedToolNames: clientTools.map(t => t.name), + reason: clientMappings + ? `Known client '${clientType}' — filtered to mapped tools only` + : "Unknown client — injecting full IDE_SAFE_TOOLS" + }, "=== INJECTING TOOLS ==="); + } + // Get or create session const session = getSession(sessionId); @@ -420,7 +450,7 @@ router.post("/chat/completions", async (req, res) => { }); // Check if we have a valid response body - logger.info({ + logger.debug({ hasResult: !!result, resultKeys: result ? Object.keys(result) : null, hasBody: result && !!result.body, @@ -442,7 +472,7 @@ router.post("/chat/completions", async (req, res) => { const openaiResponse = convertAnthropicToOpenAI(result.body, req.body.model); // Debug: Log what we're about to stream - logger.info({ + logger.debug({ openaiResponseId: openaiResponse.id, messageContent: openaiResponse.choices[0]?.message?.content?.substring(0, 100), contentLength: openaiResponse.choices[0]?.message?.content?.length || 0, @@ -454,7 +484,25 @@ router.post("/chat/completions", async (req, res) => { // Simulate streaming by sending the complete response as chunks const content = openaiResponse.choices[0].message.content || ""; - const toolCalls = openaiResponse.choices[0].message.tool_calls; + let toolCalls = openaiResponse.choices[0].message.tool_calls; + + // Map tool names for known IDE clients + if (clientType !== "unknown" && toolCalls && toolCalls.length > 0) { + toolCalls = toolCalls.map(tc => { + const mapped = mapToolForClient(tc.function?.name || "", tc.function?.arguments || "{}", clientType); + return { + ...tc, + function: { + name: mapped.name, + arguments: mapped.arguments + } + }; + }); + logger.debug({ + mappedTools: toolCalls.map(t => t.function?.name), + clientType + }, "Tool names mapped for streaming chat/completions"); + } // Send start chunk with role const startChunk = { @@ -493,7 +541,7 @@ router.post("/chat/completions", async (req, res) => { }] }; const contentWriteOk = res.write(`data: ${JSON.stringify(contentChunk)}\n\n`); - logger.info({ contentPreview: content.substring(0, 50), writeOk: contentWriteOk }, "Sent content chunk"); + logger.debug({ contentPreview: content.substring(0, 50), writeOk: contentWriteOk }, "Sent content chunk"); } // Send tool calls if present @@ -545,7 +593,7 @@ router.post("/chat/completions", async (req, res) => { res.write("data: [DONE]\n\n"); // Ensure data is flushed before ending - logger.info({ contentLength: content.length, contentPreview: content.substring(0, 50) }, "=== SSE STREAM COMPLETE ==="); + logger.debug({ contentLength: content.length, contentPreview: content.substring(0, 50) }, "=== SSE STREAM COMPLETE ==="); res.end(); logger.info({ @@ -558,10 +606,7 @@ router.post("/chat/completions", async (req, res) => { } catch (streamError) { logger.error({ error: streamError.message, - stack: streamError.stack, - resultWasNull: !result, - resultBodyWasNull: result && !result.body, - resultKeys: result ? Object.keys(result) : null + stack: streamError.stack }, "=== STREAMING ERROR ==="); // Send error in OpenAI streaming format @@ -606,6 +651,24 @@ router.post("/chat/completions", async (req, res) => { // Convert Anthropic response to OpenAI format const openaiResponse = convertAnthropicToOpenAI(result.body, req.body.model); + // Map tool names for known IDE clients + if (clientType !== "unknown" && openaiResponse.choices?.[0]?.message?.tool_calls?.length > 0) { + openaiResponse.choices[0].message.tool_calls = openaiResponse.choices[0].message.tool_calls.map(tc => { + const mapped = mapToolForClient(tc.function?.name || "", tc.function?.arguments || "{}", clientType); + return { + ...tc, + function: { + name: mapped.name, + arguments: mapped.arguments + } + }; + }); + logger.debug({ + mappedTools: openaiResponse.choices[0].message.tool_calls.map(t => t.function?.name), + clientType + }, "Tool names mapped for non-streaming chat/completions"); + } + logger.info({ duration: Date.now() - startTime, mode: "non-streaming", @@ -767,6 +830,18 @@ function getConfiguredProviders() { }); } + // Check Moonshot AI (Kimi) + if (config.moonshot?.apiKey) { + providers.push({ + name: "moonshot", + type: "moonshot-ai", + models: [ + config.moonshot.model || "kimi-k2-turbo-preview", + "kimi-k2-turbo-preview" + ] + }); + } + // Check Vertex AI (Google Cloud) if (config.vertex?.projectId) { providers.push({ @@ -1013,7 +1088,7 @@ function determineEmbeddingProvider(requestedModel = null) { async function generateOllamaEmbeddings(inputs, embeddingConfig) { const { model, endpoint } = embeddingConfig; - logger.info({ + logger.debug({ model, endpoint, inputCount: inputs.length @@ -1079,7 +1154,7 @@ async function generateOllamaEmbeddings(inputs, embeddingConfig) { async function generateLlamaCppEmbeddings(inputs, embeddingConfig) { const { model, endpoint } = embeddingConfig; - logger.info({ + logger.debug({ model, endpoint, inputCount: inputs.length @@ -1147,7 +1222,7 @@ async function generateLlamaCppEmbeddings(inputs, embeddingConfig) { async function generateOpenRouterEmbeddings(inputs, embeddingConfig) { const { model, apiKey, endpoint } = embeddingConfig; - logger.info({ + logger.debug({ model, inputCount: inputs.length }, "Generating embeddings with OpenRouter"); @@ -1181,7 +1256,7 @@ async function generateOpenRouterEmbeddings(inputs, embeddingConfig) { async function generateOpenAIEmbeddings(inputs, embeddingConfig) { const { model, apiKey, endpoint } = embeddingConfig; - logger.info({ + logger.debug({ model, inputCount: inputs.length }, "Generating embeddings with OpenAI"); @@ -1233,7 +1308,7 @@ router.post("/embeddings", async (req, res) => { // Convert input to array if string const inputs = Array.isArray(input) ? input : [input]; - logger.info({ + logger.debug({ endpoint: "/v1/embeddings", model: model || "auto-detect", inputCount: inputs.length, @@ -1335,7 +1410,7 @@ router.post("/responses", async (req, res) => { const { convertResponsesToChat, convertChatToResponses } = require("../clients/responses-format"); // Comprehensive debug logging - logger.info({ + logger.debug({ endpoint: "/v1/responses", inputType: typeof req.body.input, inputIsArray: Array.isArray(req.body.input), @@ -1354,7 +1429,7 @@ router.post("/responses", async (req, res) => { // Convert Responses API to Chat Completions format const chatRequest = convertResponsesToChat(req.body); - logger.info({ + logger.debug({ chatRequestMessageCount: chatRequest.messages?.length, chatRequestMessages: chatRequest.messages?.map(m => ({ role: m.role, @@ -1366,7 +1441,7 @@ router.post("/responses", async (req, res) => { // Convert to Anthropic format const anthropicRequest = convertOpenAIToAnthropic(chatRequest); - logger.info({ + logger.debug({ anthropicMessageCount: anthropicRequest.messages?.length, anthropicMessages: anthropicRequest.messages?.map(m => ({ role: m.role, @@ -1374,6 +1449,24 @@ router.post("/responses", async (req, res) => { })) }, "After Chat→Anthropic conversion"); + // Inject tools if client didn't send any (same two-layer filtering as chat/completions). + const clientType = detectClient(req.headers); + if (!anthropicRequest.tools || anthropicRequest.tools.length === 0) { + const clientMappings = CLIENT_TOOL_MAPPINGS[clientType]; + const clientTools = clientMappings + ? IDE_SAFE_TOOLS.filter(t => clientMappings[t.name]) + : IDE_SAFE_TOOLS; + anthropicRequest.tools = clientTools; + logger.debug({ + clientType, + injectedToolCount: clientTools.length, + injectedToolNames: clientTools.map(t => t.name), + reason: clientMappings + ? `Known client '${clientType}' — filtered to mapped tools only` + : "Unknown client — injecting full IDE_SAFE_TOOLS" + }, "=== INJECTING TOOLS (responses) ==="); + } + // Get session const session = getSession(sessionId); @@ -1400,7 +1493,7 @@ router.post("/responses", async (req, res) => { }); // Debug: Log what orchestrator returned - logger.info({ + logger.debug({ hasResult: !!result, hasBody: !!result?.body, bodyKeys: result?.body ? Object.keys(result.body) : null, @@ -1412,7 +1505,7 @@ router.post("/responses", async (req, res) => { // Convert back: Anthropic → OpenAI → Responses const chatResponse = convertAnthropicToOpenAI(result.body, req.body.model); - logger.info({ + logger.debug({ chatContent: chatResponse.choices?.[0]?.message?.content?.substring(0, 200), chatContentLength: chatResponse.choices?.[0]?.message?.content?.length || 0, hasToolCalls: !!chatResponse.choices?.[0]?.message?.tool_calls, @@ -1433,7 +1526,7 @@ router.post("/responses", async (req, res) => { // Check if client is a known AI coding tool and map tool names accordingly const clientType = detectClient(req.headers); if (clientType !== "unknown" && toolCalls.length > 0) { - logger.info({ + logger.debug({ originalTools: toolCalls.map(t => t.function?.name), clientType, userAgent: req.headers["user-agent"] @@ -1451,12 +1544,12 @@ router.post("/responses", async (req, res) => { }; }); - logger.info({ + logger.debug({ mappedTools: toolCalls.map(t => t.function?.name) }, `Tool names mapped for ${clientType}`); } - logger.info({ + logger.debug({ content: content.substring(0, 100), contentLength: content.length, toolCallCount: toolCalls.length, diff --git a/src/api/providers-handler.js b/src/api/providers-handler.js index 9b85848..73c1f56 100644 --- a/src/api/providers-handler.js +++ b/src/api/providers-handler.js @@ -179,6 +179,20 @@ function getConfiguredProviders() { }); } + // Check Moonshot AI (Kimi) + if (config.moonshot?.apiKey) { + providers.push({ + name: "moonshot", + type: "moonshot-ai", + baseUrl: config.moonshot.endpoint || "https://api.moonshot.ai/v1", + enabled: true, + models: [ + { id: config.moonshot.model || "kimi-k2-turbo-preview", name: "Configured Model" }, + { id: "kimi-k2-turbo-preview", name: "Kimi K2 Turbo Preview" }, + ] + }); + } + // Check Vertex AI (Google Cloud) if (config.vertex?.projectId) { const region = config.vertex.region || "us-east5"; @@ -369,7 +383,7 @@ router.get("/config", (req, res) => { model_provider: config.modelProvider?.type || "databricks", fallback_provider: config.modelProvider?.fallbackProvider || null, fallback_enabled: config.modelProvider?.fallbackEnabled || false, - prefer_ollama: config.modelProvider?.preferOllama || false, + tier_routing_enabled: config.modelTiers?.enabled || false, tool_execution_mode: config.toolExecutionMode || "server", configured_providers: providers.map(p => p.name), memory_enabled: config.memory?.enabled || false, diff --git a/src/api/router.js b/src/api/router.js index b3ed198..897c42c 100644 --- a/src/api/router.js +++ b/src/api/router.js @@ -2,10 +2,11 @@ const express = require("express"); const { processMessage } = require("../orchestrator"); const { getSession } = require("../sessions"); const metrics = require("../metrics"); +const logger = require("../logger"); const { createRateLimiter } = require("./middleware/rate-limiter"); const openaiRouter = require("./openai-router"); const providersRouter = require("./providers-handler"); -const { getRoutingHeaders, getRoutingStats, analyzeComplexity } = require("../routing"); +const { getRoutingHeaders, getRoutingStats, analyzeComplexity, getModelTierSelector } = require("../routing"); const { validateCwd } = require("../workspace"); const router = express.Router(); @@ -71,6 +72,99 @@ router.get("/routing/stats", (req, res) => { }); }); +// Model registry info (from LiteLLM + models.dev APIs) +router.get("/routing/models", async (req, res) => { + try { + const { getModelRegistry } = require("../routing/model-registry"); + const registry = await getModelRegistry(); + res.json({ + status: "ok", + ...registry.getStats(), + }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + +// Get specific model info +router.get("/routing/models/:model", async (req, res) => { + try { + const { getModelRegistry } = require("../routing/model-registry"); + const registry = await getModelRegistry(); + const model = registry.getModel(req.params.model); + if (!model || model.source === "default") { + return res.status(404).json({ error: "Model not found", model: req.params.model }); + } + res.json({ status: "ok", model: req.params.model, ...model }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + +// Routing tier information +router.get("/routing/tiers", (req, res) => { + try { + const { getModelTierSelector } = require("../routing/model-tiers"); + const selector = getModelTierSelector(); + res.json({ + status: "ok", + ...selector.getTierStats(), + }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + +// Cost optimization stats +router.get("/metrics/cost-optimization", (req, res) => { + try { + const { getCostOptimizer } = require("../routing/cost-optimizer"); + const optimizer = getCostOptimizer(); + res.json({ + status: "ok", + ...optimizer.getStats(), + }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + +// Request analysis test endpoint +router.post("/routing/analyze", async (req, res) => { + try { + const { getAgenticDetector } = require("../routing/agentic-detector"); + const { getModelTierSelector } = require("../routing/model-tiers"); + const { getModelRegistry } = require("../routing/model-registry"); + + const analysis = analyzeComplexity(req.body, { weighted: req.query.weighted === "true" }); + const agentic = getAgenticDetector().detect(req.body); + const selector = getModelTierSelector(); + const tier = selector.getTier(analysis.score); + + // Get recommended model for tier + const provider = req.query.provider || "openai"; + const modelSelection = selector.selectModel(tier, provider); + + // Get model cost info + let modelInfo = null; + if (modelSelection.model) { + const registry = await getModelRegistry(); + modelInfo = registry.getCost(modelSelection.model); + } + + res.json({ + status: "ok", + analysis, + agentic, + tier, + modelSelection, + modelInfo, + }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + router.get("/debug/session", (req, res) => { if (!req.sessionId) { return res.status(400).json({ error: "missing_session_id", message: "Provide x-session-id header" }); @@ -123,8 +217,19 @@ router.post("/v1/messages", rateLimiter, async (req, res, next) => { // Analyze complexity for routing headers (Phase 3) const complexity = analyzeComplexity(req.body); + let preRouteProvider = 'cloud'; + if (complexity.recommendation === 'local') { + // Use tier config to determine actual provider instead of hardcoding 'ollama' + try { + const selector = getModelTierSelector(); + const tierResult = selector.selectModel('SIMPLE', null); + preRouteProvider = tierResult.provider; + } catch (_) { + preRouteProvider = 'ollama'; + } + } const routingHeaders = getRoutingHeaders({ - provider: complexity.recommendation === 'local' ? 'ollama' : 'cloud', + provider: preRouteProvider, score: complexity.score, threshold: complexity.threshold, method: 'complexity', diff --git a/src/budget/index.js b/src/budget/index.js index ca7294e..6f687ee 100644 --- a/src/budget/index.js +++ b/src/budget/index.js @@ -26,6 +26,7 @@ class BudgetManager { } this.db = new Database(dbPath); + this.dbPath = dbPath; this.initDatabase(); logger.info({ dbPath }, 'Budget manager initialized'); } catch (err) { @@ -33,8 +34,6 @@ class BudgetManager { this.enabled = false; return; } - - } initDatabase() { diff --git a/src/clients/databricks.js b/src/clients/databricks.js index 9b536cd..f600625 100644 --- a/src/clients/databricks.js +++ b/src/clients/databricks.js @@ -6,11 +6,12 @@ const { getCircuitBreakerRegistry } = require("./circuit-breaker"); const { getMetricsCollector } = require("../observability/metrics"); const { getHealthTracker } = require("../observability/health-tracker"); const logger = require("../logger"); -const { STANDARD_TOOLS } = require("./standard-tools"); +const { STANDARD_TOOLS, STANDARD_TOOL_NAMES } = require("./standard-tools"); const { convertAnthropicToolsToOpenRouter } = require("./openrouter-utils"); const { detectModelFamily } = require("./bedrock-utils"); +const { getGPTSystemPromptAddendum } = require("./gpt-utils"); @@ -183,9 +184,9 @@ async function invokeDatabricks(body) { // Inject standard tools if client didn't send any (passthrough mode) if (!Array.isArray(databricksBody.tools) || databricksBody.tools.length === 0) { databricksBody.tools = STANDARD_TOOLS; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS (Databricks) ==="); } @@ -224,9 +225,9 @@ async function invokeAzureAnthropic(body) { // Inject standard tools if client didn't send any (passthrough mode) if (!Array.isArray(body.tools) || body.tools.length === 0) { body.tools = STANDARD_TOOLS; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS (Azure Anthropic) ==="); } @@ -248,42 +249,115 @@ async function invokeOllama(body) { throw new Error("Ollama endpoint is not configured."); } - const { convertAnthropicToolsToOllama, checkOllamaToolSupport } = require("./ollama-utils"); + const { checkOllamaToolSupport, hasAnthropicEndpoint, convertAnthropicToolsToOllama } = require("./ollama-utils"); + + const modelName = body._suggestionModeModel || body._tierModel || config.ollama.model; + + // Detect whether Ollama has the native Anthropic Messages API (v0.14.0+) + const useAnthropicApi = await hasAnthropicEndpoint(config.ollama.endpoint); + + // Check if model supports tools FIRST (before wasteful injection) + const supportsTools = await checkOllamaToolSupport(config.ollama.model); + const injectToolsOllama = process.env.INJECT_TOOLS_OLLAMA !== "false"; + + // Determine tools to send + let toolsToSend = body.tools; + let toolsInjected = false; + + if (!supportsTools) { + toolsToSend = null; + } else if (injectToolsOllama && (!Array.isArray(toolsToSend) || toolsToSend.length === 0)) { + toolsToSend = STANDARD_TOOLS; + toolsInjected = true; + } + + // Consolidated tool injection log + const toolCount = (supportsTools && Array.isArray(toolsToSend)) ? toolsToSend.length : 0; + let logMessage; + if (!supportsTools) { + logMessage = `Tools not supported (0 tools)`; + } else if (toolsInjected) { + logMessage = `injected ${toolCount} tools`; + } else if (Array.isArray(toolsToSend) && toolsToSend.length > 0) { + logMessage = `Using client-provided tools (${toolCount} tools)`; + } else if (!injectToolsOllama) { + logMessage = `Tool injection disabled (0 tools)`; + } else { + logMessage = `No tools (0 tools)`; + } + logger.debug({ + model: modelName, + apiMode: useAnthropicApi ? "anthropic" : "legacy", + toolCount, + toolsInjected, + supportsTools, + toolNames: (Array.isArray(toolsToSend) && toolsToSend.length > 0) ? toolsToSend.map(t => t.name) : [] + }, `=== Ollama STANDARD TOOLS INJECTION for ${config.ollama.model} === ${logMessage}`); + + // ---- Anthropic-native path (Ollama v0.14.0+) ---- + if (useAnthropicApi) { + const endpoint = `${config.ollama.endpoint}/v1/messages`; + const headers = { + "Content-Type": "application/json", + "anthropic-version": "2023-06-01", + }; + + // Build body with only valid Anthropic Messages API fields + const ollamaBody = { + model: modelName, + messages: body.messages, + max_tokens: body.max_tokens || 4096, + stream: false, + }; + + if (body.system) ollamaBody.system = body.system; + if (body.temperature !== undefined) ollamaBody.temperature = body.temperature; + if (body.top_p !== undefined) ollamaBody.top_p = body.top_p; + if (body.top_k !== undefined) ollamaBody.top_k = body.top_k; + if (body.stop_sequences) ollamaBody.stop_sequences = body.stop_sequences; + if (body.tool_choice) ollamaBody.tool_choice = body.tool_choice; + if (body.metadata) ollamaBody.metadata = body.metadata; + + // Tools (already Anthropic format — no conversion needed) + if (supportsTools && Array.isArray(toolsToSend) && toolsToSend.length > 0) { + ollamaBody.tools = toolsToSend; + } + + if (config.ollama.keepAlive !== undefined) { + const keepAlive = config.ollama.keepAlive; + ollamaBody.keep_alive = /^-?\d+$/.test(keepAlive) + ? parseInt(keepAlive, 10) + : keepAlive; + logger.debug({ keepAlive: ollamaBody.keep_alive }, "Ollama keep_alive configured"); + } + + return performJsonRequest(endpoint, { headers, body: ollamaBody }, "Ollama"); + } + + // ---- Legacy path (Ollama < v0.14.0, /api/chat with OpenAI format) ---- const endpoint = `${config.ollama.endpoint}/api/chat`; const headers = { "Content-Type": "application/json" }; - // Convert Anthropic messages format to Ollama format - // Ollama expects content as string, not content blocks array + // Convert Anthropic messages to Ollama format (content blocks → strings) const convertedMessages = []; - // Handle system prompt (same pattern as other providers) if (body.system && typeof body.system === "string" && body.system.trim().length > 0) { - convertedMessages.push({ - role: "system", - content: body.system.trim() - }); + convertedMessages.push({ role: "system", content: body.system.trim() }); } - // Add user/assistant messages (body.messages || []).forEach(msg => { let content = msg.content; - - // Convert content blocks array to simple string if (Array.isArray(content)) { content = content .filter(block => block.type === 'text') .map(block => block.text || '') .join('\n'); } - - convertedMessages.push({ - role: msg.role, - content: content || '' - }); + convertedMessages.push({ role: msg.role, content: content || '' }); }); - // FIX: Deduplicate consecutive messages with same role (Ollama may reject this) + // Deduplicate consecutive messages with same role const deduplicated = []; let lastRole = null; for (const msg of convertedMessages) { @@ -298,85 +372,30 @@ async function invokeOllama(body) { lastRole = msg.role; } - if (deduplicated.length !== convertedMessages.length) { - logger.info({ - originalCount: convertedMessages.length, - deduplicatedCount: deduplicated.length, - removed: convertedMessages.length - deduplicated.length, - messageRoles: convertedMessages.map(m => m.role).join(' → '), - deduplicatedRoles: deduplicated.map(m => m.role).join(' → ') - }, 'Ollama: Removed consecutive duplicate roles from message sequence'); - } - const ollamaBody = { - model: body._suggestionModeModel || config.ollama.model, + model: modelName, messages: deduplicated, - stream: false, // Force non-streaming for Ollama - streaming format conversion not yet implemented + stream: false, options: { temperature: body.temperature ?? 0.7, - num_predict: body.max_tokens ?? 4096, + num_predict: body.max_tokens ?? 16384, top_p: body.top_p ?? 1.0, }, }; - // Add keep_alive if configured (controls how long model stays loaded) - // Accepts: duration strings ("10m", "24h"), numbers (seconds), -1 (permanent), 0 (immediate unload) if (config.ollama.keepAlive !== undefined) { const keepAlive = config.ollama.keepAlive; - // Parse as number if it looks like one, otherwise use string ollamaBody.keep_alive = /^-?\d+$/.test(keepAlive) ? parseInt(keepAlive, 10) : keepAlive; logger.debug({ keepAlive: ollamaBody.keep_alive }, "Ollama keep_alive configured"); } - // Check if model supports tools FIRST (before wasteful injection) - const supportsTools = await checkOllamaToolSupport(config.ollama.model); - - // Inject standard tools if client didn't send any (passthrough mode) - let toolsToSend = body.tools; - let toolsInjected = false; - - const injectToolsOllama = process.env.INJECT_TOOLS_OLLAMA !== "false"; - - if (!supportsTools) { - // Model doesn't support tools - don't inject them - toolsToSend = null; - } else if (injectToolsOllama && (!Array.isArray(toolsToSend) || toolsToSend.length === 0)) { - // Model supports tools and none provided - inject them - toolsToSend = STANDARD_TOOLS; - toolsInjected = true; - } - - // Add tools if present AND model supports them + // Tools need conversion to OpenAI function-calling format for legacy endpoint if (supportsTools && Array.isArray(toolsToSend) && toolsToSend.length > 0) { ollamaBody.tools = convertAnthropicToolsToOllama(toolsToSend); } - // Single consolidated log message for all cases (easy to grep and compare across models) - const toolCount = (supportsTools && Array.isArray(toolsToSend)) ? toolsToSend.length : 0; - let logMessage; - - if (!supportsTools) { - logMessage = `Tools not supported (0 tools)`; - } else if (toolsInjected) { - logMessage = `injected ${toolCount} tools`; - } else if (Array.isArray(toolsToSend) && toolsToSend.length > 0) { - logMessage = `Using client-provided tools (${toolCount} tools)`; - } else if (!injectToolsOllama) { - logMessage = `Tool injection disabled (0 tools)`; - } else { - logMessage = `No tools (0 tools)`; - } - - logger.info({ - model: config.ollama.model, - toolCount, - toolsInjected, - supportsTools, - toolNames: (Array.isArray(toolsToSend) && toolsToSend.length > 0) ? toolsToSend.map(t => t.name) : [] - }, `=== Ollama STANDARD TOOLS INJECTION for ${config.ollama.model} === ${logMessage}`); - return performJsonRequest(endpoint, { headers, body: ollamaBody }, "Ollama"); } @@ -410,7 +429,7 @@ async function invokeOpenRouter(body) { } const openRouterBody = { - model: body._suggestionModeModel || config.openrouter.model, + model: body._suggestionModeModel || body._tierModel || config.openrouter.model, messages, temperature: body.temperature ?? 0.7, max_tokens: body.max_tokens ?? 4096, @@ -426,16 +445,16 @@ async function invokeOpenRouter(body) { // Client didn't send tools (likely passthrough mode) - inject standard Claude Code tools toolsToSend = STANDARD_TOOLS; toolsInjected = true; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS (OpenRouter) ==="); } if (Array.isArray(toolsToSend) && toolsToSend.length > 0) { openRouterBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend); - logger.info({ + logger.debug({ toolCount: toolsToSend.length, toolNames: toolsToSend.map(t => t.name), toolsInjected @@ -490,13 +509,16 @@ async function invokeAzureOpenAI(body) { }); } + // System prompt injection disabled - breaks model response + // Tool guidance now provided via tool descriptions instead + const azureBody = { messages, temperature: body.temperature ?? 0.3, // Lower temperature for more deterministic, action-oriented behavior max_tokens: Math.min(body.max_tokens ?? 4096, 16384), // Cap at Azure OpenAI's limit top_p: body.top_p ?? 1.0, stream: false, // Force non-streaming for Azure OpenAI - streaming format conversion not yet implemented - model: body._suggestionModeModel || config.azureOpenAI.deployment + model: body._suggestionModeModel || body._tierModel || config.azureOpenAI.deployment }; // Add tools - inject standard tools if client didn't send any (passthrough mode) @@ -507,18 +529,18 @@ async function invokeAzureOpenAI(body) { // Client didn't send tools (likely passthrough mode) - inject standard Claude Code tools toolsToSend = STANDARD_TOOLS; toolsInjected = true; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS ==="); } if (Array.isArray(toolsToSend) && toolsToSend.length > 0) { azureBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend); - azureBody.parallel_tool_calls = true; // Enable parallel tool calling for better performance + azureBody.parallel_tool_calls = true; // Enable parallel tool calls azureBody.tool_choice = "auto"; // Explicitly enable tool use (helps GPT models understand they should use tools) - logger.info({ + logger.debug({ toolCount: toolsToSend.length, toolNames: toolsToSend.map(t => t.name), toolsInjected, @@ -529,7 +551,7 @@ async function invokeAzureOpenAI(body) { }, "=== SENDING TOOLS TO AZURE OPENAI ==="); } - logger.info({ + logger.debug({ endpoint, hasTools: !!azureBody.tools, toolCount: azureBody.tools?.length || 0, @@ -563,14 +585,83 @@ async function invokeAzureOpenAI(body) { // Track function call IDs for matching with outputs const pendingCallIds = []; + // Detect if this is a continuation request (has tool results) + // Azure content filter triggers on full system prompt in continuations + // Check for: + // 1. tool_result blocks in user messages (Anthropic format) + // 2. tool messages (OpenAI format) + // 3. assistant messages with tool_use or tool_calls (indicates prior tool invocation) + // 4. Flattened continuation pattern from orchestrator (contains "IMPORTANT: Focus on") + const hasToolResults = (body.messages || []).some(msg => { + // Check for Anthropic format tool_result in user messages + if (msg.role === "user" && Array.isArray(msg.content)) { + if (msg.content.some(block => block.type === "tool_result")) return true; + } + // Check for OpenAI format tool messages + if (msg.role === "tool") return true; + // Check for assistant messages with tool_use (Anthropic) or tool_calls (OpenAI) + // If there's a prior tool use, this is a continuation + if (msg.role === "assistant") { + if (Array.isArray(msg.content)) { + if (msg.content.some(block => block.type === "tool_use")) return true; + } + if (msg.tool_calls && msg.tool_calls.length > 0) return true; + } + return false; + }) || azureBody.messages.some(msg => { + // Also check converted messages for flattened continuation pattern + // The orchestrator flattens tool results into user message with this marker + if (msg.role === "user" && typeof msg.content === "string") { + if (msg.content.includes("IMPORTANT: Focus on and respond ONLY to my most recent request")) return true; + } + return false; + }); + + if (hasToolResults) { + logger.debug({ + hasToolResults: true, + originalMessageCount: (body.messages || []).length, + convertedMessageCount: azureBody.messages.length, + messageRoles: (body.messages || []).map(m => m.role), + }, "=== CONTINUATION REQUEST DETECTED - using minimal system prompt to avoid Azure content filter ==="); + } else { + logger.debug({ + hasToolResults: false, + originalMessageCount: (body.messages || []).length, + messageRoles: (body.messages || []).map(m => m.role), + }, "Initial request - using full system prompt"); + } + + // Helper function to strip tags and meta-instructions from content + // Azure's jailbreak filter triggers on these instructions in continuation requests + const stripSystemReminders = (content) => { + if (!content || typeof content !== 'string') return content; + // Remove ... blocks + let cleaned = content.replace(/[\s\S]*?<\/system-reminder>/gi, ''); + // Remove the continuation marker that orchestrator adds + cleaned = cleaned.replace(/---\s*IMPORTANT:\s*Focus on and respond ONLY to my most recent request[^\n]*/gi, ''); + // Trim whitespace + return cleaned.trim(); + }; + for (const msg of azureBody.messages) { if (msg.role === "system") { - // System messages become developer messages - responsesInput.push({ - type: "message", - role: "developer", - content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content) - }); + // For continuation requests, use minimal system prompt to avoid content filter + // Azure's jailbreak detection triggers on security-related text in continuations + if (hasToolResults) { + responsesInput.push({ + type: "message", + role: "developer", + content: "You are a helpful coding assistant. Continue helping the user based on the tool results." + }); + } else { + // Initial request - use full system prompt + responsesInput.push({ + type: "message", + role: "developer", + content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content) + }); + } } else if (msg.role === "user") { // Check if content contains tool_result blocks (Anthropic format) if (Array.isArray(msg.content)) { @@ -585,19 +676,30 @@ async function invokeAzureOpenAI(body) { output: typeof block.content === 'string' ? block.content : JSON.stringify(block.content || "") }); } else if (block.type === "text") { - responsesInput.push({ - type: "message", - role: "user", - content: block.text || "" - }); + // For continuation requests, strip system-reminder tags to avoid jailbreak filter + const textContent = hasToolResults ? stripSystemReminders(block.text || "") : (block.text || ""); + if (textContent) { // Only add if there's content after stripping + responsesInput.push({ + type: "message", + role: "user", + content: textContent + }); + } } } } else { - responsesInput.push({ - type: "message", - role: "user", - content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content) - }); + // For continuation requests, strip system-reminder tags to avoid jailbreak filter + let userContent = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); + if (hasToolResults) { + userContent = stripSystemReminders(userContent); + } + if (userContent) { // Only add if there's content after stripping + responsesInput.push({ + type: "message", + role: "user", + content: userContent + }); + } } } else if (msg.role === "assistant") { // Assistant messages - handle tool_calls (OpenAI format) and tool_use blocks (Anthropic format) @@ -663,7 +765,7 @@ async function invokeAzureOpenAI(body) { tool_choice: azureBody.tool_choice, stream: false }; - logger.info({ + logger.debug({ format: "responses", inputCount: responsesBody.input?.length, model: responsesBody.model, @@ -681,7 +783,7 @@ async function invokeAzureOpenAI(body) { const textContent = messageOutput?.content?.find(c => c.type === "output_text")?.text || ""; // Find function_call outputs (tool calls are separate items in output array) - const toolCalls = outputArray + const rawToolCalls = outputArray .filter(o => o.type === "function_call") .map(tc => ({ id: tc.call_id || tc.id || `call_${Date.now()}`, @@ -692,7 +794,30 @@ async function invokeAzureOpenAI(body) { } })); - logger.info({ + // Deduplicate identical tool calls (GPT sometimes returns multiple identical calls) + const seenSignatures = new Set(); + const toolCalls = rawToolCalls.filter(tc => { + const signature = `${tc.function.name}:${tc.function.arguments}`; + if (seenSignatures.has(signature)) { + logger.warn({ + toolName: tc.function.name, + signature: signature.substring(0, 100), + }, "Filtered duplicate tool call from GPT response"); + return false; + } + seenSignatures.add(signature); + return true; + }); + + if (rawToolCalls.length !== toolCalls.length) { + logger.debug({ + originalCount: rawToolCalls.length, + dedupedCount: toolCalls.length, + removed: rawToolCalls.length - toolCalls.length, + }, "Deduplicated identical tool calls from single response"); + } + + logger.debug({ outputTypes: outputArray.map(o => o.type), hasMessage: !!messageOutput, toolCallCount: toolCalls.length, @@ -717,7 +842,7 @@ async function invokeAzureOpenAI(body) { usage: result.json.usage }; - logger.info({ + logger.debug({ convertedContent: textContent?.substring(0, 100), hasToolCalls: toolCalls.length > 0, toolCallCount: toolCalls.length @@ -725,7 +850,7 @@ async function invokeAzureOpenAI(body) { // Now convert from Chat Completions format to Anthropic format const anthropicJson = convertOpenAIToAnthropic(result.json); - logger.info({ + logger.debug({ anthropicContentTypes: anthropicJson.content?.map(c => c.type), stopReason: anthropicJson.stop_reason }, "Converted to Anthropic format"); @@ -747,67 +872,6 @@ async function invokeAzureOpenAI(body) { } } -/** - * Convert Azure Responses API response to Anthropic format - */ -function convertResponsesAPIToAnthropic(response, model) { - const content = []; - const outputArray = response.output || []; - - // Extract text content from message output - const messageOutput = outputArray.find(o => o.type === "message"); - if (messageOutput?.content) { - for (const item of messageOutput.content) { - if (item.type === "output_text" && item.text) { - content.push({ type: "text", text: item.text }); - } - } - } - - // Extract tool calls from function_call outputs - const toolCalls = outputArray - .filter(o => o.type === "function_call") - .map(tc => ({ - type: "tool_use", - id: tc.call_id || tc.id || `call_${Date.now()}`, - name: tc.name, - input: typeof tc.arguments === 'string' ? JSON.parse(tc.arguments || "{}") : (tc.arguments || {}) - })); - - content.push(...toolCalls); - - // Handle reasoning_content for thinking models - if (content.length === 0 && response.reasoning_content) { - content.push({ type: "text", text: response.reasoning_content }); - } - - // Ensure at least empty text if no content - if (content.length === 0) { - content.push({ type: "text", text: "" }); - } - - // Determine stop reason - let stopReason = "end_turn"; - if (toolCalls.length > 0) { - stopReason = "tool_use"; - } else if (response.status === "incomplete" && response.incomplete_details?.reason === "max_output_tokens") { - stopReason = "max_tokens"; - } - - return { - id: response.id || `msg_${Date.now()}`, - type: "message", - role: "assistant", - content, - model: model || response.model, - stop_reason: stopReason, - stop_sequence: null, - usage: { - input_tokens: response.usage?.input_tokens || 0, - output_tokens: response.usage?.output_tokens || 0, - } - }; -} async function invokeOpenAI(body) { if (!config.openai?.apiKey) { @@ -841,8 +905,10 @@ async function invokeOpenAI(body) { }); } + // System prompt injection disabled - breaks model response + const openAIBody = { - model: body._suggestionModeModel || config.openai.model || "gpt-4o", + model: body._suggestionModeModel || body._tierModel || config.openai.model || "gpt-4o", messages, temperature: body.temperature ?? 0.7, max_tokens: body.max_tokens ?? 4096, @@ -858,25 +924,25 @@ async function invokeOpenAI(body) { // Client didn't send tools (likely passthrough mode) - inject standard Claude Code tools toolsToSend = STANDARD_TOOLS; toolsInjected = true; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS (OpenAI) ==="); } if (Array.isArray(toolsToSend) && toolsToSend.length > 0) { openAIBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend); - openAIBody.parallel_tool_calls = true; // Enable parallel tool calling + openAIBody.parallel_tool_calls = false; // Disable parallel tool calls - GPT often makes duplicate calls openAIBody.tool_choice = "auto"; // Let the model decide when to use tools - logger.info({ + logger.debug({ toolCount: toolsToSend.length, toolNames: toolsToSend.map(t => t.name), toolsInjected }, "=== SENDING TOOLS TO OPENAI ==="); } - logger.info({ + logger.debug({ endpoint, model: openAIBody.model, hasTools: !!openAIBody.tools, @@ -934,7 +1000,7 @@ async function invokeLlamaCpp(body) { } if (deduplicated.length !== messages.length) { - logger.info({ + logger.debug({ originalCount: messages.length, deduplicatedCount: deduplicated.length, removed: messages.length - deduplicated.length, @@ -959,26 +1025,26 @@ async function invokeLlamaCpp(body) { if (injectToolsLlamacpp && (!Array.isArray(toolsToSend) || toolsToSend.length === 0)) { toolsToSend = STANDARD_TOOLS; toolsInjected = true; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS (llama.cpp) ==="); } else if (!injectToolsLlamacpp) { - logger.info({}, "Tool injection disabled for llama.cpp (INJECT_TOOLS_LLAMACPP=false)"); + logger.debug({}, "Tool injection disabled for llama.cpp (INJECT_TOOLS_LLAMACPP=false)"); } if (Array.isArray(toolsToSend) && toolsToSend.length > 0) { llamacppBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend); llamacppBody.tool_choice = "auto"; - logger.info({ + logger.debug({ toolCount: toolsToSend.length, toolNames: toolsToSend.map(t => t.name), toolsInjected }, "=== SENDING TOOLS TO LLAMA.CPP ==="); } - logger.info({ + logger.debug({ endpoint, hasTools: !!llamacppBody.tools, toolCount: llamacppBody.tools?.length || 0, @@ -1042,9 +1108,9 @@ async function invokeLMStudio(body) { if (!Array.isArray(toolsToSend) || toolsToSend.length === 0) { toolsToSend = STANDARD_TOOLS; toolsInjected = true; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS (LM Studio) ==="); } @@ -1052,14 +1118,14 @@ async function invokeLMStudio(body) { if (Array.isArray(toolsToSend) && toolsToSend.length > 0) { lmstudioBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend); lmstudioBody.tool_choice = "auto"; - logger.info({ + logger.debug({ toolCount: toolsToSend.length, toolNames: toolsToSend.map(t => t.name), toolsInjected }, "=== SENDING TOOLS TO LM STUDIO ==="); } - logger.info({ + logger.debug({ endpoint, hasTools: !!lmstudioBody.tools, toolCount: lmstudioBody.tools?.length || 0, @@ -1080,7 +1146,7 @@ async function invokeBedrock(body) { } const bearerToken = config.bedrock.apiKey; - logger.info({ authMethod: "Bearer Token" }, "=== BEDROCK AUTH ==="); + logger.debug({ authMethod: "Bearer Token" }, "=== BEDROCK AUTH ==="); // 2. Inject standard tools if needed let toolsToSend = body.tools; @@ -1089,9 +1155,9 @@ async function invokeBedrock(body) { if (!Array.isArray(toolsToSend) || toolsToSend.length === 0) { toolsToSend = STANDARD_TOOLS; toolsInjected = true; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS (Bedrock) ==="); } @@ -1099,10 +1165,10 @@ async function invokeBedrock(body) { const bedrockBody = { ...body, tools: toolsToSend }; // 4. Detect model family and convert format - const modelId = config.bedrock.modelId; + const modelId = body._tierModel || config.bedrock.modelId; const modelFamily = detectModelFamily(modelId); - logger.info({ + logger.debug({ modelId, modelFamily, hasTools: !!bedrockBody.tools, @@ -1167,7 +1233,7 @@ async function invokeBedrock(body) { const host = `bedrock-runtime.${config.bedrock.region}.amazonaws.com`; const endpoint = `https://${host}${path}`; - logger.info({ + logger.debug({ endpoint, authMethod: "Bearer Token", hasSystem: !!converseBody.system, @@ -1200,7 +1266,7 @@ async function invokeBedrock(body) { // Parse Converse API response (already parsed by performJsonRequest) const converseResponse = response.json; // Use property, not method - logger.info({ + logger.debug({ stopReason: converseResponse.stopReason, inputTokens: converseResponse.usage?.inputTokens || 0, outputTokens: converseResponse.usage?.outputTokens || 0, @@ -1280,7 +1346,7 @@ async function invokeZai(body) { "claude-3-haiku": "glm-4.5-air", }; - const requestedModel = body.model || config.zai.model; + const requestedModel = body._tierModel || body.model || config.zai.model; let mappedModel = modelMap[requestedModel] || config.zai.model || "glm-4.7"; mappedModel = mappedModel.toLowerCase(); @@ -1357,7 +1423,7 @@ async function invokeZai(body) { // "required" was forcing tools even for simple greetings zaiBody.tool_choice = "auto"; // Also enable parallel tool calls - zaiBody.parallel_tool_calls = true; + zaiBody.parallel_tool_calls = false; // Disable parallel tool calls - GPT often makes duplicate calls } headers = { @@ -1372,9 +1438,9 @@ async function invokeZai(body) { // Inject standard tools if client didn't send any (passthrough mode) if (!Array.isArray(zaiBody.tools) || zaiBody.tools.length === 0) { zaiBody.tools = STANDARD_TOOLS; - logger.info({ + logger.debug({ injectedToolCount: STANDARD_TOOLS.length, - injectedToolNames: STANDARD_TOOLS.map(t => t.name), + injectedToolNames: STANDARD_TOOL_NAMES, reason: "Client did not send tools (passthrough mode)" }, "=== INJECTING STANDARD TOOLS (Z.AI Anthropic) ==="); } @@ -1386,7 +1452,7 @@ async function invokeZai(body) { }; } - logger.info({ + logger.debug({ endpoint, format: isOpenAIFormat ? "openai" : "anthropic", model: zaiBody.model, @@ -1416,7 +1482,7 @@ async function invokeZai(body) { const response = await performJsonRequest(endpoint, { headers, body: zaiBody }, "Z.AI"); - logger.info({ + logger.debug({ responseOk: response?.ok, responseStatus: response?.status, hasJson: !!response?.json, @@ -1428,7 +1494,7 @@ async function invokeZai(body) { // Convert OpenAI response back to Anthropic format if needed if (isOpenAIFormat && response?.ok && response?.json) { const anthropicJson = convertOpenAIToAnthropic(response.json); - logger.info({ + logger.debug({ convertedContent: JSON.stringify(anthropicJson.content).substring(0, 200), }, "=== Z.AI CONVERTED RESPONSE ==="); // Return in the same format as other providers (with ok, status, json) @@ -1448,6 +1514,118 @@ async function invokeZai(body) { +/** + * Moonshot AI (Kimi) Provider + * + * Moonshot offers Kimi models through an OpenAI-compatible chat completions API. + * Uses native system role support (unlike Z.AI which merges into user message). + */ +async function invokeMoonshot(body) { + if (!config.moonshot?.apiKey) { + throw new Error("Moonshot API key is not configured. Set MOONSHOT_API_KEY in your .env file."); + } + + const { + convertAnthropicToolsToOpenRouter, + convertAnthropicMessagesToOpenRouter + } = require("./openrouter-utils"); + + const endpoint = config.moonshot.endpoint || "https://api.moonshot.ai/v1/chat/completions"; + + // Model mapping: Anthropic names → Moonshot/Kimi names + const modelMap = { + "claude-sonnet-4-5-20250929": "kimi-k2-turbo-preview", + "claude-sonnet-4-5": "kimi-k2-turbo-preview", + "claude-sonnet-4.5": "kimi-k2-turbo-preview", + "claude-3-5-sonnet": "kimi-k2-turbo-preview", + "claude-haiku-4-5-20251001": "kimi-k2-turbo-preview", + "claude-haiku-4-5": "kimi-k2-turbo-preview", + "claude-3-haiku": "kimi-k2-turbo-preview", + }; + + const requestedModel = body._tierModel || body.model || config.moonshot.model; + const mappedModel = modelMap[requestedModel] || config.moonshot.model || "kimi-k2-turbo-preview"; + + // Convert messages using existing utility + const messages = convertAnthropicMessagesToOpenRouter(body.messages || []); + + // Moonshot natively supports system role — add as system message + if (body.system) { + const systemContent = Array.isArray(body.system) + ? body.system.map(s => s.text || s).join("\n") + : body.system; + messages.unshift({ role: "system", content: systemContent }); + } + + const moonshotBody = { + model: mappedModel, + messages, + max_tokens: body.max_tokens || 4096, + temperature: body.temperature ?? 0.7, + top_p: body.top_p ?? 1.0, + stream: false, // Force non-streaming - OpenAI SSE to Anthropic SSE conversion not implemented + }; + + // Convert and add tools if present + if (Array.isArray(body.tools) && body.tools.length > 0) { + moonshotBody.tools = convertAnthropicToolsToOpenRouter(body.tools); + moonshotBody.tool_choice = "auto"; + moonshotBody.parallel_tool_calls = false; + } + + const headers = { + "Content-Type": "application/json", + "Authorization": `Bearer ${config.moonshot.apiKey}`, + }; + + logger.debug({ + endpoint, + model: moonshotBody.model, + originalModel: requestedModel, + messageCount: moonshotBody.messages?.length || 0, + hasTools: !!moonshotBody.tools, + toolCount: moonshotBody.tools?.length || 0, + }, "=== Moonshot REQUEST ==="); + + const response = await performJsonRequest(endpoint, { headers, body: moonshotBody }, "Moonshot"); + + const rawMsg = response?.json?.choices?.[0]?.message; + logger.debug({ + responseOk: response?.ok, + responseStatus: response?.status, + hasJson: !!response?.json, + contentType: typeof rawMsg?.content, + contentValue: typeof rawMsg?.content === 'string' ? rawMsg.content.substring(0, 300) : String(JSON.stringify(rawMsg?.content) || '').substring(0, 300), + hasReasoning: !!rawMsg?.reasoning_content, + reasoningType: typeof rawMsg?.reasoning_content, + reasoningValue: typeof rawMsg?.reasoning_content === 'string' ? rawMsg.reasoning_content.substring(0, 300) : String(JSON.stringify(rawMsg?.reasoning_content) || '').substring(0, 300), + finishReason: response?.json?.choices?.[0]?.finish_reason, + messageKeys: rawMsg ? Object.keys(rawMsg) : [], + fullRawResponse: String(JSON.stringify(response?.json) || '').substring(0, 800), + }, "=== Moonshot RAW RESPONSE ==="); + + // Convert OpenAI response back to Anthropic format + if (response?.ok && response?.json) { + const anthropicJson = convertOpenAIToAnthropic(response.json); + logger.debug({ + convertedContent: JSON.stringify(anthropicJson.content).substring(0, 500), + contentLength: anthropicJson.content?.length, + firstContentType: anthropicJson.content?.[0]?.type, + firstContentText: anthropicJson.content?.[0]?.text?.substring(0, 300), + }, "=== Moonshot CONVERTED RESPONSE ==="); + return { + ok: response.ok, + status: response.status, + json: anthropicJson, + text: JSON.stringify(anthropicJson), + contentType: "application/json", + headers: response.headers, + }; + } + + return response; +} + /** * Convert OpenAI response to Anthropic format */ @@ -1463,11 +1641,17 @@ function convertOpenAIToAnthropic(response) { // Add text content from message.content // Don't add placeholder text if there are tool_calls - tools are the actual response const hasToolCalls = Array.isArray(message.tool_calls) && message.tool_calls.length > 0; - if (message.content) { - content.push({ type: "text", text: message.content }); - } else if (message.reasoning_content && !message.content) { - // Thinking models (Kimi-K2, o1, etc.) return response in reasoning_content - content.push({ type: "text", text: message.reasoning_content }); + + // Extract text content - handle thinking models that split content/reasoning + const textContent = typeof message.content === 'string' ? message.content : ''; + const reasoningContent = typeof message.reasoning_content === 'string' ? message.reasoning_content : ''; + + if (textContent) { + // Has regular content - use it directly (ignore reasoning_content chain-of-thought) + content.push({ type: "text", text: textContent }); + } else if (reasoningContent) { + // Fallback: thinking models where content is empty but reasoning has the output + content.push({ type: "text", text: reasoningContent }); } // Convert tool calls @@ -1488,13 +1672,14 @@ function convertOpenAIToAnthropic(response) { } // Determine stop reason + // IMPORTANT: Check for actual tool_calls presence, not just finish_reason string. + // Some providers (Moonshot, etc.) return finish_reason: "stop" even when tool_calls exist. + // If we don't set stop_reason to "tool_use", the CLI won't execute the tool calls. let stopReason = "end_turn"; - if (choice.finish_reason === "tool_calls") { + if (hasToolCalls) { stopReason = "tool_use"; } else if (choice.finish_reason === "length") { stopReason = "max_tokens"; - } else if (choice.finish_reason === "stop") { - stopReason = "end_turn"; } return { @@ -1580,7 +1765,7 @@ async function invokeVertex(body) { }; // Map model name - const requestedModel = body.model || config.vertex.model; + const requestedModel = body._tierModel || body.model || config.vertex.model; const geminiModel = modelMap[requestedModel] || config.vertex.model || "gemini-2.0-flash"; // Construct Gemini API endpoint @@ -1626,7 +1811,7 @@ async function invokeVertex(body) { "Content-Type": "application/json", }; - logger.info({ + logger.debug({ endpoint: endpoint.replace(apiKey, "***"), model: geminiModel, originalModel: requestedModel, @@ -1655,7 +1840,7 @@ async function invokeVertex(body) { // Convert Gemini response to Anthropic format if (response?.json) { const anthropicJson = convertGeminiToAnthropic(response.json, requestedModel); - logger.info({ + logger.debug({ convertedContent: JSON.stringify(anthropicJson.content).substring(0, 200), }, "=== VERTEX AI (GEMINI) CONVERTED RESPONSE ==="); return { @@ -1816,35 +2001,44 @@ function convertGeminiToAnthropic(response, requestedModel) { } async function invokeModel(body, options = {}) { - const { determineProvider, isFallbackEnabled, getFallbackProvider, analyzeComplexity } = require("./routing"); + const { determineProviderSmart, isFallbackEnabled, getFallbackProvider } = require("./routing"); const metricsCollector = getMetricsCollector(); const registry = getCircuitBreakerRegistry(); const healthTracker = getHealthTracker(); - // Analyze complexity and determine provider - const complexityAnalysis = analyzeComplexity(body); - const initialProvider = options.forceProvider ?? determineProvider(body); - const preferOllama = config.modelProvider?.preferOllama ?? false; + // Determine provider via async tier routing + const routingResult = options.forceProvider + ? { provider: options.forceProvider, model: null, method: 'forced' } + : await determineProviderSmart(body); + const initialProvider = routingResult.provider; + const tierSelectedModel = routingResult.model; + + // Inject tier-selected model into body so provider functions can use it + if (tierSelectedModel) { + body._tierModel = tierSelectedModel; + } // Build routing decision object for response headers const routingDecision = { provider: initialProvider, - score: complexityAnalysis.score, - threshold: complexityAnalysis.threshold, - mode: complexityAnalysis.mode, - recommendation: complexityAnalysis.recommendation, - method: complexityAnalysis.score !== undefined ? 'complexity' : 'static', - taskType: complexityAnalysis.breakdown?.taskType?.reason, + tier: routingResult.tier || null, + model: tierSelectedModel || null, + score: routingResult.score, + threshold: routingResult.threshold, + mode: routingResult.mode, + reason: routingResult.reason, + method: routingResult.method || 'static', }; logger.debug({ initialProvider, - preferOllama, + tierSelectedModel, + tier: routingResult.tier, fallbackEnabled: isFallbackEnabled(), toolCount: Array.isArray(body?.tools) ? body.tools.length : 0, - complexityScore: complexityAnalysis.score, - complexityThreshold: complexityAnalysis.threshold, - recommendation: complexityAnalysis.recommendation, + score: routingResult.score, + reason: routingResult.reason, + method: routingResult.method, }, "Provider routing decision"); metricsCollector.recordProviderRouting(initialProvider); @@ -1885,6 +2079,8 @@ async function invokeModel(body, options = {}) { return await invokeZai(body); } else if (initialProvider === "vertex") { return await invokeVertex(body); + } else if (initialProvider === "moonshot") { + return await invokeMoonshot(body); } return await invokeDatabricks(body); }); @@ -1920,11 +2116,10 @@ async function invokeModel(body, options = {}) { metricsCollector.recordProviderFailure(initialProvider); healthTracker.recordFailure(initialProvider, err, err.status); - // Check if we should fallback + // Check if we should fallback (any provider can fall back, not just ollama) const shouldFallback = - preferOllama && - initialProvider === "ollama" && isFallbackEnabled() && + initialProvider !== getFallbackProvider() && !options.disableFallback; if (!shouldFallback) { @@ -1941,7 +2136,7 @@ async function invokeModel(body, options = {}) { fallbackProvider, reason, error: err.message, - }, "Ollama failed, attempting transparent fallback to cloud"); + }, "Primary provider failed, attempting transparent fallback"); metricsCollector.recordFallbackAttempt(initialProvider, fallbackProvider, reason); @@ -1974,6 +2169,8 @@ async function invokeModel(body, options = {}) { return await invokeZai(body); } else if (fallbackProvider === "vertex") { return await invokeVertex(body); + } else if (fallbackProvider === "moonshot") { + return await invokeMoonshot(body); } return await invokeDatabricks(body); }); @@ -2023,7 +2220,7 @@ async function invokeModel(body, options = {}) { fallbackProvider, originalError: err.message, fallbackError: fallbackErr.message, - }, "Both Ollama and fallback provider failed"); + }, "Both primary and fallback provider failed"); // Return fallback error (more actionable than Ollama error) throw fallbackErr; diff --git a/src/clients/gpt-utils.js b/src/clients/gpt-utils.js new file mode 100644 index 0000000..b5d84be --- /dev/null +++ b/src/clients/gpt-utils.js @@ -0,0 +1,181 @@ +/** + * GPT-specific utilities for handling tool calls and responses + * All settings are hardcoded - no env vars required + * + * This module addresses GPT model compatibility issues when using Azure OpenAI + * through Lynkr proxy with Claude Code: + * - GPT doesn't interpret "0 files found" as a final answer + * - GPT retries the same tool expecting different results + * - GPT needs explicit guidance on tool result interpretation + */ + +const logger = require("../logger"); + +// Hardcoded GPT settings - optimized for GPT model behavior +const GPT_SETTINGS = { + toolLoopThreshold: 2, // Lower than Claude's 3 to catch loops earlier + enhancedFormatting: true, // Always format results explicitly for GPT + similarityThreshold: 0.8, // For detecting similar (not just identical) tool calls +}; + +// Provider identifiers that use GPT models +const GPT_PROVIDERS = ['azure-openai', 'openai']; + +/** + * Check if a provider uses GPT models + * @param {string} provider - Provider type (e.g., 'azure-openai', 'databricks') + * @returns {boolean} - True if provider uses GPT models + */ +function isGPTProvider(provider) { + if (!provider) return false; + return GPT_PROVIDERS.includes(provider.toLowerCase()); +} + +/** + * Get the tool loop threshold for GPT models + * @returns {number} - Threshold (2 for GPT, lower than Claude's 3) + */ +function getGPTToolLoopThreshold() { + return GPT_SETTINGS.toolLoopThreshold; +} + +/** + * Format tool result with explicit structure for GPT models + * GPT models need clear, unambiguous formatting to understand tool results + * + * @param {string} toolName - Name of the tool that was called + * @param {string} content - The tool result content + * @param {Object} args - The arguments passed to the tool + * @returns {string} - Formatted result with explicit status and instructions + */ +function formatToolResultForGPT(toolName, content, args) { + // Handle empty/no results explicitly - add clear messaging to prevent retries + const isEmpty = !content || + content.trim() === '' || + content.includes('0 files found') || + content.includes('No matches found') || + content.includes('No results') || + content.includes('Found 0') || + /^Found \d+ files?\.$/.test(content.trim()) && content.includes('Found 0'); + + if (isEmpty) { + // Only format empty results - add explicit "don't retry" instruction + return `Tool "${toolName}" completed with no results found. +Query: ${JSON.stringify(args)} + +This is a FINAL result - do not retry this query. Respond to the user based on this outcome.`; + } + + // For successful results, return content as-is (don't add markers that might confuse GPT) + return content; +} + +/** + * Get system prompt addendum for GPT models + * This teaches GPT how to properly interpret and use tools + * + * @returns {string} - System prompt instructions for GPT + */ +function getGPTSystemPromptAddendum() { + return `Use the Bash tool with ls command for listing files. After any tool returns results, respond to the user.`; +} + +/** + * Calculate string similarity using Jaccard index + * Used to detect semantically similar tool calls + * + * @param {string} s1 - First string + * @param {string} s2 - Second string + * @returns {number} - Similarity score between 0 and 1 + */ +function stringSimilarity(s1, s2) { + if (!s1 || !s2) return 0; + if (s1 === s2) return 1; + + // Tokenize by whitespace and common delimiters + const tokenize = (s) => new Set( + s.toLowerCase() + .split(/[\s\-_\/\.\,\:\;]+/) + .filter(t => t.length > 0) + ); + + const set1 = tokenize(s1); + const set2 = tokenize(s2); + + const intersection = new Set([...set1].filter(x => set2.has(x))); + const union = new Set([...set1, ...set2]); + + return union.size > 0 ? intersection.size / union.size : 0; +} + +/** + * Check if two tool calls are semantically similar + * GPT often retries with slightly different parameters that are functionally equivalent + * + * @param {Object} call1 - First tool call {name, arguments} + * @param {Object} call2 - Second tool call {name, arguments} + * @returns {boolean} - True if calls are similar enough to be considered duplicates + */ +function areSimilarToolCalls(call1, call2) { + if (!call1 || !call2) return false; + + // Must be the same tool + const name1 = call1.function?.name ?? call1.name; + const name2 = call2.function?.name ?? call2.name; + if (name1 !== name2) return false; + + // Get arguments + const args1 = call1.function?.arguments ?? call1.arguments ?? call1.input ?? {}; + const args2 = call2.function?.arguments ?? call2.arguments ?? call2.input ?? {}; + + // Stringify for comparison + const argsStr1 = typeof args1 === 'string' ? args1 : JSON.stringify(args1); + const argsStr2 = typeof args2 === 'string' ? args2 : JSON.stringify(args2); + + // Exact match + if (argsStr1 === argsStr2) return true; + + // For search-related tools, check semantic similarity + const searchTools = ['grep', 'glob', 'search', 'find', 'read', 'bash', 'shell']; + const toolName = (name1 || '').toLowerCase(); + const isSearchTool = searchTools.some(t => toolName.includes(t)); + + if (isSearchTool) { + const similarity = stringSimilarity(argsStr1, argsStr2); + if (similarity >= GPT_SETTINGS.similarityThreshold) { + logger.debug({ + tool: name1, + similarity, + threshold: GPT_SETTINGS.similarityThreshold, + args1: argsStr1.substring(0, 100), + args2: argsStr2.substring(0, 100), + }, "Similar tool call detected"); + return true; + } + } + + return false; +} + +/** + * Get a signature for a tool call (for tracking in history) + * @param {Object} call - Tool call object + * @returns {string} - Unique signature for the call + */ +function getToolCallSignature(call) { + const name = call.function?.name ?? call.name ?? 'unknown'; + const args = call.function?.arguments ?? call.arguments ?? call.input ?? {}; + const argsStr = typeof args === 'string' ? args : JSON.stringify(args); + return `${name}:${argsStr}`; +} + +module.exports = { + GPT_SETTINGS, + isGPTProvider, + getGPTToolLoopThreshold, + formatToolResultForGPT, + getGPTSystemPromptAddendum, + stringSimilarity, + areSimilarToolCalls, + getToolCallSignature, +}; diff --git a/src/clients/ollama-utils.js b/src/clients/ollama-utils.js index 7582f05..d77bbd7 100644 --- a/src/clients/ollama-utils.js +++ b/src/clients/ollama-utils.js @@ -10,12 +10,21 @@ const modelCapabilitiesCache = new Map(); const TOOL_CAPABLE_MODELS = new Set([ "llama3.1", "llama3.2", + "llama3.3", "qwen2.5", + "qwen3", "mistral", "mistral-nemo", "firefunction-v2", "kimi-k2.5", - "nemotron" + "nemotron", + "glm-4", + "glm-4.5", + "glm-4.7", + "glm-5", + "gpt-oss", + "minimax", + "deepseek-r1", ]); /** @@ -55,25 +64,60 @@ async function checkOllamaToolSupport(modelName = config.ollama?.model) { return supportsTools; } +// --- Endpoint detection: Anthropic (/v1/messages) vs legacy (/api/chat) --- + +// null = not probed yet, true = Anthropic available, false = use legacy +let anthropicEndpointAvailable = null; + +/** + * Probe whether Ollama exposes the Anthropic-compatible /v1/messages endpoint (v0.14.0+). + * Result is cached for the process lifetime. + */ +async function hasAnthropicEndpoint(baseUrl) { + if (anthropicEndpointAvailable !== null) return anthropicEndpointAvailable; + + try { + // Send a minimal request — we only care about whether the route exists + const res = await fetch(`${baseUrl}/v1/messages`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ + model: "probe", + max_tokens: 1, + messages: [{ role: "user", content: "hi" }], + }), + }); + + // 404 → endpoint doesn't exist (old Ollama) + // Any other status (200, 400, 500) → endpoint exists + anthropicEndpointAvailable = res.status !== 404; + logger.info( + { available: anthropicEndpointAvailable, status: res.status }, + anthropicEndpointAvailable + ? "Ollama Anthropic API detected (/v1/messages) — using native passthrough" + : "Ollama Anthropic API not available — falling back to legacy /api/chat (upgrade to Ollama v0.14.0+ for best results)" + ); + } catch (err) { + // Network error — assume legacy + anthropicEndpointAvailable = false; + logger.warn({ error: err.message }, "Failed to probe Ollama Anthropic endpoint, using legacy /api/chat"); + } + + return anthropicEndpointAvailable; +} + +// Exposed for tests +function resetEndpointCache() { + anthropicEndpointAvailable = null; +} + +// --- Legacy format conversion (for Ollama < v0.14.0 using /api/chat) --- + /** - * Convert Anthropic tool format to Ollama format - * - * Anthropic format: - * { - * name: "get_weather", - * description: "Get weather", - * input_schema: { type: "object", properties: {...}, required: [...] } - * } - * - * Ollama format: - * { - * type: "function", - * function: { - * name: "get_weather", - * description: "Get weather", - * parameters: { type: "object", properties: {...}, required: [...] } - * } - * } + * Convert Anthropic tool format to Ollama/OpenAI function-calling format */ function convertAnthropicToolsToOllama(anthropicTools) { if (!Array.isArray(anthropicTools) || anthropicTools.length === 0) { @@ -93,128 +137,10 @@ function convertAnthropicToolsToOllama(anthropicTools) { })); } -/** - * Convert Ollama tool call response to Anthropic format - * - * Ollama format (actual): - * { - * message: { - * role: "assistant", - * content: "", - * tool_calls: [{ - * function: { - * name: "get_weather", - * arguments: { location: "SF" } // Already parsed object - * } - * }] - * } - * } - * - * Anthropic format: - * { - * content: [{ - * type: "tool_use", - * id: "toolu_123", - * name: "get_weather", - * input: { location: "SF" } - * }], - * stop_reason: "tool_use" - * } - */ -function convertOllamaToolCallsToAnthropic(ollamaResponse) { - const message = ollamaResponse?.message || {}; - const toolCalls = message.tool_calls || []; - const textContent = message.content || ""; - - const contentBlocks = []; - - // Add text content if present - if (textContent && textContent.trim()) { - contentBlocks.push({ - type: "text", - text: textContent, - }); - } - - // Add tool calls - for (const toolCall of toolCalls) { - const func = toolCall.function || {}; - let input = {}; - - // Handle arguments - can be string JSON or already parsed object - if (func.arguments) { - if (typeof func.arguments === "string") { - try { - input = JSON.parse(func.arguments); - } catch (err) { - logger.warn({ - error: err.message, - arguments: func.arguments - }, "Failed to parse Ollama tool arguments string"); - input = {}; - } - } else if (typeof func.arguments === "object") { - // Already an object, use directly - input = func.arguments; - } - } - - // Generate tool use ID (Ollama may or may not provide one) - const toolUseId = toolCall.id || `toolu_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - - contentBlocks.push({ - type: "tool_use", - id: toolUseId, - name: func.name || "unknown", - input, - }); - } - - // Determine stop reason - const stopReason = toolCalls.length > 0 ? "tool_use" : "end_turn"; - - return { - contentBlocks, - stopReason, - }; -} - -/** - * Build complete Anthropic response from Ollama with tool calls - */ -function buildAnthropicResponseFromOllama(ollamaResponse, requestedModel) { - const { contentBlocks, stopReason } = convertOllamaToolCallsToAnthropic(ollamaResponse); - - // Ensure at least one content block - const finalContent = contentBlocks.length > 0 - ? contentBlocks - : [{ type: "text", text: "" }]; - - // Extract token counts - const inputTokens = ollamaResponse.prompt_eval_count || 0; - const outputTokens = ollamaResponse.eval_count || 0; - - return { - id: `msg_${Date.now()}`, - type: "message", - role: "assistant", - model: requestedModel, - content: finalContent, - stop_reason: stopReason, - stop_sequence: null, - usage: { - input_tokens: inputTokens, - output_tokens: outputTokens, - cache_creation_input_tokens: 0, - cache_read_input_tokens: 0, - }, - }; -} - module.exports = { checkOllamaToolSupport, - convertAnthropicToolsToOllama, - convertOllamaToolCallsToAnthropic, - buildAnthropicResponseFromOllama, modelNameSupportsTools, + hasAnthropicEndpoint, + resetEndpointCache, + convertAnthropicToolsToOllama, }; diff --git a/src/clients/routing.js b/src/clients/routing.js index 28ab679..066ee14 100644 --- a/src/clients/routing.js +++ b/src/clients/routing.js @@ -14,7 +14,6 @@ const smartRouting = require('../routing'); // Re-export all functions from smart routing module.exports = { - determineProvider: smartRouting.determineProvider, determineProviderSmart: smartRouting.determineProviderSmart, isFallbackEnabled: smartRouting.isFallbackEnabled, getFallbackProvider: smartRouting.getFallbackProvider, diff --git a/src/clients/standard-tools.js b/src/clients/standard-tools.js index 61ac791..8efa408 100644 --- a/src/clients/standard-tools.js +++ b/src/clients/standard-tools.js @@ -76,7 +76,7 @@ const STANDARD_TOOLS = [ }, { name: "Bash", - description: "Executes a bash command in a persistent shell session. Use for terminal operations like git, npm, docker, etc. DO NOT use for file operations - use specialized tools instead.", + description: "Executes a bash command in a persistent shell session. Use for terminal operations like git, npm, docker, listing files (ls), etc. PREFERRED for listing directory contents - use 'ls' command. DO NOT use for reading file contents - use Read tool instead.", input_schema: { type: "object", properties: { @@ -98,7 +98,7 @@ const STANDARD_TOOLS = [ }, { name: "Glob", - description: "Fast file pattern matching tool. Supports glob patterns like '**/*.js' or 'src/**/*.ts'. Returns matching file paths sorted by modification time.", + description: "File pattern matching for finding files by name pattern. Use ONLY when you need to find files matching a specific pattern like '**/*.js'. For simple directory listing, use Bash with 'ls' instead.", input_schema: { type: "object", properties: { @@ -145,6 +145,66 @@ const STANDARD_TOOLS = [ required: ["pattern"] } }, + { + name: "MultiEdit", + description: "Makes multiple edits to a single file in one atomic operation. More efficient than calling Edit multiple times. Each edit is an exact string replacement.", + input_schema: { + type: "object", + properties: { + file_path: { + type: "string", + description: "Relative path within workspace. DO NOT use absolute paths." + }, + edits: { + type: "array", + description: "Array of edits to apply to the file", + items: { + type: "object", + properties: { + old_string: { + type: "string", + description: "The text to replace" + }, + new_string: { + type: "string", + description: "The text to replace it with" + } + }, + required: ["old_string", "new_string"] + } + } + }, + required: ["file_path", "edits"] + } + }, + { + name: "LS", + description: "Lists files and directories in a given path. Returns a structured listing with file types and sizes. Use for quick directory overview.", + input_schema: { + type: "object", + properties: { + path: { + type: "string", + description: "The directory to list. Defaults to current working directory." + } + }, + required: [] + } + }, + { + name: "NotebookRead", + description: "Reads and displays the contents of a Jupyter notebook (.ipynb file), including all cells with their outputs, combining code, text, and visualizations.", + input_schema: { + type: "object", + properties: { + notebook_path: { + type: "string", + description: "Relative path to the Jupyter notebook (e.g., 'analysis.ipynb'). DO NOT use absolute paths." + } + }, + required: ["notebook_path"] + } + }, { name: "TodoWrite", description: "Create and manage a structured task list for tracking progress and organizing complex tasks. Use proactively for multi-step tasks or when user provides multiple tasks.", @@ -354,4 +414,17 @@ EXAMPLE: User says "explore this project" → Call Task with subagent_type="Expl } ]; -module.exports = { STANDARD_TOOLS }; +// Pre-computed name list to avoid re-mapping on every log call +const STANDARD_TOOL_NAMES = STANDARD_TOOLS.map(t => t.name); + +// Tools that cannot work through a proxy (require bidirectional user interaction). +// All other tools are safe — per-client filtering via CLIENT_TOOL_MAPPINGS in +// openai-router.js handles excluding tools that specific clients don't support +// (e.g. Codex has no equivalent for Task, WebFetch, NotebookEdit). +const IDE_UNSUPPORTED_TOOLS = new Set(['AskUserQuestion']); + +// Filtered tool set for IDE clients — excludes tools with no IDE equivalent +const IDE_SAFE_TOOLS = STANDARD_TOOLS.filter(t => !IDE_UNSUPPORTED_TOOLS.has(t.name)); +const IDE_SAFE_TOOL_NAMES = IDE_SAFE_TOOLS.map(t => t.name); + +module.exports = { STANDARD_TOOLS, STANDARD_TOOL_NAMES, IDE_SAFE_TOOLS, IDE_SAFE_TOOL_NAMES, IDE_UNSUPPORTED_TOOLS }; diff --git a/src/config/index.js b/src/config/index.js index d75d045..a58414e 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -62,7 +62,7 @@ function resolveConfigPath(targetPath) { return path.resolve(normalised); } -const SUPPORTED_MODEL_PROVIDERS = new Set(["databricks", "azure-anthropic", "ollama", "openrouter", "azure-openai", "openai", "llamacpp", "lmstudio", "bedrock", "zai", "vertex"]); +const SUPPORTED_MODEL_PROVIDERS = new Set(["databricks", "azure-anthropic", "ollama", "openrouter", "azure-openai", "openai", "llamacpp", "lmstudio", "bedrock", "zai", "vertex", "moonshot"]); const rawModelProvider = (process.env.MODEL_PROVIDER ?? "databricks").toLowerCase(); // Validate MODEL_PROVIDER early with a clear error message @@ -132,6 +132,11 @@ const zaiApiKey = process.env.ZAI_API_KEY?.trim() || null; const zaiEndpoint = process.env.ZAI_ENDPOINT?.trim() || "https://api.z.ai/api/anthropic/v1/messages"; const zaiModel = process.env.ZAI_MODEL?.trim() || "GLM-4.7"; +// Moonshot AI (Kimi) configuration - OpenAI-compatible API +const moonshotApiKey = process.env.MOONSHOT_API_KEY?.trim() || null; +const moonshotEndpoint = process.env.MOONSHOT_ENDPOINT?.trim() || "https://api.moonshot.ai/v1/chat/completions"; +const moonshotModel = process.env.MOONSHOT_MODEL?.trim() || "kimi-k2-turbo-preview"; + // Vertex AI (Google Gemini) configuration const vertexApiKey = process.env.VERTEX_API_KEY?.trim() || process.env.GOOGLE_API_KEY?.trim() || null; const vertexModel = process.env.VERTEX_MODEL?.trim() || "gemini-2.0-flash"; @@ -144,8 +149,7 @@ const suggestionModeModel = (process.env.SUGGESTION_MODE_MODEL ?? "default").tri const hotReloadEnabled = process.env.HOT_RELOAD_ENABLED !== "false"; // default true const hotReloadDebounceMs = Number.parseInt(process.env.HOT_RELOAD_DEBOUNCE_MS ?? "1000", 10); -// Hybrid routing configuration -const preferOllama = process.env.PREFER_OLLAMA === "true"; +// Routing configuration const fallbackEnabled = process.env.FALLBACK_ENABLED !== "false"; // default true const ollamaMaxToolsForRouting = Number.parseInt( process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING ?? "3", @@ -311,37 +315,39 @@ if (modelProvider === "bedrock" && !bedrockApiKey) { ); } -// Validate hybrid routing configuration -if (preferOllama) { - if (!ollamaEndpoint) { - throw new Error("PREFER_OLLAMA is set but OLLAMA_ENDPOINT is not configured"); - } - if (fallbackEnabled && !SUPPORTED_MODEL_PROVIDERS.has(fallbackProvider)) { - throw new Error( - `FALLBACK_PROVIDER must be one of: ${Array.from(SUPPORTED_MODEL_PROVIDERS).join(", ")}` - ); - } +// Deprecation warning for PREFER_OLLAMA +if (process.env.PREFER_OLLAMA) { + console.warn('[DEPRECATION] PREFER_OLLAMA is removed. Use TIER_* env vars for routing. See documentation/routing.md'); +} - // Prevent local providers from being used as fallback (they can fail just like Ollama) +// Warn about misconfigured fallback provider (only when tier routing is active, +// since that's the only path that triggers provider fallback) +const tiersConfigured = !!( + process.env.TIER_SIMPLE?.trim() && + process.env.TIER_MEDIUM?.trim() && + process.env.TIER_COMPLEX?.trim() && + process.env.TIER_REASONING?.trim() +); +if (fallbackEnabled && tiersConfigured) { const localProviders = ["ollama", "llamacpp", "lmstudio"]; - if (fallbackEnabled && localProviders.includes(fallbackProvider)) { + if (localProviders.includes(fallbackProvider)) { throw new Error(`FALLBACK_PROVIDER cannot be '${fallbackProvider}' (local providers should not be fallbacks). Use cloud providers: databricks, azure-anthropic, azure-openai, openrouter, openai, bedrock`); } - - // Ensure fallback provider is properly configured (only if fallback is enabled) - if (fallbackEnabled) { - if (fallbackProvider === "databricks" && (!rawBaseUrl || !apiKey)) { - throw new Error("FALLBACK_PROVIDER is set to 'databricks' but DATABRICKS_API_BASE and DATABRICKS_API_KEY are not configured. Please set these environment variables or choose a different fallback provider."); - } - if (fallbackProvider === "azure-anthropic" && (!azureAnthropicEndpoint || !azureAnthropicApiKey)) { - throw new Error("FALLBACK_PROVIDER is set to 'azure-anthropic' but AZURE_ANTHROPIC_ENDPOINT and AZURE_ANTHROPIC_API_KEY are not configured. Please set these environment variables or choose a different fallback provider."); - } - if (fallbackProvider === "azure-openai" && (!azureOpenAIEndpoint || !azureOpenAIApiKey)) { - throw new Error("FALLBACK_PROVIDER is set to 'azure-openai' but AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY are not configured. Please set these environment variables or choose a different fallback provider."); - } - if (fallbackProvider === "bedrock" && !bedrockApiKey) { - throw new Error("FALLBACK_PROVIDER is set to 'bedrock' but AWS_BEDROCK_API_KEY is not configured. Please set this environment variable or choose a different fallback provider."); - } + let fallbackMisconfigured = false; + if (fallbackProvider === "databricks" && (!rawBaseUrl || !apiKey)) { + fallbackMisconfigured = true; + } + if (fallbackProvider === "azure-anthropic" && (!azureAnthropicEndpoint || !azureAnthropicApiKey)) { + fallbackMisconfigured = true; + } + if (fallbackProvider === "azure-openai" && (!azureOpenAIEndpoint || !azureOpenAIApiKey)) { + fallbackMisconfigured = true; + } + if (fallbackProvider === "bedrock" && !bedrockApiKey) { + fallbackMisconfigured = true; + } + if (fallbackMisconfigured) { + console.warn(`[WARN] FALLBACK_PROVIDER='${fallbackProvider}' is enabled but missing credentials. Fallback will not work until configured.`); } } @@ -599,6 +605,11 @@ var config = { apiKey: vertexApiKey, model: vertexModel, }, + moonshot: { + apiKey: moonshotApiKey, + endpoint: moonshotEndpoint, + model: moonshotModel, + }, hotReload: { enabled: hotReloadEnabled, debounceMs: Number.isNaN(hotReloadDebounceMs) ? 1000 : hotReloadDebounceMs, @@ -607,8 +618,6 @@ var config = { type: modelProvider, defaultModel, suggestionModeModel, - // Hybrid routing settings - preferOllama, fallbackEnabled, ollamaMaxToolsForRouting, openRouterMaxToolsForRouting, @@ -626,6 +635,13 @@ var config = { }, logger: { level: process.env.LOG_LEVEL ?? "info", + file: { + enabled: process.env.LOG_FILE_ENABLED === "true", + path: process.env.LOG_FILE_PATH ?? path.join(process.cwd(), "logs", "lynkr.log"), + level: process.env.LOG_FILE_LEVEL ?? "debug", // File captures everything + frequency: process.env.LOG_FILE_FREQUENCY ?? "daily", // daily | hourly | + maxFiles: parseInt(process.env.LOG_FILE_MAX_FILES ?? "14", 10), + }, }, sessionStore: { dbPath: sessionDbPath, @@ -710,8 +726,8 @@ var config = { semanticCache: { enabled: process.env.SEMANTIC_CACHE_ENABLED !== 'false', // Disable via env if needed similarityThreshold: parseFloat(process.env.SEMANTIC_CACHE_THRESHOLD || '0.95'), // Higher threshold - maxEntries: 500, - ttlMs: 3600000, // 1 hour + maxEntries: Number.parseInt(process.env.SEMANTIC_CACHE_MAX_ENTRIES ?? "50", 10), // Reduced from 500 to prevent memory bloat + ttlMs: Number.parseInt(process.env.SEMANTIC_CACHE_TTL_MS ?? "300000", 10), // 5 minutes (was 1 hour) }, agents: { enabled: agentsEnabled, @@ -869,6 +885,23 @@ var config = { taskTimeoutMs: Number.isNaN(workerTaskTimeoutMs) ? 5000 : workerTaskTimeoutMs, offloadThresholdBytes: Number.isNaN(workerOffloadThresholdBytes) ? 10000 : workerOffloadThresholdBytes, }, + + // Intelligent Routing + routing: { + weightedScoring: true, + costOptimization: true, + agenticDetection: true, + }, + + // Model Tier Configuration (REQUIRED) + // Format: TIER_=provider:model (e.g., TIER_SIMPLE=ollama:llama3.2) + modelTiers: { + enabled: true, + SIMPLE: process.env.TIER_SIMPLE?.trim() || null, + MEDIUM: process.env.TIER_MEDIUM?.trim() || null, + COMPLEX: process.env.TIER_COMPLEX?.trim() || null, + REASONING: process.env.TIER_REASONING?.trim() || null, + }, }; /** @@ -893,13 +926,14 @@ function reloadConfig() { config.zai.model = process.env.ZAI_MODEL?.trim() || "GLM-4.7"; config.vertex.apiKey = process.env.VERTEX_API_KEY?.trim() || process.env.GOOGLE_API_KEY?.trim() || null; config.vertex.model = process.env.VERTEX_MODEL?.trim() || "gemini-2.0-flash"; + config.moonshot.apiKey = process.env.MOONSHOT_API_KEY?.trim() || null; + config.moonshot.model = process.env.MOONSHOT_MODEL?.trim() || "kimi-k2-turbo-preview"; // Model provider settings const newProvider = (process.env.MODEL_PROVIDER ?? "databricks").toLowerCase(); if (SUPPORTED_MODEL_PROVIDERS.has(newProvider)) { config.modelProvider.type = newProvider; } - config.modelProvider.preferOllama = process.env.PREFER_OLLAMA === "true"; config.modelProvider.fallbackEnabled = process.env.FALLBACK_ENABLED !== "false"; config.modelProvider.fallbackProvider = (process.env.FALLBACK_PROVIDER ?? "databricks").toLowerCase(); config.modelProvider.suggestionModeModel = (process.env.SUGGESTION_MODE_MODEL ?? "default").trim(); @@ -920,4 +954,30 @@ function reloadConfig() { // Make config mutable for hot reload config.reloadConfig = reloadConfig; +/** + * Check if any TIER_* value references Ollama (starts with "ollama:") + * Used by server.js to decide whether to wait for Ollama at startup. + */ +config.tiersReferenceOllama = function tiersReferenceOllama() { + const tiers = config.modelTiers; + if (!tiers?.enabled) return false; + return [tiers.SIMPLE, tiers.MEDIUM, tiers.COMPLEX, tiers.REASONING] + .some(v => typeof v === 'string' && v.startsWith('ollama:')); +}; + +// Validate TIER_* configuration (warn if missing, don't crash) +const missingTiers = []; +if (!config.modelTiers.SIMPLE) missingTiers.push('TIER_SIMPLE'); +if (!config.modelTiers.MEDIUM) missingTiers.push('TIER_MEDIUM'); +if (!config.modelTiers.COMPLEX) missingTiers.push('TIER_COMPLEX'); +if (!config.modelTiers.REASONING) missingTiers.push('TIER_REASONING'); + +if (missingTiers.length > 0) { + config.modelTiers.enabled = false; + console.warn( + `[WARN] Missing tier configuration: ${missingTiers.join(', ')} — tiered routing disabled.\n` + + ` Set TIER_=provider:model to enable (e.g., TIER_SIMPLE=ollama:llama3.2)` + ); +} + module.exports = config; diff --git a/src/logger/index.js b/src/logger/index.js index 7f49d9f..85b3ea2 100644 --- a/src/logger/index.js +++ b/src/logger/index.js @@ -1,4 +1,6 @@ const pino = require("pino"); +const fs = require("fs"); +const path = require("path"); const config = require("../config"); const { createOversizedErrorStream } = require("./oversized-error-stream"); @@ -64,6 +66,27 @@ streams.push({ : process.stdout, }); +// File rotation stream (if enabled via LOG_FILE_ENABLED=true) +if (config.logger.file?.enabled) { + const fileConfig = config.logger.file; + // Ensure log directory exists + const logDir = path.dirname(fileConfig.path); + fs.mkdirSync(logDir, { recursive: true }); + + streams.push({ + level: fileConfig.level, + stream: pino.transport({ + target: "pino-roll", + options: { + file: fileConfig.path, + frequency: fileConfig.frequency, + limit: { count: fileConfig.maxFiles }, + mkdir: true, + }, + }), + }); +} + // Oversized error stream (if enabled) if (config.oversizedErrorLogging?.enabled) { streams.push({ diff --git a/src/orchestrator/index.js b/src/orchestrator/index.js index 9825c92..df0d1f6 100644 --- a/src/orchestrator/index.js +++ b/src/orchestrator/index.js @@ -1,6 +1,7 @@ const config = require("../config"); const { invokeModel } = require("../clients/databricks"); const { appendTurnToSession } = require("../sessions/record"); +const { upsertSession } = require("../sessions/store"); const { executeToolCall } = require("../tools"); const policy = require("../policy"); const logger = require("../logger"); @@ -20,6 +21,8 @@ const crypto = require("crypto"); const { asyncClone, asyncTransform, getPoolStats } = require("../workers/helpers"); const { getSemanticCache, isSemanticCacheEnabled } = require("../cache/semantic"); const lazyLoader = require("../tools/lazy-loader"); +const { areSimilarToolCalls } = require("../clients/gpt-utils"); +const { getModelRegistrySync } = require("../routing/model-registry"); /** * Get destination URL for audit logging based on provider type @@ -50,6 +53,8 @@ function getDestinationUrl(providerType) { return config.zai?.endpoint ?? 'unknown'; case 'vertex': return config.vertex?.endpoint ?? 'unknown'; + case 'moonshot': + return config.moonshot?.endpoint ?? 'unknown'; default: return 'unknown'; } @@ -456,6 +461,192 @@ function injectToolLoopStopInstruction(messages, threshold = 5) { return messages; } +// === CROSS-REQUEST TOOL CALL DEDUP TRACKING === +// These helpers track tool call signatures across multiple HTTP requests within +// the same session (client/passthrough mode). The inner-loop detection in +// runAgentLoop() only sees one request at a time, so repeated calls across +// requests escape it. + +const DEDUP_MAX_SIGNATURES = 50; +const DEDUP_WARN_THRESHOLD = 2; +const DEDUP_TERMINATE_THRESHOLD = 3; + +/** + * Initialise session.metadata.toolCallDedup if missing. + * @param {Object} session + */ +function ensureDedupStructure(session) { + if (!session || !session.metadata) return; + if (!session.metadata.toolCallDedup) { + session.metadata.toolCallDedup = { + signatures: {}, + similarGroups: {}, + lastResetAt: Date.now(), + warningInjected: false, + }; + } +} + +/** + * Record a tool call into the cross-request dedup tracker. + * Handles similarity merging and enforces the 50-entry cap. + * @param {Object} session + * @param {Object} toolCall - tool_use block (Anthropic format: { name, input, id }) + */ +function recordCrossRequestToolCall(session, toolCall) { + if (!session?.metadata) return; + ensureDedupStructure(session); + + const dedup = session.metadata.toolCallDedup; + const signature = getToolCallSignature(toolCall); + const toolName = toolCall.function?.name ?? toolCall.name ?? 'unknown'; + const args = toolCall.function?.arguments ?? toolCall.input; + const argsPreview = (typeof args === 'string' ? args : JSON.stringify(args ?? {})).substring(0, 200); + const now = Date.now(); + + // Check if this signature maps to a canonical via similarity groups + const canonicalSig = dedup.similarGroups[signature] || signature; + + if (dedup.signatures[canonicalSig]) { + dedup.signatures[canonicalSig].count += 1; + dedup.signatures[canonicalSig].lastSeen = now; + } else { + // Check for similar existing entries before creating a new one + let mergedInto = null; + for (const [existingSig, existingData] of Object.entries(dedup.signatures)) { + // Build a fake call object from stored data to compare with areSimilarToolCalls + const existingCall = { + name: existingData.toolName, + input: existingData.argsPreview, + }; + if (areSimilarToolCalls(toolCall, existingCall)) { + // Merge: map this signature to the existing canonical + dedup.similarGroups[signature] = existingSig; + dedup.signatures[existingSig].count += 1; + dedup.signatures[existingSig].lastSeen = now; + mergedInto = existingSig; + logger.debug({ + newSignature: signature, + canonicalSignature: existingSig, + toolName, + count: dedup.signatures[existingSig].count, + }, "Cross-request tool dedup: merged similar call"); + break; + } + } + + if (!mergedInto) { + // New unique signature + dedup.signatures[signature] = { + count: 1, + toolName, + firstSeen: now, + lastSeen: now, + argsPreview, + }; + } + } + + // Enforce cap: evict oldest entries if over limit + const sigKeys = Object.keys(dedup.signatures); + if (sigKeys.length > DEDUP_MAX_SIGNATURES) { + const sorted = sigKeys.sort( + (a, b) => dedup.signatures[a].lastSeen - dedup.signatures[b].lastSeen + ); + const toRemove = sorted.slice(0, sigKeys.length - DEDUP_MAX_SIGNATURES); + for (const key of toRemove) { + delete dedup.signatures[key]; + // Also clean up any similarGroups pointing to this key + for (const [groupSig, canonical] of Object.entries(dedup.similarGroups)) { + if (canonical === key) delete dedup.similarGroups[groupSig]; + } + } + } +} + +/** + * Return the highest dedup count, the associated tool name, and signature. + * @param {Object} session + * @returns {{ maxCount: number, toolName: string|null, signature: string|null }} + */ +function getMaxDedupCount(session) { + if (!session?.metadata?.toolCallDedup?.signatures) { + return { maxCount: 0, toolName: null, signature: null }; + } + const sigs = session.metadata.toolCallDedup.signatures; + let maxCount = 0; + let toolName = null; + let signature = null; + for (const [sig, data] of Object.entries(sigs)) { + if (data.count > maxCount) { + maxCount = data.count; + toolName = data.toolName; + signature = sig; + } + } + return { maxCount, toolName, signature }; +} + +/** + * Extract tool_use blocks from messages that appear after the last user text message. + * These are the tool calls from the current assistant turn that the client is sending back. + * @param {Array} messages + * @returns {Array} - Array of tool_use-like objects + */ +function extractToolUseFromCurrentTurn(messages) { + if (!Array.isArray(messages)) return []; + + // Find last user text message + let lastUserTextIndex = -1; + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg?.role !== 'user') continue; + if (typeof msg.content === 'string' && msg.content.trim().length > 0) { + lastUserTextIndex = i; + break; + } + if (Array.isArray(msg.content)) { + const hasText = msg.content.some(block => + (block?.type === 'text' && block?.text?.trim?.().length > 0) || + (block?.type === 'input_text' && block?.input_text?.trim?.().length > 0) + ); + if (hasText) { + lastUserTextIndex = i; + break; + } + } + } + + const toolUseBlocks = []; + const startIndex = lastUserTextIndex >= 0 ? lastUserTextIndex : 0; + for (let i = startIndex; i < messages.length; i++) { + const msg = messages[i]; + if (msg?.role !== 'assistant') continue; + if (!Array.isArray(msg.content)) continue; + for (const block of msg.content) { + if (block?.type === 'tool_use') { + toolUseBlocks.push(block); + } + } + } + return toolUseBlocks; +} + +/** + * Reset dedup tracking. Called when a new user question is detected. + * @param {Object} session + */ +function resetDedupTracking(session) { + if (!session?.metadata) return; + session.metadata.toolCallDedup = { + signatures: {}, + similarGroups: {}, + lastResetAt: Date.now(), + warningInjected: false, + }; + logger.debug({ sessionId: session?.id ?? null }, "Cross-request tool dedup: reset tracking for new user question"); +} + function sanitiseAzureTools(tools) { if (!Array.isArray(tools) || tools.length === 0) return undefined; const allowed = new Set([ @@ -517,13 +708,51 @@ function parseExecutionContent(content) { const trimmed = content.trim(); if (trimmed.startsWith("{") || trimmed.startsWith("[")) { try { - return JSON.parse(trimmed); + const parsed = JSON.parse(trimmed); + // Handle Anthropic content blocks array - extract text + if (Array.isArray(parsed)) { + const textParts = parsed + .filter(block => block && typeof block === 'object') + .map(block => { + if (block.type === 'text' && typeof block.text === 'string') { + return block.text; + } + // Handle other block types gracefully + if (block.text) return block.text; + if (block.content) return typeof block.content === 'string' ? block.content : JSON.stringify(block.content); + return null; + }) + .filter(text => text !== null); + + if (textParts.length > 0) { + return textParts.join('\n'); + } + } + return parsed; } catch { return content; } } return content; } + // Handle content that's already an array (content blocks) + if (Array.isArray(content)) { + const textParts = content + .filter(block => block && typeof block === 'object') + .map(block => { + if (block.type === 'text' && typeof block.text === 'string') { + return block.text; + } + if (block.text) return block.text; + if (block.content) return typeof block.content === 'string' ? block.content : JSON.stringify(block.content); + return null; + }) + .filter(text => text !== null); + + if (textParts.length > 0) { + return textParts.join('\n'); + } + } return content; } @@ -719,19 +948,17 @@ function stripThinkingBlocks(text) { return cleanedLines.join("\n").trim(); } +/** + * Convert legacy Ollama /api/chat response to Anthropic Messages format. + * Used when Ollama < v0.14.0 (no native Anthropic endpoint). + */ function ollamaToAnthropicResponse(ollamaResponse, requestedModel) { - // Ollama response format: - // { model, created_at, message: { role, content, tool_calls }, done, total_duration, ... } - // { eval_count, prompt_eval_count, ... } - const message = ollamaResponse?.message ?? {}; const rawContent = message.content || ""; const toolCalls = message.tool_calls || []; - // Build content blocks const contentItems = []; - // Add text content if present, after stripping thinking blocks if (typeof rawContent === "string" && rawContent.trim()) { const cleanedContent = stripThinkingBlocks(rawContent); if (cleanedContent) { @@ -739,18 +966,31 @@ function ollamaToAnthropicResponse(ollamaResponse, requestedModel) { } } - // Add tool calls if present + // Convert tool calls from OpenAI function-calling format to Anthropic tool_use if (Array.isArray(toolCalls) && toolCalls.length > 0) { - const { buildAnthropicResponseFromOllama } = require("../clients/ollama-utils"); - // Use the utility function for tool call conversion - return buildAnthropicResponseFromOllama(ollamaResponse, requestedModel); + for (const toolCall of toolCalls) { + const func = toolCall.function || {}; + let input = {}; + if (func.arguments) { + if (typeof func.arguments === "string") { + try { input = JSON.parse(func.arguments); } catch { input = {}; } + } else if (typeof func.arguments === "object") { + input = func.arguments; + } + } + contentItems.push({ + type: "tool_use", + id: toolCall.id || `toolu_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + name: func.name || "unknown", + input, + }); + } } if (contentItems.length === 0) { contentItems.push({ type: "text", text: "" }); } - // Ollama uses different token count fields const inputTokens = ollamaResponse.prompt_eval_count ?? 0; const outputTokens = ollamaResponse.eval_count ?? 0; @@ -760,7 +1000,8 @@ function ollamaToAnthropicResponse(ollamaResponse, requestedModel) { role: "assistant", model: requestedModel, content: contentItems, - stop_reason: ollamaResponse.done ? "end_turn" : "max_tokens", + stop_reason: toolCalls.length > 0 ? "tool_use" : + ollamaResponse.done ? "end_turn" : "max_tokens", stop_sequence: null, usage: { input_tokens: inputTokens, @@ -852,6 +1093,9 @@ function sanitizePayload(payload) { config.modelProvider?.defaultModel ?? "databricks-claude-sonnet-4-5"; clean.model = requestedModel; + if (!clean.max_tokens) { + clean.max_tokens = 16384; + } const providerType = config.modelProvider?.type ?? "databricks"; const flattenContent = providerType !== "azure-anthropic"; clean.messages = normaliseMessages(clean, { flattenContent }).filter((msg) => { @@ -996,12 +1240,10 @@ function sanitizePayload(payload) { // Check if this is a simple conversational message (no tools needed) const isConversational = (() => { if (!Array.isArray(clean.messages) || clean.messages.length === 0) { - logger.debug({ reason: "No messages array" }, "Ollama conversational check"); return false; } const lastMessage = clean.messages[clean.messages.length - 1]; if (lastMessage?.role !== "user") { - logger.debug({ role: lastMessage?.role }, "Ollama conversational check - not user"); return false; } @@ -1009,28 +1251,18 @@ function sanitizePayload(payload) { ? lastMessage.content : ""; - logger.debug({ - contentType: typeof lastMessage.content, - isString: typeof lastMessage.content === "string", - contentLength: typeof lastMessage.content === "string" ? lastMessage.content.length : "N/A", - actualContent: typeof lastMessage.content === "string" ? lastMessage.content.substring(0, 100) : JSON.stringify(lastMessage.content).substring(0, 100) - }, "Ollama conversational check - analyzing content"); - const trimmed = content.trim().toLowerCase(); // Simple greetings if (/^(hi|hello|hey|good morning|good afternoon|good evening|howdy|greetings)[\s\.\!\?]*$/.test(trimmed)) { - logger.debug({ matched: "greeting", trimmed }, "Ollama conversational check - matched"); - return true; + return "greeting"; } - // Very short messages (< 20 chars) without code/technical keywords - if (trimmed.length < 20 && !/code|file|function|error|bug|fix|write|read|create/.test(trimmed)) { - logger.debug({ matched: "short", trimmed, length: trimmed.length }, "Ollama conversational check - matched"); - return true; + // Conversational phrases that don't need tools (thanks, farewells, acknowledgements) + if (/^(thanks|thank you|thx|ty|bye|goodbye|see you|ok|okay|cool|nice|great|awesome|sure|got it|sounds good|no worries|np|cheers)[\s\.\!\?]*$/.test(trimmed)) { + return "conversational"; } - logger.debug({ trimmed: trimmed.substring(0, 50), length: trimmed.length }, "Ollama conversational check - not matched"); return false; })(); @@ -1040,37 +1272,12 @@ function sanitizePayload(payload) { delete clean.tool_choice; logger.debug({ model: config.ollama?.model, - message: "Removed tools for conversational message" - }, "Ollama conversational mode"); + reason: isConversational, + }, "Ollama conversational mode - tools removed"); } else if (modelSupportsTools && Array.isArray(clean.tools) && clean.tools.length > 0) { - // Ollama performance degrades with too many tools - // Limit to essential tools only - const OLLAMA_ESSENTIAL_TOOLS = new Set([ - "Bash", - "Read", - "Write", - "Edit", - "Glob", - "Grep", - "WebSearch", - "WebFetch" - ]); - - const limitedTools = clean.tools.filter(tool => - OLLAMA_ESSENTIAL_TOOLS.has(tool.name) - ); - - logger.debug({ - model: config.ollama?.model, - originalToolCount: clean.tools.length, - limitedToolCount: limitedTools.length, - keptTools: limitedTools.map(t => t.name) - }, "Ollama tools limited for performance"); - - clean.tools = limitedTools.length > 0 ? limitedTools : undefined; - if (!clean.tools) { - delete clean.tools; - } + // Keep all tools — Ollama receives them in Anthropic format (native API) + // or they get converted to OpenAI format in invokeOllama (legacy API) + clean.tools = ensureAnthropicToolFormat(clean.tools); } else { // Remove tools for models without tool support delete clean.tools; @@ -1098,6 +1305,14 @@ function sanitizePayload(payload) { } else { clean.tools = ensureAnthropicToolFormat(clean.tools); } + } else if (providerType === "moonshot") { + // Moonshot supports tools - keep them in Anthropic format + // They will be converted to OpenAI format in invokeMoonshot + if (!Array.isArray(clean.tools) || clean.tools.length === 0) { + delete clean.tools; + } else { + clean.tools = ensureAnthropicToolFormat(clean.tools); + } } else if (Array.isArray(clean.tools)) { // Unknown provider - remove tools for safety delete clean.tools; @@ -1215,7 +1430,7 @@ function sanitizePayload(payload) { } if (merged.length !== clean.messages.length) { - logger.info({ + logger.debug({ originalCount: clean.messages.length, mergedCount: merged.length, reduced: clean.messages.length - merged.length @@ -1225,19 +1440,11 @@ function sanitizePayload(payload) { clean.messages = merged; } - // [CONTEXT_FLOW] Log payload after sanitization logger.debug({ providerType: config.modelProvider?.type ?? "databricks", - phase: "after_sanitize", - systemField: typeof clean.system === 'string' - ? { type: 'string', length: clean.system.length } - : clean.system - ? { type: typeof clean.system, value: clean.system } - : undefined, messageCount: clean.messages?.length ?? 0, - firstMessageHasSystem: clean.messages?.[0]?.content?.includes?.('You are Claude Code') ?? false, toolCount: clean.tools?.length ?? 0 - }, '[CONTEXT_FLOW] After sanitizePayload'); + }, 'After sanitizePayload'); // === Suggestion mode: tag request and override model if configured === const { isSuggestionMode: isSuggestion } = detectSuggestionMode(clean.messages); @@ -1344,8 +1551,7 @@ async function runAgentLoop({ providerType, headers, }) { - console.log('[DEBUG] runAgentLoop ENTERED - providerType:', providerType, 'messages:', cleanPayload.messages?.length); - logger.info({ providerType, messageCount: cleanPayload.messages?.length }, 'runAgentLoop ENTERED'); + logger.debug({ providerType, messageCount: cleanPayload.messages?.length }, 'runAgentLoop entered'); const settings = resolveLoopOptions(options); // Initialize audit logger (no-op if disabled) const auditLogger = createAuditLogger(config.audit); @@ -1392,7 +1598,6 @@ async function runAgentLoop({ } steps += 1; - console.log('[LOOP DEBUG] Entered while loop - step:', steps); logger.debug( { sessionId: session?.id ?? null, @@ -1402,6 +1607,19 @@ async function runAgentLoop({ "Agent loop step", ); + // Trim messages when they grow too large to prevent OOM. + // Keep the first message (system/user) and the last MAX_LOOP_MESSAGES. + const MAX_LOOP_MESSAGES = 40; + if (cleanPayload.messages && cleanPayload.messages.length > MAX_LOOP_MESSAGES) { + const excess = cleanPayload.messages.length - MAX_LOOP_MESSAGES; + // Keep first 2 messages (system context + initial user) and trim from the middle + cleanPayload.messages.splice(2, excess); + logger.debug( + { trimmed: excess, remaining: cleanPayload.messages.length }, + "Trimmed intermediate messages to prevent memory growth", + ); + } + // Debug: Log payload before sending to Azure if (providerType === "azure-anthropic") { logger.debug( @@ -1486,14 +1704,11 @@ async function runAgentLoop({ } } - // [CONTEXT_FLOW] Log after memory injection logger.debug({ sessionId: session?.id ?? null, - phase: "after_memory", - systemPromptLength: cleanPayload.system?.length ?? 0, messageCount: cleanPayload.messages?.length ?? 0, toolCount: cleanPayload.tools?.length ?? 0 - }, '[CONTEXT_FLOW] After memory injection'); + }, 'After memory injection'); if (steps === 1 && (config.systemPrompt?.mode === 'dynamic' || config.systemPrompt?.toolDescriptions === 'minimal')) { try { @@ -1582,9 +1797,26 @@ IMPORTANT TOOL USAGE RULES: logger.debug({ sessionId: session?.id ?? null }, 'Tool termination instructions injected for non-Claude model'); } + // Compute model-aware token budget thresholds + const registry = getModelRegistrySync(); + const modelInfo = registry.getCost(requestedModel); + const modelContextWindow = modelInfo?.context || config.tokenBudget?.max || 180000; + const modelMax = Math.floor(modelContextWindow * 0.85); + const effectiveMax = Math.min(modelMax, config.tokenBudget?.max || 180000); + const effectiveWarning = Math.floor(effectiveMax * 0.65); + + logger.debug({ + sessionId: session?.id ?? null, + requestedModel, + modelContextWindow, + effectiveWarning, + effectiveMax, + source: modelInfo?.source || 'default', + }, 'Model-aware token budget computed'); + if (steps === 1 && config.tokenBudget?.enforcement !== false) { try { - const budgetCheck = tokenBudget.checkBudget(cleanPayload); + const budgetCheck = tokenBudget.checkBudget(cleanPayload, effectiveWarning, effectiveMax); if (budgetCheck.atWarning) { logger.warn({ @@ -1598,8 +1830,8 @@ IMPORTANT TOOL USAGE RULES: if (budgetCheck.overMax) { // Apply adaptive compression to fit within budget const enforcement = tokenBudget.enforceBudget(cleanPayload, { - warningThreshold: config.tokenBudget?.warning, - maxThreshold: config.tokenBudget?.max, + warningThreshold: effectiveWarning, + maxThreshold: effectiveMax, enforcement: true }); @@ -1623,7 +1855,6 @@ IMPORTANT TOOL USAGE RULES: } // Track estimated token usage before model call - console.log('[TOKEN DEBUG] About to track token usage - step:', steps); const estimatedTokens = config.tokenTracking?.enabled !== false ? tokens.countPayloadTokens(cleanPayload) : null; @@ -1637,15 +1868,6 @@ IMPORTANT TOOL USAGE RULES: } // Apply Headroom compression if enabled - const headroomEstTokens = Math.ceil(JSON.stringify(cleanPayload.messages || []).length / 4); - logger.info({ - headroomEnabled: isHeadroomEnabled(), - messageCount: cleanPayload.messages?.length ?? 0, - estimatedTokens: headroomEstTokens, - threshold: config.headroom?.minTokens || 500, - willCompress: isHeadroomEnabled() && headroomEstTokens >= (config.headroom?.minTokens || 500), - }, 'Headroom compression check'); - if (isHeadroomEnabled() && cleanPayload.messages && cleanPayload.messages.length > 0) { try { const compressionResult = await headroomCompress( @@ -1654,36 +1876,27 @@ IMPORTANT TOOL USAGE RULES: { mode: config.headroom?.mode, queryContext: cleanPayload.messages[cleanPayload.messages.length - 1]?.content, + model: requestedModel, + modelLimit: modelContextWindow, + tokenBudget: effectiveMax, } ); - logger.info({ - compressed: compressionResult.compressed, - tokensBefore: compressionResult.stats?.tokens_before, - tokensAfter: compressionResult.stats?.tokens_after, - savings: compressionResult.stats?.savings_percent ? `${compressionResult.stats.savings_percent}%` : 'N/A', - reason: compressionResult.stats?.reason || compressionResult.stats?.transforms_applied?.join(', ') || 'none', - }, 'Headroom compression result'); - if (compressionResult.compressed) { cleanPayload.messages = compressionResult.messages; if (compressionResult.tools) { cleanPayload.tools = compressionResult.tools; } - logger.info({ - sessionId: session?.id ?? null, - tokensBefore: compressionResult.stats?.tokens_before, - tokensAfter: compressionResult.stats?.tokens_after, - saved: compressionResult.stats?.tokens_saved, - savingsPercent: compressionResult.stats?.savings_percent, - transforms: compressionResult.stats?.transforms_applied, - }, 'Headroom compression applied to request'); - } else { - logger.debug({ - sessionId: session?.id ?? null, - reason: compressionResult.stats?.reason, - }, 'Headroom compression skipped'); } + + logger.debug({ + sessionId: session?.id ?? null, + outcome: compressionResult.compressed ? 'applied' : 'skipped', + tokensBefore: compressionResult.stats?.tokens_before, + tokensAfter: compressionResult.stats?.tokens_after, + savingsPercent: compressionResult.stats?.savings_percent, + reason: compressionResult.stats?.reason || compressionResult.stats?.transforms_applied?.join(', ') || 'none', + }, 'Headroom compression'); } catch (headroomErr) { logger.warn({ err: headroomErr, sessionId: session?.id ?? null }, 'Headroom compression failed, using original messages'); } @@ -1995,9 +2208,10 @@ IMPORTANT TOOL USAGE RULES: }); let assistantToolMessage; - if (providerType === "azure-anthropic") { - // For Azure Anthropic, use the content array directly from the response - // It already contains both text and tool_use blocks in the correct format + if (providerType === "azure-anthropic" || isAnthropicFormat) { + // For Anthropic-format responses (azure-anthropic, Ollama native API, + // azure-openai Responses API), use the content array directly — + // it already contains both text and tool_use blocks in the correct format assistantToolMessage = { role: "assistant", content: databricksResponse.json?.content ?? [], @@ -2010,9 +2224,9 @@ IMPORTANT TOOL USAGE RULES: }; } - // Only add fallback content for Databricks format (Azure already has content) + // Only add fallback content for OpenAI-format responses (Anthropic format already has content) if ( - providerType !== "azure-anthropic" && + providerType !== "azure-anthropic" && !isAnthropicFormat && (!assistantToolMessage.content || (typeof assistantToolMessage.content === "string" && assistantToolMessage.content.trim().length === 0)) && @@ -2048,7 +2262,7 @@ IMPORTANT TOOL USAGE RULES: // If in passthrough/client mode and there are client-side tools, return them to client // Server-side tools (Task, Web) will be executed below if ((executionMode === "passthrough" || executionMode === "client") && clientSideToolCalls.length > 0) { - logger.info( + logger.debug( { sessionId: session?.id ?? null, totalToolCount: toolCalls.length, @@ -2073,7 +2287,7 @@ IMPORTANT TOOL USAGE RULES: type: "message", role: "assistant", content: clientContent, - model: databricksResponse.json?.model || clean.model, + model: databricksResponse.json?.model || cleanPayload.model, stop_reason: "tool_use", usage: databricksResponse.json?.usage || { input_tokens: 0, @@ -2094,6 +2308,27 @@ IMPORTANT TOOL USAGE RULES: // then continue the conversation loop. For now, let's fall through to execute server-side tools. if (serverSideToolCalls.length === 0) { // No server-side tools - pure passthrough + // Record outbound client-side tool calls into cross-request dedup tracker + if (session && clientSideToolCalls.length > 0) { + ensureDedupStructure(session); + for (const call of clientSideToolCalls) { + recordCrossRequestToolCall(session, call); + } + // Persist dedup state (non-ephemeral sessions only) + if (session.id && !session._ephemeral) { + try { upsertSession(session.id, { metadata: session.metadata }); } catch (e) { + logger.debug({ err: e.message }, "Failed to persist outbound dedup state"); + } + } + const { maxCount, toolName: dedupTool } = getMaxDedupCount(session); + logger.debug({ + sessionId: session?.id ?? null, + clientToolCount: clientSideToolCalls.length, + maxDedupCount: maxCount, + maxDedupTool: dedupTool, + }, "Cross-request tool dedup: recorded outbound tool calls"); + } + return { response: { status: 200, @@ -2110,7 +2345,7 @@ IMPORTANT TOOL USAGE RULES: // Override toolCalls to only include Server-side tools for server execution toolCalls = serverSideToolCalls; - logger.info( + logger.debug( { sessionId: session?.id ?? null, serverToolCount: serverSideToolCalls.length, @@ -2119,7 +2354,7 @@ IMPORTANT TOOL USAGE RULES: ); } else if (executionMode === "passthrough" || executionMode === "client") { // Only Server-side tools, no Client-side tools - execute all server-side - logger.info( + logger.debug( { sessionId: session?.id ?? null, serverToolCount: serverSideToolCalls.length, @@ -2184,6 +2419,7 @@ IMPORTANT TOOL USAGE RULES: session, cwd, requestMessages: cleanPayload.messages, + provider: providerType, // Pass provider for GPT-specific formatting })) ); @@ -2417,10 +2653,14 @@ IMPORTANT TOOL USAGE RULES: session, cwd, requestMessages: cleanPayload.messages, + provider: providerType, // Pass provider for GPT-specific formatting }); let toolMessage; - if (providerType === "azure-anthropic") { + if (providerType === "azure-anthropic" || isAnthropicFormat) { + // Anthropic-format tool result for providers whose responses use + // Anthropic tool_use blocks (azure-anthropic, Ollama native API, + // azure-openai Responses API) const parsedContent = parseExecutionContent(execution.content); const serialisedContent = typeof parsedContent === "string" || parsedContent === null @@ -2531,34 +2771,54 @@ IMPORTANT TOOL USAGE RULES: // === TOOL CALL LOOP DETECTION === // Track tool calls to detect infinite loops where the model calls the same tool - // repeatedly with identical parameters + // repeatedly with identical or similar parameters + // All providers use threshold 2 and similarity-based detection + const loopThreshold = 2; + for (const call of toolCalls) { const signature = getToolCallSignature(call); - const count = (toolCallHistory.get(signature) || 0) + 1; - toolCallHistory.set(signature, count); + const existingEntry = toolCallHistory.get(signature); + let count = (existingEntry?.count || 0) + 1; + toolCallHistory.set(signature, { count, call }); const toolName = call.function?.name ?? call.name ?? 'unknown'; - if (count === 3 && !loopWarningInjected) { + // Check for similar (not just identical) tool calls across all providers + // This catches cases where the model slightly varies parameters but is essentially looping + for (const [existingSig, existingData] of toolCallHistory.entries()) { + if (existingSig !== signature && areSimilarToolCalls(call, existingData.call)) { + // Found a similar call - increase count to trigger loop detection earlier + count = Math.max(count, existingData.count + 1); + logger.debug({ + tool: toolName, + currentSignature: signature, + similarSignature: existingSig, + combinedCount: count, + }, "Similar tool call detected - combining counts"); + } + } + + if (count === loopThreshold && !loopWarningInjected) { logger.warn( { sessionId: session?.id ?? null, correlationId: options?.correlationId, tool: toolName, loopCount: count, + loopThreshold, signature: signature, action: 'warning_injected', totalSteps: steps, remainingSteps: settings.maxSteps - steps, }, - "Tool call loop detected - same tool called 3 times with identical parameters", + `Tool call loop detected - same tool called ${loopThreshold} times with identical/similar parameters`, ); // Inject warning message to model loopWarningInjected = true; const warningMessage = { role: "user", - content: "⚠️ System Warning: You have called the same tool with identical parameters 3 times in this request. This may indicate an infinite loop. Please provide a final answer to the user instead of calling the same tool again, or explain why you need to continue retrying with the same parameters.", + content: `⚠️ CRITICAL SYSTEM WARNING: You have called the "${toolName}" tool ${count} times with identical or similar parameters. This IS an infinite loop. STOP calling this tool immediately. You MUST now provide a direct text response to the user based on the results you have received. If the tool returned "no results" or empty output, that IS the final answer - do not retry. Summarize your findings and respond.`, }; cleanPayload.messages.push(warningMessage); @@ -2573,11 +2833,12 @@ IMPORTANT TOOL USAGE RULES: reason: "tool_call_loop_warning", toolName, loopCount: count, + loopThreshold, }, }); } - } else if (count > 3) { - // Force termination after 3 identical calls + } else if (count > loopThreshold) { + // Force termination after threshold exceeded // Log FULL context for debugging why the loop occurred logger.error( { @@ -2585,6 +2846,7 @@ IMPORTANT TOOL USAGE RULES: correlationId: options?.correlationId, tool: toolName, loopCount: count, + loopThreshold, signature: signature, action: 'request_terminated', totalSteps: steps, @@ -2605,7 +2867,7 @@ IMPORTANT TOOL USAGE RULES: body: { error: { type: "tool_call_loop_detected", - message: `Tool call loop detected: The model called the same tool ("${toolName}") with identical parameters ${count} times. This indicates an infinite loop and execution has been terminated. Please try rephrasing your request or provide different parameters.`, + message: `Tool call loop detected: The model called the same tool ("${toolName}") with identical parameters ${count} times (threshold: ${loopThreshold}). This indicates an infinite loop and execution has been terminated. Please try rephrasing your request or provide different parameters.`, }, }, terminationReason: "tool_call_loop", @@ -2637,11 +2899,19 @@ IMPORTANT TOOL USAGE RULES: anthropicPayload.content = policy.sanitiseContent(anthropicPayload.content); } } else if (actualProvider === "ollama") { - anthropicPayload = ollamaToAnthropicResponse( - databricksResponse.json, - requestedModel, - ); - anthropicPayload.content = policy.sanitiseContent(anthropicPayload.content); + const ollamaJson = databricksResponse.json; + // Detect response format: Anthropic API (v0.14.0+) has type:"message", + // legacy /api/chat has message.role + message.content + if (ollamaJson?.type === "message" && Array.isArray(ollamaJson?.content)) { + // Anthropic-native response — passthrough + anthropicPayload = ollamaJson; + } else { + // Legacy Ollama response — convert to Anthropic format + anthropicPayload = ollamaToAnthropicResponse(ollamaJson, requestedModel); + } + if (Array.isArray(anthropicPayload?.content)) { + anthropicPayload.content = policy.sanitiseContent(anthropicPayload.content); + } } else if (actualProvider === "openrouter") { const { convertOpenRouterResponseToAnthropic } = require("../clients/openrouter-utils"); @@ -2870,6 +3140,16 @@ IMPORTANT TOOL USAGE RULES: if (Array.isArray(anthropicPayload?.content)) { anthropicPayload.content = policy.sanitiseContent(anthropicPayload.content); } + } else if (actualProvider === "moonshot") { + // Moonshot responses are already converted to Anthropic format in invokeMoonshot + logger.info({ + hasJson: !!databricksResponse.json, + jsonContent: JSON.stringify(databricksResponse.json?.content)?.substring(0, 300), + }, "=== MOONSHOT ORCHESTRATOR DEBUG ==="); + anthropicPayload = databricksResponse.json; + if (Array.isArray(anthropicPayload?.content)) { + anthropicPayload.content = policy.sanitiseContent(anthropicPayload.content); + } } else { anthropicPayload = toAnthropicResponse( databricksResponse.json, @@ -3064,6 +3344,7 @@ IMPORTANT TOOL USAGE RULES: session, cwd, requestMessages: cleanPayload.messages, + provider: providerType, // Pass provider for GPT-specific formatting }); const toolResultMessage = createFallbackToolResultMessage(providerType, { @@ -3337,100 +3618,289 @@ async function processMessage({ payload, headers, session, cwd, options = {} }) // === TOOL LOOP GUARD (EARLY CHECK) === // Check BEFORE sanitization since sanitizePayload removes conversation history - const toolLoopThreshold = config.policy?.toolLoopThreshold ?? 3; + // All providers use threshold 2 to catch loops early + const providerType = config.modelProvider?.type ?? "databricks"; + const toolLoopThreshold = 2; const { toolResultCount, toolUseCount } = countToolCallsInHistory(payload?.messages); - console.log('[ToolLoopGuard EARLY] Checking ORIGINAL messages:', { - messageCount: payload?.messages?.length, - toolResultCount, - toolUseCount, - threshold: toolLoopThreshold, - }); + const executionMode = config.toolExecutionMode || "server"; + const isClientMode = executionMode === "client" || executionMode === "passthrough"; + + if (isClientMode && session) { + // === CROSS-REQUEST DEDUP (CLIENT/PASSTHROUGH MODE) === + // The inner-loop guard resets each HTTP request so repeated calls across + // requests escape detection. Track signatures in session metadata instead. + ensureDedupStructure(session); + + // Detect new user question → reset dedup tracking + const dedup = session.metadata.toolCallDedup; + const incomingToolUse = extractToolUseFromCurrentTurn(payload?.messages); + // A user text message with no preceding tool_use means a brand-new question + const hasNewUserText = (() => { + const msgs = payload?.messages || []; + for (let i = msgs.length - 1; i >= 0; i--) { + const msg = msgs[i]; + if (msg?.role === 'user') { + if (typeof msg.content === 'string' && msg.content.trim().length > 0) return true; + if (Array.isArray(msg.content)) { + return msg.content.some(block => + (block?.type === 'text' && block?.text?.trim?.().length > 0) || + (block?.type === 'input_text' && block?.input_text?.trim?.().length > 0) + ); + } + } + break; // Only check the very last message + } + return false; + })(); - if (toolResultCount >= toolLoopThreshold) { - logger.error({ - toolResultCount, - toolUseCount, - threshold: toolLoopThreshold, - sessionId: session?.id ?? null, - }, "[ToolLoopGuard] FORCE TERMINATING - too many tool calls in conversation"); + if (hasNewUserText && incomingToolUse.length === 0) { + // Pure user text with no tool results → new question + resetDedupTracking(session); + } else { + // Record each tool_use from the incoming messages into the dedup tracker + for (const toolUseBlock of incomingToolUse) { + recordCrossRequestToolCall(session, toolUseBlock); + } - // Extract tool results ONLY from CURRENT TURN (after last user text message) - // This prevents showing old results from previous questions - let toolResultsSummary = ""; - const messages = payload?.messages || []; + const { maxCount, toolName: dedupToolName, signature: dedupSig } = getMaxDedupCount(session); - // Find the last user text message index (same logic as countToolCallsInHistory) - let lastUserTextIndex = -1; - for (let i = messages.length - 1; i >= 0; i--) { - const msg = messages[i]; - if (msg?.role !== 'user') continue; - if (typeof msg.content === 'string' && msg.content.trim().length > 0) { - lastUserTextIndex = i; - break; + if (maxCount >= DEDUP_TERMINATE_THRESHOLD) { + // Force-terminate: same pattern as existing tool_loop_guard + logger.error({ + toolName: dedupToolName, + count: maxCount, + threshold: DEDUP_TERMINATE_THRESHOLD, + signature: dedupSig, + sessionId: session?.id ?? null, + }, "[CrossRequestDedup] FORCE TERMINATING - repeated tool call across requests"); + + // Extract tool results summary from current turn + let toolResultsSummary = ""; + const messages = payload?.messages || []; + const { lastUserTextIndex: luIdx } = countToolCallsInHistory(messages); + const startIdx = luIdx >= 0 ? luIdx : 0; + for (let i = startIdx; i < messages.length; i++) { + const msg = messages[i]; + if (!msg || !Array.isArray(msg.content)) continue; + for (const block of msg.content) { + if (block?.type === 'tool_result' && block?.content) { + const content = typeof block.content === 'string' + ? block.content + : JSON.stringify(block.content); + if (content && !content.includes('Found 0')) { + toolResultsSummary += content + "\n"; + } + } + } + } + + let responseText = `Based on the tool results, here's what I found:\n\n`; + if (toolResultsSummary.trim()) { + responseText += toolResultsSummary.trim(); + } else { + responseText += `The tools executed but didn't return clear results. Please check the tool output above or try a different command.`; + } + + const forcedResponse = { + id: `msg_forced_${Date.now()}`, + type: "message", + role: "assistant", + content: [{ type: "text", text: responseText }], + model: requestedModel || "unknown", + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 0, output_tokens: 100 }, + }; + + // Reset dedup after termination so next question starts fresh + resetDedupTracking(session); + // Persist to DB (non-ephemeral sessions only) + if (session.id && !session._ephemeral) { + try { upsertSession(session.id, { metadata: session.metadata }); } catch (e) { + logger.debug({ err: e.message }, "Failed to persist dedup reset"); + } + } + + return { + status: 200, + body: forcedResponse, + terminationReason: "tool_loop_guard", + }; } - if (Array.isArray(msg.content)) { - const hasText = msg.content.some(block => - (block?.type === 'text' && block?.text?.trim?.().length > 0) || - (block?.type === 'input_text' && block?.input_text?.trim?.().length > 0) - ); - if (hasText) { - lastUserTextIndex = i; - break; + + if (maxCount >= DEDUP_WARN_THRESHOLD && !dedup.warningInjected) { + logger.warn({ + toolName: dedupToolName, + count: maxCount, + threshold: DEDUP_WARN_THRESHOLD, + signature: dedupSig, + sessionId: session?.id ?? null, + }, "[CrossRequestDedup] Warning - repeated tool call detected across requests"); + + dedup.warningInjected = true; + + // Inject a strict warning into the payload so the model sees it + if (Array.isArray(payload?.messages)) { + payload.messages.push({ + role: "user", + content: `⚠️ CRITICAL SYSTEM WARNING: You have called the "${dedupToolName}" tool ${maxCount} times with identical or similar parameters across multiple requests. This IS an infinite loop. STOP calling this tool immediately. You MUST now provide a direct text response based on the results you have received. If the tool returned "no results" or empty output, that IS the final answer - do not retry. Summarize your findings and respond.`, + }); + } + } + + // Persist dedup state (non-ephemeral sessions only) + if (session.id && !session._ephemeral) { + try { upsertSession(session.id, { metadata: session.metadata }); } catch (e) { + logger.debug({ err: e.message }, "Failed to persist dedup state"); } } } - // Only extract tool results AFTER the last user text message - const startIndex = lastUserTextIndex >= 0 ? lastUserTextIndex : 0; - for (let i = startIndex; i < messages.length; i++) { - const msg = messages[i]; - if (!msg || !Array.isArray(msg.content)) continue; - for (const block of msg.content) { - if (block?.type === 'tool_result' && block?.content) { - const content = typeof block.content === 'string' - ? block.content - : JSON.stringify(block.content); - if (content && !content.includes('Found 0')) { - toolResultsSummary += content + "\n"; + // Client mode still uses the relaxed per-request threshold for the count-based guard + const effectiveThreshold = 10; + if (toolResultCount >= effectiveThreshold) { + logger.error({ + toolResultCount, + toolUseCount, + threshold: effectiveThreshold, + sessionId: session?.id ?? null, + }, "[ToolLoopGuard] FORCE TERMINATING - too many tool calls in conversation"); + + let toolResultsSummary = ""; + const messages = payload?.messages || []; + let lastUserTextIndex = -1; + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg?.role !== 'user') continue; + if (typeof msg.content === 'string' && msg.content.trim().length > 0) { + lastUserTextIndex = i; + break; + } + if (Array.isArray(msg.content)) { + const hasText = msg.content.some(block => + (block?.type === 'text' && block?.text?.trim?.().length > 0) || + (block?.type === 'input_text' && block?.input_text?.trim?.().length > 0) + ); + if (hasText) { + lastUserTextIndex = i; + break; + } + } + } + const startIndex = lastUserTextIndex >= 0 ? lastUserTextIndex : 0; + for (let i = startIndex; i < messages.length; i++) { + const msg = messages[i]; + if (!msg || !Array.isArray(msg.content)) continue; + for (const block of msg.content) { + if (block?.type === 'tool_result' && block?.content) { + const content = typeof block.content === 'string' + ? block.content + : JSON.stringify(block.content); + if (content && !content.includes('Found 0')) { + toolResultsSummary += content + "\n"; + } } } } - } - // Build response text based on actual results from CURRENT turn only - let responseText = `Based on the tool results, here's what I found:\n\n`; - if (toolResultsSummary.trim()) { - responseText += toolResultsSummary.trim(); - } else { - responseText += `The tools executed but didn't return clear results. Please check the tool output above or try a different command.`; + let responseText = `Based on the tool results, here's what I found:\n\n`; + if (toolResultsSummary.trim()) { + responseText += toolResultsSummary.trim(); + } else { + responseText += `The tools executed but didn't return clear results. Please check the tool output above or try a different command.`; + } + + const forcedResponse = { + id: `msg_forced_${Date.now()}`, + type: "message", + role: "assistant", + content: [{ type: "text", text: responseText }], + model: requestedModel || "unknown", + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 0, output_tokens: 100 }, + }; + + return { + status: 200, + body: forcedResponse, + terminationReason: "tool_loop_guard", + }; } + } else { + // Server mode: use existing threshold 2 with countToolCallsInHistory + const effectiveThreshold = toolLoopThreshold; + + if (toolResultCount >= effectiveThreshold) { + logger.error({ + toolResultCount, + toolUseCount, + threshold: effectiveThreshold, + sessionId: session?.id ?? null, + }, "[ToolLoopGuard] FORCE TERMINATING - too many tool calls in conversation"); + + let toolResultsSummary = ""; + const messages = payload?.messages || []; + let lastUserTextIndex = -1; + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg?.role !== 'user') continue; + if (typeof msg.content === 'string' && msg.content.trim().length > 0) { + lastUserTextIndex = i; + break; + } + if (Array.isArray(msg.content)) { + const hasText = msg.content.some(block => + (block?.type === 'text' && block?.text?.trim?.().length > 0) || + (block?.type === 'input_text' && block?.input_text?.trim?.().length > 0) + ); + if (hasText) { + lastUserTextIndex = i; + break; + } + } + } + const startIndex = lastUserTextIndex >= 0 ? lastUserTextIndex : 0; + for (let i = startIndex; i < messages.length; i++) { + const msg = messages[i]; + if (!msg || !Array.isArray(msg.content)) continue; + for (const block of msg.content) { + if (block?.type === 'tool_result' && block?.content) { + const content = typeof block.content === 'string' + ? block.content + : JSON.stringify(block.content); + if (content && !content.includes('Found 0')) { + toolResultsSummary += content + "\n"; + } + } + } + } - // Force return a response instead of continuing the loop - const forcedResponse = { - id: `msg_forced_${Date.now()}`, - type: "message", - role: "assistant", - content: [ - { - type: "text", - text: responseText, - }, - ], - model: requestedModel || "unknown", - stop_reason: "end_turn", - stop_sequence: null, - usage: { - input_tokens: 0, - output_tokens: 100, - }, - }; + let responseText = `Based on the tool results, here's what I found:\n\n`; + if (toolResultsSummary.trim()) { + responseText += toolResultsSummary.trim(); + } else { + responseText += `The tools executed but didn't return clear results. Please check the tool output above or try a different command.`; + } - return { - status: 200, - body: forcedResponse, - terminationReason: "tool_loop_guard", - }; + const forcedResponse = { + id: `msg_forced_${Date.now()}`, + type: "message", + role: "assistant", + content: [{ type: "text", text: responseText }], + model: requestedModel || "unknown", + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 0, output_tokens: 100 }, + }; + + return { + status: 200, + body: forcedResponse, + terminationReason: "tool_loop_guard", + }; + } } const cleanPayload = sanitizePayload(payload); diff --git a/src/routing/agentic-detector.js b/src/routing/agentic-detector.js new file mode 100644 index 0000000..889d404 --- /dev/null +++ b/src/routing/agentic-detector.js @@ -0,0 +1,320 @@ +/** + * Agentic Workflow Detector + * Detects multi-step tool chains and autonomous agent patterns + * Used to boost complexity tier for agentic workloads + */ + +const logger = require('../logger'); + +// Agent type classification with tier requirements +const AGENT_TYPES = { + SINGLE_SHOT: { + minTier: 'SIMPLE', + scoreBoost: 0, + description: 'Simple request-response, no tool chains', + }, + TOOL_CHAIN: { + minTier: 'MEDIUM', + scoreBoost: 15, + requiresToolUse: true, + description: 'Sequential tool usage (read -> edit -> test)', + }, + ITERATIVE: { + minTier: 'COMPLEX', + scoreBoost: 25, + requiresToolUse: true, + description: 'Retry loops, debugging cycles, iterative refinement', + }, + AUTONOMOUS: { + minTier: 'REASONING', + scoreBoost: 35, + requiresToolUse: true, + description: 'Open-ended tasks, full autonomy, complex decision making', + }, +}; + +// Detection patterns +const PATTERNS = { + // Tool chain indicators + toolChain: /\b(then\s+use|after\s+that|next\s+step|finally|first.*then|step\s*\d+)\b/i, + + // Iterative work indicators + iterative: /\b(keep\s+trying|until|repeat|loop|retry|iterate|fix.*again|try.*different|debug)\b/i, + + // Autonomous work indicators + autonomous: /\b(figure\s+out|solve|complete\s+the\s+task|do\s+whatever|make\s+it\s+work|find\s+a\s+way|whatever\s+it\s+takes)\b/i, + + // Multi-file work + multiFile: /\b(multiple\s+files?|across\s+(the\s+)?codebase|all\s+files?|refactor\s+entire|whole\s+project|everywhere)\b/i, + + // Planning indicators + planning: /\b(plan|design|architect|strategy|roadmap|approach|how\s+would\s+you)\b/i, + + // Implementation indicators + implementation: /\b(implement|build|create|develop|write|code|add\s+feature)\b/i, + + // Analysis indicators + analysis: /\b(analyze|investigate|understand|explain|why\s+is|what\s+causes|root\s+cause)\b/i, + + // Testing indicators + testing: /\b(test|verify|validate|check|ensure|confirm|make\s+sure)\b/i, +}; + +// High-complexity tools that indicate agentic work +const AGENTIC_TOOLS = new Set([ + // Execution tools + 'Bash', 'bash', 'shell', 'execute', 'run_command', + // Write tools + 'Write', 'write_file', 'fs_write', 'create_file', + // Edit tools + 'Edit', 'edit_file', 'fs_edit', 'edit_patch', 'str_replace_editor', + // Agent tools + 'Task', 'agent_task', 'spawn_agent', 'delegate', + // Git tools + 'Git', 'git_commit', 'git_push', 'git_create_branch', + // Test tools + 'Test', 'run_tests', 'pytest', 'jest', + // Notebook tools + 'NotebookEdit', 'notebook_edit', +]); + +// Read-only tools (lower complexity) +const READ_ONLY_TOOLS = new Set([ + 'Read', 'read_file', 'fs_read', + 'Glob', 'glob', 'find_files', + 'Grep', 'grep', 'search', 'ripgrep', + 'WebFetch', 'web_fetch', 'fetch_url', + 'WebSearch', 'web_search', +]); + +class AgenticDetector { + /** + * Detect agentic workflow patterns + * @param {Object} payload - Request payload with messages and tools + * @returns {Object} Detection result + */ + detect(payload) { + const messages = payload?.messages || []; + const tools = payload?.tools || []; + const content = this._extractContent(messages); + + let score = 0; + const signals = []; + + // Signal 1: Tool count (many tools = likely multi-step) + const toolCount = tools.length; + if (toolCount > 10) { + score += 25; + signals.push({ signal: 'very_high_tool_count', value: toolCount, weight: 25 }); + } else if (toolCount > 5) { + score += 15; + signals.push({ signal: 'high_tool_count', value: toolCount, weight: 15 }); + } else if (toolCount > 3) { + score += 8; + signals.push({ signal: 'moderate_tool_count', value: toolCount, weight: 8 }); + } + + // Signal 2: Agentic tools present (Bash, Write, Edit, Task) + const agenticToolCount = tools.filter(t => { + const name = t.name || t.function?.name || ''; + return AGENTIC_TOOLS.has(name); + }).length; + + if (agenticToolCount > 3) { + score += 25; + signals.push({ signal: 'many_agentic_tools', value: agenticToolCount, weight: 25 }); + } else if (agenticToolCount > 1) { + score += 15; + signals.push({ signal: 'has_agentic_tools', value: agenticToolCount, weight: 15 }); + } else if (agenticToolCount === 1) { + score += 8; + signals.push({ signal: 'single_agentic_tool', value: agenticToolCount, weight: 8 }); + } + + // Signal 3: Prior tool results (already in agentic loop) + const toolResultCount = this._countToolResults(messages); + if (toolResultCount > 5) { + score += 30; + signals.push({ signal: 'deep_tool_loop', value: toolResultCount, weight: 30 }); + } else if (toolResultCount > 2) { + score += 20; + signals.push({ signal: 'active_tool_loop', value: toolResultCount, weight: 20 }); + } else if (toolResultCount > 0) { + score += 10; + signals.push({ signal: 'has_tool_results', value: toolResultCount, weight: 10 }); + } + + // Signal 4: Pattern matching on content + if (PATTERNS.autonomous.test(content)) { + score += 25; + signals.push({ signal: 'autonomous_pattern', weight: 25 }); + } + + if (PATTERNS.iterative.test(content)) { + score += 20; + signals.push({ signal: 'iterative_pattern', weight: 20 }); + } + + if (PATTERNS.toolChain.test(content)) { + score += 15; + signals.push({ signal: 'tool_chain_pattern', weight: 15 }); + } + + if (PATTERNS.multiFile.test(content)) { + score += 15; + signals.push({ signal: 'multi_file_work', weight: 15 }); + } + + if (PATTERNS.planning.test(content)) { + score += 10; + signals.push({ signal: 'planning_required', weight: 10 }); + } + + if (PATTERNS.implementation.test(content) && PATTERNS.testing.test(content)) { + score += 15; + signals.push({ signal: 'implementation_with_testing', weight: 15 }); + } + + // Signal 5: Conversation depth + const messageCount = messages.length; + if (messageCount > 15) { + score += 20; + signals.push({ signal: 'very_deep_conversation', value: messageCount, weight: 20 }); + } else if (messageCount > 8) { + score += 12; + signals.push({ signal: 'deep_conversation', value: messageCount, weight: 12 }); + } else if (messageCount > 4) { + score += 6; + signals.push({ signal: 'ongoing_conversation', value: messageCount, weight: 6 }); + } + + // Signal 6: Content length (longer prompts often = more complex tasks) + if (content.length > 2000) { + score += 10; + signals.push({ signal: 'long_prompt', value: content.length, weight: 10 }); + } + + // Determine agent type + const agentType = this._classifyAgentType(score, signals); + const isAgentic = score >= 25; + + const result = { + isAgentic, + agentType, + confidence: Math.min(score / 100, 1), + score, + signals, + minTier: AGENT_TYPES[agentType].minTier, + scoreBoost: AGENT_TYPES[agentType].scoreBoost, + description: AGENT_TYPES[agentType].description, + }; + + if (isAgentic) { + logger.debug({ + agentType, + score, + signalCount: signals.length, + toolCount, + toolResultCount, + }, '[AgenticDetector] Agentic workflow detected'); + } + + return result; + } + + /** + * Classify agent type based on score and signals + */ + _classifyAgentType(score, signals) { + // Check for specific signal combinations + const hasAutonomousPattern = signals.some(s => s.signal === 'autonomous_pattern'); + const hasDeepToolLoop = signals.some(s => s.signal === 'deep_tool_loop'); + const hasManyAgenticTools = signals.some(s => s.signal === 'many_agentic_tools'); + + // Autonomous: high score + autonomous pattern or very deep tool usage + if (score >= 60 || (hasAutonomousPattern && score >= 40)) { + return 'AUTONOMOUS'; + } + + // Iterative: moderate-high score with tool loops + if (score >= 40 || (hasDeepToolLoop && score >= 30)) { + return 'ITERATIVE'; + } + + // Tool chain: some tool usage indicated + if (score >= 20 || hasManyAgenticTools) { + return 'TOOL_CHAIN'; + } + + return 'SINGLE_SHOT'; + } + + /** + * Extract user content from messages + */ + _extractContent(messages) { + const userMsgs = messages.filter(m => m?.role === 'user'); + if (userMsgs.length === 0) return ''; + + // Get last user message + const last = userMsgs[userMsgs.length - 1]; + + if (typeof last.content === 'string') { + return last.content; + } + + if (Array.isArray(last.content)) { + return last.content + .filter(block => block?.type === 'text') + .map(block => block.text || '') + .join(' '); + } + + return ''; + } + + /** + * Count tool results in conversation + */ + _countToolResults(messages) { + let count = 0; + + for (const msg of messages) { + if (msg?.role === 'user' && Array.isArray(msg.content)) { + count += msg.content.filter(c => c?.type === 'tool_result').length; + } + } + + return count; + } + + /** + * Get detection stats for debugging + */ + getPatternStats(content) { + const stats = {}; + for (const [name, pattern] of Object.entries(PATTERNS)) { + stats[name] = pattern.test(content); + } + return stats; + } +} + +// Singleton instance +let instance = null; + +function getAgenticDetector() { + if (!instance) { + instance = new AgenticDetector(); + } + return instance; +} + +module.exports = { + AgenticDetector, + getAgenticDetector, + AGENT_TYPES, + PATTERNS, + AGENTIC_TOOLS, + READ_ONLY_TOOLS, +}; diff --git a/src/routing/complexity-analyzer.js b/src/routing/complexity-analyzer.js index 0929de4..8781362 100644 --- a/src/routing/complexity-analyzer.js +++ b/src/routing/complexity-analyzer.js @@ -88,6 +88,58 @@ const FORCE_LOCAL_PATTERNS = [ /^(help|menu|commands?|options?)[\s\.\!\?]*$/i, ]; +// Weighted Scoring (15 Dimensions) +const DIMENSION_WEIGHTS = { + // Content Analysis (35%) + tokenCount: 0.08, + promptComplexity: 0.10, + technicalDepth: 0.10, + domainSpecificity: 0.07, + // Tool Analysis (25%) + toolCount: 0.08, + toolComplexity: 0.10, + toolChainPotential: 0.07, + // Reasoning Requirements (25%) + multiStepReasoning: 0.10, + codeGeneration: 0.08, + analysisDepth: 0.07, + // Context Factors (15%) + conversationDepth: 0.05, + priorToolUsage: 0.05, + ambiguity: 0.05, +}; + +// Tool complexity weights (higher = more complex) +const TOOL_COMPLEXITY_WEIGHTS = { + Bash: 0.9, + bash: 0.9, + shell: 0.9, + Write: 0.8, + write_file: 0.8, + Edit: 0.7, + edit_file: 0.7, + NotebookEdit: 0.7, + Task: 0.9, + agent_task: 0.9, + WebSearch: 0.5, + WebFetch: 0.4, + Read: 0.3, + read_file: 0.3, + Glob: 0.2, + Grep: 0.2, + default: 0.5, +}; + +// Domain-specific keywords for complexity +const DOMAIN_KEYWORDS = { + security: /\b(auth|encrypt|vulnerability|injection|xss|csrf|jwt|oauth|password|credential|secret)\b/i, + ml: /\b(model|train|inference|tensor|embedding|neural|llm|gpt|transformer|pytorch|tensorflow)\b/i, + distributed: /\b(microservice|kafka|redis|queue|scale|cluster|replicate|kubernetes|docker|container)\b/i, + database: /\b(sql|nosql|migration|index|query|transaction|orm|postgres|mongodb|mysql)\b/i, + frontend: /\b(react|vue|angular|svelte|css|html|component|state|redux|hooks)\b/i, + devops: /\b(ci\/cd|pipeline|deploy|terraform|ansible|github\s*actions|jenkins)\b/i, +}; + // ============================================================================ // PHASE 3: Metrics Tracking // ============================================================================ @@ -360,6 +412,116 @@ function scoreReasoning(content) { return { score: Math.min(score, 15), reasons }; } +// ============================================================================ +// WEIGHTED SCORING FUNCTION (15 Dimensions) +// ============================================================================ + +/** + * Calculate weighted complexity score (0-100) + * Uses 15 dimensions with configurable weights + * @param {Object} payload - Request payload + * @param {string} content - Extracted content + * @returns {Object} Weighted score result + */ +function calculateWeightedScore(payload, content) { + const dimensions = {}; + + // 1. Token count (0-100) + const tokens = estimateTokens(payload); + dimensions.tokenCount = tokens < 500 ? 10 : tokens < 2000 ? 30 : tokens < 5000 ? 50 : tokens < 10000 ? 70 : 90; + + // 2. Prompt complexity (sentence structure, avg length) + const sentences = content.split(/[.!?]+/).filter(s => s.trim().length > 0); + const avgLength = content.length / Math.max(sentences.length, 1); + dimensions.promptComplexity = Math.min(avgLength / 2, 100); + + // 3. Technical depth (keyword density) + const techMatches = (content.match(PATTERNS.technical) || []).length; + dimensions.technicalDepth = Math.min(techMatches * 15, 100); + + // 4. Domain specificity (how many domains are touched) + let domainScore = 0; + const domainsMatched = []; + for (const [domain, regex] of Object.entries(DOMAIN_KEYWORDS)) { + if (regex.test(content)) { + domainScore += 20; + domainsMatched.push(domain); + } + } + dimensions.domainSpecificity = Math.min(domainScore, 100); + + // 5. Tool count + const toolCount = payload?.tools?.length ?? 0; + dimensions.toolCount = toolCount === 0 ? 0 : + toolCount <= 3 ? 20 : + toolCount <= 6 ? 40 : + toolCount <= 10 ? 60 : + toolCount <= 15 ? 80 : 100; + + // 6. Tool complexity (weighted by tool types) + if (payload?.tools?.length > 0) { + const totalWeight = payload.tools.reduce((sum, t) => { + const name = t.name || t.function?.name || ''; + return sum + (TOOL_COMPLEXITY_WEIGHTS[name] || TOOL_COMPLEXITY_WEIGHTS.default); + }, 0); + const avgWeight = totalWeight / payload.tools.length; + dimensions.toolComplexity = avgWeight * 100; + } else { + dimensions.toolComplexity = 0; + } + + // 7. Tool chain potential (sequential operations) + dimensions.toolChainPotential = /\b(then|after|next|finally|first.*then|step\s*\d+)\b/i.test(content) ? 70 : 20; + + // 8. Multi-step reasoning + dimensions.multiStepReasoning = ADVANCED_PATTERNS.reasoning.stepByStep.test(content) ? 80 : + ADVANCED_PATTERNS.reasoning.planning.test(content) ? 60 : 20; + + // 9. Code generation requirement + dimensions.codeGeneration = /\b(write|create|implement|build|generate)\s+(a\s+)?(new\s+)?(function|class|module|api|endpoint|service|component)/i.test(content) ? 80 : 20; + + // 10. Analysis depth + dimensions.analysisDepth = ADVANCED_PATTERNS.reasoning.tradeoffs.test(content) ? 80 : + ADVANCED_PATTERNS.reasoning.analysis.test(content) ? 60 : 20; + + // 11. Conversation depth + const messageCount = payload?.messages?.length ?? 0; + dimensions.conversationDepth = messageCount < 3 ? 10 : + messageCount < 6 ? 30 : + messageCount < 10 ? 50 : 70; + + // 12. Prior tool usage (tool results in conversation) + const toolResults = (payload?.messages || []).filter(m => + m.role === 'user' && Array.isArray(m.content) && m.content.some(c => c.type === 'tool_result') + ).length; + dimensions.priorToolUsage = toolResults === 0 ? 10 : + toolResults < 3 ? 40 : + toolResults < 6 ? 60 : 80; + + // 13. Ambiguity (inverse of specificity) + const hasSpecifics = /\b(file|function|line\s*\d+|error|bug|at\s+[\w.]+:\d+|\/[\w/]+\.\w+)\b/i.test(content); + dimensions.ambiguity = hasSpecifics ? 20 : content.length < 50 ? 70 : 40; + + // Calculate weighted total + let weightedTotal = 0; + for (const [dimension, weight] of Object.entries(DIMENSION_WEIGHTS)) { + weightedTotal += (dimensions[dimension] || 0) * weight; + } + + return { + score: Math.round(weightedTotal), + dimensions, + weights: DIMENSION_WEIGHTS, + meta: { + tokens, + toolCount, + messageCount, + toolResults, + domainsMatched, + }, + }; +} + /** * Get threshold based on SMART_TOOL_SELECTION_MODE */ @@ -381,13 +543,45 @@ function getThreshold() { * Analyze request complexity and return full analysis * * @param {Object} payload - Request payload + * @param {Object} options - Analysis options * @returns {Object} Complexity analysis result */ -function analyzeComplexity(payload) { +function analyzeComplexity(payload, options = {}) { const content = extractContent(payload); const messageCount = payload?.messages?.length ?? 0; + const useWeighted = options.weighted ?? config.routing?.weightedScoring ?? false; + + // Use weighted scoring if enabled + if (useWeighted) { + const weighted = calculateWeightedScore(payload, content); + const threshold = getThreshold(); + const mode = config.smartToolSelection?.mode ?? 'heuristic'; + + // Check force patterns + const taskTypeResult = scoreTaskType(content); + let recommendation; + if (taskTypeResult.reason === 'force_local') { + recommendation = 'local'; + } else if (taskTypeResult.reason === 'force_cloud') { + recommendation = 'cloud'; + } else { + recommendation = weighted.score >= threshold ? 'cloud' : 'local'; + } - // Calculate individual scores + return { + score: weighted.score, + threshold, + mode: 'weighted', + recommendation, + breakdown: weighted.dimensions, + weights: weighted.weights, + meta: weighted.meta, + forceReason: taskTypeResult.reason?.startsWith('force_') ? taskTypeResult.reason : null, + content: content.slice(0, 100) + (content.length > 100 ? '...' : ''), + }; + } + + // Standard scoring (original logic) const tokenScore = scoreTokens(payload); const toolScore = scoreTools(payload); const taskTypeResult = scoreTaskType(content); @@ -577,6 +771,9 @@ module.exports = { scoreCodeComplexity, scoreReasoning, + // Weighted scoring + calculateWeightedScore, + // Configuration getThreshold, @@ -592,4 +789,7 @@ module.exports = { ADVANCED_PATTERNS, FORCE_CLOUD_PATTERNS, FORCE_LOCAL_PATTERNS, + DIMENSION_WEIGHTS, + TOOL_COMPLEXITY_WEIGHTS, + DOMAIN_KEYWORDS, }; diff --git a/src/routing/cost-optimizer.js b/src/routing/cost-optimizer.js new file mode 100644 index 0000000..696f73b --- /dev/null +++ b/src/routing/cost-optimizer.js @@ -0,0 +1,305 @@ +/** + * Cost Optimizer Module + * Tracks and optimizes LLM costs across providers + * Uses ModelRegistry for dynamic pricing data + */ + +const logger = require('../logger'); +const config = require('../config'); +const { getModelRegistry, getModelRegistrySync } = require('./model-registry'); +const { getModelTierSelector, TIER_DEFINITIONS } = require('./model-tiers'); + +// Session cost tracking (in-memory) +const sessionCosts = new Map(); // sessionId -> { total, requests, byModel, byProvider } + +// Global stats +const globalStats = { + totalCost: 0, + totalSavings: 0, + requestCount: 0, + byProvider: {}, + byTier: {}, +}; + +class CostOptimizer { + constructor() { + this.registry = null; + this.tierSelector = null; + } + + /** + * Initialize with registry (async) + */ + async initialize() { + this.registry = await getModelRegistry(); + this.tierSelector = getModelTierSelector(); + } + + /** + * Get registry (sync fallback) + */ + _getRegistry() { + if (!this.registry) { + this.registry = getModelRegistrySync(); + } + return this.registry; + } + + /** + * Get tier selector + */ + _getTierSelector() { + if (!this.tierSelector) { + this.tierSelector = getModelTierSelector(); + } + return this.tierSelector; + } + + /** + * Estimate cost for a request before sending + * @param {string} model - Model name + * @param {number} inputTokens - Estimated input tokens + * @param {number} outputTokens - Estimated output tokens (optional) + * @returns {Object} Cost estimate + */ + estimateCost(model, inputTokens, outputTokens = null) { + const registry = this._getRegistry(); + const costs = registry.getCost(model); + + const inputCost = (inputTokens / 1_000_000) * costs.input; + const estimatedOutputTokens = outputTokens || Math.min(inputTokens * 0.5, 4096); + const outputCost = (estimatedOutputTokens / 1_000_000) * costs.output; + + return { + inputCost: Math.round(inputCost * 1_000_000) / 1_000_000, + outputCost: Math.round(outputCost * 1_000_000) / 1_000_000, + totalEstimate: Math.round((inputCost + outputCost) * 1_000_000) / 1_000_000, + model, + inputTokens, + outputTokens: estimatedOutputTokens, + pricePerMillion: { + input: costs.input, + output: costs.output, + }, + source: costs.source, + }; + } + + /** + * Find cheapest model capable of handling a complexity tier + * @param {string} requiredTier - Minimum tier required + * @param {string[]} availableProviders - Providers to consider + * @returns {Object|null} Cheapest model info + */ + findCheapestForTier(requiredTier, availableProviders) { + const registry = this._getRegistry(); + const tierSelector = this._getTierSelector(); + + const tierOrder = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING']; + const minTierIndex = tierOrder.indexOf(requiredTier); + + if (minTierIndex === -1) { + logger.warn({ tier: requiredTier }, '[CostOptimizer] Unknown tier'); + return null; + } + + const candidates = []; + + // Collect models from all capable tiers (>= required tier) + for (let i = minTierIndex; i < tierOrder.length; i++) { + const tier = tierOrder[i]; + + for (const provider of availableProviders) { + const models = tierSelector.getPreferredModels(tier, provider); + + for (const model of models) { + const cost = registry.getCost(model); + const totalCost = cost.input + cost.output; // Simple cost metric + + candidates.push({ + model, + provider, + tier, + inputCost: cost.input, + outputCost: cost.output, + totalCost, + context: cost.context, + source: cost.source, + }); + } + } + } + + if (candidates.length === 0) { + return null; + } + + // Sort by total cost (input + output per 1M tokens) + candidates.sort((a, b) => a.totalCost - b.totalCost); + + const cheapest = candidates[0]; + + logger.debug({ + requiredTier, + selectedModel: cheapest.model, + selectedProvider: cheapest.provider, + cost: cheapest.totalCost, + candidateCount: candidates.length, + }, '[CostOptimizer] Found cheapest model'); + + return cheapest; + } + + /** + * Record actual cost after response + * @param {string} sessionId - Session identifier + * @param {string} provider - Provider used + * @param {string} model - Model used + * @param {number} inputTokens - Actual input tokens + * @param {number} outputTokens - Actual output tokens + * @param {string} tier - Complexity tier + * @returns {number} Actual cost + */ + recordCost(sessionId, provider, model, inputTokens, outputTokens, tier = 'MEDIUM') { + const registry = this._getRegistry(); + const costs = registry.getCost(model); + + const inputCost = (inputTokens / 1_000_000) * costs.input; + const outputCost = (outputTokens / 1_000_000) * costs.output; + const actualCost = inputCost + outputCost; + + // Update session costs + if (sessionId) { + if (!sessionCosts.has(sessionId)) { + sessionCosts.set(sessionId, { + total: 0, + requests: 0, + byModel: {}, + byProvider: {}, + byTier: {}, + }); + } + + const session = sessionCosts.get(sessionId); + session.total += actualCost; + session.requests++; + session.byModel[model] = (session.byModel[model] || 0) + actualCost; + session.byProvider[provider] = (session.byProvider[provider] || 0) + actualCost; + session.byTier[tier] = (session.byTier[tier] || 0) + actualCost; + } + + // Update global stats + globalStats.totalCost += actualCost; + globalStats.requestCount++; + globalStats.byProvider[provider] = (globalStats.byProvider[provider] || 0) + actualCost; + globalStats.byTier[tier] = (globalStats.byTier[tier] || 0) + actualCost; + + logger.debug({ + sessionId, + provider, + model, + inputTokens, + outputTokens, + cost: actualCost.toFixed(6), + tier, + }, '[CostOptimizer] Recorded cost'); + + return actualCost; + } + + /** + * Calculate potential savings from routing optimization + */ + calculateSavings(originalModel, optimizedModel, tokens) { + const registry = this._getRegistry(); + + const originalCost = registry.getCost(originalModel); + const optimizedCost = registry.getCost(optimizedModel); + + const originalTotal = (tokens / 1_000_000) * (originalCost.input + originalCost.output); + const optimizedTotal = (tokens / 1_000_000) * (optimizedCost.input + optimizedCost.output); + + const savings = originalTotal - optimizedTotal; + + if (savings > 0) { + globalStats.totalSavings += savings; + } + + return { + originalCost: originalTotal, + optimizedCost: optimizedTotal, + savings: Math.max(0, savings), + percentSaved: originalTotal > 0 ? (savings / originalTotal) * 100 : 0, + }; + } + + + /** + * Get session cost summary + */ + getSessionCost(sessionId) { + return sessionCosts.get(sessionId) || { + total: 0, + requests: 0, + byModel: {}, + byProvider: {}, + byTier: {}, + }; + } + + /** + * Get global stats + */ + getStats() { + return { + ...globalStats, + sessionCount: sessionCosts.size, + avgCostPerRequest: globalStats.requestCount > 0 + ? (globalStats.totalCost / globalStats.requestCount).toFixed(6) + : '0', + totalCostFormatted: `$${globalStats.totalCost.toFixed(4)}`, + totalSavingsFormatted: `$${globalStats.totalSavings.toFixed(4)}`, + }; + } + + /** + * Clear session data (for cleanup) + */ + clearSession(sessionId) { + sessionCosts.delete(sessionId); + } + + /** + * Reset all stats (for testing) + */ + resetStats() { + sessionCosts.clear(); + globalStats.totalCost = 0; + globalStats.totalSavings = 0; + globalStats.requestCount = 0; + globalStats.byProvider = {}; + globalStats.byTier = {}; + } +} + +// Singleton instance +let instance = null; + +function getCostOptimizer() { + if (!instance) { + instance = new CostOptimizer(); + } + return instance; +} + +async function getCostOptimizerAsync() { + const optimizer = getCostOptimizer(); + await optimizer.initialize(); + return optimizer; +} + +module.exports = { + CostOptimizer, + getCostOptimizer, + getCostOptimizerAsync, +}; diff --git a/src/routing/index.js b/src/routing/index.js index f47853f..789e7c8 100644 --- a/src/routing/index.js +++ b/src/routing/index.js @@ -10,7 +10,6 @@ const config = require('../config'); const logger = require('../logger'); -const { modelNameSupportsTools } = require('../clients/ollama-utils'); const { analyzeComplexity, shouldForceLocal, @@ -19,6 +18,11 @@ const { analyzeWithEmbeddings, } = require('./complexity-analyzer'); +// Intelligent routing modules +const { getAgenticDetector, AGENT_TYPES } = require('./agentic-detector'); +const { getModelTierSelector, TIER_DEFINITIONS } = require('./model-tiers'); +const { getCostOptimizer } = require('./cost-optimizer'); + // Local providers const LOCAL_PROVIDERS = ['ollama', 'llamacpp', 'lmstudio']; @@ -49,26 +53,7 @@ function getFallbackProvider() { * @param {number} options.toolCount - Number of tools in the request (for hybrid routing) * @param {boolean} options.useHybridRouting - Whether to use hybrid routing logic (default: false) */ -function getBestCloudProvider(options = {}) { - const { toolCount = 0, useHybridRouting = false } = options; - - // If hybrid routing is explicitly enabled and we have tools, use tool-based routing - const preferOllama = config.modelProvider?.preferOllama ?? false; - - if (preferOllama && useHybridRouting && toolCount > 0) { - const openRouterMaxTools = config.modelProvider?.openRouterMaxToolsForRouting ?? 15; - - // For moderate tool counts, prefer OpenRouter over Azure OpenAI - if (toolCount <= openRouterMaxTools && config.openrouter?.apiKey) { - return 'openrouter'; - } - - // For higher tool counts, use Azure OpenAI if available - if (config.azureOpenAI?.endpoint && config.azureOpenAI?.apiKey) { - return 'azure-openai'; - } - } - +function getBestCloudProvider() { // Standard priority order for cloud providers if (config.databricks?.url && config.databricks?.apiKey) return 'databricks'; if (config.azureAnthropic?.endpoint && config.azureAnthropic?.apiKey) return 'azure-anthropic'; @@ -105,23 +90,43 @@ function getBestLocalProvider() { * @returns {Object} Routing decision with provider and metadata */ async function determineProviderSmart(payload, options = {}) { - const preferOllama = config.modelProvider?.preferOllama ?? false; const primaryProvider = config.modelProvider?.type ?? 'databricks'; - // If smart routing is disabled, use static configuration - if (!preferOllama) { + // If tier routing is disabled, use static configuration + if (!config.modelTiers?.enabled) { return { provider: primaryProvider, + model: null, method: 'static', - reason: 'smart_routing_disabled', + reason: 'tier_routing_disabled', }; } // Quick check for force patterns if (shouldForceLocal(payload)) { + // When tier routing is enabled, respect TIER_SIMPLE instead of blindly choosing local + if (config.modelTiers?.enabled) { + try { + const selector = getModelTierSelector(); + const modelSelection = selector.selectModel('SIMPLE', null); + const decision = { + provider: modelSelection.provider, + model: modelSelection.model, + tier: 'SIMPLE', + method: 'force', + reason: 'force_local_pattern', + score: 0, + }; + routingMetrics.record(decision); + return decision; + } catch (err) { + logger.debug({ err: err.message }, 'Tier selection failed for force_local, falling back to local provider'); + } + } const provider = getBestLocalProvider(); const decision = { provider, + model: null, method: 'force', reason: 'force_local_pattern', score: 0, @@ -131,10 +136,10 @@ async function determineProviderSmart(payload, options = {}) { } if (shouldForceCloud(payload) && isFallbackEnabled()) { - const toolCount = payload?.tools?.length ?? 0; - const provider = getBestCloudProvider({ toolCount }); + const provider = getBestCloudProvider(); const decision = { provider, + model: null, method: 'force', reason: 'force_cloud_pattern', score: 100, @@ -143,60 +148,9 @@ async function determineProviderSmart(payload, options = {}) { return decision; } - // Check tool count thresholds for hybrid routing - const toolCount = payload?.tools?.length ?? 0; - const ollamaMaxTools = config.modelProvider?.ollamaMaxToolsForRouting ?? 3; - - // If tool count is within Ollama's threshold, route to Ollama - if (toolCount > 0 && toolCount <= ollamaMaxTools) { - const ollamaModel = config.ollama?.model; - const supportsTools = modelNameSupportsTools(ollamaModel); - - if (supportsTools) { - const provider = getBestLocalProvider(); - const decision = { - provider, - method: 'tool_threshold', - reason: 'within_ollama_tool_threshold', - score: 0, - toolCount, - threshold: ollamaMaxTools, - }; - routingMetrics.record(decision); - return decision; - } - // If Ollama doesn't support tools, fall through to cloud routing - if (isFallbackEnabled()) { - const provider = getBestCloudProvider({ toolCount }); - const decision = { - provider, - method: 'tool_support', - reason: 'local_model_no_tool_support', - score: 0, - toolCount, - }; - routingMetrics.record(decision); - return decision; - } - } - - // If tool count exceeds Ollama threshold but fallback is enabled, route to cloud - if (toolCount > ollamaMaxTools && isFallbackEnabled()) { - const provider = getBestCloudProvider({ toolCount, useHybridRouting: true }); - const decision = { - provider, - method: 'tool_threshold', - reason: 'exceeds_ollama_tool_threshold', - score: 50, - toolCount, - threshold: ollamaMaxTools, - }; - routingMetrics.record(decision); - return decision; - } - - // Full complexity analysis for non-tool requests - const analysis = analyzeComplexity(payload); + // Full complexity analysis + const useWeightedScoring = config.routing?.weightedScoring ?? false; + const analysis = analyzeComplexity(payload, { weighted: useWeightedScoring }); // Phase 4: Optional embeddings adjustment let embeddingsResult = null; @@ -214,25 +168,116 @@ async function determineProviderSmart(payload, options = {}) { } } - // Apply routing decision based on complexity + // Agentic workflow detection + let agenticResult = null; + if (config.routing?.agenticDetection !== false) { + try { + const detector = getAgenticDetector(); + agenticResult = detector.detect(payload); + + // Boost complexity score for agentic workflows + if (agenticResult.isAgentic) { + analysis.score = Math.min(100, analysis.score + agenticResult.scoreBoost); + analysis.agenticBoost = agenticResult.scoreBoost; + analysis.agentType = agenticResult.agentType; + + logger.debug({ + agentType: agenticResult.agentType, + boost: agenticResult.scoreBoost, + newScore: analysis.score, + }, '[Routing] Agentic workflow detected, boosting score'); + + // Force cloud for autonomous workflows + if (agenticResult.agentType === 'AUTONOMOUS' && isFallbackEnabled()) { + const provider = getBestCloudProvider(); + const decision = { + provider, + method: 'agentic', + reason: 'autonomous_workflow', + score: analysis.score, + agenticResult, + }; + routingMetrics.record(decision); + return decision; + } + } + } catch (err) { + logger.debug({ err: err.message }, 'Agentic detection failed'); + } + } + + // Tier-based model selection + let selectedModel = null; + let tier = null; + if (config.modelTiers?.enabled) { + try { + const selector = getModelTierSelector(); + tier = selector.getTier(analysis.score); + + // Check if agentic detection requires a higher tier + if (agenticResult?.minTier) { + const agenticTierPriority = TIER_DEFINITIONS[agenticResult.minTier]?.priority || 0; + const currentTierPriority = TIER_DEFINITIONS[tier]?.priority || 0; + if (agenticTierPriority > currentTierPriority) { + tier = agenticResult.minTier; + logger.debug({ from: selector.getTier(analysis.score), to: tier }, '[Routing] Upgrading tier for agentic workflow'); + } + } + + // Select model for the tier (will be applied after provider selection) + analysis.tier = tier; + } catch (err) { + logger.debug({ err: err.message }, 'Tier selection failed'); + } + } + + // Apply routing decision based on tier config (TIER_* env vars are mandatory) let provider; - let method = 'complexity'; - - if (analysis.recommendation === 'local') { - provider = getBestLocalProvider(); - } else { - // Cloud recommendation - if (isFallbackEnabled()) { - provider = getBestCloudProvider({ toolCount }); - } else { - // Fallback disabled, use local anyway - provider = getBestLocalProvider(); - method = 'fallback_disabled'; + let method = 'tier_config'; + + const selector = getModelTierSelector(); + const modelSelection = selector.selectModel(tier, null); + + provider = modelSelection.provider; + selectedModel = modelSelection.model; + logger.debug({ tier, provider, model: selectedModel }, '[Routing] Using tier config'); + + // Cost optimization: check if cheaper model can handle this tier + let costOptimized = false; + if (config.routing?.costOptimization && tier) { + try { + const optimizer = getCostOptimizer(); + const availableProviders = [provider]; + + // Also consider local provider if not already selected + const localProvider = getBestLocalProvider(); + if (localProvider !== provider) { + availableProviders.push(localProvider); + } + + const cheapest = optimizer.findCheapestForTier(tier, availableProviders); + if (cheapest && cheapest.provider !== provider) { + logger.debug({ + from: provider, + to: cheapest.provider, + tier, + savings: `${cheapest.model} is cheaper`, + }, '[Routing] Cost optimization: switching provider'); + + provider = cheapest.provider; + selectedModel = cheapest.model; + costOptimized = true; + method = 'cost_optimized'; + } + } catch (err) { + logger.debug({ err: err.message }, 'Cost optimization failed'); } } const decision = { provider, + model: selectedModel, + tier, method, reason: analysis.recommendation, score: analysis.score, @@ -240,6 +285,8 @@ async function determineProviderSmart(payload, options = {}) { mode: analysis.mode, analysis, embeddingsResult, + agenticResult, + costOptimized, }; // Phase 3: Record metrics @@ -251,8 +298,8 @@ async function determineProviderSmart(payload, options = {}) { score: analysis.score, threshold: analysis.threshold, recommendation: analysis.recommendation, - taskType: analysis.breakdown.taskType.reason, - toolCount, + taskType: analysis.breakdown?.taskType?.reason, + toolCount: payload?.tools?.length ?? 0, }, 'Smart routing decision' ); @@ -260,67 +307,6 @@ async function determineProviderSmart(payload, options = {}) { return decision; } -/** - * Synchronous version of determineProvider for backward compatibility - * Does not include Phase 4 embeddings analysis - */ -function determineProvider(payload) { - const preferOllama = config.modelProvider?.preferOllama ?? false; - const primaryProvider = config.modelProvider?.type ?? 'databricks'; - - // If smart routing is disabled, use static configuration - if (!preferOllama) { - return primaryProvider; - } - - // Quick check for force patterns - if (shouldForceLocal(payload)) { - return getBestLocalProvider(); - } - - if (shouldForceCloud(payload) && isFallbackEnabled()) { - const toolCount = payload?.tools?.length ?? 0; - return getBestCloudProvider({ toolCount }); - } - - // Check tool count thresholds for hybrid routing - const toolCount = payload?.tools?.length ?? 0; - const ollamaMaxTools = config.modelProvider?.ollamaMaxToolsForRouting ?? 3; - - // If tool count is within Ollama's threshold, route to Ollama - if (toolCount > 0 && toolCount <= ollamaMaxTools) { - const ollamaModel = config.ollama?.model; - const supportsTools = modelNameSupportsTools(ollamaModel); - - if (supportsTools) { - return getBestLocalProvider(); - } - // If Ollama doesn't support tools, fall through to cloud routing - if (isFallbackEnabled()) { - return getBestCloudProvider({ toolCount }); - } - } - - // If tool count exceeds Ollama threshold but fallback is enabled, route to cloud - if (toolCount > ollamaMaxTools && isFallbackEnabled()) { - return getBestCloudProvider({ toolCount, useHybridRouting: true }); - } - - // Full complexity analysis (without embeddings) for non-tool requests - const analysis = analyzeComplexity(payload); - - // Apply routing decision based on complexity - if (analysis.recommendation === 'local') { - return getBestLocalProvider(); - } - - if (isFallbackEnabled()) { - return getBestCloudProvider({ toolCount }); - } - - return getBestLocalProvider(); -} - /** * Get routing headers to include in response * Phase 3: Expose routing decision to clients @@ -343,6 +329,23 @@ function getRoutingHeaders(decision) { headers['X-Lynkr-Routing-Reason'] = decision.reason; } + // Tier and model headers + if (decision.tier) { + headers['X-Lynkr-Tier'] = decision.tier; + } + + if (decision.model) { + headers['X-Lynkr-Model'] = decision.model; + } + + if (decision.agenticResult?.isAgentic) { + headers['X-Lynkr-Agentic'] = decision.agenticResult.agentType; + } + + if (decision.costOptimized) { + headers['X-Lynkr-Cost-Optimized'] = 'true'; + } + return headers; } @@ -355,8 +358,7 @@ function getRoutingStats() { } module.exports = { - // Main routing functions - determineProvider, + // Main routing function determineProviderSmart, // Helpers @@ -372,4 +374,11 @@ module.exports = { // Re-export analyzer for direct access analyzeComplexity: require('./complexity-analyzer').analyzeComplexity, + + // Intelligent routing modules + getAgenticDetector, + getModelTierSelector, + getCostOptimizer, + AGENT_TYPES, + TIER_DEFINITIONS, }; diff --git a/src/routing/model-registry.js b/src/routing/model-registry.js new file mode 100644 index 0000000..e52258b --- /dev/null +++ b/src/routing/model-registry.js @@ -0,0 +1,437 @@ +/** + * Model Registry + * Multi-source pricing: LiteLLM -> models.dev -> Databricks fallback + * Caches data locally with 24h TTL + */ + +const fs = require('fs'); +const path = require('path'); +const logger = require('../logger'); + +// API URLs +const LITELLM_URL = 'https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json'; +const MODELS_DEV_URL = 'https://models.dev/api.json'; + +// Cache settings +const CACHE_FILE = path.join(__dirname, '../../data/model-prices-cache.json'); +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours + +// Databricks fallback pricing (based on Anthropic direct API prices) +const DATABRICKS_FALLBACK = { + // Claude models + 'databricks-claude-opus-4-6': { input: 5.0, output: 25.0, context: 1000000 }, + 'databricks-claude-opus-4-5': { input: 5.0, output: 25.0, context: 200000 }, + 'databricks-claude-opus-4-1': { input: 15.0, output: 75.0, context: 200000 }, + 'databricks-claude-sonnet-4-5': { input: 3.0, output: 15.0, context: 200000 }, + 'databricks-claude-sonnet-4': { input: 3.0, output: 15.0, context: 200000 }, + 'databricks-claude-3-7-sonnet': { input: 3.0, output: 15.0, context: 200000 }, + 'databricks-claude-haiku-4-5': { input: 1.0, output: 5.0, context: 200000 }, + + // Llama models + 'databricks-llama-4-maverick': { input: 1.0, output: 1.0, context: 128000 }, + 'databricks-meta-llama-3-3-70b-instruct': { input: 0.9, output: 0.9, context: 128000 }, + 'databricks-meta-llama-3-1-405b-instruct': { input: 2.0, output: 2.0, context: 128000 }, + 'databricks-meta-llama-3-1-8b-instruct': { input: 0.2, output: 0.2, context: 128000 }, + + // GPT models via Databricks + 'databricks-gpt-5-2': { input: 5.0, output: 15.0, context: 200000 }, + 'databricks-gpt-5-1': { input: 3.0, output: 12.0, context: 200000 }, + 'databricks-gpt-5': { input: 2.5, output: 10.0, context: 128000 }, + 'databricks-gpt-5-mini': { input: 0.5, output: 1.5, context: 128000 }, + 'databricks-gpt-5-nano': { input: 0.15, output: 0.6, context: 128000 }, + + // Gemini models via Databricks + 'databricks-gemini-3-flash': { input: 0.075, output: 0.3, context: 1000000 }, + 'databricks-gemini-3-pro': { input: 1.25, output: 5.0, context: 2000000 }, + 'databricks-gemini-2-5-pro': { input: 1.25, output: 5.0, context: 1000000 }, + 'databricks-gemini-2-5-flash': { input: 0.075, output: 0.3, context: 1000000 }, + + // DBRX + 'databricks-dbrx-instruct': { input: 0.75, output: 2.25, context: 32000 }, + + // Embedding models (price per 1M tokens) + 'databricks-gte-large-en': { input: 0.02, output: 0, context: 8192 }, + 'databricks-bge-large-en': { input: 0.02, output: 0, context: 512 }, +}; + +// Default cost for unknown models +const DEFAULT_COST = { input: 1.0, output: 3.0, context: 128000 }; + +class ModelRegistry { + constructor() { + this.litellmPrices = {}; + this.modelsDevPrices = {}; + this.loaded = false; + this.lastFetch = 0; + this.modelIndex = new Map(); + } + + /** + * Initialize registry - load from cache or fetch fresh data + */ + async initialize() { + if (this.loaded) return; + + // Try cache first + if (this._loadFromCache()) { + this.loaded = true; + // Background refresh if stale + if (Date.now() - this.lastFetch > CACHE_TTL_MS) { + this._fetchAll().catch(err => + logger.warn({ err: err.message }, '[ModelRegistry] Background refresh failed') + ); + } + return; + } + + // Fetch fresh data + await this._fetchAll(); + this.loaded = true; + } + + /** + * Fetch from both sources + */ + async _fetchAll() { + const results = await Promise.allSettled([ + this._fetchLiteLLM(), + this._fetchModelsDev(), + ]); + + const litellmOk = results[0].status === 'fulfilled'; + const modelsDevOk = results[1].status === 'fulfilled'; + + if (litellmOk || modelsDevOk) { + this._buildIndex(); + this._saveToCache(); + this.lastFetch = Date.now(); + + logger.info({ + litellm: litellmOk ? Object.keys(this.litellmPrices).length : 0, + modelsDev: modelsDevOk ? Object.keys(this.modelsDevPrices).length : 0, + total: this.modelIndex.size, + }, '[ModelRegistry] Loaded pricing data'); + } else { + logger.warn('[ModelRegistry] All sources failed, using Databricks fallback only'); + } + } + + /** + * Fetch LiteLLM pricing + */ + async _fetchLiteLLM() { + try { + const response = await fetch(LITELLM_URL, { + signal: AbortSignal.timeout(15000), + headers: { 'Accept': 'application/json' }, + }); + + if (!response.ok) throw new Error(`HTTP ${response.status}`); + + const data = await response.json(); + this.litellmPrices = this._processLiteLLM(data); + + logger.debug({ count: Object.keys(this.litellmPrices).length }, '[ModelRegistry] LiteLLM loaded'); + } catch (err) { + logger.warn({ err: err.message }, '[ModelRegistry] LiteLLM fetch failed'); + throw err; + } + } + + /** + * Process LiteLLM format into our format + * LiteLLM uses cost per token, we use cost per 1M tokens + */ + _processLiteLLM(data) { + const prices = {}; + + for (const [modelId, info] of Object.entries(data)) { + if (!info || typeof info !== 'object') continue; + + // Convert per-token to per-million-tokens + const inputCost = (info.input_cost_per_token || 0) * 1_000_000; + const outputCost = (info.output_cost_per_token || 0) * 1_000_000; + + prices[modelId.toLowerCase()] = { + input: inputCost, + output: outputCost, + context: info.max_input_tokens || info.max_tokens || 128000, + maxOutput: info.max_output_tokens || 4096, + toolCall: info.supports_function_calling ?? true, + vision: info.supports_vision ?? false, + source: 'litellm', + }; + + // Also index without provider prefix for flexible lookup + const shortName = modelId.split('/').pop().toLowerCase(); + if (shortName !== modelId.toLowerCase()) { + prices[shortName] = prices[modelId.toLowerCase()]; + } + } + + return prices; + } + + /** + * Fetch models.dev pricing + */ + async _fetchModelsDev() { + try { + const response = await fetch(MODELS_DEV_URL, { + signal: AbortSignal.timeout(15000), + headers: { 'Accept': 'application/json' }, + }); + + if (!response.ok) throw new Error(`HTTP ${response.status}`); + + const data = await response.json(); + this.modelsDevPrices = this._processModelsDev(data); + + logger.debug({ count: Object.keys(this.modelsDevPrices).length }, '[ModelRegistry] models.dev loaded'); + } catch (err) { + logger.warn({ err: err.message }, '[ModelRegistry] models.dev fetch failed'); + throw err; + } + } + + /** + * Process models.dev format into our format + */ + _processModelsDev(data) { + const prices = {}; + + for (const [providerId, providerData] of Object.entries(data)) { + if (!providerData?.models) continue; + + for (const [modelId, info] of Object.entries(providerData.models)) { + const fullId = `${providerId}/${modelId}`.toLowerCase(); + + prices[fullId] = { + input: info.cost?.input || 0, + output: info.cost?.output || 0, + cacheRead: info.cost?.cache_read, + cacheWrite: info.cost?.cache_write, + context: info.context || 128000, + maxOutput: info.output || 4096, + toolCall: info.tool_call ?? false, + reasoning: info.reasoning ?? false, + vision: Array.isArray(info.input) && info.input.includes('image'), + source: 'models.dev', + }; + + // Also index by short name + prices[modelId.toLowerCase()] = prices[fullId]; + } + } + + return prices; + } + + /** + * Build unified index from all sources + */ + _buildIndex() { + this.modelIndex.clear(); + + // Add Databricks fallback first (lowest priority) + for (const [modelId, info] of Object.entries(DATABRICKS_FALLBACK)) { + this.modelIndex.set(modelId.toLowerCase(), { ...info, source: 'databricks-fallback' }); + } + + // Add models.dev (medium priority) + for (const [modelId, info] of Object.entries(this.modelsDevPrices)) { + this.modelIndex.set(modelId, info); + } + + // Add LiteLLM (highest priority) + for (const [modelId, info] of Object.entries(this.litellmPrices)) { + this.modelIndex.set(modelId, info); + } + } + + /** + * Get cost for a model + * @param {string} modelName - Model name/ID + * @returns {Object} Cost info { input, output, context, ... } + */ + getCost(modelName) { + if (!modelName) return { ...DEFAULT_COST, source: 'default' }; + + const normalizedName = modelName.toLowerCase(); + + // Direct lookup + if (this.modelIndex.has(normalizedName)) { + return this.modelIndex.get(normalizedName); + } + + // Try common variations + const variations = [ + normalizedName, + normalizedName.replace('databricks-', ''), + normalizedName.replace('azure/', ''), + normalizedName.replace('bedrock/', ''), + normalizedName.replace('anthropic.', ''), + normalizedName.split('/').pop(), + ]; + + for (const variant of variations) { + if (this.modelIndex.has(variant)) { + return this.modelIndex.get(variant); + } + } + + // Fuzzy match for partial names + for (const [key, value] of this.modelIndex.entries()) { + if (key.includes(normalizedName) || normalizedName.includes(key)) { + return value; + } + } + + logger.debug({ model: modelName }, '[ModelRegistry] Model not found, using default'); + return { ...DEFAULT_COST, source: 'default' }; + } + + /** + * Get model info by name + */ + getModel(modelName) { + return this.getCost(modelName); + } + + /** + * Check if model is free (local) + */ + isFree(modelName) { + const cost = this.getCost(modelName); + return cost.input === 0 && cost.output === 0; + } + + /** + * Check if model supports tool calling + */ + supportsTools(modelName) { + const model = this.getCost(modelName); + return model.toolCall === true; + } + + /** + * Find models matching criteria + */ + findModels(criteria = {}) { + const results = []; + + for (const [modelId, info] of this.modelIndex.entries()) { + if (criteria.maxInputCost && info.input > criteria.maxInputCost) continue; + if (criteria.minContext && info.context < criteria.minContext) continue; + if (criteria.toolCall && !info.toolCall) continue; + if (criteria.reasoning && !info.reasoning) continue; + if (criteria.vision && !info.vision) continue; + + results.push({ modelId, ...info }); + } + + // Sort by input cost ascending + return results.sort((a, b) => a.input - b.input); + } + + /** + * Get stats for metrics endpoint + */ + getStats() { + const sources = { litellm: 0, 'models.dev': 0, 'databricks-fallback': 0, default: 0 }; + + for (const info of this.modelIndex.values()) { + const source = info.source || 'default'; + sources[source] = (sources[source] || 0) + 1; + } + + return { + totalModels: this.modelIndex.size, + bySource: sources, + lastFetch: this.lastFetch, + cacheAge: this.lastFetch ? Date.now() - this.lastFetch : null, + cacheTTL: CACHE_TTL_MS, + }; + } + + /** + * Force refresh from APIs + */ + async refresh() { + await this._fetchAll(); + } + + // Cache management + _loadFromCache() { + try { + if (!fs.existsSync(CACHE_FILE)) return false; + + const cache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); + this.litellmPrices = cache.litellm || {}; + this.modelsDevPrices = cache.modelsDev || {}; + this.lastFetch = cache.timestamp || 0; + + this._buildIndex(); + + logger.debug({ + age: Math.round((Date.now() - this.lastFetch) / 60000) + 'min', + models: this.modelIndex.size, + }, '[ModelRegistry] Loaded from cache'); + + return true; + } catch (err) { + logger.debug({ err: err.message }, '[ModelRegistry] Cache load failed'); + return false; + } + } + + _saveToCache() { + try { + const dir = path.dirname(CACHE_FILE); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const cache = { + litellm: this.litellmPrices, + modelsDev: this.modelsDevPrices, + timestamp: Date.now(), + }; + + fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2)); + logger.debug('[ModelRegistry] Cache saved'); + } catch (err) { + logger.warn({ err: err.message }, '[ModelRegistry] Cache save failed'); + } + } +} + +// Singleton with lazy initialization +let instance = null; + +async function getModelRegistry() { + if (!instance) { + instance = new ModelRegistry(); + await instance.initialize(); + } + return instance; +} + +// Sync getter (uses cache only, no network) +function getModelRegistrySync() { + if (!instance) { + instance = new ModelRegistry(); + instance._loadFromCache(); + instance._buildIndex(); + instance.loaded = true; + } + return instance; +} + +module.exports = { + ModelRegistry, + getModelRegistry, + getModelRegistrySync, + DATABRICKS_FALLBACK, + DEFAULT_COST, +}; diff --git a/src/routing/model-tiers.js b/src/routing/model-tiers.js new file mode 100644 index 0000000..9d209ab --- /dev/null +++ b/src/routing/model-tiers.js @@ -0,0 +1,365 @@ +/** + * Model Tier Selector + * Maps complexity scores to appropriate models per provider + * Uses config/model-tiers.json for tier preferences + */ + +const fs = require('fs'); +const path = require('path'); +const logger = require('../logger'); +const config = require('../config'); + +// Load tier config +const TIER_CONFIG_PATH = path.join(__dirname, '../../config/model-tiers.json'); + +// Tier definitions with complexity ranges +const TIER_DEFINITIONS = { + SIMPLE: { + description: 'Greetings, simple Q&A, confirmations', + range: [0, 25], + priority: 1, + }, + MEDIUM: { + description: 'Code reading, simple edits, research', + range: [26, 50], + priority: 2, + }, + COMPLEX: { + description: 'Multi-file changes, debugging, architecture', + range: [51, 75], + priority: 3, + }, + REASONING: { + description: 'Complex analysis, security audits, novel problems', + range: [76, 100], + priority: 4, + }, +}; + +class ModelTierSelector { + constructor() { + this.tierConfig = null; + this.localProviders = {}; + this.providerAliases = {}; + this._loadConfig(); + } + + /** + * Load tier configuration from JSON file + */ + _loadConfig() { + try { + if (fs.existsSync(TIER_CONFIG_PATH)) { + const data = JSON.parse(fs.readFileSync(TIER_CONFIG_PATH, 'utf8')); + this.tierConfig = data.tiers || {}; + this.localProviders = data.localProviders || {}; + this.providerAliases = data.providerAliases || {}; + logger.debug({ tiers: Object.keys(this.tierConfig) }, '[ModelTiers] Config loaded'); + } else { + logger.warn('[ModelTiers] Config file not found, using defaults'); + this._loadDefaults(); + } + } catch (err) { + logger.warn({ err: err.message }, '[ModelTiers] Config load failed, using defaults'); + this._loadDefaults(); + } + } + + /** + * Load default tier config + */ + _loadDefaults() { + this.tierConfig = { + SIMPLE: { preferred: { ollama: ['llama3.2'], openai: ['gpt-4o-mini'] } }, + MEDIUM: { preferred: { openai: ['gpt-4o'], anthropic: ['claude-sonnet-4-20250514'] } }, + COMPLEX: { preferred: { openai: ['o1-mini'], anthropic: ['claude-sonnet-4-20250514'] } }, + REASONING: { preferred: { openai: ['o1'], anthropic: ['claude-opus-4-20250514'] } }, + }; + this.localProviders = { + ollama: { free: true, defaultTier: 'SIMPLE' }, + llamacpp: { free: true, defaultTier: 'SIMPLE' }, + lmstudio: { free: true, defaultTier: 'SIMPLE' }, + }; + } + + /** + * Normalize provider name using aliases + */ + _normalizeProvider(provider) { + if (!provider) return 'openai'; + const lower = provider.toLowerCase(); + return this.providerAliases[lower] || lower; + } + + /** + * Get tier from complexity score + * @param {number} complexityScore - Score from 0-100 + * @returns {string} Tier name (SIMPLE, MEDIUM, COMPLEX, REASONING) + */ + getTier(complexityScore) { + const score = Math.max(0, Math.min(100, complexityScore || 0)); + + for (const [tier, def] of Object.entries(TIER_DEFINITIONS)) { + if (score >= def.range[0] && score <= def.range[1]) { + return tier; + } + } + + return score > 75 ? 'REASONING' : 'SIMPLE'; + } + + /** + * Get tier definition + */ + getTierDefinition(tier) { + return TIER_DEFINITIONS[tier] || TIER_DEFINITIONS.MEDIUM; + } + + /** + * Get tier priority (1-4) + */ + getTierPriority(tier) { + return TIER_DEFINITIONS[tier]?.priority || 2; + } + + /** + * Compare two tiers, returns positive if tier1 > tier2 + */ + compareTiers(tier1, tier2) { + return this.getTierPriority(tier1) - this.getTierPriority(tier2); + } + + /** + * Get preferred models for a tier and provider + * @param {string} tier - Tier name + * @param {string} provider - Provider name + * @returns {string[]} Array of model names + */ + getPreferredModels(tier, provider) { + const normalizedProvider = this._normalizeProvider(provider); + return this.tierConfig[tier]?.preferred?.[normalizedProvider] || []; + } + + /** + * Select model for tier from TIER_* env var (mandatory) + * @param {string} tier - Tier name (SIMPLE, MEDIUM, COMPLEX, REASONING) + * @param {string} _unused - Deprecated parameter + * @returns {Object} { model, provider, source, tier } + */ + selectModel(tier, _unused = null) { + const tierConfig = config.modelTiers?.[tier]; + if (!tierConfig) { + throw new Error(`TIER_${tier} not configured. Set TIER_${tier}=provider:model in .env`); + } + + const parsed = this._parseTierConfig(tierConfig); + if (!parsed) { + throw new Error(`Invalid TIER_${tier} format. Expected provider:model, got: ${tierConfig}`); + } + + return { + model: parsed.model, + provider: parsed.provider, + source: 'env_tier', + tier, + }; + } + + /** + * Parse tier config string (format: provider:model) + * Examples: "ollama:llama3.2", "azure-openai:gpt-5.2-chat", "openai:gpt-4o" + */ + _parseTierConfig(configStr) { + if (!configStr || typeof configStr !== 'string') return null; + + const colonIndex = configStr.indexOf(':'); + if (colonIndex === -1) { + // No colon - treat as model name, use default provider + return { + provider: config.modelProvider?.type || 'openai', + model: configStr.trim(), + }; + } + + const provider = configStr.substring(0, colonIndex).trim().toLowerCase(); + const model = configStr.substring(colonIndex + 1).trim(); + + if (!provider || !model) return null; + + return { provider, model }; + } + + /** + * Get the model configured for a provider from .env + */ + _getProviderModel(provider) { + switch (provider) { + case 'azure-openai': + case 'azureopenai': + return config.azureOpenAI?.deployment || null; + case 'openai': + return config.openai?.model || null; + case 'ollama': + return config.ollama?.model || null; + case 'openrouter': + return config.openrouter?.model || null; + case 'llamacpp': + return config.llamacpp?.model || null; + case 'lmstudio': + return config.lmstudio?.model || null; + case 'bedrock': + return config.bedrock?.modelId || null; + case 'zai': + return config.zai?.model || null; + case 'moonshot': + return config.moonshot?.model || null; + case 'vertex': + return config.vertex?.model || null; + case 'databricks': + return config.modelProvider?.defaultModel || null; + default: + return null; + } + } + + /** + * Get provider for a specific tier (from env or fallback) + */ + getProviderForTier(tier) { + const tierConfig = config.modelTiers?.[tier]; + if (tierConfig) { + const parsed = this._parseTierConfig(tierConfig); + if (parsed) return parsed.provider; + } + return config.modelProvider?.type || 'openai'; + } + + /** + * Get fallback model if provider can't handle requested tier + */ + _getFallbackModel(requestedTier, provider) { + const tierOrder = ['REASONING', 'COMPLEX', 'MEDIUM', 'SIMPLE']; + const startIndex = tierOrder.indexOf(requestedTier); + + // Try lower tiers + for (let i = startIndex + 1; i < tierOrder.length; i++) { + const fallbackTier = tierOrder[i]; + const models = this.getPreferredModels(fallbackTier, provider); + + if (models.length > 0) { + logger.debug({ + from: requestedTier, + to: fallbackTier, + provider, + model: models[0], + }, '[ModelTiers] Downgrading tier'); + + return { model: models[0], tier: fallbackTier }; + } + } + + return null; + } + + /** + * Check if provider can handle a specific tier + */ + canHandleTier(provider, tier) { + const normalizedProvider = this._normalizeProvider(provider); + const models = this.getPreferredModels(tier, normalizedProvider); + return models.length > 0; + } + + /** + * Check if provider is local/free + */ + isLocalProvider(provider) { + const normalizedProvider = this._normalizeProvider(provider); + return this.localProviders[normalizedProvider]?.free === true; + } + + /** + * Get all providers that can handle a tier + */ + getProvidersForTier(tier) { + const tierConfig = this.tierConfig[tier]; + if (!tierConfig?.preferred) return []; + return Object.keys(tierConfig.preferred); + } + + /** + * Get all tiers a provider can handle + */ + getTiersForProvider(provider) { + const normalizedProvider = this._normalizeProvider(provider); + const tiers = []; + + for (const tier of Object.keys(TIER_DEFINITIONS)) { + if (this.canHandleTier(normalizedProvider, tier)) { + tiers.push(tier); + } + } + + return tiers; + } + + /** + * Get tier stats for metrics endpoint + */ + getTierStats() { + const stats = { + tiers: {}, + providers: {}, + }; + + for (const [tier, def] of Object.entries(TIER_DEFINITIONS)) { + const providers = this.getProvidersForTier(tier); + stats.tiers[tier] = { + ...def, + providerCount: providers.length, + providers: providers, + }; + } + + // Count models per provider + const allProviders = new Set(); + for (const tierConfig of Object.values(this.tierConfig)) { + if (tierConfig.preferred) { + Object.keys(tierConfig.preferred).forEach(p => allProviders.add(p)); + } + } + + for (const provider of allProviders) { + stats.providers[provider] = { + tiers: this.getTiersForProvider(provider), + isLocal: this.isLocalProvider(provider), + }; + } + + return stats; + } + + /** + * Reload configuration (for hot reload) + */ + reload() { + this._loadConfig(); + logger.info('[ModelTiers] Configuration reloaded'); + } +} + +// Singleton instance +let instance = null; + +function getModelTierSelector() { + if (!instance) { + instance = new ModelTierSelector(); + } + return instance; +} + +module.exports = { + ModelTierSelector, + getModelTierSelector, + TIER_DEFINITIONS, +}; diff --git a/src/server.js b/src/server.js index 612af5d..df0b797 100644 --- a/src/server.js +++ b/src/server.js @@ -201,9 +201,9 @@ async function start() { const app = createApp(); - // Wait for Ollama if it's the configured provider or preferred for routing + // Wait for Ollama if it's the configured provider or referenced in tier config const provider = config.modelProvider?.type?.toLowerCase(); - if (provider === "ollama" || config.modelProvider?.preferOllama) { + if (provider === "ollama" || config.tiersReferenceOllama()) { await waitForOllama(); } diff --git a/src/sessions/cleanup.js b/src/sessions/cleanup.js index 50f0b65..c4d22a4 100644 --- a/src/sessions/cleanup.js +++ b/src/sessions/cleanup.js @@ -4,9 +4,9 @@ const { cleanupOldSessions, cleanupOldHistory } = require("./store"); class SessionCleanupManager { constructor(options = {}) { this.enabled = options.enabled !== false; - this.intervalMs = options.intervalMs || 3600000; // 1 hour - this.sessionMaxAgeMs = options.sessionMaxAgeMs || 7 * 24 * 60 * 60 * 1000; // 7 days - this.historyMaxAgeMs = options.historyMaxAgeMs || 30 * 24 * 60 * 60 * 1000; // 30 days + this.intervalMs = options.intervalMs || 300000; // 5 minutes (was 1 hour) + this.sessionMaxAgeMs = options.sessionMaxAgeMs || 24 * 60 * 60 * 1000; // 1 day (was 7 days) + this.historyMaxAgeMs = options.historyMaxAgeMs || 7 * 24 * 60 * 60 * 1000; // 7 days (was 30 days) this.timer = null; } diff --git a/src/sessions/record.js b/src/sessions/record.js index 0a5da1f..ebfa25b 100644 --- a/src/sessions/record.js +++ b/src/sessions/record.js @@ -1,5 +1,8 @@ const { appendSessionTurn } = require("./store"); +// Cap in-memory history to prevent unbounded growth during long tool loops +const MAX_IN_MEMORY_HISTORY = 100; + function ensureSessionShape(session) { if (!session) return null; if (!Array.isArray(session.history)) { @@ -19,7 +22,13 @@ function appendTurnToSession(session, entry) { target.history.push(turn); target.updatedAt = turn.timestamp; - if (target.id) { + // Trim in-memory history if it exceeds the cap + if (target.history.length > MAX_IN_MEMORY_HISTORY) { + target.history = target.history.slice(-MAX_IN_MEMORY_HISTORY); + } + + // Skip DB write for ephemeral sessions (auto-generated, no client session ID) + if (target.id && !target._ephemeral) { appendSessionTurn(target.id, turn, target.metadata ?? {}); } diff --git a/src/sessions/store.js b/src/sessions/store.js index f88238e..0c04f55 100644 --- a/src/sessions/store.js +++ b/src/sessions/store.js @@ -4,11 +4,15 @@ const logger = require("../logger"); const selectSessionStmt = db.prepare( "SELECT id, created_at, updated_at, metadata FROM sessions WHERE id = ?", ); +// Limit history to last 50 entries to prevent unbounded memory growth. +// Older entries remain in DB for auditing but aren't loaded into memory. +const MAX_HISTORY_ROWS = 50; const selectHistoryStmt = db.prepare( `SELECT role, type, status, content, metadata, timestamp FROM session_history WHERE session_id = ? - ORDER BY timestamp ASC, id ASC`, + ORDER BY timestamp DESC, id DESC + LIMIT ${MAX_HISTORY_ROWS}`, ); const insertSessionStmt = db.prepare( "INSERT INTO sessions (id, created_at, updated_at, metadata) VALUES (@id, @created_at, @updated_at, @metadata)", @@ -75,7 +79,8 @@ function getSession(sessionId) { if (!sessionId) return null; const sessionRow = selectSessionStmt.get(sessionId); if (!sessionRow) return null; - const historyRows = selectHistoryStmt.all(sessionId); + // Query returns rows in DESC order (for LIMIT to grab newest), reverse to ASC + const historyRows = selectHistoryStmt.all(sessionId).reverse(); return toSession(sessionRow, historyRows); } diff --git a/src/tools/agent-task.js b/src/tools/agent-task.js index 4e69e22..74700d4 100644 --- a/src/tools/agent-task.js +++ b/src/tools/agent-task.js @@ -2,6 +2,50 @@ const { registerTool } = require("."); const { spawnAgent, autoSelectAgent } = require("../agents"); const logger = require("../logger"); +/** + * Extract text from Anthropic content blocks format + * Handles: [{"type":"text","text":"..."}] -> "..." + */ +function extractTextFromContentBlocks(content) { + if (typeof content !== 'string') { + return content; + } + + const trimmed = content.trim(); + if (!trimmed.startsWith('[')) { + return content; + } + + try { + const parsed = JSON.parse(trimmed); + if (!Array.isArray(parsed)) { + return content; + } + + // Extract text from content blocks + const textParts = parsed + .filter(block => block && typeof block === 'object') + .map(block => { + if (block.type === 'text' && typeof block.text === 'string') { + return block.text; + } + if (typeof block.text === 'string') { + return block.text; + } + return null; + }) + .filter(text => text !== null); + + if (textParts.length > 0) { + return textParts.join('\n\n'); + } + + return content; + } catch { + return content; + } +} + function registerAgentTaskTool() { registerTool( "Task", @@ -49,10 +93,13 @@ function registerAgentTaskTool() { }); if (result.success) { + // Extract text from Anthropic content blocks if present + const cleanContent = extractTextFromContentBlocks(result.result); + return { ok: true, status: 200, - content: result.result, + content: cleanContent, metadata: { agentType: subagentType, agentId: result.stats.agentId, diff --git a/src/tools/index.js b/src/tools/index.js index 11227f0..1983c00 100644 --- a/src/tools/index.js +++ b/src/tools/index.js @@ -1,5 +1,6 @@ const logger = require("../logger"); const { truncateToolOutput } = require("./truncate"); +const { isGPTProvider, formatToolResultForGPT } = require("../clients/gpt-utils"); const registry = new Map(); const registryLowercase = new Map(); @@ -254,7 +255,18 @@ async function executeToolCall(call, context = {}) { const formatted = normalizeHandlerResult(result); // Apply tool output truncation for token efficiency - const truncatedContent = truncateToolOutput(normalisedCall.name, formatted.content); + let truncatedContent = truncateToolOutput(normalisedCall.name, formatted.content); + + // GPT-specific formatting temporarily disabled for testing + // const isGPT = context?.provider && isGPTProvider(context.provider); + // if (isGPT) { + // truncatedContent = formatToolResultForGPT( + // normalisedCall.name, + // truncatedContent, + // normalisedCall.arguments + // ); + // } + const isGPT = false; // Disabled for testing return { id: normalisedCall.id, @@ -267,7 +279,8 @@ async function executeToolCall(call, context = {}) { registered: true, truncated: truncatedContent !== formatted.content, originalLength: formatted.content?.length, - truncatedLength: truncatedContent?.length + truncatedLength: truncatedContent?.length, + gptFormatted: isGPT, }, }; } catch (err) { diff --git a/test/README.md b/test/README.md index eb481eb..66e0eb9 100644 --- a/test/README.md +++ b/test/README.md @@ -9,7 +9,7 @@ All tests for the Lynkr project are consolidated in this `test/` directory. **Purpose**: Tests the hybrid routing logic in isolation **Run**: `DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/routing.test.js` **Coverage**: 10 tests -- Routing with PREFER_OLLAMA disabled +- Routing with tier-based routing disabled (no TIER_* vars set) - Simple requests → Ollama - Complex requests → Cloud - Tool capability checks diff --git a/test/azure-openai-config.test.js b/test/azure-openai-config.test.js index 76b026d..75246d0 100644 --- a/test/azure-openai-config.test.js +++ b/test/azure-openai-config.test.js @@ -17,6 +17,12 @@ describe("Azure OpenAI Configuration Tests", () => { process.env.AZURE_OPENAI_API_KEY = ""; process.env.AZURE_OPENAI_DEPLOYMENT = ""; process.env.AZURE_OPENAI_API_VERSION = ""; + + // Prevent .env TIER_* values from being picked up by dotenv + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -119,7 +125,7 @@ describe("Azure OpenAI Configuration Tests", () => { describe("Fallback Provider Validation", () => { it("should accept azure-openai as fallback provider with credentials", () => { - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "true"; @@ -134,9 +140,8 @@ describe("Azure OpenAI Configuration Tests", () => { assert.strictEqual(config.modelProvider.fallbackProvider, "azure-openai"); }); - it("should reject when azure-openai is fallback but credentials missing", () => { - process.env.MODEL_PROVIDER = "ollama"; // Set to ollama for hybrid routing scenario - process.env.PREFER_OLLAMA = "true"; + it("should warn when azure-openai is fallback but credentials missing", () => { + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "true"; @@ -146,11 +151,15 @@ describe("Azure OpenAI Configuration Tests", () => { process.env.AZURE_OPENAI_API_KEY = ""; process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.com"; + // Enable tier routing so fallback validation runs + process.env.TIER_SIMPLE = "ollama:llama3.2"; + process.env.TIER_MEDIUM = "ollama:llama3.2"; + process.env.TIER_COMPLEX = "ollama:llama3.2"; + process.env.TIER_REASONING = "ollama:llama3.2"; - // Should throw error about missing Azure OpenAI credentials (fail-fast validation) - assert.throws(() => { - require("../src/config"); - }, /AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY/); + // Should warn but not throw (fallback misconfigured warning) + const config = require("../src/config"); + assert.strictEqual(config.modelProvider.fallbackProvider, "azure-openai"); }); }); diff --git a/test/azure-openai-integration.test.js b/test/azure-openai-integration.test.js index 8202f83..5554b96 100644 --- a/test/azure-openai-integration.test.js +++ b/test/azure-openai-integration.test.js @@ -17,6 +17,12 @@ describe("Azure OpenAI Integration Tests", () => { process.env.MODEL_PROVIDER = "databricks"; process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.com"; + + // Prevent .env TIER_* values from being picked up by dotenv + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -169,7 +175,7 @@ describe("Azure OpenAI Integration Tests", () => { }); it("should select azure-openai as fallback provider", () => { - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "true"; diff --git a/test/azure-openai-routing.test.js b/test/azure-openai-routing.test.js index 5212722..a45ce51 100644 --- a/test/azure-openai-routing.test.js +++ b/test/azure-openai-routing.test.js @@ -9,6 +9,11 @@ describe("Azure OpenAI Routing Tests", () => { // Clear module cache delete require.cache[require.resolve("../src/config")]; delete require.cache[require.resolve("../src/clients/routing")]; + delete require.cache[require.resolve("../src/routing/index.js")]; + delete require.cache[require.resolve("../src/routing/model-tiers")]; + delete require.cache[require.resolve("../src/routing/complexity-analyzer")]; + delete require.cache[require.resolve("../src/routing/cost-optimizer")]; + delete require.cache[require.resolve("../src/routing/agentic-detector")]; // Store original config originalConfig = { ...process.env }; @@ -20,9 +25,15 @@ describe("Azure OpenAI Routing Tests", () => { process.env.MODEL_PROVIDER = "databricks"; // Set default to avoid validation errors process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.com"; - + // Explicitly set valid fallback to override any local .env pollution (e.g. lmstudio) process.env.FALLBACK_PROVIDER = "databricks"; + + // Ensure no TIER_* vars leak between tests + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -31,32 +42,25 @@ describe("Azure OpenAI Routing Tests", () => { }); describe("Primary Provider Routing", () => { - it("should route to azure-openai when set as MODEL_PROVIDER", () => { + it("should route to azure-openai when set as MODEL_PROVIDER", async () => { process.env.MODEL_PROVIDER = "azure-openai"; process.env.AZURE_OPENAI_ENDPOINT = "https://test.openai.azure.com"; process.env.AZURE_OPENAI_API_KEY = "test-key"; - process.env.PREFER_OLLAMA = "false"; routing = require("../src/clients/routing"); - const provider = routing.determineProvider({ tools: [] }); + const result = await routing.determineProviderSmart({ + messages: [{ role: "user", content: "test" }], + tools: [] + }); - assert.strictEqual(provider, "azure-openai"); + assert.strictEqual(result.provider, "azure-openai"); }); }); - describe("Hybrid Routing with Azure OpenAI", () => { - it("should route moderate tool requests to azure-openai when available", () => { - // Explicitly unset OpenRouter to ensure it's not available - // Set to empty string instead of delete to prevent dotenv from reloading it - process.env.OPENROUTER_API_KEY = ""; - - process.env.PREFER_OLLAMA = "true"; - process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; - process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; - process.env.FALLBACK_ENABLED = "true"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "3"; - process.env.OPENROUTER_MAX_TOOLS_FOR_ROUTING = "15"; + describe("Static Routing with Azure OpenAI", () => { + it("should return primary provider regardless of tool count (tier routing disabled)", async () => { + process.env.MODEL_PROVIDER = "azure-openai"; process.env.AZURE_OPENAI_ENDPOINT = "https://test.openai.azure.com"; process.env.AZURE_OPENAI_API_KEY = "test-key"; @@ -67,24 +71,19 @@ describe("Azure OpenAI Routing Tests", () => { routing = require("../src/clients/routing"); - // 5 tools: more than Ollama threshold (3), less than OpenRouter threshold (15) - const provider = routing.determineProvider({ + const result = await routing.determineProviderSmart({ + messages: [{ role: "user", content: "test" }], tools: [{}, {}, {}, {}, {}] }); - assert.strictEqual(provider, "azure-openai"); + assert.strictEqual(result.provider, "azure-openai"); + assert.strictEqual(result.method, "static"); }); - it("should prefer OpenRouter over Azure OpenAI when both configured", () => { - process.env.PREFER_OLLAMA = "true"; - process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; - process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; - process.env.FALLBACK_ENABLED = "true"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "3"; - process.env.OPENROUTER_MAX_TOOLS_FOR_ROUTING = "15"; - process.env.OPENROUTER_API_KEY = "openrouter-key"; + it("should return primary provider for simple requests", async () => { + process.env.MODEL_PROVIDER = "azure-openai"; process.env.AZURE_OPENAI_ENDPOINT = "https://test.openai.azure.com"; - process.env.AZURE_OPENAI_API_KEY = "azure-key"; + process.env.AZURE_OPENAI_API_KEY = "test-key"; // Clear cache after env setup delete require.cache[require.resolve("../src/config/index.js")]; @@ -93,20 +92,17 @@ describe("Azure OpenAI Routing Tests", () => { routing = require("../src/clients/routing"); - // 5 tools: should prefer OpenRouter - const provider = routing.determineProvider({ - tools: [{}, {}, {}, {}, {}] + const result = await routing.determineProviderSmart({ + messages: [{ role: "user", content: "test" }], + tools: [{}, {}] }); - assert.strictEqual(provider, "openrouter"); + assert.strictEqual(result.provider, "azure-openai"); + assert.strictEqual(result.method, "static"); }); - it("should route simple requests to Ollama even when Azure OpenAI configured", () => { - process.env.PREFER_OLLAMA = "true"; - process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; - process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; - process.env.FALLBACK_ENABLED = "true"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "3"; + it("should return static routing from determineProviderSmart when tiers disabled", async () => { + process.env.MODEL_PROVIDER = "azure-openai"; process.env.AZURE_OPENAI_ENDPOINT = "https://test.openai.azure.com"; process.env.AZURE_OPENAI_API_KEY = "test-key"; @@ -117,18 +113,20 @@ describe("Azure OpenAI Routing Tests", () => { routing = require("../src/clients/routing"); - // 2 tools: under Ollama threshold - const provider = routing.determineProvider({ + const result = await routing.determineProviderSmart({ + messages: [{ role: "user", content: "test" }], tools: [{}, {}] }); - assert.strictEqual(provider, "ollama"); + assert.strictEqual(result.provider, "azure-openai"); + assert.strictEqual(result.method, "static"); + assert.strictEqual(result.reason, "tier_routing_disabled"); }); }); describe("Fallback Configuration", () => { it("should support azure-openai as fallback provider", () => { - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "true"; diff --git a/test/bedrock-integration.test.js b/test/bedrock-integration.test.js index e0e5c96..4435b4e 100644 --- a/test/bedrock-integration.test.js +++ b/test/bedrock-integration.test.js @@ -11,6 +11,12 @@ describe("AWS Bedrock Integration", () => { delete require.cache[require.resolve("../src/config")]; delete require.cache[require.resolve("../src/clients/routing")]; delete require.cache[require.resolve("../src/clients/bedrock-utils")]; + + // Prevent .env TIER_* values from being picked up by dotenv + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -374,62 +380,45 @@ describe("AWS Bedrock Integration", () => { process.env.MODEL_PROVIDER = "bedrock"; process.env.AWS_ACCESS_KEY_ID = "AKIATEST123"; process.env.AWS_SECRET_ACCESS_KEY = "testSecretKey123"; - process.env.PREFER_OLLAMA = "false"; const config = require("../src/config"); const routing = require("../src/clients/routing"); const payload = { messages: [{ role: "user", content: "test" }] }; - const provider = routing.determineProvider(payload); + const provider = routing.determineProviderSync(payload); - // When not in hybrid mode, should use primary provider + // determineProviderSync returns static MODEL_PROVIDER assert.strictEqual(provider, "bedrock"); }); - it("should route to bedrock in hybrid mode for moderate tool counts", () => { - process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; - process.env.OLLAMA_MODEL = "llama3.1"; - process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; + it("should return static routing from determineProviderSmart when tiers disabled", async () => { + process.env.MODEL_PROVIDER = "bedrock"; process.env.AWS_ACCESS_KEY_ID = "AKIATEST123"; process.env.AWS_SECRET_ACCESS_KEY = "testSecretKey123"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "2"; - process.env.FALLBACK_ENABLED = "true"; - process.env.FALLBACK_PROVIDER = "bedrock"; - - // Clear other providers to ensure bedrock is chosen - delete process.env.OPENROUTER_API_KEY; - delete process.env.OPENAI_API_KEY; - delete process.env.AZURE_OPENAI_API_KEY; - delete process.env.AZURE_OPENAI_ENDPOINT; - delete process.env.LLAMACPP_ENDPOINT; - delete process.env.LMSTUDIO_ENDPOINT; - delete process.env.DATABRICKS_API_KEY; - delete process.env.DATABRICKS_API_BASE; const config = require("../src/config"); const routing = require("../src/clients/routing"); - // 20 tools should exceed both Ollama and OpenRouter limits, routing to fallback provider (bedrock) + // Many tools -- but without TIER_* vars, determineProviderSmart returns static routing const payload = { messages: [{ role: "user", content: "test" }], tools: Array(20).fill({ name: "tool" }), }; - const provider = routing.determineProvider(payload); + const result = await routing.determineProviderSmart(payload); - assert.strictEqual(provider, "bedrock"); + assert.strictEqual(result.provider, "bedrock"); + assert.strictEqual(result.method, "static"); + assert.strictEqual(result.reason, "tier_routing_disabled"); }); }); describe("Fallback Provider", () => { it("should allow bedrock as fallback provider", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "llama3.1"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.FALLBACK_PROVIDER = "bedrock"; - process.env.AWS_ACCESS_KEY_ID = "AKIATEST123"; - process.env.AWS_SECRET_ACCESS_KEY = "testSecretKey123"; + process.env.AWS_BEDROCK_API_KEY = "test-bedrock-key"; process.env.FALLBACK_ENABLED = "true"; // Should not throw @@ -439,24 +428,21 @@ describe("AWS Bedrock Integration", () => { it("should validate bedrock credentials when used as fallback", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "llama3.1"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.FALLBACK_PROVIDER = "bedrock"; process.env.FALLBACK_ENABLED = "true"; // Set to empty string to override .env file values - process.env.AWS_ACCESS_KEY_ID = ""; - process.env.AWS_SECRET_ACCESS_KEY = ""; + process.env.AWS_BEDROCK_API_KEY = ""; assert.throws( () => require("../src/config"), - /FALLBACK_PROVIDER is set to 'bedrock' but AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are not configured/ + /FALLBACK_PROVIDER is set to 'bedrock' but AWS_BEDROCK_API_KEY is not configured/ ); }); it("should not allow local providers as fallback", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "llama3.1"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.FALLBACK_PROVIDER = "llamacpp"; diff --git a/test/hybrid-routing-integration.test.js b/test/hybrid-routing-integration.test.js index e39e29a..88986e4 100644 --- a/test/hybrid-routing-integration.test.js +++ b/test/hybrid-routing-integration.test.js @@ -21,6 +21,12 @@ describe("Hybrid Routing Integration Tests", () => { process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.databricks.com"; process.env.MODEL_PROVIDER = "databricks"; + + // Set TIER_* to empty to prevent .env file values from being picked up by dotenv + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -30,7 +36,7 @@ describe("Hybrid Routing Integration Tests", () => { describe("Configuration Validation", () => { it("should use default OLLAMA_ENDPOINT when not specified", () => { - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; delete process.env.OLLAMA_ENDPOINT; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.DATABRICKS_API_KEY = "test-key"; @@ -43,7 +49,7 @@ describe("Hybrid Routing Integration Tests", () => { }); it("should reject invalid FALLBACK_PROVIDER", () => { - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "true"; @@ -55,20 +61,24 @@ describe("Hybrid Routing Integration Tests", () => { }); it("should reject circular fallback (ollama -> ollama)", () => { - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "true"; process.env.FALLBACK_PROVIDER = "ollama"; + // Enable tier routing so fallback validation runs + process.env.TIER_SIMPLE = "ollama:llama3.2"; + process.env.TIER_MEDIUM = "ollama:llama3.2"; + process.env.TIER_COMPLEX = "ollama:llama3.2"; + process.env.TIER_REASONING = "ollama:llama3.2"; assert.throws(() => { require("../src/config"); }, /FALLBACK_PROVIDER cannot be 'ollama'/); }); - it("should reject PREFER_OLLAMA with databricks fallback but no databricks credentials", () => { - process.env.MODEL_PROVIDER = "ollama"; // Set to ollama for hybrid routing scenario - process.env.PREFER_OLLAMA = "true"; + it("should warn when databricks fallback has no credentials", () => { + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "true"; @@ -76,39 +86,44 @@ describe("Hybrid Routing Integration Tests", () => { // Set to empty strings instead of deleting (dotenv.config() in config module would reload from .env) process.env.DATABRICKS_API_KEY = ""; process.env.DATABRICKS_API_BASE = ""; + // Enable tier routing so fallback validation runs + process.env.TIER_SIMPLE = "ollama:llama3.2"; + process.env.TIER_MEDIUM = "ollama:llama3.2"; + process.env.TIER_COMPLEX = "ollama:llama3.2"; + process.env.TIER_REASONING = "ollama:llama3.2"; - // Should throw error about missing databricks credentials - // (Either from standard validation or hybrid routing validation) - assert.throws(() => { - require("../src/config"); - }, /DATABRICKS_API_BASE and DATABRICKS_API_KEY/); + // Should warn but not throw (fallback misconfigured) + const config = require("../src/config"); + assert.strictEqual(config.modelProvider.fallbackProvider, "databricks"); }); - it("should accept valid hybrid routing configuration", () => { - process.env.PREFER_OLLAMA = "true"; + it("should accept valid tier routing configuration", () => { + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "true"; process.env.FALLBACK_PROVIDER = "databricks"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "3"; // Override .env which sets it to 2 process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.com"; + process.env.TIER_SIMPLE = "ollama:llama3.2"; + process.env.TIER_MEDIUM = "ollama:llama3.2"; + process.env.TIER_COMPLEX = "databricks:claude-sonnet"; + process.env.TIER_REASONING = "databricks:claude-sonnet"; const config = require("../src/config"); - assert.strictEqual(config.modelProvider.preferOllama, true); + assert.strictEqual(config.modelProvider.type, "ollama"); assert.strictEqual(config.modelProvider.fallbackEnabled, true); - assert.strictEqual(config.modelProvider.ollamaMaxToolsForRouting, 3); assert.strictEqual(config.modelProvider.fallbackProvider, "databricks"); + assert.strictEqual(config.modelTiers.enabled, true); }); }); describe("Metrics Recording", () => { beforeEach(() => { - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; - process.env.OLLAMA_FALLBACK_PROVIDER = "databricks"; config = require("../src/config"); const metricsModule = require("../src/observability/metrics"); @@ -207,7 +222,7 @@ describe("Hybrid Routing Integration Tests", () => { it("should categorize circuit breaker errors", () => { // This would need to be tested by importing the function if exported // For now, we test via the integrated behavior - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; @@ -231,7 +246,7 @@ describe("Hybrid Routing Integration Tests", () => { }); it("should estimate cost savings correctly", () => { - process.env.PREFER_OLLAMA = "true"; + process.env.MODEL_PROVIDER = "ollama"; process.env.OLLAMA_ENDPOINT = "http://localhost:11434"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; diff --git a/test/hybrid-routing-performance.test.js b/test/hybrid-routing-performance.test.js index 71fb552..331f237 100644 --- a/test/hybrid-routing-performance.test.js +++ b/test/hybrid-routing-performance.test.js @@ -56,15 +56,20 @@ function testRoutingDecisionPerformance() { delete require.cache[require.resolve('../src/config')]; delete require.cache[require.resolve('../src/clients/routing')]; - process.env.PREFER_OLLAMA = 'true'; + process.env.MODEL_PROVIDER = 'ollama'; process.env.OLLAMA_ENDPOINT = 'http://localhost:11434'; process.env.OLLAMA_MODEL = 'qwen2.5-coder:latest'; process.env.DATABRICKS_API_KEY = 'test-key'; process.env.DATABRICKS_API_BASE = 'http://test.com'; + // Set TIER_* to empty = tier routing disabled, determineProviderSync returns static provider + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; const routing = require('../src/clients/routing'); - log('\n📊 Benchmarking routing decisions...', 'cyan'); + log('\n Benchmarking routing decisions...', 'cyan'); // Test 1: Simple request (0 tools) const simplePayload = { @@ -75,10 +80,10 @@ function testRoutingDecisionPerformance() { const { duration: simpleTime, throughput: simpleThroughput } = benchmark( 'Simple request routing', 100000, - () => routing.determineProvider(simplePayload) + () => routing.determineProviderSync(simplePayload) ); - log(`⏱️ Simple request: ${simpleTime.toFixed(2)}ms for 100k decisions`, 'cyan'); + log(` Simple request: ${simpleTime.toFixed(2)}ms for 100k decisions`, 'cyan'); log(` Average: ${(simpleTime / 100000).toFixed(6)}ms per decision`, 'blue'); log(` Throughput: ${simpleThroughput.toLocaleString()} decisions/sec`, 'green'); @@ -94,10 +99,10 @@ function testRoutingDecisionPerformance() { const { duration: complexTime, throughput: complexThroughput } = benchmark( 'Complex request routing', 100000, - () => routing.determineProvider(complexPayload) + () => routing.determineProviderSync(complexPayload) ); - log(`⏱️ Complex request: ${complexTime.toFixed(2)}ms for 100k decisions`, 'cyan'); + log(` Complex request: ${complexTime.toFixed(2)}ms for 100k decisions`, 'cyan'); log(` Average: ${(complexTime / 100000).toFixed(6)}ms per decision`, 'blue'); log(` Throughput: ${complexThroughput.toLocaleString()} decisions/sec`, 'green'); @@ -110,18 +115,18 @@ function testRoutingDecisionPerformance() { const { duration: toolCheckTime, throughput: toolCheckThroughput } = benchmark( 'Tool capability check', 100000, - () => routing.determineProvider(toolCapabilityPayload) + () => routing.determineProviderSync(toolCapabilityPayload) ); - log(`⏱️ Tool capability check: ${toolCheckTime.toFixed(2)}ms for 100k decisions`, 'cyan'); + log(` Tool capability check: ${toolCheckTime.toFixed(2)}ms for 100k decisions`, 'cyan'); log(` Average: ${(toolCheckTime / 100000).toFixed(6)}ms per decision`, 'blue'); log(` Throughput: ${toolCheckThroughput.toLocaleString()} decisions/sec`, 'green'); // Analysis - log('\n📈 Analysis:', 'yellow'); + log('\n Analysis:', 'yellow'); log(` Routing adds <0.01ms per request (negligible overhead)`, 'green'); log(` Throughput: ${simpleThroughput.toLocaleString()} decisions/sec`, 'green'); - log(` ✅ Routing is extremely fast and won't impact request latency`, 'green'); + log(` Routing is extremely fast and won't impact request latency`, 'green'); return { simpleTime, @@ -141,7 +146,7 @@ function testMetricsOverhead() { const { getMetricsCollector } = require('../src/observability/metrics'); const metrics = getMetricsCollector(); - log('\n📊 Benchmarking metrics operations...', 'cyan'); + log('\n Benchmarking metrics operations...', 'cyan'); // Test recording provider routing const { duration: routingTime, throughput: routingThroughput } = benchmark( @@ -150,7 +155,7 @@ function testMetricsOverhead() { () => metrics.recordProviderRouting('ollama') ); - log(`⏱️ Provider routing: ${routingTime.toFixed(2)}ms for 100k recordings`, 'cyan'); + log(` Provider routing: ${routingTime.toFixed(2)}ms for 100k recordings`, 'cyan'); log(` Average: ${(routingTime / 100000).toFixed(6)}ms per record`, 'blue'); log(` Throughput: ${routingThroughput.toLocaleString()} ops/sec`, 'green'); @@ -161,7 +166,7 @@ function testMetricsOverhead() { () => metrics.recordProviderSuccess('ollama', 450) ); - log(`⏱️ Provider success: ${successTime.toFixed(2)}ms for 100k recordings`, 'cyan'); + log(` Provider success: ${successTime.toFixed(2)}ms for 100k recordings`, 'cyan'); log(` Average: ${(successTime / 100000).toFixed(6)}ms per record`, 'blue'); log(` Throughput: ${successThroughput.toLocaleString()} ops/sec`, 'green'); @@ -172,7 +177,7 @@ function testMetricsOverhead() { () => metrics.recordFallbackAttempt('ollama', 'databricks', 'timeout') ); - log(`⏱️ Fallback attempts: ${fallbackTime.toFixed(2)}ms for 100k recordings`, 'cyan'); + log(` Fallback attempts: ${fallbackTime.toFixed(2)}ms for 100k recordings`, 'cyan'); log(` Average: ${(fallbackTime / 100000).toFixed(6)}ms per record`, 'blue'); log(` Throughput: ${fallbackThroughput.toLocaleString()} ops/sec`, 'green'); @@ -183,15 +188,15 @@ function testMetricsOverhead() { () => metrics.recordCostSavings(0.001) ); - log(`⏱️ Cost savings: ${costTime.toFixed(2)}ms for 100k recordings`, 'cyan'); + log(` Cost savings: ${costTime.toFixed(2)}ms for 100k recordings`, 'cyan'); log(` Average: ${(costTime / 100000).toFixed(6)}ms per record`, 'blue'); log(` Throughput: ${costThroughput.toLocaleString()} ops/sec`, 'green'); // Analysis const avgMetricsTime = (routingTime + successTime + fallbackTime + costTime) / 4 / 100000; - log('\n📈 Analysis:', 'yellow'); + log('\n Analysis:', 'yellow'); log(` Average metrics overhead: ${avgMetricsTime.toFixed(6)}ms per operation`, 'green'); - log(` ✅ Metrics collection is extremely lightweight`, 'green'); + log(` Metrics collection is extremely lightweight`, 'green'); return { routingTime, @@ -212,14 +217,19 @@ function testCombinedStack() { delete require.cache[require.resolve('../src/clients/routing')]; delete require.cache[require.resolve('../src/observability/metrics')]; - process.env.PREFER_OLLAMA = 'true'; + process.env.MODEL_PROVIDER = 'ollama'; process.env.OLLAMA_ENDPOINT = 'http://localhost:11434'; process.env.OLLAMA_MODEL = 'qwen2.5-coder:latest'; + // Set TIER_* to empty = static routing via determineProviderSync + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; const routing = require('../src/clients/routing'); const { getMetricsCollector } = require('../src/observability/metrics'); - log('\n📊 Benchmarking complete routing + metrics stack...', 'cyan'); + log('\n Benchmarking complete routing + metrics stack...', 'cyan'); // Simulate full routing decision + metrics recording const payload = { @@ -232,21 +242,21 @@ function testCombinedStack() { 50000, () => { const metrics = getMetricsCollector(); - const provider = routing.determineProvider(payload); + const provider = routing.determineProviderSync(payload); metrics.recordProviderRouting(provider); metrics.recordProviderSuccess(provider, 450); } ); - log(`⏱️ Full stack: ${fullTime.toFixed(2)}ms for 50k operations`, 'cyan'); + log(` Full stack: ${fullTime.toFixed(2)}ms for 50k operations`, 'cyan'); log(` Average: ${(fullTime / 50000).toFixed(6)}ms per request`, 'blue'); log(` Throughput: ${fullThroughput.toLocaleString()} ops/sec`, 'green'); // Analysis - log('\n📈 Analysis:', 'yellow'); + log('\n Analysis:', 'yellow'); const overhead = (fullTime / 50000); log(` Total routing + metrics overhead: ${overhead.toFixed(6)}ms`, 'green'); - log(` ✅ Negligible impact on request latency (<0.02ms)`, 'green'); + log(` Negligible impact on request latency (<0.02ms)`, 'green'); return { fullTime, @@ -263,7 +273,7 @@ function testHelperFunctions() { delete require.cache[require.resolve('../src/clients/databricks')]; - log('\n📊 Benchmarking helper functions...', 'cyan'); + log('\n Benchmarking helper functions...', 'cyan'); // Test categorizeFailure (we'll simulate it) const categorizeFailure = (error) => { @@ -296,7 +306,7 @@ function testHelperFunctions() { } ); - log(`⏱️ Categorize failure: ${categorizeTime.toFixed(2)}ms for 400k operations`, 'cyan'); + log(` Categorize failure: ${categorizeTime.toFixed(2)}ms for 400k operations`, 'cyan'); log(` Average: ${(categorizeTime / 400000).toFixed(6)}ms per categorization`, 'blue'); log(` Throughput: ${(categorizeThroughput * 4).toLocaleString()} ops/sec`, 'green'); @@ -315,13 +325,13 @@ function testHelperFunctions() { () => estimateCostSavings(1000, 500) ); - log(`⏱️ Cost estimation: ${costCalcTime.toFixed(2)}ms for 100k calculations`, 'cyan'); + log(` Cost estimation: ${costCalcTime.toFixed(2)}ms for 100k calculations`, 'cyan'); log(` Average: ${(costCalcTime / 100000).toFixed(6)}ms per calculation`, 'blue'); log(` Throughput: ${costCalcThroughput.toLocaleString()} ops/sec`, 'green'); - log('\n📈 Analysis:', 'yellow'); + log('\n Analysis:', 'yellow'); log(` Helper functions add negligible overhead (<0.001ms)`, 'green'); - log(` ✅ No performance impact from utility functions`, 'green'); + log(` No performance impact from utility functions`, 'green'); return { categorizeTime, @@ -333,54 +343,54 @@ function testHelperFunctions() { // FINAL REPORT // ============================================================================= function printFinalReport(results) { - section('📊 HYBRID ROUTING PERFORMANCE SUMMARY'); + section('HYBRID ROUTING PERFORMANCE SUMMARY'); console.log('\n'); - console.log('┌────────────────────────────────────────────────────────────┐'); - console.log('│ HYBRID ROUTING PERFORMANCE │'); - console.log('├────────────────────────────────────────────────────────────┤'); + console.log('+---------------------------------------------------------+'); + console.log('| HYBRID ROUTING PERFORMANCE |'); + console.log('+---------------------------------------------------------+'); - log(`│ 1. Routing Decisions │`, 'bright'); - log(`│ Average: ${results.routing.avgDecisionTime.toFixed(6)}ms per decision │`, 'cyan'); - log(`│ Overhead: ${colors.green}Negligible (<0.01ms)${colors.reset} │`); + log(`| 1. Routing Decisions |`, 'bright'); + log(`| Average: ${results.routing.avgDecisionTime.toFixed(6)}ms per decision |`, 'cyan'); + log(`| Overhead: ${colors.green}Negligible (<0.01ms)${colors.reset} |`); - console.log('├────────────────────────────────────────────────────────────┤'); + console.log('+---------------------------------------------------------+'); - log(`│ 2. Metrics Collection │`, 'bright'); - log(`│ Average: ${results.metrics.avgMetricsTime.toFixed(6)}ms per operation │`, 'cyan'); - log(`│ Overhead: ${colors.green}Negligible (<0.01ms)${colors.reset} │`); + log(`| 2. Metrics Collection |`, 'bright'); + log(`| Average: ${results.metrics.avgMetricsTime.toFixed(6)}ms per operation |`, 'cyan'); + log(`| Overhead: ${colors.green}Negligible (<0.01ms)${colors.reset} |`); - console.log('├────────────────────────────────────────────────────────────┤'); + console.log('+---------------------------------------------------------+'); - log(`│ 3. Full Routing Stack │`, 'bright'); - log(`│ Average: ${results.combined.overhead.toFixed(6)}ms per request │`, 'cyan'); - log(`│ Throughput: ${results.combined.fullThroughput.toLocaleString()} ops/sec │`, 'cyan'); - log(`│ Impact: ${colors.green}Negligible (<0.02ms)${colors.reset} │`); + log(`| 3. Full Routing Stack |`, 'bright'); + log(`| Average: ${results.combined.overhead.toFixed(6)}ms per request |`, 'cyan'); + log(`| Throughput: ${results.combined.fullThroughput.toLocaleString()} ops/sec |`, 'cyan'); + log(`| Impact: ${colors.green}Negligible (<0.02ms)${colors.reset} |`); - console.log('├────────────────────────────────────────────────────────────┤'); + console.log('+---------------------------------------------------------+'); - log(`│ 4. Helper Functions │`, 'bright'); - log(`│ Overhead: ${colors.green}Negligible (<0.001ms)${colors.reset} │`); + log(`| 4. Helper Functions |`, 'bright'); + log(`| Overhead: ${colors.green}Negligible (<0.001ms)${colors.reset} |`); - console.log('└────────────────────────────────────────────────────────────┘'); + console.log('+---------------------------------------------------------+'); // Overall assessment console.log('\n'); - log('🏆 Overall Performance Assessment:', 'bright'); - log(' ✅ Routing overhead: <0.01ms per request', 'green'); - log(' ✅ Metrics overhead: <0.01ms per request', 'green'); - log(' ✅ Combined overhead: <0.02ms per request', 'green'); - log(' ✅ No measurable impact on API latency', 'green'); - - console.log('\n📈 Expected Real-World Performance:'); - log(' • Ollama (local): ~500-1000ms per request', 'cyan'); - log(' • Cloud (Databricks): ~1500-2000ms per request', 'cyan'); - log(' • Routing overhead: ~0.02ms (0.001-0.002% of total)', 'cyan'); - log(' • Latency savings with Ollama: 40-60% faster', 'green'); - log(' • Cost savings with Ollama: 100% (free)', 'green'); + log('Overall Performance Assessment:', 'bright'); + log(' Routing overhead: <0.01ms per request', 'green'); + log(' Metrics overhead: <0.01ms per request', 'green'); + log(' Combined overhead: <0.02ms per request', 'green'); + log(' No measurable impact on API latency', 'green'); + + console.log('\n Expected Real-World Performance:'); + log(' Ollama (local): ~500-1000ms per request', 'cyan'); + log(' Cloud (Databricks): ~1500-2000ms per request', 'cyan'); + log(' Routing overhead: ~0.02ms (0.001-0.002% of total)', 'cyan'); + log(' Latency savings with Ollama: 40-60% faster', 'green'); + log(' Cost savings with Ollama: 100% (free)', 'green'); console.log('\n'); - log('🚀 Conclusion: Hybrid routing adds negligible overhead while', 'bright'); + log('Conclusion: Hybrid routing adds negligible overhead while', 'bright'); log(' providing significant latency and cost improvements!', 'bright'); console.log('\n'); } @@ -389,7 +399,7 @@ function printFinalReport(results) { // RUN ALL TESTS // ============================================================================= async function runAllTests() { - log('\n🚀 Starting Hybrid Routing Performance Test Suite\n', 'bright'); + log('\n Starting Hybrid Routing Performance Test Suite\n', 'bright'); try { const results = { @@ -401,10 +411,10 @@ async function runAllTests() { printFinalReport(results); - log('\n✅ All performance tests completed successfully!\n', 'green'); + log('\n All performance tests completed successfully!\n', 'green'); process.exit(0); } catch (error) { - log(`\n❌ Performance test suite failed: ${error.message}\n`, 'red'); + log(`\n Performance test suite failed: ${error.message}\n`, 'red'); console.error(error); process.exit(1); } diff --git a/test/llamacpp-integration.test.js b/test/llamacpp-integration.test.js index 1b6006a..13e105e 100644 --- a/test/llamacpp-integration.test.js +++ b/test/llamacpp-integration.test.js @@ -11,6 +11,12 @@ describe("llama.cpp Integration", () => { delete require.cache[require.resolve("../src/config")]; delete require.cache[require.resolve("../src/clients/routing")]; delete require.cache[require.resolve("../src/clients/openrouter-utils")]; + + // Prevent .env TIER_* values from being picked up by dotenv + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -104,33 +110,46 @@ describe("llama.cpp Integration", () => { }); describe("Routing", () => { - it("should route to llamacpp when MODEL_PROVIDER is llamacpp", () => { + it("should route to llamacpp when MODEL_PROVIDER is llamacpp", async () => { process.env.MODEL_PROVIDER = "llamacpp"; process.env.LLAMACPP_ENDPOINT = "http://localhost:8080"; - process.env.PREFER_OLLAMA = "false"; const config = require("../src/config"); const routing = require("../src/clients/routing"); const payload = { messages: [{ role: "user", content: "test" }] }; - const provider = routing.determineProvider(payload); + const result = await routing.determineProviderSmart(payload); - assert.strictEqual(provider, "llamacpp"); + assert.strictEqual(result.provider, "llamacpp"); + assert.strictEqual(result.method, "static"); }); - it("should route to llamacpp for moderate tool count when other providers not configured", () => { - // This test is skipped because llamacpp is checked AFTER openrouter/openai/azure in routing - // and those providers may be present in the test environment - // llama.cpp will be used when it's the PRIMARY provider or when it's the only option + it("should return static routing from determineProviderSmart when tiers disabled", async () => { + process.env.MODEL_PROVIDER = "llamacpp"; + process.env.LLAMACPP_ENDPOINT = "http://localhost:8080"; + + const config = require("../src/config"); + const routing = require("../src/clients/routing"); + + const payload = { messages: [{ role: "user", content: "test" }] }; + const result = await routing.determineProviderSmart(payload); + + assert.strictEqual(result.provider, "llamacpp"); + assert.strictEqual(result.method, "static"); + assert.strictEqual(result.reason, "tier_routing_disabled"); }); it("should throw error when llamacpp is set as FALLBACK_PROVIDER", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_PROVIDER = "llamacpp"; process.env.LLAMACPP_ENDPOINT = "http://localhost:8080"; process.env.FALLBACK_ENABLED = "true"; + // Enable tier routing so fallback validation runs + process.env.TIER_SIMPLE = "ollama:llama3.2"; + process.env.TIER_MEDIUM = "ollama:llama3.2"; + process.env.TIER_COMPLEX = "ollama:llama3.2"; + process.env.TIER_REASONING = "ollama:llama3.2"; assert.throws( () => require("../src/config"), diff --git a/test/lmstudio-integration.test.js b/test/lmstudio-integration.test.js index a1a3b92..93fd1cc 100644 --- a/test/lmstudio-integration.test.js +++ b/test/lmstudio-integration.test.js @@ -11,6 +11,12 @@ describe("LM Studio Integration", () => { delete require.cache[require.resolve("../src/config")]; delete require.cache[require.resolve("../src/clients/routing")]; delete require.cache[require.resolve("../src/clients/openrouter-utils")]; + + // Prevent .env TIER_* values from being picked up by dotenv + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -105,26 +111,33 @@ describe("LM Studio Integration", () => { it("should route to lmstudio when MODEL_PROVIDER is lmstudio", () => { process.env.MODEL_PROVIDER = "lmstudio"; process.env.LMSTUDIO_ENDPOINT = "http://localhost:1234"; - process.env.PREFER_OLLAMA = "false"; const config = require("../src/config"); const routing = require("../src/clients/routing"); const payload = { messages: [{ role: "user", content: "test" }] }; - const provider = routing.determineProvider(payload); + const provider = routing.determineProviderSync(payload); assert.strictEqual(provider, "lmstudio"); }); - it("should route to lmstudio for moderate tool count when other providers not configured", () => { - // This test is skipped because lmstudio is the LAST option in routing - // and other providers (openrouter, openai, azure, llamacpp) take precedence - // LM Studio will be used when it's the PRIMARY provider, not in routing fallback + it("should return static routing from determineProviderSmart when tiers disabled", async () => { + process.env.MODEL_PROVIDER = "lmstudio"; + process.env.LMSTUDIO_ENDPOINT = "http://localhost:1234"; + + const config = require("../src/config"); + const routing = require("../src/clients/routing"); + + const payload = { messages: [{ role: "user", content: "test" }] }; + const result = await routing.determineProviderSmart(payload); + + assert.strictEqual(result.provider, "lmstudio"); + assert.strictEqual(result.method, "static"); + assert.strictEqual(result.reason, "tier_routing_disabled"); }); it("should throw error when lmstudio is set as FALLBACK_PROVIDER", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_PROVIDER = "lmstudio"; process.env.LMSTUDIO_ENDPOINT = "http://localhost:1234"; @@ -312,7 +325,6 @@ describe("LM Studio Integration", () => { describe("Fallback Prevention", () => { it("should prevent lmstudio from being used as fallback provider", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_PROVIDER = "lmstudio"; process.env.LMSTUDIO_ENDPOINT = "http://localhost:1234"; diff --git a/test/openai-integration.test.js b/test/openai-integration.test.js index f85b976..1ed2282 100644 --- a/test/openai-integration.test.js +++ b/test/openai-integration.test.js @@ -11,6 +11,12 @@ describe("OpenAI Integration", () => { delete require.cache[require.resolve("../src/config")]; delete require.cache[require.resolve("../src/clients/routing")]; delete require.cache[require.resolve("../src/clients/openrouter-utils")]; + + // Prevent .env TIER_* values from being picked up by dotenv + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -92,50 +98,41 @@ describe("OpenAI Integration", () => { }); describe("Routing", () => { - it("should route to openai when MODEL_PROVIDER is openai", () => { + it("should route to openai when MODEL_PROVIDER is openai", async () => { process.env.MODEL_PROVIDER = "openai"; process.env.OPENAI_API_KEY = "sk-test-key"; - process.env.PREFER_OLLAMA = "false"; const config = require("../src/config"); const routing = require("../src/clients/routing"); const payload = { messages: [{ role: "user", content: "test" }] }; - const provider = routing.determineProvider(payload); + const result = await routing.determineProviderSmart(payload); - assert.strictEqual(provider, "openai"); + assert.strictEqual(result.provider, "openai"); + assert.strictEqual(result.method, "static"); }); - it("should route to openai as fallback when heavy tool count", () => { - // Clear any existing OpenRouter key to ensure fallback to OpenAI - delete process.env.OPENROUTER_API_KEY; - - process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; - process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "2"; - process.env.OPENROUTER_MAX_TOOLS_FOR_ROUTING = "5"; + it("should return static routing from determineProviderSmart when tiers disabled", async () => { + process.env.MODEL_PROVIDER = "openai"; process.env.OPENAI_API_KEY = "sk-test-key"; - process.env.FALLBACK_ENABLED = "true"; - process.env.FALLBACK_PROVIDER = "openai"; const config = require("../src/config"); const routing = require("../src/clients/routing"); - // 10 tools - above both Ollama and OpenRouter thresholds, should go to fallback const payload = { messages: [{ role: "user", content: "test" }], tools: Array.from({ length: 10 }, (_, i) => ({ name: `tool${i}`, description: "test" })), }; - const provider = routing.determineProvider(payload); - // Should route to openai as the configured fallback provider - assert.strictEqual(provider, "openai"); + const result = await routing.determineProviderSmart(payload); + // No TIER_* vars = static routing + assert.strictEqual(result.provider, "openai"); + assert.strictEqual(result.method, "static"); + assert.strictEqual(result.reason, "tier_routing_disabled"); }); it("should use openai as fallback provider when configured", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_PROVIDER = "openai"; process.env.OPENAI_API_KEY = "sk-test-key"; diff --git a/test/routing.test.js b/test/routing.test.js index 5e9ba03..88afc30 100644 --- a/test/routing.test.js +++ b/test/routing.test.js @@ -11,13 +11,22 @@ describe("Routing Logic", () => { delete require.cache[require.resolve("../src/config/index.js")]; delete require.cache[require.resolve("../src/clients/routing")]; delete require.cache[require.resolve("../src/routing/index.js")]; - delete require.cache[require.resolve("../src/clients/ollama-utils")]; + delete require.cache[require.resolve("../src/routing/model-tiers")]; + delete require.cache[require.resolve("../src/routing/complexity-analyzer")]; + delete require.cache[require.resolve("../src/routing/cost-optimizer")]; + delete require.cache[require.resolve("../src/routing/agentic-detector")]; // Store original config originalConfig = { ...process.env }; - + // Explicitly set valid fallback to override any local .env pollution (e.g. lmstudio) process.env.FALLBACK_PROVIDER = "databricks"; + + // Ensure no TIER_* vars leak between tests + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; }); afterEach(() => { @@ -25,23 +34,24 @@ describe("Routing Logic", () => { process.env = originalConfig; }); - describe("determineProvider()", () => { - it("should return configured provider when PREFER_OLLAMA is false", () => { + describe("static routing (tier routing disabled)", () => { + it("should return configured provider when tier routing is disabled", async () => { process.env.MODEL_PROVIDER = "databricks"; - process.env.PREFER_OLLAMA = "false"; + process.env.DATABRICKS_API_KEY = "test-key"; + process.env.DATABRICKS_API_BASE = "http://test.com"; config = require("../src/config"); routing = require("../src/clients/routing"); const payload = { messages: [{ role: "user", content: "test" }] }; - const provider = routing.determineProvider(payload); + const result = await routing.determineProviderSmart(payload); - assert.strictEqual(provider, "databricks"); + assert.strictEqual(result.provider, "databricks"); + assert.strictEqual(result.method, "static"); }); - it("should route to ollama when no tools and PREFER_OLLAMA is true", () => { + it("should return ollama when MODEL_PROVIDER is ollama", async () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; config = require("../src/config"); @@ -52,15 +62,14 @@ describe("Routing Logic", () => { tools: [], }; - const provider = routing.determineProvider(payload); - assert.strictEqual(provider, "ollama"); + const result = await routing.determineProviderSmart(payload); + assert.strictEqual(result.provider, "ollama"); + assert.strictEqual(result.method, "static"); }); - it("should route to ollama when tool count < threshold", () => { + it("should return primary provider regardless of tool count", async () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "3"; config = require("../src/config"); routing = require("../src/clients/routing"); @@ -73,25 +82,15 @@ describe("Routing Logic", () => { ], }; - const provider = routing.determineProvider(payload); - assert.strictEqual(provider, "ollama"); + const result = await routing.determineProviderSmart(payload); + assert.strictEqual(result.provider, "ollama"); + assert.strictEqual(result.method, "static"); }); - it("should route to cloud when tool count >= threshold", () => { - process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; - process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "3"; - process.env.OPENROUTER_MAX_TOOLS_FOR_ROUTING = "3"; // Set same as ollama to skip openrouter tier - process.env.FALLBACK_PROVIDER = "databricks"; - process.env.FALLBACK_ENABLED = "true"; // Ensure fallback is enabled + it("should return primary provider even with many tools", async () => { + process.env.MODEL_PROVIDER = "databricks"; process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.com"; - // Set Azure OpenAI to empty to prevent dotenv from loading .env values - // dotenv won't override existing vars, even if empty - process.env.AZURE_OPENAI_ENDPOINT = ""; - process.env.AZURE_OPENAI_API_KEY = ""; - process.env.OPENROUTER_API_KEY = ""; config = require("../src/config"); routing = require("../src/clients/routing"); @@ -107,16 +106,13 @@ describe("Routing Logic", () => { ], }; - const provider = routing.determineProvider(payload); - assert.strictEqual(provider, "databricks"); + const result = await routing.determineProviderSmart(payload); + assert.strictEqual(result.provider, "databricks"); + assert.strictEqual(result.method, "static"); }); - it("should route to cloud when model doesn't support tools", () => { - process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; - process.env.OLLAMA_MODEL = "llama3:latest"; // Non-tool-capable model - process.env.OLLAMA_FALLBACK_PROVIDER = "databricks"; - process.env.FALLBACK_ENABLED = "true"; // Ensure fallback is enabled + it("should return configured MODEL_PROVIDER", async () => { + process.env.MODEL_PROVIDER = "databricks"; process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.com"; @@ -128,42 +124,55 @@ describe("Routing Logic", () => { tools: [{ name: "tool1", description: "test" }], }; - const provider = routing.determineProvider(payload); - assert.strictEqual(provider, "databricks"); + const result = await routing.determineProviderSmart(payload); + assert.strictEqual(result.provider, "databricks"); + assert.strictEqual(result.method, "static"); }); + }); + + describe("determineProviderSmart()", () => { + it("should return static routing when tier routing is disabled (no TIER_* vars)", async () => { + process.env.MODEL_PROVIDER = "databricks"; + process.env.DATABRICKS_API_KEY = "test-key"; + process.env.DATABRICKS_API_BASE = "http://test.com"; - it("should use custom max tools threshold", () => { + config = require("../src/config"); + routing = require("../src/clients/routing"); + + const payload = { messages: [{ role: "user", content: "test" }] }; + const result = await routing.determineProviderSmart(payload); + + assert.strictEqual(result.provider, "databricks"); + assert.strictEqual(result.method, "static"); + assert.strictEqual(result.reason, "tier_routing_disabled"); + assert.strictEqual(result.model, null); + }); + + it("should use tier routing when TIER_* vars are set", async () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; - process.env.OLLAMA_MAX_TOOLS_FOR_ROUTING = "5"; - process.env.OLLAMA_FALLBACK_PROVIDER = "databricks"; + process.env.TIER_SIMPLE = "ollama:llama3.2"; + process.env.TIER_MEDIUM = "ollama:llama3.2"; + process.env.TIER_COMPLEX = "databricks:claude-sonnet"; + process.env.TIER_REASONING = "databricks:claude-sonnet"; process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.com"; config = require("../src/config"); routing = require("../src/clients/routing"); - const payload = { - messages: [{ role: "user", content: "test" }], - tools: [ - { name: "tool1", description: "test" }, - { name: "tool2", description: "test" }, - { name: "tool3", description: "test" }, - { name: "tool4", description: "test" }, - ], - }; + const payload = { messages: [{ role: "user", content: "test" }] }; + const result = await routing.determineProviderSmart(payload); - // 4 tools < 5, should route to ollama - const provider = routing.determineProvider(payload); - assert.strictEqual(provider, "ollama"); + // When tier routing is enabled, method should not be 'static' + assert.notStrictEqual(result.method, "static"); + assert.ok(result.provider, "provider should be set"); }); }); describe("isFallbackEnabled()", () => { it("should return true by default", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; // Override .env file which sets FALLBACK_ENABLED=false // Test default behavior when not set to "false" @@ -177,7 +186,6 @@ describe("Routing Logic", () => { it("should return false when explicitly disabled", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_ENABLED = "false"; @@ -191,7 +199,6 @@ describe("Routing Logic", () => { describe("getFallbackProvider()", () => { it("should return databricks by default", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.DATABRICKS_API_KEY = "test-key"; process.env.DATABRICKS_API_BASE = "http://test.com"; @@ -204,7 +211,6 @@ describe("Routing Logic", () => { it("should return configured fallback provider", () => { process.env.MODEL_PROVIDER = "ollama"; - process.env.PREFER_OLLAMA = "true"; process.env.OLLAMA_MODEL = "qwen2.5-coder:latest"; process.env.FALLBACK_PROVIDER = "azure-anthropic"; process.env.AZURE_ANTHROPIC_ENDPOINT = "http://test.com";