Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -283,10 +283,21 @@ Creates and configures an intelligent model router.

```typescript
interface ModelRouterOptions {
strategy: "fallback" | "context" | "cost"; // Routing strategy
strategy: "fallback" | "context" | "cost" | "smart"; // Routing strategy
smart?: SmartRouterOptions; // Smart routing options
fallbackMap?: Record<string, string[]>; // Fallback model map
maxRetries?: number; // Max retry attempts (default: 1)
}

interface SmartRouterOptions {
defaultModel?: string; // Fallback when confidence is low
confidenceThreshold?: number; // Minimum task confidence
modelOverrides?: Record<string, string>; // Override task-to-model mapping
taskClassification?: Record<string, TaskClassificationConfig>;
cacheDetections?: boolean; // Cache repeated prompt detections
detectionCacheTtlMs?: number; // Detection cache TTL in ms
enableAnalytics?: boolean; // Record task routing analytics
}
```

**Example:**
Expand All @@ -309,6 +320,21 @@ patchGlobalFetch();
// API calls will automatically retry with fallback models on failure
```

Smart routing type definitions are available for the upcoming task classifier:

```typescript
import type { SmartRouterOptions, TaskClassification } from "tokenfirewall";

const smartOptions: SmartRouterOptions = {
defaultModel: "gpt-4o-mini",
confidenceThreshold: 0.75,
modelOverrides: {
code_generation: "claude-3-5-sonnet-20241022",
math_reasoning: "o1-mini"
}
};
```

### Routing Strategies

**1. Fallback Strategy** - Uses predefined fallback map
Expand Down
9 changes: 8 additions & 1 deletion src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,14 @@ export type {
FailureContext,
RoutingDecision,
RouterEvent,
ApiKeyConfig
ApiKeyConfig,
BuiltInTaskType,
TaskType,
TaskDetectionMethod,
TaskModelPreference,
TaskClassificationConfig,
TaskClassification,
SmartRouterOptions
} from "./router/types";

/**
Expand Down
111 changes: 110 additions & 1 deletion src/router/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,114 @@
/**
* Routing strategy types
*/
export type RoutingStrategy = "fallback" | "context" | "cost";
export type RoutingStrategy = "fallback" | "context" | "cost" | "smart";

/**
* Built-in smart routing task types documented for model selection.
*/
export type BuiltInTaskType =
| "code_generation"
| "code_review"
| "math_reasoning"
| "complex_reasoning"
| "document_analysis"
| "creative_writing"
| "translation"
| "simple_chat"
| "data_extraction"
| "chinese_language"
| "factual_qa"
| "technical_documentation";

/**
* Task type accepted by the smart router. String intersections preserve
* autocomplete for built-ins while allowing user-defined task names.
*/
export type TaskType = BuiltInTaskType | (string & {});

/**
* Detection method that produced a smart routing classification.
*/
export type TaskDetectionMethod =
| "keyword"
| "pattern"
| "language"
| "context"
| "manual"
| "custom";

/**
* Model preference for a task type.
*/
export interface TaskModelPreference {
/** Primary model recommended for this task */
model: string;
/** Optional provider name when the model can be ambiguous */
provider?: string;
/** Optional priority for tie-breaking; higher wins */
priority?: number;
/** Optional reason shown in analytics/debug output */
reason?: string;
}

/**
* Custom task definition used by the smart router.
*/
export interface TaskClassificationConfig {
/** Keywords that indicate this task */
keywords?: string[];
/** String or RegExp patterns that indicate this task */
patterns?: Array<string | RegExp>;
/** Preferred model for this task */
model: string;
/** Optional fallback models for this task */
alternatives?: TaskModelPreference[];
/** Minimum confidence required before selecting this task */
confidenceThreshold?: number;
}

/**
* Result produced by task classification.
*/
export interface TaskClassification {
/** Detected task type */
taskType: TaskType;
/** Confidence from 0 to 1 */
confidence: number;
/** Selected model for the request */
selectedModel: string;
/** Human-readable reason for the classification */
reason: string;
/** Detection method that produced the classification */
method?: TaskDetectionMethod;
/** Alternative model candidates */
alternatives?: TaskModelPreference[];
}

/**
* Options for the smart routing strategy.
*/
export interface SmartRouterOptions {
/** Default model used when confidence is below the threshold */
defaultModel?: string;
/** Minimum confidence required to use a classified task route */
confidenceThreshold?: number;
/** Per-task model overrides */
modelOverrides?: Partial<Record<BuiltInTaskType, string>> & Record<string, string>;
/** Custom task definitions keyed by task type */
taskClassification?: Record<string, TaskClassificationConfig>;
/** Cache task detections for repeated prompts (default: true for implementations) */
cacheDetections?: boolean;
/** Detection cache TTL in milliseconds */
detectionCacheTtlMs?: number;
/** Enable task analytics collection */
enableAnalytics?: boolean;
/** Optional custom async detector invoked before built-in detection */
customDetector?: (
prompt: string,
context?: Record<string, unknown>
) => Promise<TaskClassification | null> | TaskClassification | null;
}

/**
* Failure types detected by error detector
Expand Down Expand Up @@ -35,6 +142,8 @@ export interface ApiKeyConfig {
export interface ModelRouterOptions {
/** Routing strategy to use */
strategy: RoutingStrategy;
/** Smart routing options used when strategy is "smart" */
smart?: SmartRouterOptions;
/** Map of primary models to fallback models */
fallbackMap?: Record<string, string[]>;
/** Maximum number of retry attempts (default: 1) */
Expand Down
48 changes: 48 additions & 0 deletions tests/router-types.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/**
* Router type definition smoke tests.
*
* Run: node tests/router-types.test.js
*/

const { createModelRouter, disableModelRouter } = require("../dist/index.js");

function assert(condition, message) {
if (!condition) {
throw new Error(message);
}
}

try {
const smartRouter = createModelRouter({
strategy: "smart",
smart: {
defaultModel: "gpt-4o-mini",
confidenceThreshold: 0.75,
modelOverrides: {
code_generation: "claude-3-5-sonnet-20241022",
math_reasoning: "o1-mini",
},
cacheDetections: true,
enableAnalytics: true,
},
});

assert(smartRouter.getStrategy() === "smart", "smart strategy should be accepted");
disableModelRouter();

const fallbackRouter = createModelRouter({
strategy: "fallback",
fallbackMap: {
"gpt-4o": ["gpt-4o-mini"],
},
});

assert(fallbackRouter.getStrategy() === "fallback", "existing strategies should still work");
disableModelRouter();

console.log("Router type definition smoke tests passed.");
} catch (error) {
disableModelRouter();
console.error("Router type definition smoke tests failed:", error.message);
process.exit(1);
}