Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions src/services/llm/adapters/google/GoogleModels.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
/**
* Google Model Specifications
* Updated March 2026 with Gemini 3.1 models
* Updated May 2026 with Gemini 3.5 models
*/

import { ModelSpec } from '../modelTypes';

export const GOOGLE_MODELS: ModelSpec[] = [
// Gemini 3.1 models (latest)
// Gemini 3.1 models
{
provider: 'google',
name: 'Gemini 3.1 Pro Preview',
Expand Down Expand Up @@ -40,6 +40,24 @@ export const GOOGLE_MODELS: ModelSpec[] = [
}
},

// Gemini 3.5 models
{
provider: 'google',
name: 'Gemini 3.5 Flash',
apiName: 'gemini-3.5-flash',
contextWindow: 1048576,
maxTokens: 65536,
inputCostPerMillion: 1.50,
outputCostPerMillion: 9.00,
capabilities: {
supportsJSON: true,
supportsImages: true,
supportsFunctions: true,
supportsStreaming: true,
supportsThinking: true
}
},

// Gemini 3.0 models
{
provider: 'google',
Expand Down Expand Up @@ -108,4 +126,4 @@ export const GOOGLE_MODELS: ModelSpec[] = [
}
];

export const GOOGLE_DEFAULT_MODEL = 'gemini-3-pro-preview';
export const GOOGLE_DEFAULT_MODEL = 'gemini-3-pro-preview';
16 changes: 16 additions & 0 deletions src/services/llm/adapters/openrouter/OpenRouterModels.ts
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,22 @@ export const OPENROUTER_MODELS: ModelSpec[] = [
supportsThinking: true
}
},
{
provider: 'openrouter',
name: 'Gemini 3.5 Flash',
apiName: 'google/gemini-3.5-flash',
contextWindow: 1048576,
maxTokens: 65536,
inputCostPerMillion: 1.50,
outputCostPerMillion: 9.00,
capabilities: {
supportsJSON: true,
supportsImages: true,
supportsFunctions: true,
supportsStreaming: true,
supportsThinking: true
}
},
{
provider: 'openrouter',
name: 'GPT-5',
Expand Down
22 changes: 18 additions & 4 deletions tests/debug/provider-model-live-smoke.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@ import { DEFAULT_MODELS } from '../../src/services/llm/adapters/ModelRegistry';
import { OpenAIAdapter } from '../../src/services/llm/adapters/openai/OpenAIAdapter';
import { OpenRouterAdapter } from '../../src/services/llm/adapters/openrouter/OpenRouterAdapter';
import { OpenAICodexAdapter, type CodexOAuthTokens } from '../../src/services/llm/adapters/openai-codex/OpenAICodexAdapter';
import { GoogleAdapter } from '../../src/services/llm/adapters/google/GoogleAdapter';
import type { GenerateOptions, LLMResponse } from '../../src/services/llm/adapters/types';

jest.setTimeout(240_000);

type SmokeProvider = 'openai' | 'openrouter' | 'openai-codex';
type SmokeProvider = 'openai' | 'openrouter' | 'openai-codex' | 'google';

interface ProviderSettingsShape {
llmProviders?: {
Expand All @@ -58,7 +59,7 @@ interface SmokeTarget {
}

const RUN_LIVE = process.env.RUN_MODEL_SMOKE === '1';
const PROVIDERS: SmokeProvider[] = ['openai', 'openrouter', 'openai-codex'];
const PROVIDERS: SmokeProvider[] = ['openai', 'openrouter', 'openai-codex', 'google'];

function readDotEnv(): Map<string, string> {
const envPath = path.join(process.cwd(), '.env');
Expand Down Expand Up @@ -123,6 +124,10 @@ function normalizeModelForProvider(provider: SmokeProvider, model: string): stri
return model.slice('openai/'.length);
}

if (provider === 'google' && model.startsWith('google/')) {
return model.slice('google/'.length);
}

return model;
}

Expand All @@ -132,6 +137,7 @@ function getProviderModel(provider: SmokeProvider): string {
openai: getEnv('OPENAI_SMOKE_MODEL'),
openrouter: getEnv('OPENROUTER_SMOKE_MODEL'),
'openai-codex': getEnv('CODEX_SMOKE_MODEL'),
google: getEnv('GOOGLE_SMOKE_MODEL'),
}[provider];

const model = providerOverride || sharedOverride || DEFAULT_MODELS[provider];
Expand Down Expand Up @@ -184,7 +190,7 @@ function setRequestUrlToRealFetch(): void {
});
}

function createAdapter(provider: SmokeProvider): OpenAIAdapter | OpenRouterAdapter | OpenAICodexAdapter {
function createAdapter(provider: SmokeProvider): OpenAIAdapter | OpenRouterAdapter | OpenAICodexAdapter | GoogleAdapter {
if (provider === 'openai') {
const apiKey = getEnv('OPENAI_API_KEY');
if (!apiKey) {
Expand All @@ -201,6 +207,14 @@ function createAdapter(provider: SmokeProvider): OpenAIAdapter | OpenRouterAdapt
return new OpenRouterAdapter(apiKey);
}

if (provider === 'google') {
const apiKey = getEnv('GEMINI_API_KEY');
if (!apiKey) {
throw new Error('GEMINI_API_KEY is required for Google smoke tests');
}
return new GoogleAdapter(apiKey);
}

const tokens = loadCodexTokensFromLocalDataJson();
if (!tokens) {
throw new Error('Codex OAuth tokens are required in data.json for Codex smoke tests');
Expand All @@ -217,7 +231,7 @@ async function callModel(target: SmokeTarget): Promise<LLMResponse> {

// Codex currently rejects max_output_tokens on the OAuth endpoint.
if (target.provider !== 'openai-codex') {
options.maxTokens = Number(getEnv('MODEL_SMOKE_MAX_TOKENS') || 16);
options.maxTokens = Number(getEnv('MODEL_SMOKE_MAX_TOKENS') || 64);
}

return adapter.generateUncached(
Expand Down
23 changes: 23 additions & 0 deletions tests/eval/configs/gemini-3.5-flash.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
mode: live
testVaultPath: tests/eval/test-vault/

providers:
openrouter:
apiKeyEnv: OPENROUTER_API_KEY
models:
- google/gemini-3.5-flash
enabled: true

defaults:
temperature: 0
maxRetries: 1
retryDelayMs: 2000
timeout: 120000
systemPrompt: default

capture:
enabled: true
dumpOnFailure: true
artifactsDir: test-artifacts/

scenarios: tests/eval/scenarios/**/*.eval.yaml
22 changes: 22 additions & 0 deletions tests/unit/ModelRegistry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,25 @@ describe('ModelRegistry GPT-5.5 models', () => {
expect(DEFAULT_MODELS['openai-codex']).toBe('gpt-5.5');
});
});

describe('ModelRegistry Gemini 3.5 Flash models', () => {
it('registers Gemini 3.5 Flash for Google', () => {
expect(ModelRegistry.findModel('google', 'gemini-3.5-flash')).toEqual(expect.objectContaining({
name: 'Gemini 3.5 Flash',
contextWindow: 1048576,
maxTokens: 65536,
inputCostPerMillion: 1.5,
outputCostPerMillion: 9
}));
});

it('registers Gemini 3.5 Flash for OpenRouter with the provider namespace', () => {
expect(ModelRegistry.findModel('openrouter', 'google/gemini-3.5-flash')).toEqual(expect.objectContaining({
name: 'Gemini 3.5 Flash',
contextWindow: 1048576,
maxTokens: 65536,
inputCostPerMillion: 1.5,
outputCostPerMillion: 9
}));
});
});