diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index ee9cddf949c..0459382ba6a 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -30,6 +30,7 @@ import { ProviderDriverError } from "../Errors.ts"; import { makeClaudeAdapter } from "../Layers/ClaudeAdapter.ts"; import { checkClaudeProviderStatus, + getClaudeModelCapabilities, makePendingClaudeProvider, probeClaudeCapabilities, } from "../Layers/ClaudeProvider.ts"; @@ -141,14 +142,6 @@ export const ClaudeDriver: ProviderDriver = { continuationGroupKey, }); - const adapterOptions = { - instanceId, - environment: processEnv, - ...(eventLoggers.native ? { nativeEventLogger: eventLoggers.native } : {}), - }; - const adapter = yield* makeClaudeAdapter(effectiveConfig, adapterOptions); - const textGeneration = yield* makeClaudeTextGeneration(effectiveConfig, processEnv); - // Per-instance capabilities cache: keyed on binary + resolved HOME so // account-specific probes never share auth metadata across instances. const capabilitiesProbeCache = yield* Cache.make({ @@ -160,6 +153,23 @@ export const ClaudeDriver: ProviderDriver = { ), }); const capabilitiesCacheKey = yield* makeClaudeCapabilitiesCacheKey(effectiveConfig, cwd); + const resolveModelCapabilities = (model: string | null | undefined) => + Cache.get(capabilitiesProbeCache, capabilitiesCacheKey).pipe( + Effect.map((capabilities) => getClaudeModelCapabilities(model, capabilities?.models)), + ); + + const adapterOptions = { + instanceId, + environment: processEnv, + resolveModelCapabilities, + ...(eventLoggers.native ? { nativeEventLogger: eventLoggers.native } : {}), + }; + const adapter = yield* makeClaudeAdapter(effectiveConfig, adapterOptions); + const textGeneration = yield* makeClaudeTextGeneration( + effectiveConfig, + processEnv, + resolveModelCapabilities, + ); const checkProvider = checkClaudeProviderStatus( effectiveConfig, diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index 17aeff2d0e3..61e1d54feb5 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -14,6 +14,7 @@ import type { import { ApprovalRequestId, ClaudeSettings, + type ModelCapabilities, ProviderDriverKind, ProviderItemId, ProviderRuntimeEvent, @@ -21,7 +22,7 @@ import { ThreadId, ProviderInstanceId, } from "@t3tools/contracts"; -import { createModelSelection } from "@t3tools/shared/model"; +import { createModelCapabilities, createModelSelection } from "@t3tools/shared/model"; import { assert, describe, it } from "@effect/vitest"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; @@ -156,6 +157,7 @@ function makeHarness(config?: { readonly baseDir?: string; readonly claudeConfig?: Partial; readonly instanceId?: ProviderInstanceId; + readonly resolveModelCapabilities?: ClaudeAdapterLiveOptions["resolveModelCapabilities"]; }) { const query = new FakeClaudeQuery(); let createInput: @@ -171,6 +173,9 @@ function makeHarness(config?: { createInput = input; return query; }, + ...(config?.resolveModelCapabilities + ? { resolveModelCapabilities: config.resolveModelCapabilities } + : {}), ...(config?.nativeEventLogger ? { nativeEventLogger: config.nativeEventLogger, @@ -205,6 +210,19 @@ function makeHarness(config?: { }; } +function customEffortCapabilities(...efforts: ReadonlyArray): ModelCapabilities { + return createModelCapabilities({ + optionDescriptors: [ + { + id: "effort", + label: "Reasoning", + type: "select", + options: efforts.map((effort) => ({ id: effort, label: effort })), + }, + ], + }); +} + function makeDeterministicRandomService(seed = 0x1234_5678): { nextIntUnsafe: () => number; nextDoubleUnsafe: () => number; @@ -350,6 +368,7 @@ describe("ClaudeAdapterLive", () => { assert.deepEqual(createInput?.options.settingSources, ["user", "project", "local"]); assert.equal(createInput?.options.permissionMode, "bypassPermissions"); assert.equal(createInput?.options.allowDangerouslySkipPermissions, true); + assert.equal(createInput?.options.effort, undefined); }).pipe( Effect.provideService(Random.Random, makeDeterministicRandomService()), Effect.provide(harness.layer), @@ -418,6 +437,75 @@ describe("ClaudeAdapterLive", () => { ); }); + it.effect("does not default effort for custom Claude models", () => { + const harness = makeHarness(); + return Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + modelSelection: createModelSelection(ProviderInstanceId.make("claudeAgent"), "gpt-5.6-sol"), + runtimeMode: "full-access", + }); + + const createInput = harness.getLastCreateQueryInput(); + assert.equal(createInput?.options.effort, undefined); + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }); + + it.effect("preserves xhigh effort for custom Claude models", () => { + const harness = makeHarness({ + resolveModelCapabilities: () => Effect.succeed(customEffortCapabilities("xhigh")), + }); + return Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + modelSelection: createModelSelection( + ProviderInstanceId.make("claudeAgent"), + "gpt-5.6-sol", + [{ id: "effort", value: "xhigh" }], + ), + runtimeMode: "full-access", + }); + + const createInput = harness.getLastCreateQueryInput(); + assert.equal(createInput?.options.effort, "xhigh"); + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }); + + it.effect("ignores custom Claude effort levels not advertised by the SDK", () => { + const harness = makeHarness({ + resolveModelCapabilities: () => Effect.succeed(customEffortCapabilities("low")), + }); + return Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + modelSelection: createModelSelection( + ProviderInstanceId.make("claudeAgent"), + "gpt-5.6-sol", + [{ id: "effort", value: "xhigh" }], + ), + runtimeMode: "full-access", + }); + + const createInput = harness.getLastCreateQueryInput(); + assert.equal(createInput?.options.effort, undefined); + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }); + it.effect("runs Claude SDK sessions with the configured CLAUDE_CONFIG_DIR", () => { const harness = makeHarness({ claudeConfig: { homePath: "~/.claude-work" } }); return Effect.gen(function* () { diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index f6e63eeffad..28dd79f047a 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -27,6 +27,7 @@ import { type CanonicalRequestType, type ClaudeSettings, EventId, + type ModelCapabilities, type ProviderApprovalDecision, ProviderDriverKind, ProviderInstanceId, @@ -221,6 +222,9 @@ export interface ClaudeAdapterLiveOptions { }) => ClaudeQueryRuntime; readonly nativeEventLogPath?: string; readonly nativeEventLogger?: EventNdjsonLogger; + readonly resolveModelCapabilities?: ( + model: string | null | undefined, + ) => Effect.Effect; } function isUuid(value: string): boolean { @@ -1359,6 +1363,9 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( stream: "native", }) : undefined); + const resolveModelCapabilities = + options?.resolveModelCapabilities ?? + ((model: string | null | undefined) => Effect.succeed(getClaudeModelCapabilities(model))); const createQuery = options?.createQuery ?? @@ -3414,7 +3421,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( const extraArgs = parseCliArgs(claudeSettings.launchArgs).flags; const modelSelection = input.modelSelection?.instanceId === boundInstanceId ? input.modelSelection : undefined; - const caps = getClaudeModelCapabilities(modelSelection?.model); + const caps = yield* resolveModelCapabilities(modelSelection?.model); const descriptors = getProviderOptionDescriptors({ caps }); const apiModelId = modelSelection ? resolveClaudeApiModelId(modelSelection) : undefined; const initialContextWindow = selectedClaudeContextWindow(modelSelection); diff --git a/apps/server/src/provider/Layers/ClaudeProvider.test.ts b/apps/server/src/provider/Layers/ClaudeProvider.test.ts new file mode 100644 index 00000000000..cae3ef5a71e --- /dev/null +++ b/apps/server/src/provider/Layers/ClaudeProvider.test.ts @@ -0,0 +1,98 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { ClaudeSettings } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Schema from "effect/Schema"; +import * as TestClock from "effect/testing/TestClock"; +import { beforeEach, vi } from "vite-plus/test"; + +import { probeClaudeCapabilities } from "./ClaudeProvider.ts"; + +type ClaudeQuery = typeof import("@anthropic-ai/claude-agent-sdk").query; +type ClaudeInitialization = Awaited["initializationResult"]>>; + +const claudeQueryMock = vi.hoisted(() => vi.fn()); +const decodeClaudeSettings = Schema.decodeEffect(ClaudeSettings); + +vi.mock("@anthropic-ai/claude-agent-sdk", () => ({ + query: claudeQueryMock, +})); + +beforeEach(() => { + claudeQueryMock.mockReset(); +}); + +it.layer(NodeServices.layer)("probeClaudeCapabilities", (it) => { + it.effect("keeps completed probes when another custom model times out", () => + Effect.gen(function* () { + let activeProbes = 0; + let peakActiveProbes = 0; + const closedModels: Array = []; + + claudeQueryMock.mockImplementation((input: Parameters[0]) => { + assert.ok(input.options); + assert.ok(input.options.abortController); + const model = input.options.model ?? "default"; + const abort = input.options.abortController; + activeProbes += 1; + peakActiveProbes = Math.max(peakActiveProbes, activeProbes); + let closed = false; + const close = () => { + if (!closed) { + closed = true; + activeProbes -= 1; + closedModels.push(model); + } + }; + + const initializationResult = () => { + if (model === "slow") { + return new Promise((_resolve, reject) => { + abort.signal.addEventListener("abort", () => reject(new Error("aborted")), { + once: true, + }); + }); + } + return Promise.resolve({ + account: { email: `${model}@example.com` }, + commands: [], + models: [ + { + value: model, + displayName: model, + description: "Custom model", + supportsEffort: true, + supportedEffortLevels: ["low", "high"], + }, + ], + } as unknown as ClaudeInitialization); + }; + + return { close, initializationResult } as ReturnType; + }); + + const settings = yield* decodeClaudeSettings({ + customModels: ["slow", "fast", "later"], + }); + const probe = yield* probeClaudeCapabilities(settings).pipe( + Effect.forkChild({ startImmediately: true }), + ); + + while (claudeQueryMock.mock.calls.length < 3) { + yield* Effect.yieldNow; + } + yield* TestClock.adjust("25 seconds"); + + const result = yield* Fiber.join(probe); + assert.deepStrictEqual(result?.models.map((model) => model.value).toSorted(), [ + "fast", + "later", + ]); + assert.strictEqual(result?.email, "fast@example.com"); + assert.strictEqual(peakActiveProbes, 2); + assert.strictEqual(activeProbes, 0); + assert.deepStrictEqual(closedModels.toSorted(), ["fast", "later", "slow"]); + }), + ); +}); diff --git a/apps/server/src/provider/Layers/ClaudeProvider.ts b/apps/server/src/provider/Layers/ClaudeProvider.ts index ff0d1992454..e7b9f4b3a42 100644 --- a/apps/server/src/provider/Layers/ClaudeProvider.ts +++ b/apps/server/src/provider/Layers/ClaudeProvider.ts @@ -21,6 +21,7 @@ import { resolveSpawnCommand } from "@t3tools/shared/shell"; import { compareSemverVersions } from "@t3tools/shared/semver"; import { query as claudeQuery, + type ModelInfo as ClaudeModelInfo, type SlashCommand as ClaudeSlashCommand, type SDKUserMessage, } from "@anthropic-ai/claude-agent-sdk"; @@ -39,7 +40,7 @@ import { import { resolveClaudeSdkExecutablePath } from "../Drivers/ClaudeExecutable.ts"; import { makeClaudeEnvironment } from "../Drivers/ClaudeHome.ts"; -const DEFAULT_CLAUDE_MODEL_CAPABILITIES: ModelCapabilities = createModelCapabilities({ +const EMPTY_CLAUDE_MODEL_CAPABILITIES: ModelCapabilities = createModelCapabilities({ optionDescriptors: [], }); @@ -295,6 +296,78 @@ function getBuiltInClaudeModelsForVersion( }); } +function claudeEffortLabel(effort: string): string { + switch (effort) { + case "low": + return "Low"; + case "medium": + return "Medium"; + case "high": + return "High"; + case "xhigh": + return "Extra High"; + case "max": + return "Max"; + default: + return effort; + } +} + +function customClaudeModelCapabilities(model: ClaudeModelInfo | undefined): ModelCapabilities { + const effortLevels = model?.supportedEffortLevels ?? []; + if (effortLevels.length === 0) { + return EMPTY_CLAUDE_MODEL_CAPABILITIES; + } + + return createModelCapabilities({ + optionDescriptors: [ + buildSelectOptionDescriptor({ + id: "effort", + label: "Reasoning", + options: effortLevels.map((effort) => ({ + value: effort, + label: claudeEffortLabel(effort), + })), + }), + ], + }); +} + +function indexClaudeModels(models: Iterable): Map { + const modelsBySlug = new Map(); + for (const model of models) { + const existing = modelsBySlug.get(model.value); + if ( + (existing?.supportedEffortLevels?.length ?? 0) > 0 && + (model.supportedEffortLevels?.length ?? 0) === 0 + ) { + continue; + } + modelsBySlug.set(model.value, model); + } + return modelsBySlug; +} + +function claudeModelsFromSettings( + builtInModels: ReadonlyArray, + customModels: ReadonlyArray, + availableModels: ReadonlyArray = [], +): ReadonlyArray { + const availableModelsBySlug = indexClaudeModels(availableModels); + return providerModelsFromSettings( + builtInModels, + customModels, + EMPTY_CLAUDE_MODEL_CAPABILITIES, + ).map((model) => + model.isCustom + ? { + ...model, + capabilities: customClaudeModelCapabilities(availableModelsBySlug.get(model.slug)), + } + : model, + ); +} + function formatClaudeFable5UpgradeMessage(version: string | null): string { const versionLabel = version ? `v${version}` : "the installed version"; return `Claude Code ${versionLabel} is too old for Claude Fable 5. Upgrade to v${MINIMUM_CLAUDE_FABLE_5_VERSION} or newer to access it.`; @@ -310,11 +383,17 @@ function formatClaudeOpus47UpgradeMessage(version: string | null): string { return `Claude Code ${versionLabel} is too old for Claude Opus 4.7. Upgrade to v${MINIMUM_CLAUDE_OPUS_4_7_VERSION} or newer to access it.`; } -export function getClaudeModelCapabilities(model: string | null | undefined): ModelCapabilities { +export function getClaudeModelCapabilities( + model: string | null | undefined, + availableModels: ReadonlyArray = [], +): ModelCapabilities { const slug = model?.trim(); + if (!slug) { + return EMPTY_CLAUDE_MODEL_CAPABILITIES; + } return ( BUILT_IN_MODELS.find((candidate) => candidate.slug === slug)?.capabilities ?? - DEFAULT_CLAUDE_MODEL_CAPABILITIES + customClaudeModelCapabilities(indexClaudeModels(availableModels).get(slug)) ); } @@ -353,6 +432,7 @@ export function normalizeClaudeCliEffort( } if ( effort === "xhigh" && + BUILT_IN_MODELS.some((candidate) => candidate.slug === model) && model !== "claude-fable-5" && model !== "claude-opus-4-8" && model !== "claude-sonnet-5" @@ -491,6 +571,7 @@ function apiProviderAuthMetadata( // account info. The previous 8s budget expired mid-init, so the probe returned // `undefined` and left the provider unverified and unselectable in the picker. const CAPABILITIES_PROBE_TIMEOUT_MS = 25_000; +const CAPABILITIES_PROBE_CONCURRENCY = 2; function nonEmptyProbeString(value: string): string | undefined { const candidate = value.trim(); @@ -508,6 +589,7 @@ type ClaudeCapabilitiesProbe = { */ readonly apiProvider: string | undefined; readonly slashCommands: ReadonlyArray; + readonly models: ReadonlyArray; }; function parseClaudeInitializationCommands( @@ -589,8 +671,10 @@ function waitForAbortSignal(signal: AbortSignal): Promise { * We pass a never-yielding AsyncIterable as the prompt so that no user * message is ever written to the subprocess stdin. This means the Claude * Code subprocess completes its local initialization IPC (returning - * account info and slash commands) but never starts an API request to - * Anthropic. We read the init data and then abort the subprocess. + * account info, slash commands, and model metadata) but never starts an API + * request to Anthropic. Custom models are initialized individually because + * Claude only advertises their supported effort levels when selected. We read + * the init data and then abort every subprocess. * * This is used as a fallback when `claude auth status` does not include * subscription type information. @@ -600,60 +684,89 @@ const probeClaudeCapabilities = ( environment?: NodeJS.ProcessEnv, cwd?: string, ) => { - const abort = new AbortController(); return Effect.gen(function* () { const claudeEnvironment = yield* makeClaudeEnvironment(claudeSettings, environment); const executablePath = yield* resolveClaudeSdkExecutablePath( claudeSettings.binaryPath, claudeEnvironment, ); - return yield* Effect.tryPromise(async () => { - const q = claudeQuery({ - // Never yield — we only need initialization data, not a conversation. - // This prevents any prompt from reaching the Anthropic API. - // oxlint-disable-next-line require-yield - prompt: (async function* (): AsyncGenerator { - await waitForAbortSignal(abort.signal); - })(), - options: { - persistSession: false, - pathToClaudeCodeExecutable: executablePath, - abortController: abort, - settingSources: ["user", "project", "local"], - allowedTools: [], - env: claudeEnvironment, - ...(cwd ? { cwd } : {}), - stderr: () => {}, - }, - }); - const init = await q.initializationResult(); - const account = init.account as - | { - readonly email?: string; - readonly subscriptionType?: string; - readonly tokenSource?: string; - readonly apiProvider?: string; - } - | undefined; - return { - email: account?.email, - subscriptionType: account?.subscriptionType, - tokenSource: account?.tokenSource, - apiProvider: account?.apiProvider, - slashCommands: parseClaudeInitializationCommands(init.commands), - } satisfies ClaudeCapabilitiesProbe; - }); + const initialize = (model?: string) => { + const abort = new AbortController(); + let query: ReturnType | undefined; + return Effect.tryPromise(() => { + query = claudeQuery({ + // Never yield — we only need initialization data, not a conversation. + // This prevents any prompt from reaching the Anthropic API. + // oxlint-disable-next-line require-yield + prompt: (async function* (): AsyncGenerator { + await waitForAbortSignal(abort.signal); + })(), + options: { + persistSession: false, + pathToClaudeCodeExecutable: executablePath, + abortController: abort, + settingSources: ["user", "project", "local"], + allowedTools: [], + env: claudeEnvironment, + ...(model ? { model } : {}), + ...(cwd ? { cwd } : {}), + stderr: () => {}, + }, + }); + return query.initializationResult(); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + if (!abort.signal.aborted) abort.abort(); + query?.close(); + }), + ), + Effect.timeoutOption(CAPABILITIES_PROBE_TIMEOUT_MS), + Effect.result, + Effect.map((result) => + Result.isFailure(result) ? undefined : Option.getOrUndefined(result.success), + ), + ); + }; + const customModels = [ + ...new Set(claudeSettings.customModels.map((model) => model.trim())), + ].filter((model) => model.length > 0); + const initializations = (yield* Effect.forEach( + customModels.length > 0 ? customModels : [undefined], + initialize, + { concurrency: CAPABILITIES_PROBE_CONCURRENCY }, + )).filter((initialization) => initialization !== undefined); + if (initializations.length === 0 && customModels.length > 0) { + const fallback = yield* initialize(); + if (fallback) initializations.push(fallback); + } + const init = initializations[0]; + if (!init) return undefined; + + const modelsByValue = indexClaudeModels( + initializations.flatMap((initialization) => initialization.models), + ); + const account = init.account as + | { + readonly email?: string; + readonly subscriptionType?: string; + readonly tokenSource?: string; + readonly apiProvider?: string; + } + | undefined; + return { + email: account?.email, + subscriptionType: account?.subscriptionType, + tokenSource: account?.tokenSource, + apiProvider: account?.apiProvider, + slashCommands: parseClaudeInitializationCommands(init.commands), + models: [...modelsByValue.values()], + } satisfies ClaudeCapabilitiesProbe; }).pipe( - Effect.ensuring( - Effect.sync(() => { - if (!abort.signal.aborted) abort.abort(); - }), - ), - Effect.timeoutOption(CAPABILITIES_PROBE_TIMEOUT_MS), Effect.result, Effect.map((result) => { if (Result.isFailure(result)) return undefined; - return Option.isSome(result.success) ? result.success.value : undefined; + return result.success; }), ); }; @@ -687,11 +800,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( > { const resolvedEnvironment = environment ?? process.env; const checkedAt = DateTime.formatIso(yield* DateTime.now); - const allModels = providerModelsFromSettings( - BUILT_IN_MODELS, - claudeSettings.customModels, - DEFAULT_CLAUDE_MODEL_CAPABILITIES, - ); + const allModels = claudeModelsFromSettings(BUILT_IN_MODELS, claudeSettings.customModels); if (!claudeSettings.enabled) { return buildServerProvider({ @@ -777,11 +886,6 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( }); } - const models = providerModelsFromSettings( - getBuiltInClaudeModelsForVersion(parsedVersion), - claudeSettings.customModels, - DEFAULT_CLAUDE_MODEL_CAPABILITIES, - ); const versionUpgradeMessage = supportsClaudeFable5(parsedVersion) ? undefined : supportsClaudeOpus48(parsedVersion) @@ -793,6 +897,11 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( const capabilities = resolveCapabilities ? yield* resolveCapabilities(claudeSettings).pipe(Effect.orElseSucceed(() => undefined)) : undefined; + const models = claudeModelsFromSettings( + getBuiltInClaudeModelsForVersion(parsedVersion), + claudeSettings.customModels, + capabilities?.models, + ); const slashCommands = capabilities?.slashCommands ?? []; const dedupedSlashCommands = dedupeSlashCommands(slashCommands); @@ -845,11 +954,7 @@ export const makePendingClaudeProvider = ( ): Effect.Effect => Effect.gen(function* () { const checkedAt = yield* nowIso; - const models = providerModelsFromSettings( - BUILT_IN_MODELS, - claudeSettings.customModels, - DEFAULT_CLAUDE_MODEL_CAPABILITIES, - ); + const models = claudeModelsFromSettings(BUILT_IN_MODELS, claudeSettings.customModels); if (!claudeSettings.enabled) { return buildServerProvider({ diff --git a/apps/server/src/provider/Layers/ProviderRegistry.test.ts b/apps/server/src/provider/Layers/ProviderRegistry.test.ts index 159d853121c..3ad23d04a23 100644 --- a/apps/server/src/provider/Layers/ProviderRegistry.test.ts +++ b/apps/server/src/provider/Layers/ProviderRegistry.test.ts @@ -12,6 +12,7 @@ import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as CodexErrors from "effect-codex-app-server/errors"; +import type { ModelInfo as ClaudeModelInfo } from "@anthropic-ai/claude-agent-sdk"; import { ClaudeSettings, CodexSettings, @@ -103,6 +104,7 @@ type TestClaudeCapabilities = { readonly tokenSource: string | undefined; readonly apiProvider: string | undefined; readonly slashCommands: ReadonlyArray; + readonly models: ReadonlyArray; }; function claudeCapabilities(overrides: Partial = {}) { @@ -113,6 +115,7 @@ function claudeCapabilities(overrides: Partial = {}) { tokenSource: undefined, apiProvider: undefined, slashCommands: [], + models: [], ...overrides, }); } @@ -1510,6 +1513,127 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te ), ); + it.effect("uses advertised reasoning effort options for custom Claude models", () => + Effect.gen(function* () { + const status = yield* checkClaudeProviderStatus( + { + ...defaultClaudeSettings, + customModels: ["gpt-5.6-sol"], + }, + claudeCapabilities({ + models: [ + { + value: "gpt-5.6-sol", + displayName: "GPT 5.6 Sol", + description: "Custom model", + supportsEffort: true, + supportedEffortLevels: ["low", "high", "xhigh"], + }, + ], + }), + ); + const customModel = status.models.find((model) => model.slug === "gpt-5.6-sol"); + const effortDescriptor = customModel?.capabilities?.optionDescriptors?.find( + (descriptor) => descriptor.type === "select" && descriptor.id === "effort", + ); + + assert.strictEqual(customModel?.isCustom, true); + assert.deepStrictEqual(effortDescriptor, { + id: "effort", + label: "Reasoning", + type: "select", + options: [ + { id: "low", label: "Low" }, + { id: "high", label: "High" }, + { id: "xhigh", label: "Extra High" }, + ], + }); + }).pipe( + Effect.provide( + mockSpawnerLayer((args) => { + const joined = args.join(" "); + if (joined === "--version") return { stdout: "2.1.215\n", stderr: "", code: 0 }; + throw new Error(`Unexpected args: ${joined}`); + }), + ), + ), + ); + + it.effect("does not guess reasoning effort options for custom Claude models", () => + Effect.gen(function* () { + const status = yield* checkClaudeProviderStatus( + { + ...defaultClaudeSettings, + customModels: ["custom-without-effort"], + }, + claudeCapabilities({ + models: [ + { + value: "custom-without-effort", + displayName: "Custom Without Effort", + description: "Custom model", + supportsEffort: false, + }, + ], + }), + ); + const customModel = status.models.find((model) => model.slug === "custom-without-effort"); + + assert.strictEqual(customModel?.isCustom, true); + assert.deepStrictEqual(customModel?.capabilities?.optionDescriptors, []); + }).pipe( + Effect.provide( + mockSpawnerLayer((args) => { + const joined = args.join(" "); + if (joined === "--version") return { stdout: "2.1.215\n", stderr: "", code: 0 }; + throw new Error(`Unexpected args: ${joined}`); + }), + ), + ), + ); + + it.effect("preserves advertised effort levels across custom model probes", () => + Effect.gen(function* () { + const advertisedModel: ClaudeModelInfo = { + value: "gpt-5.6-sol", + displayName: "GPT 5.6 Sol", + description: "Custom model", + supportsEffort: true, + supportedEffortLevels: ["low", "xhigh"], + }; + const status = yield* checkClaudeProviderStatus( + { + ...defaultClaudeSettings, + customModels: [advertisedModel.value], + }, + claudeCapabilities({ + models: [advertisedModel, { ...advertisedModel, supportedEffortLevels: [] }], + }), + ); + const effortDescriptor = status.models + .find((model) => model.slug === advertisedModel.value) + ?.capabilities?.optionDescriptors?.find((descriptor) => descriptor.id === "effort"); + + assert.deepStrictEqual(effortDescriptor, { + id: "effort", + label: "Reasoning", + type: "select", + options: [ + { id: "low", label: "Low" }, + { id: "xhigh", label: "Extra High" }, + ], + }); + }).pipe( + Effect.provide( + mockSpawnerLayer((args) => { + const joined = args.join(" "); + if (joined === "--version") return { stdout: "2.1.215\n", stderr: "", code: 0 }; + throw new Error(`Unexpected args: ${joined}`); + }), + ), + ), + ); + it.effect("returns ready and labels Bedrock-backed Claude as authenticated", () => Effect.gen(function* () { // Bedrock authenticates via external AWS credentials, so the SDK init diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts index 0f3905a0cb1..ff8b384d59e 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts @@ -1,4 +1,4 @@ -import { ClaudeSettings, ProviderInstanceId } from "@t3tools/contracts"; +import { ClaudeSettings, type ModelCapabilities, ProviderInstanceId } from "@t3tools/contracts"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { it } from "@effect/vitest"; import * as Effect from "effect/Effect"; @@ -6,7 +6,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; -import { createModelSelection } from "@t3tools/shared/model"; +import { createModelCapabilities, createModelSelection } from "@t3tools/shared/model"; import { expect } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; @@ -78,6 +78,7 @@ function withFakeClaudeEnv( stdinMustContain?: string; configDirMustBe?: string; claudeConfig?: Partial; + modelCapabilities?: ModelCapabilities; }, effectFn: (textGeneration: TextGeneration.TextGeneration["Service"]) => Effect.Effect, ) { @@ -184,7 +185,12 @@ function withFakeClaudeEnv( ); const config = decodeClaudeSettings(input.claudeConfig ?? {}); - const textGeneration = yield* makeClaudeTextGeneration(config); + const modelCapabilities = input.modelCapabilities; + const textGeneration = yield* makeClaudeTextGeneration( + config, + undefined, + modelCapabilities ? () => Effect.succeed(modelCapabilities) : undefined, + ); return yield* effectFn(textGeneration); }).pipe(Effect.scoped); } @@ -255,6 +261,108 @@ it.layer(ClaudeTextGenerationTestLayer)("ClaudeTextGeneration", (it) => { ), ); + it.effect("preserves xhigh effort for custom Claude models", () => + withFakeClaudeEnv( + { + output: JSON.stringify({ + structured_output: { + title: "Use custom reasoning effort", + }, + }), + argsMustContain: "--model gpt-5.6-sol --effort xhigh", + modelCapabilities: createModelCapabilities({ + optionDescriptors: [ + { + id: "effort", + label: "Reasoning", + type: "select", + options: [{ id: "xhigh", label: "Extra High" }], + }, + ], + }), + }, + (textGeneration) => + Effect.gen(function* () { + const generated = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Name this thread.", + modelSelection: createModelSelection( + ProviderInstanceId.make("claudeAgent"), + "gpt-5.6-sol", + [{ id: "effort", value: "xhigh" }], + ), + }); + + expect(generated.title).toBe("Use custom reasoning effort"); + }), + ), + ); + + it.effect("ignores custom Claude effort levels not advertised by the SDK", () => + withFakeClaudeEnv( + { + output: JSON.stringify({ + structured_output: { + title: "Use advertised custom reasoning", + }, + }), + argsMustContain: "--model gpt-5.6-sol", + argsMustNotContain: "--effort", + modelCapabilities: createModelCapabilities({ + optionDescriptors: [ + { + id: "effort", + label: "Reasoning", + type: "select", + options: [{ id: "low", label: "Low" }], + }, + ], + }), + }, + (textGeneration) => + Effect.gen(function* () { + const generated = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Name this thread.", + modelSelection: createModelSelection( + ProviderInstanceId.make("claudeAgent"), + "gpt-5.6-sol", + [{ id: "effort", value: "xhigh" }], + ), + }); + + expect(generated.title).toBe("Use advertised custom reasoning"); + }), + ), + ); + + it.effect("does not default effort for custom Claude models", () => + withFakeClaudeEnv( + { + output: JSON.stringify({ + structured_output: { + title: "Use provider reasoning default", + }, + }), + argsMustContain: "--model gpt-5.6-sol", + argsMustNotContain: "--effort", + }, + (textGeneration) => + Effect.gen(function* () { + const generated = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Name this thread.", + modelSelection: createModelSelection( + ProviderInstanceId.make("claudeAgent"), + "gpt-5.6-sol", + ), + }); + + expect(generated.title).toBe("Use provider reasoning default"); + }), + ), + ); + it.effect("generates thread titles through the Claude provider", () => withFakeClaudeEnv( { diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.ts index 453bb62b728..16800f9a4e9 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.ts @@ -13,7 +13,11 @@ import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { type ClaudeSettings, type ModelSelection } from "@t3tools/contracts"; +import { + type ClaudeSettings, + type ModelCapabilities, + type ModelSelection, +} from "@t3tools/contracts"; import { sanitizeBranchFragment, sanitizeFeatureBranchName } from "@t3tools/shared/git"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; @@ -61,6 +65,10 @@ const decodeClaudeOutputEnvelope = Schema.decodeEffect(Schema.fromJsonString(Cla export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(function* ( claudeSettings: ClaudeSettings, environment?: NodeJS.ProcessEnv, + resolveModelCapabilities: ( + model: string | null | undefined, + ) => Effect.Effect = (model) => + Effect.succeed(getClaudeModelCapabilities(model)), ) { const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const claudeEnvironment = yield* makeClaudeEnvironment(claudeSettings, environment); @@ -126,7 +134,7 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu toJsonSchemaObject(outputSchemaJson), "Failed to encode structured output schema.", ); - const caps = getClaudeModelCapabilities(modelSelection.model); + const caps = yield* resolveModelCapabilities(modelSelection.model); const descriptors = getProviderOptionDescriptors({ caps, selections: modelSelection.options,