From e0827ccd1dde5b289a445176792183b5bf390ad3 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 13:44:14 +0000 Subject: [PATCH 1/2] fix(ai-gateway): populate is_byok from Vercel gateway modelAttempts --- .../ai-gateway/processUsage.messages.test.ts | 95 +++++++++++++++++++ .../lib/ai-gateway/processUsage.messages.ts | 3 +- .../ai-gateway/processUsage.responses.test.ts | 32 +++++++ .../lib/ai-gateway/processUsage.responses.ts | 3 +- .../src/lib/ai-gateway/processUsage.shared.ts | 23 +++++ apps/web/src/lib/ai-gateway/processUsage.ts | 27 ++++-- .../src/lib/ai-gateway/processUsage.types.ts | 20 +++- ...ercel-messages.log.resp.json.approved.json | 2 +- ...vercel-messages.log.resp.sse.approved.json | 2 +- ...rcel-responses.log.resp.json.approved.json | 2 +- ...ercel-responses.log.resp.sse.approved.json | 2 +- 11 files changed, 198 insertions(+), 13 deletions(-) diff --git a/apps/web/src/lib/ai-gateway/processUsage.messages.test.ts b/apps/web/src/lib/ai-gateway/processUsage.messages.test.ts index 6661be2768..dc9c09ae59 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.messages.test.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.messages.test.ts @@ -91,6 +91,101 @@ describe('processMessagesApiUsage', () => { expect(result.cacheWriteTokens).toBe(0); }); + test('extracts is_byok=true from Vercel modelAttempts credentialType', () => { + const usage = { + input_tokens: 10, + output_tokens: 5, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + server_tool_use: { input_tokens: 0, web_fetch_requests: 0, web_search_requests: 0 }, + }; + const providerMetadata = { + gateway: { + routing: { + finalProvider: 'bedrock', + modelAttempts: [ + { + success: true, + providerAttempts: [ + { provider: 'bedrock', credentialType: 'byok', success: true }, + ], + }, + ], + }, + cost: '0', + marketCost: '0.000402', + }, + }; + + const result = processMessagesApiUsage(usage, providerMetadata, coreProps); + + expect(result.is_byok).toBe(true); + expect(result.cost_mUsd).toBe(402); + }); + + test('extracts is_byok=false from Vercel system credentialType', () => { + const usage = { + input_tokens: 10, + output_tokens: 5, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + server_tool_use: { input_tokens: 0, web_fetch_requests: 0, web_search_requests: 0 }, + }; + const providerMetadata = { + gateway: { + routing: { + finalProvider: 'bedrock', + modelAttempts: [ + { + success: true, + providerAttempts: [ + { provider: 'bedrock', credentialType: 'system', success: true }, + ], + }, + ], + }, + cost: '0', + marketCost: '0.000402', + }, + }; + + const result = processMessagesApiUsage(usage, providerMetadata, coreProps); + + expect(result.is_byok).toBe(false); + }); + + test('picks the successful provider attempt when earlier attempts failed', () => { + const usage = { + input_tokens: 10, + output_tokens: 5, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + server_tool_use: { input_tokens: 0, web_fetch_requests: 0, web_search_requests: 0 }, + }; + const providerMetadata = { + gateway: { + routing: { + finalProvider: 'anthropic', + modelAttempts: [ + { + success: true, + providerAttempts: [ + { provider: 'bedrock', credentialType: 'byok', success: false }, + { provider: 'anthropic', credentialType: 'system', success: true }, + ], + }, + ], + }, + cost: '0', + marketCost: '0.000402', + }, + }; + + const result = processMessagesApiUsage(usage, providerMetadata, coreProps); + + expect(result.is_byok).toBe(false); + }); + test('returns zero cost when no usage or metadata is provided', () => { const result = processMessagesApiUsage(null, null, coreProps); diff --git a/apps/web/src/lib/ai-gateway/processUsage.messages.ts b/apps/web/src/lib/ai-gateway/processUsage.messages.ts index 4102742cf4..67af350043 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.messages.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.messages.ts @@ -15,6 +15,7 @@ import { computeOpenRouterCostFields, computeVercelCostMicrodollars, drainSseStream, + extractVercelIsByok, } from '@/lib/ai-gateway/processUsage.shared'; import type Anthropic from '@anthropic-ai/sdk'; @@ -61,7 +62,7 @@ export function processMessagesApiUsage( cacheHitTokens, cacheWriteTokens, cost_mUsd, - is_byok: null, + is_byok: extractVercelIsByok(vercelGateway), }; } diff --git a/apps/web/src/lib/ai-gateway/processUsage.responses.test.ts b/apps/web/src/lib/ai-gateway/processUsage.responses.test.ts index 470074171d..bd517c47cb 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.responses.test.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.responses.test.ts @@ -97,6 +97,38 @@ describe('processResponsesApiUsage', () => { expect(result.inputTokens).toBe(0); expect(result.outputTokens).toBe(0); }); + + test('extracts is_byok=true from Vercel modelAttempts credentialType', () => { + const usage = { + input_tokens: 10, + output_tokens: 5, + total_tokens: 15, + input_tokens_details: { cached_tokens: 0 }, + output_tokens_details: { reasoning_tokens: 0 }, + }; + const providerMetadata = { + gateway: { + routing: { + finalProvider: 'openai', + modelAttempts: [ + { + success: true, + providerAttempts: [ + { provider: 'openai', credentialType: 'byok', success: true }, + ], + }, + ], + }, + cost: '0', + marketCost: '0.0001', + }, + }; + + const result = processResponsesApiUsage(usage, providerMetadata, coreProps); + + expect(result.is_byok).toBe(true); + expect(result.cost_mUsd).toBe(100); + }); }); describe('parseMicrodollarUsageFromStream approval tests', () => { diff --git a/apps/web/src/lib/ai-gateway/processUsage.responses.ts b/apps/web/src/lib/ai-gateway/processUsage.responses.ts index ce90ff9dee..eb8e2eae3d 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.responses.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.responses.ts @@ -16,6 +16,7 @@ import { computeOpenRouterCostFields, computeVercelCostMicrodollars, drainSseStream, + extractVercelIsByok, } from '@/lib/ai-gateway/processUsage.shared'; // OpenRouter adds cost fields to the standard Responses API usage object. @@ -72,7 +73,7 @@ export function processResponsesApiUsage( cacheHitTokens, cacheWriteTokens: 0, cost_mUsd, - is_byok: null, + is_byok: extractVercelIsByok(vercelGateway), }; } diff --git a/apps/web/src/lib/ai-gateway/processUsage.shared.ts b/apps/web/src/lib/ai-gateway/processUsage.shared.ts index 39c2443697..aafd404042 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.shared.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.shared.ts @@ -61,6 +61,29 @@ export function computeVercelCostMicrodollars( return toMicrodollars(isNaN(marketCost_USD) ? 0 : marketCost_USD); } +/** + * Extracts whether the Vercel AI Gateway served the request with BYOK credentials. + * + * The gateway reports per-attempt `credentialType` ("byok" | "system") in + * `provider_metadata.gateway.routing.modelAttempts[].providerAttempts[]`. We look + * at the successful provider attempt within the successful model attempt. Returns + * `null` when credentialType is absent or no successful attempt is found. + */ +export function extractVercelIsByok( + vercelGateway: NonNullable | undefined | null +): boolean | null { + const modelAttempts = vercelGateway?.routing?.modelAttempts; + if (!modelAttempts) return null; + const successfulModel = modelAttempts.find(m => m.success) ?? modelAttempts.at(-1); + const providerAttempts = successfulModel?.providerAttempts; + if (!providerAttempts) return null; + const successfulProvider = providerAttempts.find(p => p.success) ?? providerAttempts.at(-1); + const credentialType = successfulProvider?.credentialType; + if (credentialType === 'byok') return true; + if (credentialType === 'system') return false; + return null; +} + /** * Drains a ReadableStream of binary chunks, calling `onTextChunk` for each * decoded piece of text. Handles client-abort (`ResponseAborted`) gracefully diff --git a/apps/web/src/lib/ai-gateway/processUsage.ts b/apps/web/src/lib/ai-gateway/processUsage.ts index 4a5e4d1b2f..f45ff1cd00 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.ts @@ -43,6 +43,7 @@ import type { OpenRouterUsage, PromptInfo, UsageMetaData, + VercelProviderMetaData, } from '@/lib/ai-gateway/processUsage.types'; import { parseResponsesMicrodollarUsageFromStream, @@ -53,7 +54,11 @@ import { parseMessagesMicrodollarUsageFromString, } from '@/lib/ai-gateway/processUsage.messages'; import { OPENROUTER_BYOK_COST_MULTIPLIER } from '@/lib/ai-gateway/processUsage.constants'; -import { computeOpenRouterCostFields, drainSseStream } from '@/lib/ai-gateway/processUsage.shared'; +import { + computeOpenRouterCostFields, + drainSseStream, + extractVercelIsByok, +} from '@/lib/ai-gateway/processUsage.shared'; import { isClaudeModel } from '@/lib/ai-gateway/providers/anthropic.constants'; import { isMinimaxModel } from '@/lib/ai-gateway/providers/minimax'; import type { KiloExclusiveModel } from '@/lib/ai-gateway/providers/kilo-exclusive-model'; @@ -653,7 +658,8 @@ export function countAndStoreUsage( export function processOpenRouterUsage( usage: OpenRouterUsage | null | undefined, - coreProps: NotYetCostedUsageStats + coreProps: NotYetCostedUsageStats, + vercelProviderMetadata?: VercelProviderMetaData | null ): JustTheCostsUsageStats { // usage may be null when there's no response (e.g. error), so default to empty object const { cost_mUsd, is_byok } = computeOpenRouterCostFields( @@ -671,7 +677,7 @@ export function processOpenRouterUsage( 0, outputTokens: usage?.completion_tokens ?? 0, cost_mUsd, - is_byok, + is_byok: is_byok ?? extractVercelIsByok(vercelProviderMetadata?.gateway), }; } @@ -703,6 +709,7 @@ export async function parseMicrodollarUsageFromStream( let usage: OpenRouterUsage | null = null; let inference_provider: string | null = null; let finish_reason: string | null = null; + let vercelProviderMetadata: VercelProviderMetaData | null = null; const sseStreamParser = createParser({ onEvent(event: EventSourceMessage) { @@ -744,9 +751,13 @@ export async function parseMicrodollarUsageFromStream( messageId = json.id ?? messageId; usage = json.usage ?? usage; const choice = json.choices?.[0]; + const chunkProviderMetadata = choice?.delta?.provider_metadata; + if (chunkProviderMetadata) { + vercelProviderMetadata = chunkProviderMetadata; + } inference_provider = json.provider ?? - choice?.delta?.provider_metadata?.gateway?.routing?.finalProvider ?? + chunkProviderMetadata?.gateway?.routing?.finalProvider ?? inference_provider; finish_reason = choice?.finish_reason ?? finish_reason; @@ -788,7 +799,7 @@ export async function parseMicrodollarUsageFromStream( status_code: effectiveStatusCode, }; - const costs = processOpenRouterUsage(usage, coreProps); + const costs = processOpenRouterUsage(usage, coreProps, vercelProviderMetadata); return { ...coreProps, ...costs }; } @@ -831,7 +842,11 @@ export function parseMicrodollarUsageFromString( status_code: statusCode, }; - const costs = processOpenRouterUsage(responseJson?.usage, coreProps); + const costs = processOpenRouterUsage( + responseJson?.usage, + coreProps, + choice?.message?.provider_metadata ?? null + ); return { ...coreProps, ...costs }; } diff --git a/apps/web/src/lib/ai-gateway/processUsage.types.ts b/apps/web/src/lib/ai-gateway/processUsage.types.ts index 4586c61c66..f88ea58ec6 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.types.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.types.ts @@ -19,8 +19,26 @@ export type OpenRouterUsage = { total_tokens: number; }; //ref: https://openrouter.ai/docs/use-cases/usage-accounting#response-format +export type VercelProviderAttempt = { + provider?: string; + credentialType?: string; + success?: boolean; +}; + +export type VercelModelAttempt = { + success?: boolean; + providerAttempts?: VercelProviderAttempt[]; +}; + export type VercelProviderMetaData = { - gateway?: { routing?: { finalProvider?: string }; cost?: string; marketCost?: string }; + gateway?: { + routing?: { + finalProvider?: string; + modelAttempts?: VercelModelAttempt[]; + }; + cost?: string; + marketCost?: string; + }; }; export type MaybeHasVercelProviderMetaData = { diff --git a/apps/web/src/tests/sample/vercel-messages.log.resp.json.approved.json b/apps/web/src/tests/sample/vercel-messages.log.resp.json.approved.json index caa324d4b7..747e80249a 100644 --- a/apps/web/src/tests/sample/vercel-messages.log.resp.json.approved.json +++ b/apps/web/src/tests/sample/vercel-messages.log.resp.json.approved.json @@ -17,5 +17,5 @@ "cacheHitTokens": 0, "cacheWriteTokens": 0, "cost_mUsd": 402, - "is_byok": null + "is_byok": true } \ No newline at end of file diff --git a/apps/web/src/tests/sample/vercel-messages.log.resp.sse.approved.json b/apps/web/src/tests/sample/vercel-messages.log.resp.sse.approved.json index b4a387010b..7326191747 100644 --- a/apps/web/src/tests/sample/vercel-messages.log.resp.sse.approved.json +++ b/apps/web/src/tests/sample/vercel-messages.log.resp.sse.approved.json @@ -17,5 +17,5 @@ "cacheHitTokens": 0, "cacheWriteTokens": 0, "cost_mUsd": 402, - "is_byok": null + "is_byok": true } \ No newline at end of file diff --git a/apps/web/src/tests/sample/vercel-responses.log.resp.json.approved.json b/apps/web/src/tests/sample/vercel-responses.log.resp.json.approved.json index aff2eb4bbd..3208289a7c 100644 --- a/apps/web/src/tests/sample/vercel-responses.log.resp.json.approved.json +++ b/apps/web/src/tests/sample/vercel-responses.log.resp.json.approved.json @@ -17,5 +17,5 @@ "cacheHitTokens": 0, "cacheWriteTokens": 0, "cost_mUsd": 6138, - "is_byok": null + "is_byok": true } \ No newline at end of file diff --git a/apps/web/src/tests/sample/vercel-responses.log.resp.sse.approved.json b/apps/web/src/tests/sample/vercel-responses.log.resp.sse.approved.json index 6fa5bddc9b..a9a01fcafd 100644 --- a/apps/web/src/tests/sample/vercel-responses.log.resp.sse.approved.json +++ b/apps/web/src/tests/sample/vercel-responses.log.resp.sse.approved.json @@ -17,5 +17,5 @@ "cacheHitTokens": 0, "cacheWriteTokens": 0, "cost_mUsd": 6138, - "is_byok": null + "is_byok": true } \ No newline at end of file From be79110063ed4e74148ccad7d40da91aae290dff Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 14:18:12 +0000 Subject: [PATCH 2/2] chore: pnpm format --- apps/web/src/lib/ai-gateway/processUsage.messages.test.ts | 8 ++------ .../web/src/lib/ai-gateway/processUsage.responses.test.ts | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/apps/web/src/lib/ai-gateway/processUsage.messages.test.ts b/apps/web/src/lib/ai-gateway/processUsage.messages.test.ts index dc9c09ae59..35ef9f284a 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.messages.test.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.messages.test.ts @@ -106,9 +106,7 @@ describe('processMessagesApiUsage', () => { modelAttempts: [ { success: true, - providerAttempts: [ - { provider: 'bedrock', credentialType: 'byok', success: true }, - ], + providerAttempts: [{ provider: 'bedrock', credentialType: 'byok', success: true }], }, ], }, @@ -138,9 +136,7 @@ describe('processMessagesApiUsage', () => { modelAttempts: [ { success: true, - providerAttempts: [ - { provider: 'bedrock', credentialType: 'system', success: true }, - ], + providerAttempts: [{ provider: 'bedrock', credentialType: 'system', success: true }], }, ], }, diff --git a/apps/web/src/lib/ai-gateway/processUsage.responses.test.ts b/apps/web/src/lib/ai-gateway/processUsage.responses.test.ts index bd517c47cb..3bf4659207 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.responses.test.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.responses.test.ts @@ -113,9 +113,7 @@ describe('processResponsesApiUsage', () => { modelAttempts: [ { success: true, - providerAttempts: [ - { provider: 'openai', credentialType: 'byok', success: true }, - ], + providerAttempts: [{ provider: 'openai', credentialType: 'byok', success: true }], }, ], },