diff --git a/.changeset/ai-sdk-usage-contract.md b/.changeset/ai-sdk-usage-contract.md new file mode 100644 index 000000000..b6018e910 --- /dev/null +++ b/.changeset/ai-sdk-usage-contract.md @@ -0,0 +1,5 @@ +--- +"@voltagent/core": minor +--- + +Expose AI SDK usage semantics in agent `onEnd` outputs: `usage` now represents final-step usage, `totalUsage` represents aggregate usage across all steps, and `steps` is included when available. diff --git a/packages/core/src/agent/agent.spec-d.ts b/packages/core/src/agent/agent.spec-d.ts index d531cc234..51c0ef040 100644 --- a/packages/core/src/agent/agent.spec-d.ts +++ b/packages/core/src/agent/agent.spec-d.ts @@ -714,10 +714,27 @@ describe("Agent Type System", () => { }); describe("Response Type Tests", () => { + const usage = (inputTokens: number, outputTokens: number, totalTokens: number) => ({ + inputTokens, + outputTokens, + totalTokens, + inputTokenDetails: { + noCacheTokens: inputTokens, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + outputTokenDetails: { + textTokens: outputTokens, + reasoningTokens: 0, + }, + }); + it("should validate StandardizedTextResult", () => { const textResult: StandardizedTextResult = { text: "Generated text", - usage: { promptTokens: 50, completionTokens: 50, totalTokens: 100 }, + usage: usage(50, 50, 100), + totalUsage: usage(150, 50, 200), + steps: [], providerResponse: {}, finishReason: "stop", warnings: ["Warning 1"], @@ -730,7 +747,9 @@ describe("Agent Type System", () => { it("should validate StreamTextFinishResult", () => { const streamTextResult: StreamTextFinishResult = { text: "Streamed text", - usage: { promptTokens: 75, completionTokens: 75, totalTokens: 150 }, + usage: usage(75, 75, 150), + totalUsage: usage(175, 75, 250), + steps: [], finishReason: "length", providerResponse: {}, warnings: [], @@ -743,7 +762,9 @@ describe("Agent Type System", () => { it("should validate StandardizedObjectResult", () => { const objectResult: StandardizedObjectResult<{ name: string; age: number }> = { object: { name: "John", age: 30 }, - usage: { promptTokens: 100, completionTokens: 100, totalTokens: 200 }, + usage: usage(100, 100, 200), + totalUsage: usage(200, 100, 300), + steps: [], providerResponse: {}, finishReason: "stop", warnings: undefined, @@ -758,7 +779,9 @@ describe("Agent Type System", () => { it("should validate StreamObjectFinishResult", () => { const streamObjectResult: StreamObjectFinishResult<{ items: string[] }> = { object: { items: ["a", "b", "c"] }, - usage: { promptTokens: 125, completionTokens: 125, totalTokens: 250 }, + usage: usage(125, 125, 250), + totalUsage: usage(225, 125, 350), + steps: [], providerResponse: {}, warnings: [], finishReason: "stop", diff --git a/packages/core/src/agent/agent.spec.ts b/packages/core/src/agent/agent.spec.ts index 20ceedfe6..e2572f4fd 100644 --- a/packages/core/src/agent/agent.spec.ts +++ b/packages/core/src/agent/agent.spec.ts @@ -1204,6 +1204,8 @@ Use pandas and summarize findings.`.split("\n"), }; })(), usage: Promise.resolve(lastStepUsage), + totalUsage: Promise.resolve(summedUsage), + steps: Promise.resolve([{ usage: lastStepUsage }]), finishReason: Promise.resolve("stop"), warnings: [], toUIMessageStream: vi.fn(), @@ -1224,6 +1226,8 @@ Use pandas and summarize findings.`.split("\n"), const finishPart = parts.find((part) => part.type === "finish"); expect(finishPart?.totalUsage).toEqual(summedUsage); + await expect(result.totalUsage).resolves.toEqual(summedUsage); + await expect(result.steps).resolves.toEqual([{ usage: lastStepUsage }]); }); it("keeps fullStream intact after probe for ReadableStream-based providers", async () => { @@ -3210,6 +3214,78 @@ Use pandas and summarize findings.`.split("\n"), expect(arg.output.text).toBe("Success response"); }); + it("should expose AI SDK final-step usage and aggregate totalUsage in onEnd", async () => { + const onEnd = vi.fn(); + const agent = new Agent({ + name: "TestAgent", + instructions: "Test", + model: mockModel as any, + hooks: { onEnd }, + }); + + const stepUsages = [ + { inputTokens: 100, outputTokens: 19, totalTokens: 119 }, + { inputTokens: 140, outputTokens: 27, totalTokens: 167 }, + { inputTokens: 190, outputTokens: 32, totalTokens: 222 }, + ]; + const totalUsage = { inputTokens: 430, outputTokens: 78, totalTokens: 508 }; + const steps = stepUsages.map((usage, index) => ({ + text: index === stepUsages.length - 1 ? "Final response" : "", + content: [], + reasoning: [], + reasoningText: undefined, + files: [], + sources: [], + toolCalls: [], + staticToolCalls: [], + dynamicToolCalls: [], + toolResults: [], + staticToolResults: [], + dynamicToolResults: [], + finishReason: index === stepUsages.length - 1 ? "stop" : "tool-calls", + usage, + warnings: [], + request: {}, + response: { + id: `step-${index}`, + modelId: "test-model", + timestamp: new Date(), + messages: [], + }, + providerMetadata: undefined, + })); + + vi.mocked(ai.generateText).mockResolvedValue({ + text: "Final response", + content: [], + reasoning: [], + files: [], + sources: [], + toolCalls: [], + toolResults: [], + finishReason: "stop", + usage: stepUsages[2], + totalUsage, + warnings: [], + request: {}, + response: { + id: "test", + modelId: "test-model", + timestamp: new Date(), + messages: [], + }, + steps, + } as any); + + await agent.generateText("Test"); + + expect(onEnd).toHaveBeenCalledTimes(1); + const output = onEnd.mock.calls[0]?.[0].output; + expect(output?.usage).toEqual(stepUsages[2]); + expect(output?.totalUsage).toEqual(totalUsage); + expect(output?.steps?.map((step: any) => step.usage)).toEqual(stepUsages); + }); + it("should call onStepFinish for multi-step generation", async () => { const onStepFinish = vi.fn(); const agent = new Agent({ diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts index c7fd9bb49..9bc2ed8b2 100644 --- a/packages/core/src/agent/agent.ts +++ b/packages/core/src/agent/agent.ts @@ -700,6 +700,8 @@ export type StreamTextResultWithContext< readonly textStream: AIStreamTextResult["textStream"]; readonly fullStream: AsyncIterable>; readonly usage: AIStreamTextResult["usage"]; + readonly totalUsage: AIStreamTextResult["totalUsage"]; + readonly steps: AIStreamTextResult["steps"]; readonly finishReason: AIStreamTextResult["finishReason"]; // Partial output stream for streaming structured objects readonly partialOutputStream?: AIStreamTextResult["partialOutputStream"]; @@ -1504,7 +1506,9 @@ export class Agent { agent: this, output: { text: finalText, - usage: usageInfo, + usage: providerUsage, + totalUsage: (result as { totalUsage?: LanguageModelUsage }).totalUsage, + steps: result.steps, providerResponse: result.response, finishReason: result.finishReason, warnings: result.warnings, @@ -1662,6 +1666,20 @@ export class Agent { completionTokens: 0, totalTokens: 0, }; + const providerUsage: LanguageModelUsage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + inputTokenDetails: { + noCacheTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + outputTokenDetails: { + textTokens: 0, + reasoningTokens: 0, + }, + }; // Apply guardrails to bailed result const finalText = await executeOutputGuardrails({ @@ -1683,7 +1701,8 @@ export class Agent { agent: this, output: { text: finalText, - usage: usageInfo, + usage: providerUsage, + totalUsage: providerUsage, providerResponse: undefined as any, finishReason: "bail" as any, warnings: undefined, @@ -1698,7 +1717,8 @@ export class Agent { // Return bailed result as successful generation return { text: finalText, - usage: usageInfo, + usage: providerUsage, + totalUsage: providerUsage, finishReason: "bail" as any, warnings: undefined, response: {} as any, @@ -2184,10 +2204,10 @@ export class Agent { } } - const usage = convertUsage(usageForFinish); + const usageInfo = convertUsage(usageForFinish); const persistedAssistantMetadata = this.buildPersistedAssistantMessageMetadata({ oc, - usage, + usage: usageInfo, finishReason: finalResult.finishReason ?? null, }); this.applyMetadataToLastAssistantMessage({ @@ -2227,7 +2247,7 @@ export class Agent { operation: "streamText", agent: this, metadata: { - usage, + usage: usageInfo, finishReason: "bail" as any, warnings: finalResult.warnings ?? null, }, @@ -2245,7 +2265,7 @@ export class Agent { operation: "streamText", agent: this, metadata: { - usage, + usage: usageInfo, finishReason: finalResult.finishReason ?? null, warnings: finalResult.warnings ?? null, }, @@ -2284,7 +2304,9 @@ export class Agent { agent: this, output: { text: finalText, - usage, + usage: providerUsage, + totalUsage: finalResult.totalUsage, + steps: finalResult.steps, providerResponse: finalResult.response, finishReason: finalResult.finishReason, warnings: finalResult.warnings, @@ -2299,7 +2321,7 @@ export class Agent { await userOnFinish(guardrailedResult); } - const tokenInfo = usage ? `${usage.totalTokens} tokens` : "no usage data"; + const tokenInfo = usageInfo ? `${usageInfo.totalTokens} tokens` : "no usage data"; methodLogger.debug( buildAgentLogMessage( this.name, @@ -2794,6 +2816,12 @@ export class Agent { get usage() { return result.usage; }, + get totalUsage() { + return result.totalUsage; + }, + get steps() { + return result.steps; + }, get finishReason() { return result.finishReason; }, @@ -3094,7 +3122,9 @@ export class Agent { agent: this, output: { object: finalObject, - usage: usageInfo, + usage: providerUsage, + totalUsage: (result as { totalUsage?: LanguageModelUsage }).totalUsage, + steps: (result as { steps?: ReadonlyArray> }).steps, providerResponse: (result as any).response, finishReason: result.finishReason, warnings: result.warnings, @@ -3527,7 +3557,9 @@ export class Agent { agent: this, output: { object: finalObject, - usage: usageInfo, + usage: providerUsage, + totalUsage: (finalResult as { totalUsage?: LanguageModelUsage }).totalUsage, + steps: (finalResult as { steps?: ReadonlyArray> }).steps, providerResponse: finalResult.response, finishReason: finalResult.finishReason, warnings: finalResult.warnings, diff --git a/packages/core/src/agent/hooks/index.spec.ts b/packages/core/src/agent/hooks/index.spec.ts index 60b6bb5d2..00ca7a048 100644 --- a/packages/core/src/agent/hooks/index.spec.ts +++ b/packages/core/src/agent/hooks/index.spec.ts @@ -95,8 +95,15 @@ describe("Agent Hooks Functionality", () => { expect(arg.output.finishReason).toBe("stop"); expect(arg.output.usage).toEqual( expect.objectContaining({ - promptTokens: 10, - completionTokens: 5, + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + }), + ); + expect(arg.output.totalUsage).toEqual( + expect.objectContaining({ + inputTokens: 10, + outputTokens: 5, totalTokens: 15, }), ); diff --git a/packages/core/src/agent/subagent/test-utils.ts b/packages/core/src/agent/subagent/test-utils.ts index c2bf651fd..7cb1557c1 100644 --- a/packages/core/src/agent/subagent/test-utils.ts +++ b/packages/core/src/agent/subagent/test-utils.ts @@ -249,6 +249,8 @@ export function createMockAgentWithStubs(options: CreateMockAgentOptions = {}) { textStream: textStream as any, text: Promise.resolve(textContent), usage: Promise.resolve(createMockUsage()), + totalUsage: Promise.resolve(createMockUsage()), + steps: Promise.resolve([]), finishReason: Promise.resolve("stop"), context: new Map(), partialOutputStream: undefined, diff --git a/packages/core/src/agent/types.ts b/packages/core/src/agent/types.ts index 24ea5a84d..77be82be3 100644 --- a/packages/core/src/agent/types.ts +++ b/packages/core/src/agent/types.ts @@ -13,7 +13,7 @@ import type { } from "../agent/providers/base/types"; import type { PrepareStep, StopWhen } from "../ai-types"; -import type { LanguageModel, UIMessage } from "ai"; +import type { LanguageModel, LanguageModelUsage, StepResult, ToolSet, UIMessage } from "ai"; import type { Memory } from "../memory"; import type { BaseRetriever } from "../retriever/retriever"; import type { ProviderTool, Tool, Toolkit, VercelTool } from "../tool"; @@ -1407,8 +1407,14 @@ export interface StreamTextFinishResult { /** The final, consolidated text output from the stream. */ text: string; - /** Token usage information (if available). */ - usage?: UsageInfo; + /** Token usage for the final model step (if available). */ + usage?: LanguageModelUsage; + + /** Aggregate token usage across all model steps (if available). */ + totalUsage?: LanguageModelUsage; + + /** AI SDK step results for the generation (if available). */ + steps?: ReadonlyArray>; /** Feedback metadata for the trace, if enabled. */ feedback?: AgentFeedbackMetadata | null; @@ -1440,8 +1446,14 @@ export interface StreamObjectFinishResult { /** The final, fully formed object from the stream. */ object: TObject; - /** Token usage information (if available). */ - usage?: UsageInfo; + /** Token usage for the final model step (if available). */ + usage?: LanguageModelUsage; + + /** Aggregate token usage across all model steps (if available). */ + totalUsage?: LanguageModelUsage; + + /** AI SDK step results for the generation (if available). */ + steps?: ReadonlyArray>; /** The original completion response object from the provider (if available). */ providerResponse?: unknown; @@ -1470,8 +1482,12 @@ export type StreamObjectOnFinishCallback = ( export interface StandardizedTextResult { /** The generated text. */ text: string; - /** Token usage information (if available). */ - usage?: UsageInfo; + /** Token usage for the final model step (if available). */ + usage?: LanguageModelUsage; + /** Aggregate token usage across all model steps (if available). */ + totalUsage?: LanguageModelUsage; + /** AI SDK step results for the generation (if available). */ + steps?: ReadonlyArray>; /** Feedback metadata for the trace, if enabled. */ feedback?: AgentFeedbackMetadata | null; /** Original provider response (if needed). */ @@ -1491,8 +1507,12 @@ export interface StandardizedTextResult { export interface StandardizedObjectResult { /** The generated object. */ object: TObject; - /** Token usage information (if available). */ - usage?: UsageInfo; + /** Token usage for the final model step (if available). */ + usage?: LanguageModelUsage; + /** Aggregate token usage across all model steps (if available). */ + totalUsage?: LanguageModelUsage; + /** AI SDK step results for the generation (if available). */ + steps?: ReadonlyArray>; /** Original provider response (if needed). */ providerResponse?: unknown; /** Finish reason (if available from provider). */ diff --git a/website/docs/agents/context.md b/website/docs/agents/context.md index 241e0ed12..aa1ef2418 100644 --- a/website/docs/agents/context.md +++ b/website/docs/agents/context.md @@ -390,7 +390,8 @@ const auditHooks = createHooks({ conversationId: context.conversationId, input: context.input, output: context.output, - usage: output?.usage, + finalStepUsage: output?.usage, + totalUsage: output?.totalUsage, }); }, }); diff --git a/website/docs/agents/hooks.md b/website/docs/agents/hooks.md index a6388936a..62bb18561 100644 --- a/website/docs/agents/hooks.md +++ b/website/docs/agents/hooks.md @@ -93,9 +93,9 @@ const myAgentHooks = createHooks({ console.error(`[Hook] Error Details:`, JSON.stringify(error, null, 2)); } else if (output) { console.log(`[Hook] Agent ${agent.name} finished successfully.`); - // Log usage or inspect output type - if ("usage" in output && output.usage) { - console.log(`[Hook] Token Usage: ${output.usage.totalTokens}`); + // Log aggregate usage or inspect output type + if ("totalUsage" in output && output.totalUsage) { + console.log(`[Hook] Total Token Usage: ${output.totalUsage.totalTokens}`); } if ("text" in output && output.text) { console.log(`[Hook] Final text length: ${output.text.length}`); @@ -355,8 +355,8 @@ onEnd: async ({ agent, output, error, conversationId, context }) => { }); // Log usage if available - if (output?.usage) { - console.log(` Usage: ${output.usage.totalTokens} tokens`); + if (output?.totalUsage) { + console.log(` Total usage: ${output.totalUsage.totalTokens} tokens`); } } }; @@ -588,8 +588,8 @@ const enhancedHooks = createHooks({ onEnd: async ({ output, context }) => { console.log(`Messages processed for operation ${context.operationId}`); - if (output?.usage) { - console.log(`Tokens used: ${output.usage.totalTokens}`); + if (output?.totalUsage) { + console.log(`Tokens used: ${output.totalUsage.totalTokens}`); } }, }); @@ -609,15 +609,28 @@ const agent = new Agent({ The `output` parameter structure depends on the agent method called. Check for `text` or `object` fields: +For token accounting, VoltAgent exposes the AI SDK usage contract directly: + +- `output.usage` is the final step usage, matching AI SDK `result.usage`. +- `output.totalUsage` is the aggregate usage across all steps, matching AI SDK `result.totalUsage`. +- `output.steps?.[index].usage` is the per-step usage, matching AI SDK `result.steps[].usage`. + +In a single-step run, `usage` and `totalUsage` are usually the same. In a multi-step tool run, use `totalUsage` for total cost or billing, `usage` for the final model step, and `steps[].usage` for step-level analysis. + ```ts const hooks = createHooks({ onEnd: async ({ output }) => { if (!output) return; // operation failed or was aborted - // Log usage if available - if (output.usage) { - console.log(`Total tokens: ${output.usage.totalTokens}`); - } + const finalStepUsage = output.usage; + const aggregateUsage = output.totalUsage; + const perStepUsage = output.steps?.map((step) => step.usage); + + console.log({ + finalStepTokens: finalStepUsage?.totalTokens, + aggregateTokens: aggregateUsage?.totalTokens, + stepTokens: perStepUsage?.map((usage) => usage.totalTokens), + }); // Handle text results if ("text" in output && output.text) { diff --git a/website/docs/agents/overview.md b/website/docs/agents/overview.md index cffc6b8b8..4d3bcae98 100644 --- a/website/docs/agents/overview.md +++ b/website/docs/agents/overview.md @@ -121,13 +121,15 @@ const response = await agent.streamText("Explain async/await"); })(); // Access final values (resolve when stream completes) -const [fullText, usage, finishReason] = await Promise.all([ +const [fullText, finalStepUsage, totalUsage, finishReason] = await Promise.all([ response.text, // Promise - response.usage, // Promise + response.usage, // Promise for the final step + response.totalUsage, // Promise across all steps response.finishReason, // Promise ]); -console.log(`\nTotal: ${fullText.length} chars, ${usage?.totalTokens} tokens`); +console.log(`\nTotal: ${fullText.length} chars, ${totalUsage?.totalTokens} tokens`); +console.log(`Final step tokens: ${finalStepUsage?.totalTokens}`); ``` ### Feedback (optional)