Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/ai-sdk-usage-contract.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@voltagent/core": minor
---

Expose AI SDK usage semantics in agent `onEnd` outputs: `usage` now represents final-step usage, `totalUsage` represents aggregate usage across all steps, and `steps` is included when available.
31 changes: 27 additions & 4 deletions packages/core/src/agent/agent.spec-d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -714,10 +714,27 @@ describe("Agent Type System", () => {
});

describe("Response Type Tests", () => {
const usage = (inputTokens: number, outputTokens: number, totalTokens: number) => ({
inputTokens,
outputTokens,
totalTokens,
inputTokenDetails: {
noCacheTokens: inputTokens,
cacheReadTokens: 0,
cacheWriteTokens: 0,
},
outputTokenDetails: {
textTokens: outputTokens,
reasoningTokens: 0,
},
});

it("should validate StandardizedTextResult", () => {
const textResult: StandardizedTextResult = {
text: "Generated text",
usage: { promptTokens: 50, completionTokens: 50, totalTokens: 100 },
usage: usage(50, 50, 100),
totalUsage: usage(150, 50, 200),
steps: [],
providerResponse: {},
finishReason: "stop",
warnings: ["Warning 1"],
Expand All @@ -730,7 +747,9 @@ describe("Agent Type System", () => {
it("should validate StreamTextFinishResult", () => {
const streamTextResult: StreamTextFinishResult = {
text: "Streamed text",
usage: { promptTokens: 75, completionTokens: 75, totalTokens: 150 },
usage: usage(75, 75, 150),
totalUsage: usage(175, 75, 250),
steps: [],
finishReason: "length",
providerResponse: {},
warnings: [],
Expand All @@ -743,7 +762,9 @@ describe("Agent Type System", () => {
it("should validate StandardizedObjectResult", () => {
const objectResult: StandardizedObjectResult<{ name: string; age: number }> = {
object: { name: "John", age: 30 },
usage: { promptTokens: 100, completionTokens: 100, totalTokens: 200 },
usage: usage(100, 100, 200),
totalUsage: usage(200, 100, 300),
steps: [],
providerResponse: {},
finishReason: "stop",
warnings: undefined,
Expand All @@ -758,7 +779,9 @@ describe("Agent Type System", () => {
it("should validate StreamObjectFinishResult", () => {
const streamObjectResult: StreamObjectFinishResult<{ items: string[] }> = {
object: { items: ["a", "b", "c"] },
usage: { promptTokens: 125, completionTokens: 125, totalTokens: 250 },
usage: usage(125, 125, 250),
totalUsage: usage(225, 125, 350),
steps: [],
providerResponse: {},
warnings: [],
finishReason: "stop",
Expand Down
76 changes: 76 additions & 0 deletions packages/core/src/agent/agent.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1204,6 +1204,8 @@ Use pandas and summarize findings.`.split("\n"),
};
})(),
usage: Promise.resolve(lastStepUsage),
totalUsage: Promise.resolve(summedUsage),
steps: Promise.resolve([{ usage: lastStepUsage }]),
finishReason: Promise.resolve("stop"),
warnings: [],
toUIMessageStream: vi.fn(),
Expand All @@ -1224,6 +1226,8 @@ Use pandas and summarize findings.`.split("\n"),

const finishPart = parts.find((part) => part.type === "finish");
expect(finishPart?.totalUsage).toEqual(summedUsage);
await expect(result.totalUsage).resolves.toEqual(summedUsage);
await expect(result.steps).resolves.toEqual([{ usage: lastStepUsage }]);
});

it("keeps fullStream intact after probe for ReadableStream-based providers", async () => {
Expand Down Expand Up @@ -3210,6 +3214,78 @@ Use pandas and summarize findings.`.split("\n"),
expect(arg.output.text).toBe("Success response");
});

it("should expose AI SDK final-step usage and aggregate totalUsage in onEnd", async () => {
const onEnd = vi.fn();
const agent = new Agent({
name: "TestAgent",
instructions: "Test",
model: mockModel as any,
hooks: { onEnd },
});

const stepUsages = [
{ inputTokens: 100, outputTokens: 19, totalTokens: 119 },
{ inputTokens: 140, outputTokens: 27, totalTokens: 167 },
{ inputTokens: 190, outputTokens: 32, totalTokens: 222 },
];
const totalUsage = { inputTokens: 430, outputTokens: 78, totalTokens: 508 };
const steps = stepUsages.map((usage, index) => ({
text: index === stepUsages.length - 1 ? "Final response" : "",
content: [],
reasoning: [],
reasoningText: undefined,
files: [],
sources: [],
toolCalls: [],
staticToolCalls: [],
dynamicToolCalls: [],
toolResults: [],
staticToolResults: [],
dynamicToolResults: [],
finishReason: index === stepUsages.length - 1 ? "stop" : "tool-calls",
usage,
warnings: [],
request: {},
response: {
id: `step-${index}`,
modelId: "test-model",
timestamp: new Date(),
messages: [],
},
providerMetadata: undefined,
}));

vi.mocked(ai.generateText).mockResolvedValue({
text: "Final response",
content: [],
reasoning: [],
files: [],
sources: [],
toolCalls: [],
toolResults: [],
finishReason: "stop",
usage: stepUsages[2],
totalUsage,
warnings: [],
request: {},
response: {
id: "test",
modelId: "test-model",
timestamp: new Date(),
messages: [],
},
steps,
} as any);

await agent.generateText("Test");

expect(onEnd).toHaveBeenCalledTimes(1);
const output = onEnd.mock.calls[0]?.[0].output;
expect(output?.usage).toEqual(stepUsages[2]);
expect(output?.totalUsage).toEqual(totalUsage);
expect(output?.steps?.map((step: any) => step.usage)).toEqual(stepUsages);
});

it("should call onStepFinish for multi-step generation", async () => {
const onStepFinish = vi.fn();
const agent = new Agent({
Expand Down
54 changes: 43 additions & 11 deletions packages/core/src/agent/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -700,6 +700,8 @@ export type StreamTextResultWithContext<
readonly textStream: AIStreamTextResult<TOOLS, any>["textStream"];
readonly fullStream: AsyncIterable<VoltAgentTextStreamPart<TOOLS>>;
readonly usage: AIStreamTextResult<TOOLS, any>["usage"];
readonly totalUsage: AIStreamTextResult<TOOLS, any>["totalUsage"];
readonly steps: AIStreamTextResult<TOOLS, any>["steps"];
readonly finishReason: AIStreamTextResult<TOOLS, any>["finishReason"];
// Partial output stream for streaming structured objects
readonly partialOutputStream?: AIStreamTextResult<TOOLS, any>["partialOutputStream"];
Expand Down Expand Up @@ -1504,7 +1506,9 @@ export class Agent {
agent: this,
output: {
text: finalText,
usage: usageInfo,
usage: providerUsage,
totalUsage: (result as { totalUsage?: LanguageModelUsage }).totalUsage,
steps: result.steps,
providerResponse: result.response,
finishReason: result.finishReason,
warnings: result.warnings,
Expand Down Expand Up @@ -1662,6 +1666,20 @@ export class Agent {
completionTokens: 0,
totalTokens: 0,
};
const providerUsage: LanguageModelUsage = {
inputTokens: 0,
outputTokens: 0,
totalTokens: 0,
inputTokenDetails: {
noCacheTokens: 0,
cacheReadTokens: 0,
cacheWriteTokens: 0,
},
outputTokenDetails: {
textTokens: 0,
reasoningTokens: 0,
},
};

// Apply guardrails to bailed result
const finalText = await executeOutputGuardrails({
Expand All @@ -1683,7 +1701,8 @@ export class Agent {
agent: this,
output: {
text: finalText,
usage: usageInfo,
usage: providerUsage,
totalUsage: providerUsage,
providerResponse: undefined as any,
finishReason: "bail" as any,
warnings: undefined,
Expand All @@ -1698,7 +1717,8 @@ export class Agent {
// Return bailed result as successful generation
return {
text: finalText,
usage: usageInfo,
usage: providerUsage,
totalUsage: providerUsage,
finishReason: "bail" as any,
warnings: undefined,
response: {} as any,
Expand Down Expand Up @@ -2184,10 +2204,10 @@ export class Agent {
}
}

const usage = convertUsage(usageForFinish);
const usageInfo = convertUsage(usageForFinish);
const persistedAssistantMetadata = this.buildPersistedAssistantMessageMetadata({
oc,
usage,
usage: usageInfo,
finishReason: finalResult.finishReason ?? null,
});
this.applyMetadataToLastAssistantMessage({
Expand Down Expand Up @@ -2227,7 +2247,7 @@ export class Agent {
operation: "streamText",
agent: this,
metadata: {
usage,
usage: usageInfo,
finishReason: "bail" as any,
warnings: finalResult.warnings ?? null,
},
Expand All @@ -2245,7 +2265,7 @@ export class Agent {
operation: "streamText",
agent: this,
metadata: {
usage,
usage: usageInfo,
finishReason: finalResult.finishReason ?? null,
warnings: finalResult.warnings ?? null,
},
Expand Down Expand Up @@ -2284,7 +2304,9 @@ export class Agent {
agent: this,
output: {
text: finalText,
usage,
usage: providerUsage,
totalUsage: finalResult.totalUsage,
steps: finalResult.steps,
providerResponse: finalResult.response,
finishReason: finalResult.finishReason,
warnings: finalResult.warnings,
Expand All @@ -2299,7 +2321,7 @@ export class Agent {
await userOnFinish(guardrailedResult);
}

const tokenInfo = usage ? `${usage.totalTokens} tokens` : "no usage data";
const tokenInfo = usageInfo ? `${usageInfo.totalTokens} tokens` : "no usage data";
methodLogger.debug(
buildAgentLogMessage(
this.name,
Expand Down Expand Up @@ -2794,6 +2816,12 @@ export class Agent {
get usage() {
return result.usage;
},
get totalUsage() {
return result.totalUsage;
},
get steps() {
return result.steps;
},
get finishReason() {
return result.finishReason;
},
Expand Down Expand Up @@ -3094,7 +3122,9 @@ export class Agent {
agent: this,
output: {
object: finalObject,
usage: usageInfo,
usage: providerUsage,
totalUsage: (result as { totalUsage?: LanguageModelUsage }).totalUsage,
steps: (result as { steps?: ReadonlyArray<StepResult<ToolSet>> }).steps,
providerResponse: (result as any).response,
finishReason: result.finishReason,
warnings: result.warnings,
Expand Down Expand Up @@ -3527,7 +3557,9 @@ export class Agent {
agent: this,
output: {
object: finalObject,
usage: usageInfo,
usage: providerUsage,
totalUsage: (finalResult as { totalUsage?: LanguageModelUsage }).totalUsage,
steps: (finalResult as { steps?: ReadonlyArray<StepResult<ToolSet>> }).steps,
providerResponse: finalResult.response,
finishReason: finalResult.finishReason,
warnings: finalResult.warnings,
Expand Down
11 changes: 9 additions & 2 deletions packages/core/src/agent/hooks/index.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,15 @@ describe("Agent Hooks Functionality", () => {
expect(arg.output.finishReason).toBe("stop");
expect(arg.output.usage).toEqual(
expect.objectContaining({
promptTokens: 10,
completionTokens: 5,
inputTokens: 10,
outputTokens: 5,
totalTokens: 15,
}),
);
expect(arg.output.totalUsage).toEqual(
expect.objectContaining({
inputTokens: 10,
outputTokens: 5,
totalTokens: 15,
}),
);
Expand Down
2 changes: 2 additions & 0 deletions packages/core/src/agent/subagent/test-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ export function createMockAgentWithStubs(options: CreateMockAgentOptions = {}) {
textStream: textStream as any,
text: Promise.resolve(textContent),
usage: Promise.resolve(createMockUsage()),
totalUsage: Promise.resolve(createMockUsage()),
steps: Promise.resolve([]),
finishReason: Promise.resolve("stop"),
context: new Map(),
partialOutputStream: undefined,
Expand Down
Loading
Loading