-
Notifications
You must be signed in to change notification settings - Fork 31
feat(agent): emit per-category context token breakdown #2352
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| import type { AcpMessage } from "@shared/types/session-events"; | ||
| import { describe, expect, it } from "vitest"; | ||
| import { extractContextUsage } from "./useContextUsage"; | ||
|
|
||
| function usageUpdateEvent(used: number, size: number): AcpMessage { | ||
| return { | ||
| type: "acp_message", | ||
| ts: 1, | ||
| message: { | ||
| jsonrpc: "2.0", | ||
| method: "session/update", | ||
| params: { | ||
| sessionId: "s1", | ||
| update: { sessionUpdate: "usage_update", used, size }, | ||
| }, | ||
| }, | ||
| }; | ||
| } | ||
|
|
||
| function breakdownEvent( | ||
| breakdown: Record<string, number>, | ||
| method = "_posthog/usage_update", | ||
| ): AcpMessage { | ||
| return { | ||
| type: "acp_message", | ||
| ts: 1, | ||
| message: { jsonrpc: "2.0", method, params: { sessionId: "s1", breakdown } }, | ||
| }; | ||
| } | ||
|
|
||
| describe("extractContextUsage", () => { | ||
| it("returns null with no usage event", () => { | ||
| expect(extractContextUsage([])).toBeNull(); | ||
| }); | ||
|
|
||
| it("derives aggregate from the latest session/update", () => { | ||
| const result = extractContextUsage([usageUpdateEvent(50_000, 200_000)]); | ||
| expect(result?.used).toBe(50_000); | ||
| expect(result?.size).toBe(200_000); | ||
| expect(result?.percentage).toBe(25); | ||
| expect(result?.breakdown).toBeNull(); | ||
| }); | ||
|
|
||
| it("merges breakdown from a _posthog/usage_update notification", () => { | ||
| const result = extractContextUsage([ | ||
| usageUpdateEvent(50_000, 200_000), | ||
| breakdownEvent({ | ||
| systemPrompt: 4000, | ||
| tools: 500, | ||
| rules: 0, | ||
| skills: 0, | ||
| mcp: 0, | ||
| subagents: 0, | ||
| conversation: 45_500, | ||
| }), | ||
| ]); | ||
| expect(result?.breakdown?.systemPrompt).toBe(4000); | ||
| expect(result?.breakdown?.conversation).toBe(45_500); | ||
| }); | ||
|
|
||
| it("tolerates the double-underscore method prefix from extNotification", () => { | ||
| const result = extractContextUsage([ | ||
| usageUpdateEvent(50_000, 200_000), | ||
| breakdownEvent( | ||
| { | ||
| systemPrompt: 4000, | ||
| tools: 0, | ||
| rules: 0, | ||
| skills: 0, | ||
| mcp: 0, | ||
| subagents: 0, | ||
| conversation: 46_000, | ||
| }, | ||
| "__posthog/usage_update", | ||
| ), | ||
| ]); | ||
| expect(result?.breakdown?.systemPrompt).toBe(4000); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,84 @@ | ||||||||||||||||||||||
| import { describe, expect, it } from "vitest"; | ||||||||||||||||||||||
| import { | ||||||||||||||||||||||
| buildBreakdown, | ||||||||||||||||||||||
| emptyBaseline, | ||||||||||||||||||||||
| estimateJsonTokens, | ||||||||||||||||||||||
| estimateSystemPrompt, | ||||||||||||||||||||||
| estimateTokens, | ||||||||||||||||||||||
| } from "./context-breakdown"; | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| describe("estimateTokens", () => { | ||||||||||||||||||||||
| it("returns 0 for empty input", () => { | ||||||||||||||||||||||
| expect(estimateTokens("")).toBe(0); | ||||||||||||||||||||||
| expect(estimateTokens(undefined)).toBe(0); | ||||||||||||||||||||||
| expect(estimateTokens(null)).toBe(0); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| it("scales roughly with input length", () => { | ||||||||||||||||||||||
| expect(estimateTokens("a".repeat(35))).toBe(10); | ||||||||||||||||||||||
| expect(estimateTokens("a".repeat(350))).toBe(100); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
Comment on lines
+17
to
+20
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Context Used: Do not attempt to comment on incorrect alphabetica... (source) Prompt To Fix With AIThis is a comment left during a code review.
Path: packages/agent/src/adapters/claude/context-breakdown.test.ts
Line: 17-20
Comment:
The two assertions here are separate data points for the same behaviour and should use `it.each` per the team's preference for parameterised tests. The same applies to the "returns 0 for empty input" block above.
```suggestion
it.each([
[35, 10],
[350, 100],
])("scales roughly with input length (%i chars → %i tokens)", (chars, expected) => {
expect(estimateTokens("a".repeat(chars))).toBe(expected);
});
```
**Context Used:** Do not attempt to comment on incorrect alphabetica... ([source](https://app.greptile.com/review/custom-context?memory=instruction-0))
How can I resolve this? If you propose a fix, please make it concise.Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time! |
||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| describe("estimateJsonTokens", () => { | ||||||||||||||||||||||
| it("counts JSON representation of objects", () => { | ||||||||||||||||||||||
| const tokens = estimateJsonTokens({ name: "Read", schema: { foo: 1 } }); | ||||||||||||||||||||||
| expect(tokens).toBeGreaterThan(0); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| it("returns 0 for non-serializable values", () => { | ||||||||||||||||||||||
| const circular: Record<string, unknown> = {}; | ||||||||||||||||||||||
| circular.self = circular; | ||||||||||||||||||||||
| expect(estimateJsonTokens(circular)).toBe(0); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| describe("estimateSystemPrompt", () => { | ||||||||||||||||||||||
| it("includes the Claude preset budget when preset is used", () => { | ||||||||||||||||||||||
| const noAppend = estimateSystemPrompt({ type: "preset" }); | ||||||||||||||||||||||
| expect(noAppend).toBeGreaterThan(0); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| it("adds the append portion on top of the preset", () => { | ||||||||||||||||||||||
| const append = "a".repeat(350); | ||||||||||||||||||||||
| const result = estimateSystemPrompt({ type: "preset", append }); | ||||||||||||||||||||||
| const presetOnly = estimateSystemPrompt({ type: "preset" }); | ||||||||||||||||||||||
| expect(result - presetOnly).toBe(100); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| it("counts a raw string verbatim with no preset overhead", () => { | ||||||||||||||||||||||
| expect(estimateSystemPrompt("a".repeat(350))).toBe(100); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| it("treats undefined as the bare preset", () => { | ||||||||||||||||||||||
| expect(estimateSystemPrompt(undefined)).toBe( | ||||||||||||||||||||||
| estimateSystemPrompt({ type: "preset" }), | ||||||||||||||||||||||
| ); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| describe("buildBreakdown", () => { | ||||||||||||||||||||||
| it("derives conversation from input - stable sum", () => { | ||||||||||||||||||||||
| const baseline = { | ||||||||||||||||||||||
| ...emptyBaseline(), | ||||||||||||||||||||||
| systemPrompt: 4000, | ||||||||||||||||||||||
| tools: 500, | ||||||||||||||||||||||
| }; | ||||||||||||||||||||||
| const result = buildBreakdown(baseline, 10_000); | ||||||||||||||||||||||
| expect(result.systemPrompt).toBe(4000); | ||||||||||||||||||||||
| expect(result.tools).toBe(500); | ||||||||||||||||||||||
| expect(result.conversation).toBe(5500); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| it("floors conversation at 0 when stable pieces exceed input", () => { | ||||||||||||||||||||||
| const baseline = { ...emptyBaseline(), systemPrompt: 5000 }; | ||||||||||||||||||||||
| expect(buildBreakdown(baseline, 1000).conversation).toBe(0); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| it("includes zero categories", () => { | ||||||||||||||||||||||
| const result = buildBreakdown(emptyBaseline(), 100); | ||||||||||||||||||||||
| expect(result.mcp).toBe(0); | ||||||||||||||||||||||
| expect(result.skills).toBe(0); | ||||||||||||||||||||||
| expect(result.subagents).toBe(0); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
| }); | ||||||||||||||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
it.eachto give each input its own test name.Context Used: Do not attempt to comment on incorrect alphabetica... (source)
Prompt To Fix With AI
Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!