From 54873106c58838b022a17b4a21af8ee957d18545 Mon Sep 17 00:00:00 2001 From: CHAEWAN KIM <66085474+amsminn@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:31:11 +0900 Subject: [PATCH] feat(coding-agent): cap inline transport images at 24 MiB --- packages/coding-agent/src/core/changes.md | 22 ++ packages/coding-agent/src/core/messages.ts | 129 +++++++++++ packages/coding-agent/src/core/sdk.ts | 43 +--- .../coding-agent/test/image-elision.test.ts | 205 ++++++++++++++++++ packages/coding-agent/test/suite/harness.ts | 9 +- .../test/transport-image-budget.test.ts | 104 +++++++++ 6 files changed, 474 insertions(+), 38 deletions(-) create mode 100644 packages/coding-agent/test/image-elision.test.ts create mode 100644 packages/coding-agent/test/transport-image-budget.test.ts diff --git a/packages/coding-agent/src/core/changes.md b/packages/coding-agent/src/core/changes.md index 94ab111c8..116eea332 100644 --- a/packages/coding-agent/src/core/changes.md +++ b/packages/coding-agent/src/core/changes.md @@ -1,5 +1,27 @@ # changes +## Provider-bound inline image budget (2026-07-22) + +### What changed + +- `messages.ts`: added a transport-only 24 MiB inline image budget. Provider-bound conversion keeps the newest image + block, counts it against the budget, and replaces images older than the hard recency cutoff with a re-read + placeholder while preserving all text and leaving the persisted session untouched. +- `sdk.ts`: routes the main agent loop through the shared transport conversion while preserving the dynamic + `images.blockImages` kill switch and its existing placeholder/deduplication behavior. +- `test/suite/harness.ts`: uses the same transport conversion and accepts a small injectable image budget for + deterministic first-request integration coverage. + +### Why extension system couldn't handle this alone + +- Inline images must be bounded after session messages are converted but before every main-loop provider request, + including resumed sessions and provider fallbacks. That conversion boundary is owned by the core Agent wiring. + +### Expected merge conflict zones + +- MEDIUM: `sdk.ts` around the Agent `convertToLlm` wiring. +- LOW: the transport helpers at the end of `messages.ts` and the Agent construction in `test/suite/harness.ts`. + ## Streaming steer/followUp submissions bypass the session-work barrier (2026-07-21) ### What changed diff --git a/packages/coding-agent/src/core/messages.ts b/packages/coding-agent/src/core/messages.ts index 17d65c6eb..a35b0389d 100644 --- a/packages/coding-agent/src/core/messages.ts +++ b/packages/coding-agent/src/core/messages.ts @@ -207,3 +207,132 @@ export function convertToLlm(messages: AgentMessage[]): Message[] { }) .filter((m) => m !== undefined); } + +// ============================================================================ +// Transport image budget +// ============================================================================ + +/** + * Cap on inline image base64 (characters approximately equal wire bytes) per + * provider request. Anthropic documents a 32 MB request limit; reserving the + * remainder for text and request overhead keeps the transport below that wall. + */ +export const TRANSPORT_IMAGE_BUDGET_BYTES = 24 * 1024 * 1024; + +export const IMAGE_ELISION_PLACEHOLDER = + "[Image elided: an older image was removed to keep the request within the provider's size limit. Re-read the source file if you need to view it again.]"; + +export const BLOCKED_IMAGE_PLACEHOLDER = "Image reading is disabled."; + +export interface ElideOldImagesOptions { + /** Cumulative inline image base64 budget. Defaults to TRANSPORT_IMAGE_BUDGET_BYTES. */ + budgetBytes?: number; + /** Newest image blocks always kept regardless of budget. They still consume it. Defaults to 1. */ + alwaysKeepNewest?: number; +} + +export interface TransportConvertOptions extends ElideOldImagesOptions { + /** Replace every image when the images.blockImages setting is enabled. */ + blockImages: boolean; +} + +/** Drop consecutive duplicates of a placeholder produced by adjacent image replacements. */ +export function dedupeConsecutivePlaceholder( + content: (TextContent | ImageContent)[], + placeholder: string, +): (TextContent | ImageContent)[] { + return content.filter( + (block, index, blocks) => + !( + block.type === "text" && + block.text === placeholder && + index > 0 && + blocks[index - 1].type === "text" && + (blocks[index - 1] as TextContent).text === placeholder + ), + ); +} + +/** + * Bound inline image payload at request-build time. Images are considered from + * newest to oldest. Once an image would exceed the budget, it and every older + * image are replaced by a text placeholder. The newest protected blocks are + * kept regardless of size, but still consume the budget. + * + * The input and persisted session remain untouched. If every image fits, the + * original array reference is returned. + */ +export function elideOldImages(messages: Message[], options?: ElideOldImagesOptions): Message[] { + const budgetBytes = options?.budgetBytes ?? TRANSPORT_IMAGE_BUDGET_BYTES; + const alwaysKeepNewest = options?.alwaysKeepNewest ?? 1; + const imagesToElide = new Set(); + let keptImages = 0; + let imageBytes = 0; + let cutoffReached = false; + + for (let messageIndex = messages.length - 1; messageIndex >= 0; messageIndex--) { + const message = messages[messageIndex]; + if ((message.role !== "user" && message.role !== "toolResult") || !Array.isArray(message.content)) { + continue; + } + + for (let blockIndex = message.content.length - 1; blockIndex >= 0; blockIndex--) { + const block = message.content[blockIndex]; + if (block.type !== "image") continue; + + if (cutoffReached) { + imagesToElide.add(`${messageIndex}:${blockIndex}`); + continue; + } + + if (keptImages < alwaysKeepNewest || imageBytes + block.data.length <= budgetBytes) { + keptImages++; + imageBytes += block.data.length; + continue; + } + + cutoffReached = true; + imagesToElide.add(`${messageIndex}:${blockIndex}`); + } + } + + if (imagesToElide.size === 0) return messages; + + return messages.map((message, messageIndex) => { + if ((message.role !== "user" && message.role !== "toolResult") || !Array.isArray(message.content)) { + return message; + } + if (!message.content.some((_block, blockIndex) => imagesToElide.has(`${messageIndex}:${blockIndex}`))) { + return message; + } + + const replaced = message.content.map((block, blockIndex): TextContent | ImageContent => + imagesToElide.has(`${messageIndex}:${blockIndex}`) ? { type: "text", text: IMAGE_ELISION_PLACEHOLDER } : block, + ); + return { + ...message, + content: dedupeConsecutivePlaceholder(replaced, IMAGE_ELISION_PLACEHOLDER) as typeof message.content, + }; + }); +} + +/** Convert messages for the main provider transport and apply its image policy. */ +export function convertToLlmForTransport(messages: AgentMessage[], options: TransportConvertOptions): Message[] { + const converted = convertToLlm(messages); + if (!options.blockImages) return elideOldImages(converted, options); + + return converted.map((message) => { + if ((message.role !== "user" && message.role !== "toolResult") || !Array.isArray(message.content)) { + return message; + } + if (!message.content.some((block) => block.type === "image")) return message; + + const replaced = message.content.map((block): TextContent | ImageContent => + block.type === "image" ? { type: "text", text: BLOCKED_IMAGE_PLACEHOLDER } : block, + ); + return { + ...message, + content: dedupeConsecutivePlaceholder(replaced, BLOCKED_IMAGE_PLACEHOLDER) as typeof message.content, + }; + }); +} diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 6be984407..5328a768c 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -9,7 +9,7 @@ import { AuthStorage } from "./auth-storage.ts"; import { DEFAULT_THINKING_LEVEL } from "./defaults.ts"; import type { ServiceTier } from "./extensions/builtin/service-tier.ts"; import type { ExtensionRunner, LoadExtensionsResult, SessionStartEvent, ToolDefinition } from "./extensions/index.ts"; -import { convertToLlm } from "./messages.ts"; +import { convertToLlmForTransport, TRANSPORT_IMAGE_BUDGET_BYTES } from "./messages.ts"; import { ModelRegistry } from "./model-registry.ts"; import { findInitialModel, getModelNarrowingPatterns, resolveModelScope } from "./model-resolver.ts"; import { ModelRuntime } from "./model-runtime.ts"; @@ -293,42 +293,13 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} let agent: Agent; - // Create convertToLlm wrapper that filters images if blockImages is enabled (defense-in-depth) - const convertToLlmWithBlockImages = (messages: AgentMessage[]): Message[] => { - const converted = convertToLlm(messages); - // Check setting dynamically so mid-session changes take effect - if (!settingsManager.getBlockImages()) { - return converted; - } - // Filter out ImageContent from all messages, replacing with text placeholder - return converted.map((msg) => { - if (msg.role === "user" || msg.role === "toolResult") { - const content = msg.content; - if (Array.isArray(content)) { - const hasImages = content.some((c) => c.type === "image"); - if (hasImages) { - const filteredContent = content - .map((c) => - c.type === "image" ? { type: "text" as const, text: "Image reading is disabled." } : c, - ) - .filter( - (c, i, arr) => - // Dedupe consecutive "Image reading is disabled." texts - !( - c.type === "text" && - c.text === "Image reading is disabled." && - i > 0 && - arr[i - 1].type === "text" && - (arr[i - 1] as { type: "text"; text: string }).text === "Image reading is disabled." - ), - ); - return { ...msg, content: filteredContent }; - } - } - } - return msg; + // Read blockImages per request so a mid-session settings change takes effect. + const convertToLlmWithBlockImages = (messages: AgentMessage[]): Message[] => + convertToLlmForTransport(messages, { + blockImages: settingsManager.getBlockImages(), + budgetBytes: TRANSPORT_IMAGE_BUDGET_BYTES, + alwaysKeepNewest: 1, }); - }; const extensionRunnerRef: { current?: ExtensionRunner } = {}; diff --git a/packages/coding-agent/test/image-elision.test.ts b/packages/coding-agent/test/image-elision.test.ts new file mode 100644 index 000000000..40ab46f1e --- /dev/null +++ b/packages/coding-agent/test/image-elision.test.ts @@ -0,0 +1,205 @@ +import type { AgentMessage } from "@earendil-works/pi-agent-core"; +import type { Message } from "@earendil-works/pi-ai"; +import { describe, expect, it } from "vitest"; +import { + BLOCKED_IMAGE_PLACEHOLDER, + convertToLlmForTransport, + elideOldImages, + IMAGE_ELISION_PLACEHOLDER, + TRANSPORT_IMAGE_BUDGET_BYTES, +} from "../src/core/messages.ts"; + +function img(data: string) { + return { type: "image", data, mimeType: "image/png" } as const; +} + +function toolResult(...blocks: unknown[]): Message { + return { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: blocks, + } as unknown as Message; +} + +function agentToolResult(...blocks: unknown[]): AgentMessage { + return { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: blocks, + timestamp: 1, + } as unknown as AgentMessage; +} + +describe("elideOldImages", () => { + it("returns the same reference when there are no images", () => { + const messages = [{ role: "user", content: "hi" }] as unknown as Message[]; + + expect(elideOldImages(messages, { budgetBytes: 0, alwaysKeepNewest: 0 })).toBe(messages); + }); + + it("returns the same reference when all images fit the budget", () => { + const messages = [toolResult(img("A".repeat(100)))]; + + expect(elideOldImages(messages, { budgetBytes: 1000, alwaysKeepNewest: 0 })).toBe(messages); + }); + + it("uses a hard recency cutoff after the budget is exceeded", () => { + const messages = [ + toolResult(img("A".repeat(60))), + toolResult(img("B".repeat(60))), + toolResult(img("C".repeat(60))), + ]; + + const result = elideOldImages(messages, { budgetBytes: 130, alwaysKeepNewest: 0 }); + + expect(result.map((message) => (message.content as Array<{ type: string }>)[0].type)).toEqual([ + "text", + "image", + "image", + ]); + expect((result[0].content as Array<{ type: string; text?: string }>)[0].text).toBe(IMAGE_ELISION_PLACEHOLDER); + }); + + it("keeps the newest block regardless of size and charges it to the budget", () => { + const messages = [ + toolResult(img("A".repeat(10))), + toolResult(img("B".repeat(10))), + toolResult(img("HUGE".repeat(100))), + ]; + + const result = elideOldImages(messages, { budgetBytes: 50, alwaysKeepNewest: 1 }); + + expect(result.map((message) => (message.content as Array<{ type: string }>)[0].type)).toEqual([ + "text", + "text", + "image", + ]); + }); + + it("preserves sibling text blocks verbatim when eliding", () => { + const messages = [ + toolResult({ type: "text", text: "Read image file [image/png]" }, img("A".repeat(60))), + toolResult(img("B".repeat(60))), + ]; + + const result = elideOldImages(messages, { budgetBytes: 60, alwaysKeepNewest: 0 }); + + expect(result[0].content).toEqual([ + { type: "text", text: "Read image file [image/png]" }, + { type: "text", text: IMAGE_ELISION_PLACEHOLDER }, + ]); + }); + + it("dedupes consecutive placeholders within one message", () => { + const messages = [ + { role: "user", content: [img("A".repeat(60)), img("B".repeat(60))] } as unknown as Message, + toolResult(img("C".repeat(60))), + ]; + + const result = elideOldImages(messages, { budgetBytes: 60, alwaysKeepNewest: 0 }); + + expect(result[0].content).toEqual([{ type: "text", text: IMAGE_ELISION_PLACEHOLDER }]); + }); + + it("treats later blocks within a message as newer", () => { + const messages = [{ role: "user", content: [img("A".repeat(60)), img("B".repeat(60))] } as unknown as Message]; + + const result = elideOldImages(messages, { budgetBytes: 60, alwaysKeepNewest: 0 }); + const content = result[0].content as Array<{ type: string; data?: string }>; + + expect(content.map((block) => block.type)).toEqual(["text", "image"]); + expect(content[1].data).toBe("B".repeat(60)); + }); + + it("tracks repeated image objects by block position", () => { + const repeated = img("A".repeat(60)); + const messages = [{ role: "user", content: [repeated, repeated] } as unknown as Message]; + + const result = elideOldImages(messages, { budgetBytes: 60, alwaysKeepNewest: 0 }); + + expect((result[0].content as Array<{ type: string }>).map((block) => block.type)).toEqual(["text", "image"]); + }); + + it("leaves assistant messages and string-content user messages untouched", () => { + const assistant = { + role: "assistant", + content: [{ type: "text", text: "hi" }], + } as unknown as Message; + const stringUser = { role: "user", content: "plain" } as unknown as Message; + const messages = [assistant, stringUser, toolResult(img("A".repeat(10)))]; + + expect(elideOldImages(messages, { budgetBytes: 1000, alwaysKeepNewest: 0 })).toBe(messages); + }); + + it("exposes the documented production budget", () => { + expect(TRANSPORT_IMAGE_BUDGET_BYTES).toBe(24 * 1024 * 1024); + }); + + it("uses the production budget by default", () => { + const messages = [toolResult(img("A".repeat(10)))]; + + expect(elideOldImages(messages)).toBe(messages); + }); + + it("keeps one newest image by default", () => { + const messages = [toolResult(img("A")), toolResult(img("B"))]; + + const result = elideOldImages(messages, { budgetBytes: 0 }); + + expect(result.map((message) => (message.content as Array<{ type: string }>)[0].type)).toEqual(["text", "image"]); + }); +}); + +describe("convertToLlmForTransport", () => { + it("passes convertToLlm output through when images fit", () => { + const messages = [agentToolResult({ type: "text", text: "n" }, img("A".repeat(10)))]; + + const result = convertToLlmForTransport(messages, { + blockImages: false, + budgetBytes: 1000, + alwaysKeepNewest: 0, + }); + + expect(result[0].content).toEqual([{ type: "text", text: "n" }, img("A".repeat(10))]); + }); + + it("elides old images and keeps the newest when over budget", () => { + const messages = [agentToolResult(img("A".repeat(60))), agentToolResult(img("B".repeat(60)))]; + + const result = convertToLlmForTransport(messages, { + blockImages: false, + budgetBytes: 60, + alwaysKeepNewest: 1, + }); + + expect((result[0].content as Array<{ type: string; text?: string }>)[0]).toEqual({ + type: "text", + text: IMAGE_ELISION_PLACEHOLDER, + }); + expect((result[1].content as Array<{ type: string }>)[0].type).toBe("image"); + }); + + it("makes blockImages override budget options across messages while preserving text and dedupe", () => { + const messages = [ + agentToolResult({ type: "text", text: "first" }, img("A".repeat(10)), img("B".repeat(10))), + agentToolResult(img("C".repeat(10)), { type: "text", text: "last" }), + ]; + + const result = convertToLlmForTransport(messages, { + blockImages: true, + budgetBytes: 0, + alwaysKeepNewest: 1, + }); + + expect(result[0].content).toEqual([ + { type: "text", text: "first" }, + { type: "text", text: BLOCKED_IMAGE_PLACEHOLDER }, + ]); + expect(result[1].content).toEqual([ + { type: "text", text: BLOCKED_IMAGE_PLACEHOLDER }, + { type: "text", text: "last" }, + ]); + }); +}); diff --git a/packages/coding-agent/test/suite/harness.ts b/packages/coding-agent/test/suite/harness.ts index 26a4f300f..bec285b60 100644 --- a/packages/coding-agent/test/suite/harness.ts +++ b/packages/coding-agent/test/suite/harness.ts @@ -18,7 +18,7 @@ import { registerFauxProvider } from "@earendil-works/pi-ai/compat"; import { AgentSession, type AgentSessionEvent } from "../../src/core/agent-session.ts"; import { AuthStorage } from "../../src/core/auth-storage.ts"; import type { ExtensionRunner } from "../../src/core/extensions/index.ts"; -import { convertToLlm } from "../../src/core/messages.ts"; +import { convertToLlmForTransport } from "../../src/core/messages.ts"; import { SessionManager } from "../../src/core/session-manager.ts"; import type { Settings } from "../../src/core/settings-manager.ts"; import { SettingsManager } from "../../src/core/settings-manager.ts"; @@ -81,6 +81,7 @@ export interface HarnessOptions { persistSession?: boolean; autoTitleSessions?: boolean; fallbackNow?: () => number; + transportImageBudget?: { budgetBytes: number; alwaysKeepNewest: number }; } export interface Harness { @@ -163,7 +164,11 @@ export async function createHarness(options: HarnessOptions = {}): Promise + convertToLlmForTransport(messages, { + blockImages: settingsManager.getBlockImages(), + ...options.transportImageBudget, + }), onPayload: async (payload) => { options.onPayload?.(payload); const runner = extensionRunnerRef.current; diff --git a/packages/coding-agent/test/transport-image-budget.test.ts b/packages/coding-agent/test/transport-image-budget.test.ts new file mode 100644 index 000000000..132d4aba4 --- /dev/null +++ b/packages/coding-agent/test/transport-image-budget.test.ts @@ -0,0 +1,104 @@ +import type { AgentMessage } from "@earendil-works/pi-agent-core"; +import { fauxAssistantMessage, type Message } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { IMAGE_ELISION_PLACEHOLDER } from "../src/core/messages.ts"; +import { createHarness, type Harness } from "./suite/harness.ts"; + +function image(data: string) { + return { type: "image", data, mimeType: "image/png" } as const; +} + +type ContentMessage = AgentMessage & { + content: string | Array<{ type: string; text?: string; data?: string; mimeType?: string }>; +}; + +function hasContent(message: AgentMessage): message is ContentMessage { + return "content" in message; +} + +function findSeededMessage(messages: AgentMessage[]): ContentMessage | undefined { + return messages.find( + (message): message is ContentMessage => + hasContent(message) && + message.role === "user" && + Array.isArray(message.content) && + message.content.some((block) => block.type === "text" && block.text === "seed-note"), + ); +} + +function countImages(messages: AgentMessage[]): number { + let count = 0; + for (const message of messages) { + if (!hasContent(message) || !Array.isArray(message.content)) continue; + for (const block of message.content) { + if (block.type === "image") count++; + } + } + return count; +} + +describe("transport image budget (main loop)", () => { + const harnesses: Harness[] = []; + + afterEach(() => { + while (harnesses.length > 0) harnesses.pop()?.cleanup(); + }); + + it("elides old images from the first request when over budget", async () => { + const harness = await createHarness({ + transportImageBudget: { budgetBytes: 100, alwaysKeepNewest: 1 }, + }); + harnesses.push(harness); + const seededMessage: Message = { + role: "user", + content: [ + { type: "text", text: "seed-note" }, + image("A".repeat(60)), + image("B".repeat(60)), + image("C".repeat(60)), + ], + timestamp: 1, + }; + harness.sessionManager.appendMessage(seededMessage); + harness.session.agent.state.messages = harness.sessionManager.buildSessionContext().messages; + harness.setResponses([fauxAssistantMessage("done")]); + + await harness.session.prompt("continue"); + + const callLog = harness.faux.getCallLog(); + expect(callLog).toHaveLength(1); + expect(countImages(callLog[0].context.messages)).toBe(1); + const transported = findSeededMessage(callLog[0].context.messages); + expect(transported?.content).toEqual([ + { type: "text", text: "seed-note" }, + { type: "text", text: IMAGE_ELISION_PLACEHOLDER }, + image("C".repeat(60)), + ]); + + const persisted = findSeededMessage(harness.sessionManager.buildSessionContext().messages); + expect(persisted?.content).toEqual(seededMessage.content); + }); + + it("does not touch requests under the budget", async () => { + const harness = await createHarness({ + transportImageBudget: { budgetBytes: 100, alwaysKeepNewest: 1 }, + }); + harnesses.push(harness); + const seededMessage: Message = { + role: "user", + content: [{ type: "text", text: "seed-note" }, image("A".repeat(10)), image("B".repeat(10))], + timestamp: 1, + }; + harness.sessionManager.appendMessage(seededMessage); + harness.session.agent.state.messages = harness.sessionManager.buildSessionContext().messages; + harness.setResponses([fauxAssistantMessage("done")]); + + await harness.session.prompt("continue"); + + const callLog = harness.faux.getCallLog(); + expect(callLog).toHaveLength(1); + expect(countImages(callLog[0].context.messages)).toBe(2); + expect(findSeededMessage(callLog[0].context.messages)?.content).toEqual(seededMessage.content); + expect(JSON.stringify(callLog[0].context.messages)).not.toContain(IMAGE_ELISION_PLACEHOLDER); + }); +});