diff --git a/.changeset/multimodal-tool-results.md b/.changeset/multimodal-tool-results.md new file mode 100644 index 000000000..f4e414043 --- /dev/null +++ b/.changeset/multimodal-tool-results.md @@ -0,0 +1,17 @@ +--- +'@tanstack/ai': patch +'@tanstack/ai-openai': patch +'@tanstack/ai-anthropic': patch +'@tanstack/ai-client': patch +'@tanstack/ai-react-ui': patch +'@tanstack/ai-solid-ui': patch +'@tanstack/ai-vue-ui': patch +--- + +Preserve multimodal tool results across chat history and provider adapters. + +This fixes tool result handling so `ContentPart[]` outputs are preserved instead +of being stringified in core chat flows, OpenAI Responses tool outputs, and +Anthropic tool results. It also updates the client-side tool result type and +the default React, Solid, and Vue chat message renderers to handle text and +image tool result content. diff --git a/packages/typescript/ai-anthropic/src/adapters/text.ts b/packages/typescript/ai-anthropic/src/adapters/text.ts index 235d9f5b5..e87e305d9 100644 --- a/packages/typescript/ai-anthropic/src/adapters/text.ts +++ b/packages/typescript/ai-anthropic/src/adapters/text.ts @@ -379,7 +379,13 @@ export class AnthropicTextAdapter< type: 'tool_result', tool_use_id: message.toolCallId, content: - typeof message.content === 'string' ? message.content : '', + typeof message.content === 'string' + ? message.content + : Array.isArray(message.content) + ? message.content.map((part) => + this.convertContentPartToAnthropic(part), + ) + : '', }, ], }) diff --git a/packages/typescript/ai-anthropic/tests/anthropic-adapter.test.ts b/packages/typescript/ai-anthropic/tests/anthropic-adapter.test.ts index 76c50ccf2..dc28c8d9f 100644 --- a/packages/typescript/ai-anthropic/tests/anthropic-adapter.test.ts +++ b/packages/typescript/ai-anthropic/tests/anthropic-adapter.test.ts @@ -184,6 +184,95 @@ describe('Anthropic adapter option mapping', () => { }) }) + it('preserves multimodal tool result content for tool_result blocks', async () => { + const mockStream = (async function* () { + yield { + type: 'message_start', + message: { + id: 'msg_123', + model: 'claude-3-7-sonnet-20250219', + role: 'assistant', + type: 'message', + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 0 }, + }, + } + yield { + type: 'message_delta', + delta: { stop_reason: 'end_turn' }, + usage: { output_tokens: 1 }, + } + yield { + type: 'message_stop', + } + })() + + mocks.betaMessagesCreate.mockResolvedValueOnce(mockStream as any) + + const adapter = createAdapter('claude-3-7-sonnet-20250219') + const toolResult = [ + { + type: 'text' as const, + content: 'Screenshot of the current state', + }, + { + type: 'image' as const, + source: { + type: 'url' as const, + value: 'https://example.com/screenshot.png', + }, + }, + ] + + for await (const _chunk of chat({ + adapter, + messages: [ + { role: 'user', content: 'What changed?' }, + { + role: 'assistant', + content: 'Checking', + toolCalls: [ + { + id: 'call_canvas', + type: 'function', + function: { name: 'view_canvas', arguments: '{}' }, + }, + ], + }, + { role: 'tool', toolCallId: 'call_canvas', content: toolResult }, + ], + tools: [weatherTool], + })) { + // drain + } + + const [payload] = mocks.betaMessagesCreate.mock.calls.at(-1) as [any] + expect(payload.messages[2]).toEqual({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call_canvas', + content: [ + { + type: 'text', + text: 'Screenshot of the current state', + }, + { + type: 'image', + source: { + type: 'url', + url: 'https://example.com/screenshot.png', + }, + }, + ], + }, + ], + }) + }) + it('merges consecutive user messages when tool results precede a follow-up user message', async () => { // This is the core multi-turn bug: after a tool call + result, the next user message // creates consecutive role:'user' messages (tool_result as user + new user message). diff --git a/packages/typescript/ai-client/src/types.ts b/packages/typescript/ai-client/src/types.ts index b705ebbdf..9b1a55a52 100644 --- a/packages/typescript/ai-client/src/types.ts +++ b/packages/typescript/ai-client/src/types.ts @@ -13,6 +13,8 @@ import type { } from '@tanstack/ai' import type { ConnectionAdapter } from './connection-adapters' +type ToolResultContent = string | Array + /** * Tool call states - track the lifecycle of a tool call */ @@ -152,7 +154,7 @@ export type ToolCallPart = any> = export interface ToolResultPart { type: 'tool-result' toolCallId: string - content: string + content: ToolResultContent state: ToolResultState error?: string // Error message if state is "error" } diff --git a/packages/typescript/ai-code-mode/models-eval/metrics.ts b/packages/typescript/ai-code-mode/models-eval/metrics.ts index 97a9d2944..3468f612c 100644 --- a/packages/typescript/ai-code-mode/models-eval/metrics.ts +++ b/packages/typescript/ai-code-mode/models-eval/metrics.ts @@ -1,4 +1,4 @@ -import type { UIMessage } from '@tanstack/ai' +import type { ToolResultPart, UIMessage } from '@tanstack/ai' export interface TypeScriptAttempt { toolCallId: string @@ -44,11 +44,33 @@ function safeJsonParse(value: string): unknown { } } +function parseToolResultContent( + content: ToolResultPart['content'], +): + | { + success?: boolean + error?: { name?: string; message?: string } + } + | undefined { + return typeof content === 'string' + ? (safeJsonParse(content) as + | { + success?: boolean + error?: { name?: string; message?: string } + } + | undefined) + : undefined +} + export function computeMetrics(messages: Array): ComputedMetrics { const toolCallLookup = new Map() const toolResultLookup = new Map< string, - { content: string; state?: string; error?: string } + { + content: ToolResultPart['content'] + state?: string + error?: string + } >() let totalToolCalls = 0 @@ -95,13 +117,8 @@ export function computeMetrics(messages: Array): ComputedMetrics { | { typescriptCode?: string } | undefined const result = toolResultLookup.get(toolCallId) - const parsedResult = result?.content - ? (safeJsonParse(result.content) as - | { - success?: boolean - error?: { name?: string; message?: string } - } - | undefined) + const parsedResult = result + ? parseToolResultContent(result.content) : undefined const success = parsedResult?.success diff --git a/packages/typescript/ai-openai/src/adapters/text.ts b/packages/typescript/ai-openai/src/adapters/text.ts index 1747ce4ec..9ca00d0f5 100644 --- a/packages/typescript/ai-openai/src/adapters/text.ts +++ b/packages/typescript/ai-openai/src/adapters/text.ts @@ -703,13 +703,24 @@ export class OpenAITextAdapter< for (const message of messages) { // Handle tool messages - convert to FunctionToolCallOutput if (message.role === 'tool') { + const output = + typeof message.content === 'string' + ? message.content + : this.normalizeContent(message.content).map((part) => + this.convertContentPartToOpenAI( + part as ContentPart< + unknown, + OpenAIImageMetadata, + OpenAIAudioMetadata, + unknown, + unknown + >, + ), + ) result.push({ type: 'function_call_output', call_id: message.toolCallId || '', - output: - typeof message.content === 'string' - ? message.content - : JSON.stringify(message.content), + output, }) continue } diff --git a/packages/typescript/ai-openai/tests/openai-adapter.test.ts b/packages/typescript/ai-openai/tests/openai-adapter.test.ts index 552793a2e..ab883292d 100644 --- a/packages/typescript/ai-openai/tests/openai-adapter.test.ts +++ b/packages/typescript/ai-openai/tests/openai-adapter.test.ts @@ -129,4 +129,85 @@ describe('OpenAI adapter option mapping', () => { expect(Array.isArray(payload.tools)).toBe(true) expect(payload.tools.length).toBeGreaterThan(0) }) + + it('preserves multimodal tool result content for function_call_output', async () => { + const mockStream = createMockChatCompletionsStream([ + { + type: 'response.done', + response: { + id: 'resp-123', + model: 'gpt-4o-mini', + status: 'completed', + created_at: 1234567891, + usage: { + input_tokens: 12, + output_tokens: 0, + }, + }, + }, + ]) + + const responsesCreate = vi.fn().mockResolvedValueOnce(mockStream) + + const adapter = createAdapter('gpt-4o-mini') + ;(adapter as any).client = { + responses: { + create: responsesCreate, + }, + } + + const toolResult = [ + { + type: 'text' as const, + content: 'Screenshot of the current state', + }, + { + type: 'image' as const, + source: { + type: 'url' as const, + value: 'https://example.com/screenshot.png', + }, + }, + ] + + for await (const _chunk of chat({ + adapter, + messages: [ + { role: 'user', content: 'What changed?' }, + { + role: 'assistant', + content: 'Checking', + toolCalls: [ + { + id: 'call_canvas', + type: 'function', + function: { name: 'view_canvas', arguments: '{}' }, + }, + ], + }, + { role: 'tool', toolCallId: 'call_canvas', content: toolResult }, + ], + tools: [weatherTool], + })) { + // drain + } + + const [payload] = responsesCreate.mock.calls[0] + const toolOutput = payload.input.find( + (item: any) => + item.type === 'function_call_output' && item.call_id === 'call_canvas', + ) + + expect(toolOutput.output).toEqual([ + { + type: 'input_text', + text: 'Screenshot of the current state', + }, + { + type: 'input_image', + image_url: 'https://example.com/screenshot.png', + detail: 'auto', + }, + ]) + }) }) diff --git a/packages/typescript/ai-react-ui/src/chat-message.tsx b/packages/typescript/ai-react-ui/src/chat-message.tsx index beacdfae3..18c5d22da 100644 --- a/packages/typescript/ai-react-ui/src/chat-message.tsx +++ b/packages/typescript/ai-react-ui/src/chat-message.tsx @@ -2,6 +2,22 @@ import { ThinkingPart } from './thinking-part' import type { ReactNode } from 'react' import type { UIMessage } from '@tanstack/ai-react' +type ToolResultContent = Extract< + UIMessage['parts'][number], + { type: 'tool-result' } +>['content'] +type ToolResultContentSource = + | { + type: 'url' + value: string + mimeType?: string + } + | { + type: 'data' + value: string + mimeType: string + } + export interface ToolCallRenderProps { id: string name: string @@ -34,11 +50,55 @@ export interface ChatMessageProps { /** Custom renderer for tool result parts */ toolResultRenderer?: (props: { toolCallId: string - content: string + content: ToolResultContent state: string }) => ReactNode } +function getContentPartSourceUrl( + source: ToolResultContentSource, +): string { + if (source.type === 'url') { + return source.value + } + + return source.value.startsWith('data:') + ? source.value + : `data:${source.mimeType};base64,${source.value}` +} + +function renderToolResultContent(content: ToolResultContent): ReactNode { + if (typeof content === 'string') { + return content + } + + return content.map((part, index) => { + switch (part.type) { + case 'text': + return ( +
+ {part.content} +
+ ) + case 'image': + return ( + Tool result + ) + default: + return ( +
+ Unsupported tool result content: {part.type} +
+ ) + } + }) +} + /** * Message component - renders a single message with all its parts * @@ -258,7 +318,7 @@ function MessagePart({ data-tool-call-id={part.toolCallId} data-tool-result-state={part.state} > -
{part.content}
+
{renderToolResultContent(part.content)}
) } diff --git a/packages/typescript/ai-solid-ui/src/chat-message.tsx b/packages/typescript/ai-solid-ui/src/chat-message.tsx index 10b3c5715..8ac782080 100644 --- a/packages/typescript/ai-solid-ui/src/chat-message.tsx +++ b/packages/typescript/ai-solid-ui/src/chat-message.tsx @@ -3,6 +3,22 @@ import { ThinkingPart } from './thinking-part' import type { JSX } from 'solid-js' import type { UIMessage } from '@tanstack/ai-solid' +type ToolResultContent = Extract< + UIMessage['parts'][number], + { type: 'tool-result' } +>['content'] +type ToolResultContentSource = + | { + type: 'url' + value: string + mimeType?: string + } + | { + type: 'data' + value: string + mimeType: string + } + export interface ToolCallRenderProps { id: string name: string @@ -35,11 +51,50 @@ export interface ChatMessageProps { /** Custom renderer for tool result parts */ toolResultRenderer?: (props: { toolCallId: string - content: string + content: ToolResultContent state: string }) => JSX.Element } +function getContentPartSourceUrl( + source: ToolResultContentSource, +): string { + if (source.type === 'url') { + return source.value + } + + return source.value.startsWith('data:') + ? source.value + : `data:${source.mimeType};base64,${source.value}` +} + +function renderToolResultContent(content: ToolResultContent): JSX.Element { + if (typeof content === 'string') { + return <>{content} + } + + return ( + + {(part) => { + switch (part.type) { + case 'text': + return
{part.content}
+ case 'image': + return ( + Tool result + ) + default: + return null + } + }} +
+ ) +} + /** * Message component - renders a single message with all its parts * @@ -249,7 +304,9 @@ function MessagePart(props: { data-tool-call-id={props.part.toolCallId} data-tool-result-state={props.part.state} > -
{props.part.content}
+
+ {renderToolResultContent(props.part.content)} +
) } diff --git a/packages/typescript/ai-vue-ui/src/chat-message.vue b/packages/typescript/ai-vue-ui/src/chat-message.vue index deeaeb03a..0a9cc59ad 100644 --- a/packages/typescript/ai-vue-ui/src/chat-message.vue +++ b/packages/typescript/ai-vue-ui/src/chat-message.vue @@ -1,7 +1,7 @@ diff --git a/packages/typescript/ai-vue-ui/src/types.ts b/packages/typescript/ai-vue-ui/src/types.ts index d4e07419c..1caada595 100644 --- a/packages/typescript/ai-vue-ui/src/types.ts +++ b/packages/typescript/ai-vue-ui/src/types.ts @@ -1,5 +1,10 @@ import type { ConnectionAdapter, UIMessage } from '@tanstack/ai-vue' +export type ToolResultContent = Extract< + UIMessage['parts'][number], + { type: 'tool-result' } +>['content'] + export interface ChatProps { /** CSS class name for the root element */ class?: string diff --git a/packages/typescript/ai/src/activities/chat/index.ts b/packages/typescript/ai/src/activities/chat/index.ts index c7ec866d6..bfe6da197 100644 --- a/packages/typescript/ai/src/activities/chat/index.ts +++ b/packages/typescript/ai/src/activities/chat/index.ts @@ -19,7 +19,11 @@ import { parseWithStandardSchema, } from './tools/schema-converter' import { maxIterations as maxIterationsStrategy } from './agent-loop-strategies' -import { convertMessagesToModelMessages } from './messages' +import { + convertMessagesToModelMessages, + normalizeToolResultContent, + parseToolResultValue, +} from './messages' import { MiddlewareRunner } from './middleware/compose' import type { ApprovalRequest, @@ -1002,12 +1006,7 @@ class TextEngine< // This handles results sent back from the client after executing client-side tools if (message.role === 'tool' && message.toolCallId) { // Parse content back to original output (was stringified by uiMessageToModelMessages) - let output: unknown - try { - output = JSON.parse(message.content as string) - } catch { - output = message.content - } + const output = parseToolResultValue(message.content) // Skip approval response messages (they have pendingExecution marker) // These are NOT real client tool results — they are synthetic tool messages // created by uiMessageToModelMessages for approved-but-not-yet-executed tools. @@ -1084,7 +1083,7 @@ class TextEngine< const chunks: Array = [] for (const result of results) { - const content = JSON.stringify(result.result) + const content = normalizeToolResultContent(result.result) chunks.push({ type: 'TOOL_CALL_END', @@ -1116,17 +1115,11 @@ class TextEngine< for (const message of this.messages) { if (message.role === 'tool' && message.toolCallId) { // Check if this is an approval response with pendingExecution marker - let hasPendingExecution = false - if (typeof message.content === 'string') { - try { - const parsed = JSON.parse(message.content) - if (parsed.pendingExecution === true) { - hasPendingExecution = true - } - } catch { - // Not JSON, treat as regular tool result - } - } + const parsedContent = parseToolResultValue(message.content) + const hasPendingExecution = + parsedContent && + typeof parsedContent === 'object' && + (parsedContent as any).pendingExecution === true // Only mark as complete if NOT pending execution if (!hasPendingExecution) { diff --git a/packages/typescript/ai/src/activities/chat/messages.ts b/packages/typescript/ai/src/activities/chat/messages.ts index b7f97b880..687cefe18 100644 --- a/packages/typescript/ai/src/activities/chat/messages.ts +++ b/packages/typescript/ai/src/activities/chat/messages.ts @@ -1,9 +1,11 @@ import type { ContentPart, + ContentPartSource, MessagePart, ModelMessage, TextPart, ToolCallPart, + ToolResultContent, UIMessage, } from '../../types' // =========================== @@ -24,6 +26,92 @@ function isContentPart(part: MessagePart): part is ContentPart { ) } +function isContentPartSource( + source: unknown, +): source is ContentPartSource { + if (!source || typeof source !== 'object') { + return false + } + + const type = (source as { type?: unknown }).type + const value = (source as { value?: unknown }).value + const mimeType = (source as { mimeType?: unknown }).mimeType + + if (type === 'url') { + return typeof value === 'string' + } + + if (type === 'data') { + return typeof value === 'string' && typeof mimeType === 'string' + } + + return false +} + +function isUnknownContentPart(part: unknown): part is ContentPart { + if (!part || typeof part !== 'object') { + return false + } + + const type = (part as { type?: unknown }).type + if (type === 'text') { + return typeof (part as { content?: unknown }).content === 'string' + } + + if ( + type === 'image' || + type === 'audio' || + type === 'video' || + type === 'document' + ) { + return isContentPartSource((part as { source?: unknown }).source) + } + + return false +} + +export function isContentPartArray( + value: unknown, +): value is Array { + return Array.isArray(value) && value.every(isUnknownContentPart) +} + +export function normalizeToolResultContent( + value: unknown, +): ToolResultContent { + if (typeof value === 'string') { + return value + } + + if (isContentPartArray(value)) { + return value + } + + return JSON.stringify(value) +} + +export function parseToolResultValue(value: unknown): unknown { + if (typeof value !== 'string') { + return value + } + + try { + return JSON.parse(value) + } catch { + return value + } +} + +export function decodeToolResultContent(value: unknown): ToolResultContent { + const parsed = parseToolResultValue(value) + + if (isContentPartArray(parsed)) { + return parsed + } + + return normalizeToolResultContent(value) +} + /** * Collapse an array of ContentParts into the most compact ModelMessage content: * - Empty array → null @@ -57,6 +145,20 @@ function getTextContent(content: string | null | Array): string { .join('') } +function getToolResultContent( + content: string | null | Array, +): ToolResultContent { + if (typeof content === 'string') { + return content + } + + if (Array.isArray(content)) { + return content + } + + return getTextContent(content) +} + /** * Convert UIMessages or ModelMessages to ModelMessages */ @@ -248,7 +350,7 @@ function buildAssistantMessages(uiMessage: UIMessage): Array { if (part.output !== undefined && !emittedToolResultIds.has(part.id)) { messageList.push({ role: 'tool', - content: JSON.stringify(part.output), + content: normalizeToolResultContent(part.output), toolCallId: part.id, }) emittedToolResultIds.add(part.id) @@ -312,7 +414,7 @@ export function modelMessageToUIMessage( parts.push({ type: 'tool-result', toolCallId: modelMessage.toolCallId, - content: getTextContent(modelMessage.content), + content: getToolResultContent(modelMessage.content), state: 'complete', }) } else if (Array.isArray(modelMessage.content)) { @@ -375,7 +477,7 @@ export function modelMessagesToUIMessages( currentAssistantMessage.parts.push({ type: 'tool-result', toolCallId: msg.toolCallId!, - content: getTextContent(msg.content), + content: getToolResultContent(msg.content), state: 'complete', }) } else { diff --git a/packages/typescript/ai/src/activities/chat/stream/message-updaters.ts b/packages/typescript/ai/src/activities/chat/stream/message-updaters.ts index 80b94d59a..e16bfbe6c 100644 --- a/packages/typescript/ai/src/activities/chat/stream/message-updaters.ts +++ b/packages/typescript/ai/src/activities/chat/stream/message-updaters.ts @@ -8,6 +8,7 @@ import type { ThinkingPart, ToolCallPart, + ToolResultContent, ToolResultPart, UIMessage, } from '../../../types' @@ -97,7 +98,7 @@ export function updateToolResultPart( messages: Array, messageId: string, toolCallId: string, - content: string, + content: ToolResultContent, state: ToolResultState, error?: string, ): Array { diff --git a/packages/typescript/ai/src/activities/chat/stream/processor.ts b/packages/typescript/ai/src/activities/chat/stream/processor.ts index caed6a74d..6166a8850 100644 --- a/packages/typescript/ai/src/activities/chat/stream/processor.ts +++ b/packages/typescript/ai/src/activities/chat/stream/processor.ts @@ -17,7 +17,13 @@ * @see docs/chat-architecture.md — Canonical reference for AG-UI chunk ordering, * adapter contract, single-shot flows, and expected UIMessage output. */ -import { generateMessageId, uiMessageToModelMessages } from '../messages.js' +import { + decodeToolResultContent, + generateMessageId, + normalizeToolResultContent, + parseToolResultValue, + uiMessageToModelMessages, +} from '../messages.js' import { defaultJSONParser } from './json-parser' import { updateTextPart, @@ -291,7 +297,7 @@ export class StreamProcessor { ) // Step 2: Create a tool-result part (for LLM conversation history) - const content = typeof output === 'string' ? output : JSON.stringify(output) + const content = normalizeToolResultContent(output) const toolResultState: ToolResultState = error ? 'error' : 'complete' updatedMessages = updateToolResultPart( @@ -984,15 +990,12 @@ export class StreamProcessor { } // Update UIMessage if there's a result - if (chunk.result) { + if (chunk.result !== undefined) { // Step 1: Update the tool-call part's output field (for UI consistency // with client tools — see GitHub issue #176) - let output: unknown - try { - output = JSON.parse(chunk.result) - } catch { - output = chunk.result - } + const output = parseToolResultValue(chunk.result) + const content = decodeToolResultContent(chunk.result) + this.messages = updateToolCallWithOutput( this.messages, chunk.toolCallId, @@ -1005,7 +1008,7 @@ export class StreamProcessor { this.messages, messageId, chunk.toolCallId, - chunk.result, + content, resultState, ) this.emitMessagesChange() diff --git a/packages/typescript/ai/src/activities/chat/tools/tool-calls.ts b/packages/typescript/ai/src/activities/chat/tools/tool-calls.ts index c05b759cf..57cdd73f5 100644 --- a/packages/typescript/ai/src/activities/chat/tools/tool-calls.ts +++ b/packages/typescript/ai/src/activities/chat/tools/tool-calls.ts @@ -1,3 +1,4 @@ +import { normalizeToolResultContent } from '../messages' import { isStandardSchema, parseWithStandardSchema } from './schema-converter' import type { CustomEvent, @@ -9,6 +10,7 @@ import type { ToolCallEndEvent, ToolCallStartEvent, ToolExecutionContext, + ToolResultContent, } from '../../../types' import type { AfterToolCallInfo, @@ -164,7 +166,7 @@ export class ToolCallManager { for (const toolCall of toolCallsArray) { const tool = this.tools.find((t) => t.name === toolCall.function.name) - let toolResultContent: string + let toolResultContent: ToolResultContent if (tool?.execute) { try { // Parse arguments (normalize "null" to "{}" for empty tool_use blocks) @@ -216,8 +218,7 @@ export class ToolCallManager { } } - toolResultContent = - typeof result === 'string' ? result : JSON.stringify(result) + toolResultContent = normalizeToolResultContent(result) } catch (error: unknown) { // If tool execution fails, add error message const message = diff --git a/packages/typescript/ai/src/types.ts b/packages/typescript/ai/src/types.ts index 984e15125..d83ed59dd 100644 --- a/packages/typescript/ai/src/types.ts +++ b/packages/typescript/ai/src/types.ts @@ -259,6 +259,8 @@ export type ConstrainedContent< | null | Array> +export type ToolResultContent = string | Array + export interface ModelMessage< TContent extends string | null | Array = | string @@ -300,7 +302,7 @@ export interface ToolCallPart { export interface ToolResultPart { type: 'tool-result' toolCallId: string - content: string + content: ToolResultContent state: ToolResultState error?: string // Error message if state is "error" } @@ -880,7 +882,7 @@ export interface ToolCallEndEvent extends BaseAGUIEvent { /** Final parsed input arguments */ input?: unknown /** Tool execution result (if executed) */ - result?: string + result?: ToolResultContent } /** diff --git a/packages/typescript/ai/tests/message-converters.test.ts b/packages/typescript/ai/tests/message-converters.test.ts index 76f55270a..d7fe6275d 100644 --- a/packages/typescript/ai/tests/message-converters.test.ts +++ b/packages/typescript/ai/tests/message-converters.test.ts @@ -1066,6 +1066,48 @@ describe('Message Converters', () => { { type: 'text', content: 'The temperature is 72F.' }, ]) }) + + it('should preserve multimodal tool result content', () => { + const imageResult: Array = [ + { + type: 'text', + content: 'Screenshot of the current state', + }, + { + type: 'image', + source: { + type: 'url', + value: 'https://example.com/screenshot.png', + }, + }, + ] + + const modelMessages: Array = [ + { role: 'user', content: 'Check the screenshot' }, + { + role: 'assistant', + content: 'Inspecting', + toolCalls: [ + { + id: 'tc-1', + type: 'function', + function: { name: 'view_canvas', arguments: '{}' }, + }, + ], + }, + { role: 'tool', content: imageResult, toolCallId: 'tc-1' }, + ] + + const result = modelMessagesToUIMessages(modelMessages) + const assistantParts = result[1]?.parts || [] + + expect(assistantParts).toContainEqual({ + type: 'tool-result', + toolCallId: 'tc-1', + content: imageResult, + state: 'complete', + }) + }) }) describe('convertMessagesToModelMessages', () => {