Preserve multimodal tool results

.changeset/multimodal-tool-results.md

@@ -0,0 +1,17 @@
+---
+'@tanstack/ai': patch
+'@tanstack/ai-openai': patch
+'@tanstack/ai-anthropic': patch
+'@tanstack/ai-client': patch
+'@tanstack/ai-react-ui': patch
+'@tanstack/ai-solid-ui': patch
+'@tanstack/ai-vue-ui': patch
+---
+
+Preserve multimodal tool results across chat history and provider adapters.
+
+This fixes tool result handling so `ContentPart[]` outputs are preserved instead
+of being stringified in core chat flows, OpenAI Responses tool outputs, and
+Anthropic tool results. It also updates the client-side tool result type and
+the default React, Solid, and Vue chat message renderers to handle text and
+image tool result content.

packages/typescript/ai-anthropic/src/adapters/text.ts

@@ -379,7 +379,13 @@ export class AnthropicTextAdapter<
               type: 'tool_result',
               tool_use_id: message.toolCallId,
               content:
-                typeof message.content === 'string' ? message.content : '',
+                typeof message.content === 'string'
+                  ? message.content
+                  : Array.isArray(message.content)
+                    ? message.content.map((part) =>
+                        this.convertContentPartToAnthropic(part),
+                      )
+                    : '',
             },
           ],
         })

packages/typescript/ai-anthropic/tests/anthropic-adapter.test.ts

@@ -184,6 +184,95 @@ describe('Anthropic adapter option mapping', () => {
     })
   })
 
+  it('preserves multimodal tool result content for tool_result blocks', async () => {
+    const mockStream = (async function* () {
+      yield {
+        type: 'message_start',
+        message: {
+          id: 'msg_123',
+          model: 'claude-3-7-sonnet-20250219',
+          role: 'assistant',
+          type: 'message',
+          content: [],
+          stop_reason: null,
+          stop_sequence: null,
+          usage: { input_tokens: 10, output_tokens: 0 },
+        },
+      }
+      yield {
+        type: 'message_delta',
+        delta: { stop_reason: 'end_turn' },
+        usage: { output_tokens: 1 },
+      }
+      yield {
+        type: 'message_stop',
+      }
+    })()
+
+    mocks.betaMessagesCreate.mockResolvedValueOnce(mockStream as any)
+
+    const adapter = createAdapter('claude-3-7-sonnet-20250219')
+    const toolResult = [
+      {
+        type: 'text' as const,
+        content: 'Screenshot of the current state',
+      },
+      {
+        type: 'image' as const,
+        source: {
+          type: 'url' as const,
+          value: 'https://example.com/screenshot.png',
+        },
+      },
+    ]
+
+    for await (const _chunk of chat({
+      adapter,
+      messages: [
+        { role: 'user', content: 'What changed?' },
+        {
+          role: 'assistant',
+          content: 'Checking',
+          toolCalls: [
+            {
+              id: 'call_canvas',
+              type: 'function',
+              function: { name: 'view_canvas', arguments: '{}' },
+            },
+          ],
+        },
+        { role: 'tool', toolCallId: 'call_canvas', content: toolResult },
+      ],
+      tools: [weatherTool],
+    })) {
+      // drain
+    }
+
+    const [payload] = mocks.betaMessagesCreate.mock.calls.at(-1) as [any]
+    expect(payload.messages[2]).toEqual({
+      role: 'user',
+      content: [
+        {
+          type: 'tool_result',
+          tool_use_id: 'call_canvas',
+          content: [
+            {
+              type: 'text',
+              text: 'Screenshot of the current state',
+            },
+            {
+              type: 'image',
+              source: {
+                type: 'url',
+                url: 'https://example.com/screenshot.png',
+              },
+            },
+          ],
+        },
+      ],
+    })
+  })
+
   it('merges consecutive user messages when tool results precede a follow-up user message', async () => {
     // This is the core multi-turn bug: after a tool call + result, the next user message
     // creates consecutive role:'user' messages (tool_result as user + new user message).

packages/typescript/ai-client/src/types.ts

@@ -13,6 +13,8 @@ import type {
 } from '@tanstack/ai'
 import type { ConnectionAdapter } from './connection-adapters'
 
+type ToolResultContent = string | Array<ContentPart>
+
 /**
  * Tool call states - track the lifecycle of a tool call
  */
@@ -152,7 +154,7 @@ export type ToolCallPart<TTools extends ReadonlyArray<AnyClientTool> = any> =
 export interface ToolResultPart {
   type: 'tool-result'
   toolCallId: string
-  content: string
+  content: ToolResultContent
   state: ToolResultState
   error?: string // Error message if state is "error"
 }

packages/typescript/ai-code-mode/models-eval/metrics.ts

@@ -1,4 +1,4 @@
-import type { UIMessage } from '@tanstack/ai'
+import type { ToolResultPart, UIMessage } from '@tanstack/ai'
 
 export interface TypeScriptAttempt {
   toolCallId: string
@@ -44,11 +44,33 @@ function safeJsonParse(value: string): unknown {
   }
 }
 
+function parseToolResultContent(
+  content: ToolResultPart['content'],
+):
+  | {
+      success?: boolean
+      error?: { name?: string; message?: string }
+    }
+  | undefined {
+  return typeof content === 'string'
+    ? (safeJsonParse(content) as
+        | {
+            success?: boolean
+            error?: { name?: string; message?: string }
+          }
+        | undefined)
+    : undefined
+}
+
 export function computeMetrics(messages: Array<UIMessage>): ComputedMetrics {
   const toolCallLookup = new Map<string, { name: string; arguments: string }>()
   const toolResultLookup = new Map<
     string,
-    { content: string; state?: string; error?: string }
+    {
+      content: ToolResultPart['content']
+      state?: string
+      error?: string
+    }
   >()
 
   let totalToolCalls = 0
@@ -95,13 +117,8 @@ export function computeMetrics(messages: Array<UIMessage>): ComputedMetrics {
       | { typescriptCode?: string }
       | undefined
     const result = toolResultLookup.get(toolCallId)
-    const parsedResult = result?.content
-      ? (safeJsonParse(result.content) as
-          | {
-              success?: boolean
-              error?: { name?: string; message?: string }
-            }
-          | undefined)
+    const parsedResult = result
+      ? parseToolResultContent(result.content)
       : undefined
 
     const success = parsedResult?.success

packages/typescript/ai-openai/src/adapters/text.ts

@@ -703,13 +703,24 @@ export class OpenAITextAdapter<
     for (const message of messages) {
       // Handle tool messages - convert to FunctionToolCallOutput
       if (message.role === 'tool') {
+        const output =
+          typeof message.content === 'string'
+            ? message.content
+            : this.normalizeContent(message.content).map((part) =>
+                this.convertContentPartToOpenAI(
+                  part as ContentPart<
+                    unknown,
+                    OpenAIImageMetadata,
+                    OpenAIAudioMetadata,
+                    unknown,
+                    unknown
+                  >,
+                ),
+              )
         result.push({
           type: 'function_call_output',
           call_id: message.toolCallId || '',
-          output:
-            typeof message.content === 'string'
-              ? message.content
-              : JSON.stringify(message.content),
+          output,
         })
         continue
       }

packages/typescript/ai-openai/tests/openai-adapter.test.ts

@@ -129,4 +129,85 @@ describe('OpenAI adapter option mapping', () => {
     expect(Array.isArray(payload.tools)).toBe(true)
     expect(payload.tools.length).toBeGreaterThan(0)
   })
+
+  it('preserves multimodal tool result content for function_call_output', async () => {
+    const mockStream = createMockChatCompletionsStream([
+      {
+        type: 'response.done',
+        response: {
+          id: 'resp-123',
+          model: 'gpt-4o-mini',
+          status: 'completed',
+          created_at: 1234567891,
+          usage: {
+            input_tokens: 12,
+            output_tokens: 0,
+          },
+        },
+      },
+    ])
+
+    const responsesCreate = vi.fn().mockResolvedValueOnce(mockStream)
+
+    const adapter = createAdapter('gpt-4o-mini')
+    ;(adapter as any).client = {
+      responses: {
+        create: responsesCreate,
+      },
+    }
+
+    const toolResult = [
+      {
+        type: 'text' as const,
+        content: 'Screenshot of the current state',
+      },
+      {
+        type: 'image' as const,
+        source: {
+          type: 'url' as const,
+          value: 'https://example.com/screenshot.png',
+        },
+      },
+    ]
+
+    for await (const _chunk of chat({
+      adapter,
+      messages: [
+        { role: 'user', content: 'What changed?' },
+        {
+          role: 'assistant',
+          content: 'Checking',
+          toolCalls: [
+            {
+              id: 'call_canvas',
+              type: 'function',
+              function: { name: 'view_canvas', arguments: '{}' },
+            },
+          ],
+        },
+        { role: 'tool', toolCallId: 'call_canvas', content: toolResult },
+      ],
+      tools: [weatherTool],
+    })) {
+      // drain
+    }
+
+    const [payload] = responsesCreate.mock.calls[0]
+    const toolOutput = payload.input.find(
+      (item: any) =>
+        item.type === 'function_call_output' && item.call_id === 'call_canvas',
+    )
+
+    expect(toolOutput.output).toEqual([
+      {
+        type: 'input_text',
+        text: 'Screenshot of the current state',
+      },
+      {
+        type: 'input_image',
+        image_url: 'https://example.com/screenshot.png',
+        detail: 'auto',
+      },
+    ])
+  })
 })