Skip to content

Commit f74dade

Browse files
committed
Support files in agent block
1 parent 176fa64 commit f74dade

28 files changed

Lines changed: 1239 additions & 51 deletions

File tree

apps/sim/blocks/blocks.test.ts

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,42 @@ describe.concurrent('Blocks Module', () => {
163163
})
164164
})
165165

166+
describe('Agent block', () => {
167+
it('should expose canonical file attachments and normalize file params', () => {
168+
const block = getBlock('agent')
169+
170+
expect(block).toBeDefined()
171+
const uploadSubBlock = block?.subBlocks.find((subBlock) => subBlock.id === 'attachmentFiles')
172+
const advancedSubBlock = block?.subBlocks.find((subBlock) => subBlock.id === 'files')
173+
174+
expect(uploadSubBlock?.type).toBe('file-upload')
175+
expect(uploadSubBlock?.canonicalParamId).toBe('files')
176+
expect(uploadSubBlock?.multiple).toBe(true)
177+
expect(advancedSubBlock?.canonicalParamId).toBe('files')
178+
expect(block?.inputs.files).toEqual({
179+
type: 'array',
180+
description: 'Files to include with the latest user message',
181+
})
182+
183+
expect(
184+
block?.tools.config?.params?.({
185+
model: 'gpt-4o',
186+
files:
187+
'[{"id":"file-1","key":"workspace/ws-1/example.png","name":"example.png","url":"/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace","size":123,"type":"image/png"}]',
188+
})
189+
).toMatchObject({
190+
files: [
191+
{
192+
id: 'file-1',
193+
key: 'workspace/ws-1/example.png',
194+
name: 'example.png',
195+
type: 'image/png',
196+
},
197+
],
198+
})
199+
})
200+
})
201+
166202
describe('getBlocksByCategory', () => {
167203
it('should return blocks in the "blocks" category', () => {
168204
const blocks = getBlocksByCategory('blocks')

apps/sim/blocks/blocks/agent.ts

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { AuthMode, IntegrationType } from '@/blocks/types'
55
import {
66
getModelOptions,
77
getProviderCredentialSubBlocks,
8+
normalizeFileInput,
89
RESPONSE_FORMAT_WAND_CONFIG,
910
} from '@/blocks/utils'
1011
import {
@@ -133,6 +134,25 @@ Return ONLY the JSON array.`,
133134
defaultValue: 'claude-sonnet-4-6',
134135
options: getModelOptions,
135136
},
137+
{
138+
id: 'attachmentFiles',
139+
title: 'Files',
140+
type: 'file-upload',
141+
canonicalParamId: 'files',
142+
placeholder: 'Upload files for the agent',
143+
multiple: true,
144+
mode: 'basic',
145+
required: false,
146+
},
147+
{
148+
id: 'files',
149+
title: 'Files',
150+
type: 'short-input',
151+
canonicalParamId: 'files',
152+
placeholder: 'Reference files from previous blocks',
153+
mode: 'advanced',
154+
required: false,
155+
},
136156
{
137157
id: 'reasoningEffort',
138158
title: 'Reasoning Effort',
@@ -472,6 +492,9 @@ Return ONLY the JSON array.`,
472492
return tool
473493
},
474494
params: (params: Record<string, any>) => {
495+
const normalizedFiles = normalizeFileInput(params.files)
496+
const baseParams = normalizedFiles ? { ...params, files: normalizedFiles } : params
497+
475498
// If tools array is provided, handle tool usage control
476499
if (params.tools && Array.isArray(params.tools)) {
477500
// Transform tools to include usageControl
@@ -506,9 +529,9 @@ Return ONLY the JSON array.`,
506529
logger.info('Filtered out tools set to none', { tools: filteredOutTools.join(', ') })
507530
}
508531

509-
return { ...params, tools: transformedTools }
532+
return { ...baseParams, tools: transformedTools }
510533
}
511-
return params
534+
return baseParams
512535
},
513536
},
514537
},
@@ -518,6 +541,7 @@ Return ONLY the JSON array.`,
518541
description:
519542
'Array of message objects with role and content: [{ role: "system", content: "..." }, { role: "user", content: "..." }]',
520543
},
544+
files: { type: 'array', description: 'Files to include with the latest user message' },
521545
memoryType: {
522546
type: 'string',
523547
description:

apps/sim/executor/execution/engine.ts

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -449,19 +449,6 @@ export class ExecutionEngine {
449449

450450
const readyNodes = this.edgeManager.processOutgoingEdges(node, output, false)
451451

452-
this.execLogger.info('Processing outgoing edges', {
453-
nodeId,
454-
outgoingEdgesCount: node.outgoingEdges.size,
455-
outgoingEdges: Array.from(node.outgoingEdges.entries()).map(([id, e]) => ({
456-
id,
457-
target: e.target,
458-
sourceHandle: e.sourceHandle,
459-
})),
460-
output,
461-
readyNodesCount: readyNodes.length,
462-
readyNodes,
463-
})
464-
465452
this.addMultipleToQueue(readyNodes)
466453

467454
if (this.context.pendingDynamicNodes && this.context.pendingDynamicNodes.length > 0) {

apps/sim/executor/handlers/agent/agent-handler.test.ts

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,78 @@ describe('AgentBlockHandler', () => {
264264
expect(result).toEqual(expectedOutput)
265265
})
266266

267+
it('should attach files to the last user message only', async () => {
268+
const inputs = {
269+
model: 'gpt-4o',
270+
messages: [
271+
{ role: 'system' as const, content: 'You are helpful.' },
272+
{ role: 'user' as const, content: 'Earlier question' },
273+
{ role: 'assistant' as const, content: 'Earlier answer' },
274+
{ role: 'user' as const, content: 'Analyze this file' },
275+
],
276+
files: [
277+
{
278+
id: 'file-1',
279+
key: 'workspace/ws-1/example.png',
280+
name: 'example.png',
281+
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
282+
size: 128,
283+
type: 'image/png',
284+
base64: 'aW1hZ2U=',
285+
},
286+
],
287+
apiKey: 'test-api-key',
288+
}
289+
290+
mockGetProviderFromModel.mockReturnValue('openai')
291+
292+
await handler.execute(mockContext, mockBlock, inputs)
293+
294+
const requestBody = mockExecuteProviderRequest.mock.calls[0][1]
295+
expect(requestBody.messages[1]).toMatchObject({
296+
role: 'user',
297+
content: 'Earlier question',
298+
})
299+
expect(requestBody.messages[1].files).toBeUndefined()
300+
expect(requestBody.messages[3]).toMatchObject({
301+
role: 'user',
302+
content: 'Analyze this file',
303+
files: [
304+
{
305+
id: 'file-1',
306+
name: 'example.png',
307+
type: 'image/png',
308+
base64: 'aW1hZ2U=',
309+
},
310+
],
311+
})
312+
})
313+
314+
it('should reject files for providers without attachment support', async () => {
315+
const inputs = {
316+
model: 'deepseek-chat',
317+
messages: [{ role: 'user' as const, content: 'Analyze this file' }],
318+
files: [
319+
{
320+
id: 'file-1',
321+
key: 'workspace/ws-1/example.png',
322+
name: 'example.png',
323+
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
324+
size: 128,
325+
type: 'image/png',
326+
base64: 'aW1hZ2U=',
327+
},
328+
],
329+
apiKey: 'test-api-key',
330+
}
331+
332+
mockGetProviderFromModel.mockReturnValue('deepseek')
333+
334+
await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow(
335+
'File attachments are not supported for provider "deepseek"'
336+
)
337+
})
338+
267339
it('should preserve usageControl for custom tools and filter out "none"', async () => {
268340
const inputs = {
269341
model: 'gpt-4o',

apps/sim/executor/handlers/agent/agent-handler.ts

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@ import { sleep } from '@sim/utils/helpers'
66
import { and, eq, inArray, isNull } from 'drizzle-orm'
77
import { normalizeStringRecord, normalizeWorkflowVariables } from '@/lib/core/utils/records'
88
import { createMcpToolId } from '@/lib/mcp/utils'
9+
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
10+
import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server'
911
import { getCustomToolById } from '@/lib/workflows/custom-tools/operations'
1012
import { getAllBlocks } from '@/blocks'
1113
import type { BlockOutput } from '@/blocks/types'
14+
import { normalizeFileInput } from '@/blocks/utils'
1215
import {
1316
validateBlockType,
1417
validateCustomToolsAllowed,
@@ -36,6 +39,7 @@ import { buildAPIUrl, buildAuthHeaders } from '@/executor/utils/http'
3639
import { stringifyJSON } from '@/executor/utils/json'
3740
import { resolveVertexCredential } from '@/executor/utils/vertex-credential'
3841
import { executeProviderRequest } from '@/providers'
42+
import { getAttachmentProvider, getProviderAttachmentMaxBytes } from '@/providers/attachments'
3943
import { getProviderFromModel, transformBlockTool } from '@/providers/utils'
4044
import type { SerializedBlock } from '@/serializer/types'
4145
import { filterSchemaForLLM, type ToolSchema } from '@/tools/params'
@@ -87,12 +91,18 @@ export class AgentBlockHandler implements BlockHandler {
8791

8892
const streamingConfig = this.getStreamingConfig(ctx, block)
8993
const messages = await this.buildMessages(ctx, filteredInputs, skillMetadata)
94+
const messagesWithFiles = await this.attachFilesToLastUserMessage(
95+
ctx,
96+
messages,
97+
filteredInputs.files,
98+
providerId
99+
)
90100

91101
const providerRequest = this.buildProviderRequest({
92102
ctx,
93103
providerId,
94104
model,
95-
messages,
105+
messages: messagesWithFiles,
96106
inputs: filteredInputs,
97107
formattedTools,
98108
responseFormat,
@@ -672,6 +682,62 @@ export class AgentBlockHandler implements BlockHandler {
672682
return messages.length > 0 ? messages : undefined
673683
}
674684

685+
private async attachFilesToLastUserMessage(
686+
ctx: ExecutionContext,
687+
messages: Message[] | undefined,
688+
filesInput: unknown,
689+
providerId: string
690+
): Promise<Message[] | undefined> {
691+
const normalizedFiles = normalizeFileInput(filesInput)
692+
if (!normalizedFiles || normalizedFiles.length === 0) {
693+
return messages
694+
}
695+
696+
if (!messages || messages.length === 0) {
697+
throw new Error('Files require at least one user message in the agent prompt')
698+
}
699+
700+
if (!getAttachmentProvider(providerId)) {
701+
throw new Error(`File attachments are not supported for provider "${providerId}"`)
702+
}
703+
704+
let lastUserMessageIndex = -1
705+
for (let index = messages.length - 1; index >= 0; index--) {
706+
if (messages[index].role === 'user') {
707+
lastUserMessageIndex = index
708+
break
709+
}
710+
}
711+
if (lastUserMessageIndex === -1) {
712+
throw new Error('Files require at least one user message in the agent prompt')
713+
}
714+
715+
const requestId = ctx.executionId || ctx.workflowId || 'agent-files'
716+
const userFiles = processFilesToUserFiles(normalizedFiles as RawFileInput[], requestId, logger)
717+
if (userFiles.length === 0) {
718+
throw new Error('Files must include at least one valid file object')
719+
}
720+
721+
const hydratedFiles = await hydrateUserFilesWithBase64(userFiles, {
722+
requestId,
723+
workspaceId: ctx.workspaceId,
724+
workflowId: ctx.workflowId,
725+
executionId: ctx.executionId,
726+
userId: ctx.userId,
727+
logger,
728+
maxBytes: getProviderAttachmentMaxBytes(providerId),
729+
})
730+
731+
const lastUserMessage = messages[lastUserMessageIndex]
732+
const nextMessages = [...messages]
733+
nextMessages[lastUserMessageIndex] = {
734+
...lastUserMessage,
735+
files: [...(lastUserMessage.files ?? []), ...hydratedFiles],
736+
}
737+
738+
return nextMessages
739+
}
740+
675741
private extractValidMessages(messages?: Message[]): Message[] {
676742
if (!messages || !Array.isArray(messages)) return []
677743

apps/sim/executor/handlers/agent/types.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import type { UserFile } from '@/executor/types'
2+
13
export interface SkillInput {
24
skillId: string
35
name?: string
@@ -37,6 +39,7 @@ export interface AgentInputs {
3739
reasoningEffort?: string
3840
verbosity?: string
3941
thinkingLevel?: string
42+
files?: unknown
4043
}
4144

4245
/**
@@ -66,6 +69,7 @@ export interface ToolInput {
6669
export interface Message {
6770
role: 'system' | 'user' | 'assistant'
6871
content: string
72+
files?: UserFile[]
6973
executionId?: string
7074
function_call?: any
7175
tool_calls?: any[]

apps/sim/lib/uploads/utils/file-utils.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M
172172
return null
173173
}
174174

175-
if (contentType === 'image' && !CLAUDE_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType.toLowerCase())) {
175+
if (contentType === 'image' && !MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType.toLowerCase())) {
176176
return null
177177
}
178178

@@ -186,7 +186,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M
186186
}
187187
}
188188

189-
const CLAUDE_SUPPORTED_IMAGE_MIME_TYPES = new Set([
189+
export const MODEL_SUPPORTED_IMAGE_MIME_TYPES = new Set([
190190
'image/jpeg',
191191
'image/jpg',
192192
'image/png',

apps/sim/providers/anthropic/core.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
checkForForcedToolUsage,
1010
createReadableStreamFromAnthropicStream,
1111
} from '@/providers/anthropic/utils'
12+
import { buildAnthropicMessageContent } from '@/providers/attachments'
1213
import {
1314
getMaxOutputTokensForModel,
1415
getThinkingCapability,
@@ -229,9 +230,10 @@ export async function executeAnthropicProviderRequest(
229230
],
230231
})
231232
} else {
233+
const content = buildAnthropicMessageContent(msg.content, msg.files, config.providerId)
232234
messages.push({
233235
role: msg.role === 'assistant' ? 'assistant' : 'user',
234-
content: msg.content ? [{ type: 'text', text: msg.content }] : [],
236+
content: content as unknown as Anthropic.Messages.ContentBlockParam[],
235237
})
236238
}
237239
})

0 commit comments

Comments
 (0)