Skip to content

Commit 887d2cf

Browse files
mesrefoglumesrefoglu
committed
Address comments and add allowed file types to OpenAI
1 parent 61a7710 commit 887d2cf

5 files changed

Lines changed: 171 additions & 40 deletions

File tree

‎apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/messages-input/messages-input.tsx‎

Lines changed: 45 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ import {
2020
} from '@/components/emcn'
2121
import { Trash } from '@/components/emcn/icons/trash'
2222
import { cn } from '@/lib/core/utils/cn'
23+
import {
24+
CLAUDE_SUPPORTED_IMAGE_MIME_TYPES,
25+
MIME_TYPE_MAPPING,
26+
} from '@/lib/uploads/utils/file-utils'
2327
import { EnvVarDropdown } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/env-var-dropdown'
2428
import {
2529
FileUpload,
@@ -38,30 +42,43 @@ import { getProviderFromModel, supportsFileAttachments } from '@/providers/model
3842
const MIN_TEXTAREA_HEIGHT_PX = 80
3943
const MAX_TEXTAREA_HEIGHT_PX = 320
4044

41-
const ANTHROPIC_SUPPORTED_IMAGE_TYPES = new Set([
45+
const ANTHROPIC_SUPPORTED_DOCUMENT_TYPES = new Set(
46+
Object.entries(MIME_TYPE_MAPPING)
47+
.filter(([, contentType]) => contentType === 'document')
48+
.map(([mimeType]) => mimeType)
49+
)
50+
51+
const OPENAI_SUPPORTED_IMAGE_TYPES = new Set([
4252
'image/jpeg',
4353
'image/jpg',
4454
'image/png',
4555
'image/gif',
4656
'image/webp',
4757
])
4858

49-
const ANTHROPIC_SUPPORTED_DOCUMENT_TYPES = new Set([
50-
'application/pdf',
51-
'text/plain',
52-
'text/csv',
53-
'application/json',
54-
'application/xml',
55-
'text/xml',
56-
'text/html',
57-
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
58-
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
59-
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
59+
const OPENAI_SUPPORTED_FILE_TYPES = new Set([
60+
'text/x-c',
61+
'text/x-c++',
62+
'text/x-csharp',
63+
'text/css',
6064
'application/msword',
61-
'application/vnd.ms-excel',
62-
'application/vnd.ms-powerpoint',
65+
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
66+
'text/x-golang',
67+
'text/html',
68+
'text/x-java',
69+
'text/javascript',
70+
'application/json',
6371
'text/markdown',
64-
'application/rtf',
72+
'application/pdf',
73+
'text/x-php',
74+
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
75+
'text/x-python',
76+
'text/x-script.python',
77+
'text/x-ruby',
78+
'application/x-sh',
79+
'text/x-tex',
80+
'application/typescript',
81+
'text/plain',
6582
])
6683

6784
/** Pattern to match complete message objects in JSON */
@@ -97,11 +114,19 @@ function hasAnthropicUnsupportedFiles(value: FileUploadValue | null | undefined)
97114
const type = file.type.toLowerCase()
98115
if (type === 'image/svg+xml') return false
99116
return (
100-
!ANTHROPIC_SUPPORTED_IMAGE_TYPES.has(type) && !ANTHROPIC_SUPPORTED_DOCUMENT_TYPES.has(type)
117+
!CLAUDE_SUPPORTED_IMAGE_MIME_TYPES.has(type) && !ANTHROPIC_SUPPORTED_DOCUMENT_TYPES.has(type)
101118
)
102119
})
103120
}
104121

122+
function hasOpenAIUnsupportedFiles(value: FileUploadValue | null | undefined): boolean {
123+
const files = Array.isArray(value) ? value : value ? [value] : []
124+
return files.some((file) => {
125+
const type = file.type.toLowerCase()
126+
return !OPENAI_SUPPORTED_IMAGE_TYPES.has(type) && !OPENAI_SUPPORTED_FILE_TYPES.has(type)
127+
})
128+
}
129+
105130
/**
106131
* Interface for individual message in the messages array
107132
*/
@@ -648,8 +673,10 @@ export function MessagesInput({
648673
const showUnsupportedFilesWarning =
649674
isFilesMessage &&
650675
fileAttachmentsSupported &&
651-
(selectedProvider === 'anthropic' || selectedProvider === 'azure-anthropic') &&
652-
hasAnthropicUnsupportedFiles(message.files)
676+
(((selectedProvider === 'anthropic' || selectedProvider === 'azure-anthropic') &&
677+
hasAnthropicUnsupportedFiles(message.files)) ||
678+
((selectedProvider === 'openai' || selectedProvider === 'azure-openai') &&
679+
hasOpenAIUnsupportedFiles(message.files)))
653680
const fieldState = subBlockInput.fieldHelpers.getFieldState(fieldId)
654681
const fieldHandlers = subBlockInput.fieldHelpers.createFieldHandlers(
655682
fieldId,

‎apps/sim/executor/handlers/agent/types.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import type { FileUploadValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload'
2+
13
export interface SkillInput {
24
skillId: string
35
name?: string
@@ -73,7 +75,7 @@ export interface TextMessage {
7375

7476
export interface FilesMessage {
7577
role: 'files'
76-
files?: unknown
78+
files?: FileUploadValue | null
7779
}
7880

7981
export type Message = TextMessage | FilesMessage

‎apps/sim/lib/uploads/utils/file-utils.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,10 @@ export function createFileContent(fileBuffer: Buffer, mimeType: string): Message
153153
/**
154154
* Create message content from base64-encoded file data.
155155
*/
156-
export function createFileContentFromBase64(base64: string, mimeType: string): MessageContent | null {
156+
export function createFileContentFromBase64(
157+
base64: string,
158+
mimeType: string
159+
): MessageContent | null {
157160
// SVG is XML text — Claude only supports raster image formats (JPEG, PNG, GIF, WebP),
158161
// so send SVGs as an XML document instead
159162
if (mimeType.toLowerCase() === 'image/svg+xml') {
@@ -186,7 +189,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M
186189
}
187190
}
188191

189-
const CLAUDE_SUPPORTED_IMAGE_MIME_TYPES = new Set([
192+
export const CLAUDE_SUPPORTED_IMAGE_MIME_TYPES = new Set([
190193
'image/jpeg',
191194
'image/jpg',
192195
'image/png',

‎apps/sim/providers/openai/utils.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,60 @@ describe('buildResponsesInputFromMessages', () => {
3535
],
3636
})
3737
})
38+
39+
it('ignores files with unsupported MIME types', () => {
40+
const fileAttachments: ProviderFileAttachment[] = [
41+
{
42+
name: 'brief.pdf',
43+
type: 'application/pdf',
44+
base64: 'cGRm',
45+
},
46+
{
47+
name: 'archive.bin',
48+
type: 'application/octet-stream',
49+
base64: 'YmluYXJ5',
50+
},
51+
{
52+
name: 'unknown',
53+
type: '',
54+
base64: 'dW5rbm93bg==',
55+
},
56+
]
57+
58+
const input = buildResponsesInputFromMessages(
59+
[{ role: 'user', content: 'Use the supported file only' }],
60+
fileAttachments
61+
)
62+
63+
expect(input).toEqual([
64+
{
65+
role: 'user',
66+
content: [
67+
{ type: 'input_text', text: 'Use the supported file only' },
68+
{
69+
type: 'input_file',
70+
file_data: 'data:application/pdf;base64,cGRm',
71+
filename: 'brief.pdf',
72+
},
73+
],
74+
},
75+
])
76+
})
77+
78+
it('leaves messages unchanged when all file attachments are unsupported', () => {
79+
const fileAttachments: ProviderFileAttachment[] = [
80+
{
81+
name: 'spreadsheet.xlsx',
82+
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
83+
base64: 'eGxzeA==',
84+
},
85+
]
86+
87+
const input = buildResponsesInputFromMessages(
88+
[{ role: 'user', content: 'This should stay text-only' }],
89+
fileAttachments
90+
)
91+
92+
expect(input).toEqual([{ role: 'user', content: 'This should stay text-only' }])
93+
})
3894
})

‎apps/sim/providers/openai/utils.ts‎

Lines changed: 62 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,15 @@ export interface ResponsesToolCall {
1818
arguments: string
1919
}
2020

21+
type ResponsesInputContentPart =
22+
| { type: 'input_text'; text: string }
23+
| { type: 'input_image'; image_url: string; detail: 'auto' }
24+
| { type: 'input_file'; file_data: string; filename?: string }
25+
2126
export type ResponsesInputItem =
2227
| {
2328
role: 'system' | 'user' | 'assistant'
24-
content:
25-
| string
26-
| Array<
27-
| { type: 'input_text'; text: string }
28-
| { type: 'input_image'; image_url: string; detail: 'auto' }
29-
| { type: 'input_file'; file_data: string; filename?: string }
30-
>
29+
content: string | ResponsesInputContentPart[]
3130
}
3231
| {
3332
type: 'function_call'
@@ -48,28 +47,72 @@ export interface ResponsesToolDefinition {
4847
parameters?: Record<string, unknown>
4948
}
5049

50+
const OPENAI_SUPPORTED_FILE_MIME_TYPES = new Set([
51+
'text/x-c',
52+
'text/x-c++',
53+
'text/x-csharp',
54+
'text/css',
55+
'application/msword',
56+
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
57+
'text/x-golang',
58+
'text/html',
59+
'text/x-java',
60+
'text/javascript',
61+
'application/json',
62+
'text/markdown',
63+
'application/pdf',
64+
'text/x-php',
65+
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
66+
'text/x-python',
67+
'text/x-script.python',
68+
'text/x-ruby',
69+
'application/x-sh',
70+
'text/x-tex',
71+
'application/typescript',
72+
'text/plain',
73+
])
74+
75+
const OPENAI_SUPPORTED_IMAGE_MIME_TYPES = new Set([
76+
'image/jpeg',
77+
'image/jpg',
78+
'image/png',
79+
'image/gif',
80+
'image/webp',
81+
])
82+
5183
function toDataUrl(file: ProviderFileAttachment): string {
5284
return `data:${file.type};base64,${file.base64}`
5385
}
5486

55-
function buildResponsesFileParts(fileAttachments?: ProviderFileAttachment[]) {
87+
function buildResponsesFileParts(
88+
fileAttachments?: ProviderFileAttachment[]
89+
): ResponsesInputContentPart[] {
5690
if (!fileAttachments?.length) return []
5791

58-
return fileAttachments.map((file) => {
92+
return fileAttachments.flatMap<ResponsesInputContentPart>((file) => {
93+
const type = file.type.toLowerCase()
5994
const dataUrl = toDataUrl(file)
60-
if (file.type.toLowerCase().startsWith('image/')) {
61-
return {
62-
type: 'input_image' as const,
63-
image_url: dataUrl,
64-
detail: 'auto' as const,
65-
}
95+
if (OPENAI_SUPPORTED_IMAGE_MIME_TYPES.has(type)) {
96+
return [
97+
{
98+
type: 'input_image' as const,
99+
image_url: dataUrl,
100+
detail: 'auto' as const,
101+
},
102+
]
66103
}
67104

68-
return {
69-
type: 'input_file' as const,
70-
file_data: dataUrl,
71-
filename: file.name,
105+
if (!OPENAI_SUPPORTED_FILE_MIME_TYPES.has(type)) {
106+
return []
72107
}
108+
109+
return [
110+
{
111+
type: 'input_file' as const,
112+
file_data: dataUrl,
113+
filename: file.name,
114+
},
115+
]
73116
})
74117
}
75118

0 commit comments

Comments
 (0)