Support Azure chat completions file attachments

Author: mesrefoglu

apps/sim/providers/azure-openai/index.ts

@@ -15,6 +15,7 @@ import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server'
 import type { StreamingExecution } from '@/executor/types'
 import { MAX_TOOL_ITERATIONS } from '@/providers'
 import {
+  appendFileAttachmentsToChatCompletionMessages,
   checkForForcedToolUsage,
   createReadableStreamFromAzureOpenAIStream,
   extractApiVersionFromUrl,
@@ -93,6 +94,8 @@ async function executeChatCompletionsRequest(
     allMessages.push(...(request.messages as ChatCompletionMessageParam[]))
   }
 
+  appendFileAttachmentsToChatCompletionMessages(allMessages, request.fileAttachments)
+
   const tools: ChatCompletionTool[] | undefined = request.tools?.length
     ? request.tools.map((tool) => ({
         type: 'function' as const,

apps/sim/providers/azure-openai/utils.test.ts

@@ -0,0 +1,94 @@
+import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions'
+import { describe, expect, it } from 'vitest'
+import { appendFileAttachmentsToChatCompletionMessages } from '@/providers/azure-openai/utils'
+import type { ProviderFileAttachment } from '@/providers/types'
+
+describe('appendFileAttachmentsToChatCompletionMessages', () => {
+  it('adds supported file attachments to the latest user message', () => {
+    const messages: ChatCompletionMessageParam[] = [
+      { role: 'user', content: 'Earlier request' },
+      { role: 'assistant', content: 'Earlier response' },
+      { role: 'user', content: 'Use these attachments' },
+    ]
+    const fileAttachments: ProviderFileAttachment[] = [
+      {
+        name: 'brief.pdf',
+        type: 'application/pdf',
+        base64: 'cGRm',
+      },
+      {
+        name: 'photo.png',
+        type: 'image/png',
+        base64: 'cG5n',
+      },
+    ]
+
+    appendFileAttachmentsToChatCompletionMessages(messages, fileAttachments)
+
+    expect(messages[0]).toEqual({ role: 'user', content: 'Earlier request' })
+    expect(messages[1]).toEqual({ role: 'assistant', content: 'Earlier response' })
+    expect(messages[2]).toEqual({
+      role: 'user',
+      content: [
+        { type: 'text', text: 'Use these attachments' },
+        {
+          type: 'file',
+          file: {
+            file_data: 'cGRm',
+            filename: 'brief.pdf',
+          },
+        },
+        {
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,cG5n',
+            detail: 'auto',
+          },
+        },
+      ],
+    })
+  })
+
+  it('adds a user message when there is no existing user message', () => {
+    const messages: ChatCompletionMessageParam[] = [{ role: 'system', content: 'You are helpful' }]
+
+    appendFileAttachmentsToChatCompletionMessages(messages, [
+      {
+        name: 'brief.pdf',
+        type: 'application/pdf',
+        base64: 'cGRm',
+      },
+    ])
+
+    expect(messages).toEqual([
+      { role: 'system', content: 'You are helpful' },
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Please use the attached files.' },
+          {
+            type: 'file',
+            file: {
+              file_data: 'cGRm',
+              filename: 'brief.pdf',
+            },
+          },
+        ],
+      },
+    ])
+  })
+
+  it('leaves messages unchanged when attachments have unsupported MIME types', () => {
+    const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'Text only' }]
+
+    appendFileAttachmentsToChatCompletionMessages(messages, [
+      {
+        name: 'spreadsheet.xlsx',
+        type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        base64: 'eGxzeA==',
+      },
+    ])
+
+    expect(messages).toEqual([{ role: 'user', content: 'Text only' }])
+  })
+})

apps/sim/providers/azure-openai/utils.ts

@@ -1,8 +1,15 @@
 import type { Logger } from '@sim/logger'
 import type OpenAI from 'openai'
-import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
+import type {
+  ChatCompletionChunk,
+  ChatCompletionContentPart,
+  ChatCompletionMessageParam,
+  ChatCompletionUserMessageParam,
+} from 'openai/resources/chat/completions'
 import type { CompletionUsage } from 'openai/resources/completions'
 import type { Stream } from 'openai/streaming'
+import { buildChatCompletionFileParts } from '@/providers/openai/utils'
+import type { ProviderFileAttachment } from '@/providers/types'
 import { checkForForcedToolUsageOpenAI, createOpenAICompatibleStream } from '@/providers/utils'
 
 /**
@@ -37,6 +44,43 @@ export function checkForForcedToolUsage(
   )
 }
 
+function isUserMessage(
+  message: ChatCompletionMessageParam
+): message is ChatCompletionUserMessageParam {
+  return message.role === 'user'
+}
+
+function createTextPart(text: string): ChatCompletionContentPart {
+  return {
+    type: 'text',
+    text,
+  }
+}
+
+/**
+ * Adds supported file attachments to the latest user message for Azure Chat Completions.
+ */
+export function appendFileAttachmentsToChatCompletionMessages(
+  messages: ChatCompletionMessageParam[],
+  fileAttachments?: ProviderFileAttachment[]
+): void {
+  const fileParts = buildChatCompletionFileParts(fileAttachments)
+  if (fileParts.length === 0) return
+
+  const lastUserMessage = [...messages].reverse().find(isUserMessage)
+  if (!lastUserMessage) {
+    messages.push({
+      role: 'user',
+      content: [createTextPart('Please use the attached files.'), ...fileParts],
+    })
+    return
+  }
+
+  lastUserMessage.content = Array.isArray(lastUserMessage.content)
+    ? [...lastUserMessage.content, ...fileParts]
+    : [createTextPart(lastUserMessage.content || 'Please use the attached files.'), ...fileParts]
+}
+
 /**
  * Determines if an Azure OpenAI endpoint URL is for the chat completions API.
  * Returns true for URLs containing /chat/completions pattern.

apps/sim/providers/openai/utils.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import type OpenAI from 'openai'
+import type { ChatCompletionContentPart } from 'openai/resources/chat/completions'
 import type { Message, ProviderFileAttachment } from '@/providers/types'
 
 const logger = createLogger('ResponsesUtils')
@@ -116,6 +117,41 @@ function buildResponsesFileParts(
   })
 }
 
+export function buildChatCompletionFileParts(
+  fileAttachments?: ProviderFileAttachment[]
+): ChatCompletionContentPart[] {
+  if (!fileAttachments?.length) return []
+
+  return fileAttachments.flatMap<ChatCompletionContentPart>((file) => {
+    const type = file.type.toLowerCase()
+    if (OPENAI_SUPPORTED_IMAGE_MIME_TYPES.has(type)) {
+      return [
+        {
+          type: 'image_url',
+          image_url: {
+            url: toDataUrl(file),
+            detail: 'auto',
+          },
+        },
+      ]
+    }
+
+    if (!OPENAI_SUPPORTED_FILE_MIME_TYPES.has(type)) {
+      return []
+    }
+
+    return [
+      {
+        type: 'file',
+        file: {
+          file_data: file.base64,
+          filename: file.name,
+        },
+      },
+    ]
+  })
+}
+
 /**
  * Converts chat-style messages into Responses API input items.
  */