fix(connectors): shared streaming size-cap reader for ado file hydration (promote from s3)

Author: waleedlatif1

apps/sim/connectors/azure-devops/azure-devops.ts

@@ -3,7 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
 import { AzureDevOpsIcon } from '@/components/icons'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
-import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils'
+import { htmlToPlainText, joinTagArray, parseTagDate, readBodyWithLimit } from '@/connectors/utils'
 
 const logger = createLogger('AzureDevOpsConnector')
 
@@ -1032,13 +1032,13 @@ async function getFileDocument(
     throw new Error(`Failed to fetch repository file content: ${contentResponse.status}`)
   }
 
-  const buffer = Buffer.from(await contentResponse.arrayBuffer())
-  if (isBinaryBuffer(buffer)) {
-    logger.info('Skipping binary Azure DevOps file', { path })
+  const buffer = await readBodyWithLimit(contentResponse, MAX_FILE_SIZE)
+  if (buffer === null) {
+    logger.info('Skipping oversized Azure DevOps file', { path })
     return null
   }
-  if (buffer.byteLength > MAX_FILE_SIZE) {
-    logger.info('Skipping oversized Azure DevOps file', { path, size: buffer.byteLength })
+  if (isBinaryBuffer(buffer)) {
+    logger.info('Skipping binary Azure DevOps file', { path })
     return null
   }
 

apps/sim/connectors/s3/s3.ts

@@ -4,7 +4,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
 import { S3Icon } from '@/components/icons'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
-import { parseTagDate } from '@/connectors/utils'
+import { parseTagDate, readBodyWithLimit } from '@/connectors/utils'
 import { encodeS3PathComponent, getSignatureKey } from '@/tools/s3/utils'
 
 const logger = createLogger('S3Connector')
@@ -329,36 +329,6 @@ function buildListQueryString(params: Record<string, string>): string {
     .join('&')
 }
 
-/**
- * Reads a response body as UTF-8 text while enforcing a hard byte cap. The
- * declared `content-length` header cannot be trusted as the sole guard:
- * S3-compatible stores (MinIO, Cloudflare R2) may use chunked transfer
- * encoding and omit the header entirely. Bytes are accumulated from the
- * stream and reading aborts as soon as the cap is exceeded, so an oversized
- * body is never fully buffered. Returns null when the cap is exceeded.
- */
-async function readBodyWithLimit(response: Response, maxBytes: number): Promise<string | null> {
-  if (!response.body) {
-    const text = await response.text()
-    return Buffer.byteLength(text) > maxBytes ? null : text
-  }
-
-  const reader = response.body.getReader()
-  const chunks: Uint8Array[] = []
-  let total = 0
-  while (true) {
-    const { done, value } = await reader.read()
-    if (done) break
-    total += value.byteLength
-    if (total > maxBytes) {
-      await reader.cancel().catch(() => {})
-      return null
-    }
-    chunks.push(value)
-  }
-  return Buffer.concat(chunks).toString('utf-8')
-}
-
 /**
  * Decodes XML entities found in S3 response text values. `&amp;` is decoded
  * last so sequences like `&amp;lt;` resolve to `&lt;` rather than `<`.
@@ -663,11 +633,12 @@ export const s3Connector: ConnectorConfig = {
         return null
       }
 
-      const content = await readBodyWithLimit(response, MAX_FILE_SIZE)
-      if (content === null) {
+      const body = await readBodyWithLimit(response, MAX_FILE_SIZE)
+      if (body === null) {
         logger.warn('Skipping oversized S3 object (size cap exceeded while streaming)', { key })
         return null
       }
+      const content = body.toString('utf-8')
       if (!content.trim()) return null
 
       const entry: S3ObjectEntry = {

apps/sim/connectors/utils.ts

@@ -78,3 +78,36 @@ export function parseMultiValue(value: unknown): string[] {
   }
   return []
 }
+
+/**
+ * Reads a response body into a Buffer while enforcing a hard byte cap. The
+ * declared `content-length` header cannot be trusted as the sole guard —
+ * chunked transfer encoding may omit it entirely — so bytes are accumulated
+ * from the stream and reading aborts as soon as the cap is exceeded, ensuring
+ * an oversized (or hostile) body is never fully buffered into memory.
+ * Returns null when the cap is exceeded.
+ */
+export async function readBodyWithLimit(
+  response: Response,
+  maxBytes: number
+): Promise<Buffer | null> {
+  if (!response.body) {
+    const buffer = Buffer.from(await response.arrayBuffer())
+    return buffer.byteLength > maxBytes ? null : buffer
+  }
+
+  const reader = response.body.getReader()
+  const chunks: Uint8Array[] = []
+  let total = 0
+  while (true) {
+    const { done, value } = await reader.read()
+    if (done) break
+    total += value.byteLength
+    if (total > maxBytes) {
+      await reader.cancel().catch(() => {})
+      return null
+    }
+    chunks.push(value)
+  }
+  return Buffer.concat(chunks)
+}