fix(connectors): tighten listingCapped semantics per review (WIQL cap, batch omissions, cap-vs-exhaustion)

Author: waleedlatif1

apps/docs/app/global.css

@@ -510,6 +510,13 @@ figure[data-rehype-pretty-code-figure],
   max-width: 480px !important;
 }
 
+/* Search dialog overlay + panel must cover the sticky navbar — both default to z-50,
+   and the navbar wins the tie by DOM order, leaving it unblurred above the overlay */
+.bg-fd-overlay,
+[role="dialog"][data-state] {
+  z-index: 60 !important;
+}
+
 pre {
   font-size: 0.875rem;
   line-height: 1.7;

apps/sim/connectors/azure-devops/azure-devops.ts

@@ -871,9 +871,14 @@ async function listRepoFiles(
   const chunk = entries.slice(offset, offset + FILE_BATCH_SIZE)
   const documents = chunk.map((entry) => fileToStub(organization, project, entry))
 
-  const { documents: capped, capped: hitLimit } = applyMaxItemsCap(documents, maxItems, syncContext)
-
   const nextOffset = offset + FILE_BATCH_SIZE
+  const { documents: capped, capped: hitLimit } = applyMaxItemsCap(
+    documents,
+    maxItems,
+    syncContext,
+    nextOffset < entries.length
+  )
+
   const hasMore = !hitLimit && nextOffset < entries.length
 
   return {
@@ -1003,24 +1008,32 @@ async function getFileDocument(
 
 /**
  * Applies the optional maxItems cap to a batch, tracking the running total in
- * syncContext and flagging `listingCapped` when the cap is hit. The sync engine
- * reads `listingCapped` to suppress deletion reconciliation on a truncated
- * listing — without it, a capped full sync would wrongly delete every source
- * document beyond the cap.
+ * syncContext and flagging `listingCapped` when the cap actually truncated the
+ * listing. The sync engine reads `listingCapped` to suppress deletion
+ * reconciliation on a truncated listing — without it, a capped full sync would
+ * wrongly delete every source document beyond the cap.
+ *
+ * `moreAvailable` tells the helper whether the current phase has further items
+ * beyond this page. The flag is only set when documents were actually dropped
+ * (this page was sliced, or more pages exist) — when the cap merely coincides
+ * with source exhaustion, reconciliation stays enabled so deleted source
+ * documents are still cleaned up.
  */
 function applyMaxItemsCap(
   documents: ExternalDocument[],
   maxItems: number,
-  syncContext: Record<string, unknown> | undefined
+  syncContext: Record<string, unknown> | undefined,
+  moreAvailable: boolean
 ): { documents: ExternalDocument[]; capped: boolean } {
   if (maxItems <= 0) return { documents, capped: false }
   const prevTotal = (syncContext?.totalDocsFetched as number) ?? 0
   const remaining = Math.max(0, maxItems - prevTotal)
-  const sliced = documents.length > remaining ? documents.slice(0, remaining) : documents
+  const slicedSome = documents.length > remaining
+  const sliced = slicedSome ? documents.slice(0, remaining) : documents
   const newTotal = prevTotal + sliced.length
   if (syncContext) syncContext.totalDocsFetched = newTotal
   const capped = newTotal >= maxItems
-  if (capped && syncContext) syncContext.listingCapped = true
+  if (capped && (slicedSome || moreAvailable) && syncContext) syncContext.listingCapped = true
   return { documents: sliced, capped }
 }
 
@@ -1104,7 +1117,12 @@ async function listWikiPages(
     documents.push(...stubs)
   }
 
-  const { documents: capped, capped: hitLimit } = applyMaxItemsCap(documents, maxItems, syncContext)
+  const { documents: capped, capped: hitLimit } = applyMaxItemsCap(
+    documents,
+    maxItems,
+    syncContext,
+    Boolean(nextContinuation) || wikiIndex + 1 < wikis.length
+  )
   if (hitLimit) {
     return { documents: capped, hasMore: false }
   }
@@ -1145,6 +1163,10 @@ async function listWorkItems(
     if (syncContext) syncContext.workItemIds = ids
   }
 
+  if (ids.length >= WIQL_MAX_RESULTS && syncContext) {
+    syncContext.listingCapped = true
+  }
+
   if (ids.length === 0) {
     return { documents: [], hasMore: false }
   }
@@ -1157,11 +1179,23 @@ async function listWorkItems(
 
   const chunk = ids.slice(offset, offset + WORK_ITEM_BATCH_SIZE)
   const raw = await fetchWorkItemsBatch(accessToken, organization, project, chunk)
+  if (raw.length < chunk.length && syncContext) {
+    syncContext.listingCapped = true
+    logger.warn(
+      'workitemsbatch omitted ids that WIQL returned; flagging listing as incomplete so reconciliation skips deletion',
+      { requested: chunk.length, returned: raw.length, organization, project }
+    )
+  }
   const documents = raw.map((item) => workItemToDocument(organization, project, item))
 
-  const { documents: capped, capped: hitLimit } = applyMaxItemsCap(documents, maxItems, syncContext)
-
   const nextOffset = offset + WORK_ITEM_BATCH_SIZE
+  const { documents: capped, capped: hitLimit } = applyMaxItemsCap(
+    documents,
+    maxItems,
+    syncContext,
+    nextOffset < ids.length
+  )
+
   const hasMore = !hitLimit && nextOffset < ids.length
 
   return {
@@ -1426,6 +1460,10 @@ export const azureDevopsConnector: ConnectorConfig = {
         return { documents, nextCursor: result.nextCursor, hasMore: true }
       }
       if (capReached()) {
+        const remainingPhase = nextPhase(current, contentType)
+        if (remainingPhase && syncContext) {
+          syncContext.listingCapped = true
+        }
         return { documents, hasMore: false }
       }
       current = nextPhase(current, contentType)

apps/sim/connectors/google-forms/google-forms.ts

@@ -550,9 +550,13 @@ export const googleFormsConnector: ConnectorConfig = {
     const data = await response.json()
     let files = (data.files || []) as DriveFormFile[]
 
+    let slicedSome = false
     if (maxForms > 0) {
       const remaining = maxForms - previouslyFetched
-      if (files.length > remaining) files = files.slice(0, remaining)
+      if (files.length > remaining) {
+        slicedSome = true
+        files = files.slice(0, remaining)
+      }
     }
 
     /**
@@ -597,11 +601,13 @@ export const googleFormsConnector: ConnectorConfig = {
     /**
      * Mark the listing as incomplete so the sync engine skips deletion
      * reconciliation. This applies when the `maxForms` cap truncates results
-     * (forms beyond the cap are not absent from the source) or when a transient
-     * error caused a still-present form to be dropped from this page — deleting
-     * those would wipe valid documents from the knowledge base.
+     * while more forms exist (forms beyond the cap are not absent from the
+     * source) or when a transient error caused a still-present form to be
+     * dropped from this page — deleting those would wipe valid documents from
+     * the knowledge base. When the cap merely coincides with source exhaustion
+     * (no next page), reconciliation stays enabled so deleted forms are cleaned up.
      */
-    if (syncContext && (hitLimit || skippedOnError)) {
+    if (syncContext && ((hitLimit && (slicedSome || Boolean(nextPageToken))) || skippedOnError)) {
       syncContext.listingCapped = true
     }
 

apps/sim/connectors/s3/s3.ts

@@ -581,17 +581,20 @@ export const s3Connector: ConnectorConfig = {
       .filter((entry) => entry.size > 0 && entry.size <= MAX_FILE_SIZE)
       .map((entry) => objectToStub(ctx, entry))
 
+    let slicedSome = false
     if (maxObjects > 0) {
       const remaining = maxObjects - previouslyFetched
       if (documents.length > remaining) {
+        slicedSome = true
         documents = documents.slice(0, remaining)
       }
     }
 
     const totalFetched = previouslyFetched + documents.length
     if (syncContext) syncContext.totalDocsFetched = totalFetched
     const hitLimit = maxObjects > 0 && totalFetched >= maxObjects
-    if (hitLimit && syncContext) syncContext.listingCapped = true
+    const moreAvailable = slicedSome || (isTruncated && Boolean(nextContinuationToken))
+    if (hitLimit && moreAvailable && syncContext) syncContext.listingCapped = true
 
     return {
       documents,

apps/sim/connectors/sentry/sentry.ts

@@ -525,19 +525,23 @@ export const sentryConnector: ConnectorConfig = {
 
     const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
     let documents = issues.map(issueToStub)
+    let slicedSome = false
     if (maxIssues > 0) {
       const remaining = Math.max(0, maxIssues - prevFetched)
       if (documents.length > remaining) {
+        slicedSome = true
         documents = documents.slice(0, remaining)
       }
     }
 
     const totalFetched = prevFetched + documents.length
     if (syncContext) syncContext.totalDocsFetched = totalFetched
     const hitLimit = maxIssues > 0 && totalFetched >= maxIssues
-    if (hitLimit && syncContext) syncContext.listingCapped = true
 
     const nextCursor = parseNextCursor(response.headers.get('Link'))
+    if (hitLimit && (slicedSome || Boolean(nextCursor)) && syncContext) {
+      syncContext.listingCapped = true
+    }
     const hasMore = !hitLimit && Boolean(nextCursor)
 
     return {

apps/sim/connectors/typeform/typeform.ts

@@ -424,9 +424,13 @@ export const typeformConnector: ConnectorConfig = {
      * cap is hit because `hasMore` becomes false.
      */
     let cappedItems = items
+    let slicedSome = false
     if (maxResponses > 0) {
       const remaining = Math.max(0, maxResponses - prevTotal)
-      if (items.length > remaining) cappedItems = items.slice(0, remaining)
+      if (items.length > remaining) {
+        slicedSome = true
+        cappedItems = items.slice(0, remaining)
+      }
     }
 
     const documents: ExternalDocument[] = cappedItems.map((item) =>
@@ -437,13 +441,6 @@ export const typeformConnector: ConnectorConfig = {
     if (syncContext) syncContext.totalDocsFetched = totalFetched
     const hitLimit = maxResponses > 0 && totalFetched >= maxResponses
 
-    /**
-     * Signal a truncated listing so the engine skips deletion reconciliation:
-     * a capped page does not represent the full set of responses, and deleting
-     * everything past the cap would wipe still-present docs from the KB.
-     */
-    if (hitLimit && syncContext) syncContext.listingCapped = true
-
     /**
      * The `before` cursor is the response `token` (not `response_id`). Each full
      * page advances to the oldest token seen so the next request pages strictly
@@ -452,7 +449,19 @@ export const typeformConnector: ConnectorConfig = {
      */
     const lastItem = items[items.length - 1]
     const nextCursor = lastItem?.token
-    const hasMore = !hitLimit && items.length === RESPONSES_PER_PAGE && Boolean(nextCursor)
+    const sourceHasMore = items.length === RESPONSES_PER_PAGE && Boolean(nextCursor)
+
+    /**
+     * Signal a truncated listing so the engine skips deletion reconciliation —
+     * but only when the cap actually dropped responses (this page was sliced, or
+     * the source had more pages). If the cap merely coincides with source
+     * exhaustion, reconciliation stays enabled so deleted responses are cleaned up.
+     */
+    if (hitLimit && (slicedSome || sourceHasMore) && syncContext) {
+      syncContext.listingCapped = true
+    }
+
+    const hasMore = !hitLimit && sourceHasMore
 
     return {
       documents,