improvement(redis): strip idempotency body and cap mothership stream zsets

Author: waleedlatif1

apps/sim/lib/copilot/request/session/buffer.test.ts

@@ -149,7 +149,7 @@ describe('mothership-stream-outbox', () => {
     expect(replayed.map((entry) => entry.payload.text)).toEqual(['world'])
   })
 
-  it('does not trim active stream history while appending events', async () => {
+  it('trims active stream history to eventLimit on every append', async () => {
     const cursor = await allocateCursor('stream-1')
 
     await appendEvent(
@@ -163,7 +163,11 @@ describe('mothership-stream-outbox', () => {
       })
     )
 
-    expect(mockRedis.zremrangebyrank).not.toHaveBeenCalled()
+    expect(mockRedis.zremrangebyrank).toHaveBeenCalledWith(
+      'mothership_stream:stream-1:events',
+      0,
+      expect.any(Number)
+    )
   })
 
   it('clears persisted stream state during teardown cleanup', async () => {

apps/sim/lib/copilot/request/session/buffer.ts

@@ -144,6 +144,7 @@ export async function appendEvents(
       zaddArgs.push(envelope.seq, JSON.stringify(envelope))
     }
     pipeline.zadd(key, ...(zaddArgs as [number, string, ...Array<number | string>]))
+    pipeline.zremrangebyrank(key, 0, -config.eventLimit - 1)
     pipeline.expire(key, config.ttlSeconds)
     pipeline.set(seqKey, String(envelopes[envelopes.length - 1].seq), 'EX', config.ttlSeconds)
     await pipeline.exec()

apps/sim/lib/core/idempotency/service.ts

@@ -16,6 +16,16 @@ export interface IdempotencyConfig {
   namespace?: string
   /** When true, failed keys are deleted rather than stored so the operation is retried on the next attempt. */
   retryFailures?: boolean
+  /**
+   * When false, the operation's return value is not persisted alongside
+   * the dedupe marker — only `{ success, status, error? }` is stored.
+   * Duplicate calls still short-circuit, but `executeWithIdempotency`
+   * resolves to `undefined` on the dedupe path. Use for webhook/polling
+   * flows where the cached body is large (multi-KB execution results)
+   * and callers don't consume the value of a duplicated delivery.
+   * Defaults to true.
+   */
+  storeResultBody?: boolean
   /**
    * Force a specific storage backend regardless of the environment's
    * auto-detection. Use `'database'` for correctness-critical flows
@@ -77,6 +87,7 @@ export class IdempotencyService {
       ttlSeconds: config.ttlSeconds ?? DEFAULT_TTL,
       namespace: config.namespace ?? 'default',
       retryFailures: config.retryFailures ?? false,
+      storeResultBody: config.storeResultBody ?? true,
     }
     this.storageMethod = config.forceStorage ?? getStorageMethod()
     logger.info(`IdempotencyService using ${this.storageMethod} storage`, {
@@ -441,7 +452,9 @@ export class IdempotencyService {
 
       await this.storeResult(
         claimResult.normalizedKey,
-        { success: true, result, status: 'completed' },
+        this.config.storeResultBody
+          ? { success: true, result, status: 'completed' }
+          : { success: true, status: 'completed' },
         claimResult.storageMethod
       )
 
@@ -510,15 +523,29 @@ export class IdempotencyService {
   }
 }
 
+/**
+ * Webhook idempotency. We're the receiver of provider-initiated webhooks,
+ * not the originator — duplicate deliveries from the provider's retry
+ * machinery just need a "we saw this" marker, not a replayable response
+ * body. `storeResultBody: false` drops the cached workflow result from
+ * each key, eliminating the long tail of large gmail/outlook payloads
+ * that pushed Redis Cloud into OOM on 2026-05-15.
+ *
+ * TTL stays at 7 days because that's the longest provider retry window
+ * we care about (Gmail / Pub/Sub). With body-stripping the per-key cost
+ * is ~150 bytes, so the long TTL is essentially free.
+ */
 export const webhookIdempotency = new IdempotencyService({
   namespace: 'webhook',
-  ttlSeconds: 60 * 60 * 24 * 7, // 7 days
+  ttlSeconds: 60 * 60 * 24 * 7, // 7 days — must exceed Gmail/Pub-Sub retry window
+  storeResultBody: false,
 })
 
 export const pollingIdempotency = new IdempotencyService({
   namespace: 'polling',
   ttlSeconds: 60 * 60 * 24 * 3, // 3 days
   retryFailures: true,
+  storeResultBody: false,
 })
 
 /**