Improve HTTP response handling and retry behavior

.changeset/batching-protocol-consistency.md

@@ -0,0 +1,20 @@
+---
+'@segment/analytics-next': minor
+'@segment/analytics-node': minor
+'@segment/analytics-core': patch
+---
+
+Unify and harden HTTP response handling and retry behavior across browser and node SDKs.
+
+- Browser (`@segment/analytics-next`)
+	- Add config-driven response handling for Segment.io delivery (`httpConfig` with rate-limit/backoff controls).
+	- Improve batching/dispatcher retry semantics for 429 and transient failures.
+	- Use configured `protocol` for batching requests when `apiHost` has no scheme, while preserving compatibility for `apiHost` values that already include `http://` or `https://`.
+
+- Node (`@segment/analytics-node`)
+	- Align publisher retry/status behavior with updated response handling rules.
+	- Add `maxTotalBackoffDuration` and `maxRateLimitDuration` settings to control retry ceilings.
+	- Update default retry configuration to increase resilience under transient failures.
+
+- Core (`@segment/analytics-core`)
+	- Standardize backoff defaults used by retry queues.

.github/workflows/e2e-browser-tests.yml

@@ -47,11 +47,9 @@ jobs:
         with:
           node-version: '20'
 
-      - name: Apply HTTP patch for testing
-        working-directory: sdk
-        run: |
-          git apply ../sdk-e2e-tests/patches/analytics-browser-http.patch
-          echo "HTTP patch applied successfully"
+      # The batched-dispatcher double-scheme bug is fixed in the SDK source,
+      # so the HTTP patch is no longer needed. run-tests.sh will gracefully
+      # skip it via --check if the e2e-config.json still references it.
 
       - name: Install SDK dependencies
         working-directory: sdk

packages/browser/e2e-cli/e2e-config.json

@@ -1,10 +1,12 @@
 {
   "sdk": "browser",
-  "test_suites": "basic,settings",
+  "test_suites": "basic,settings,retry,retry-settings",
   "auto_settings": true,
   "patch": "analytics-browser-http.patch",
   "env": {
     "BROWSER_BATCHING": "false",
-    "SETTINGS_ERROR_FALLBACK": "false"
+    "HTTP_CONFIG_SETTINGS": "true",
+    "SETTINGS_ERROR_FALLBACK": "false",
+    "E2E_TEST_SKIP": "settings-enabled-flag"
   }
 }

packages/browser/e2e-cli/src/cli.ts

@@ -52,6 +52,97 @@ interface CLIInput {
   config?: CLIConfig
 }
 
+// --- Fetch Monitor ---
+// The browser SDK's Segment.io plugin handles retries internally and swallows
+// all errors (never fires delivery_failure events). We monitor fetch calls to
+// detect when delivery activity has settled and to observe final HTTP statuses.
+
+let lastApiResponseTime = 0
+let inflightApiRequests = 0
+let lastApiStatus = 0
+let firstApiErrorStatus = 0
+let apiHostPattern = ''
+
+function installFetchMonitor(apiHost: string): void {
+  apiHostPattern = apiHost.replace(/^https?:\/\//, '')
+  const nativeFetch = globalThis.fetch
+
+  ;(globalThis as any).fetch = async function monitoredFetch(
+    input: RequestInfo | URL,
+    init?: RequestInit
+  ): Promise<Response> {
+    const url =
+      typeof input === 'string'
+        ? input
+        : input instanceof URL
+        ? input.href
+        : (input as Request).url
+
+    // Only monitor API requests, not CDN settings/project requests
+    const isApi =
+      apiHostPattern &&
+      url.includes(apiHostPattern) &&
+      !url.includes('/settings') &&
+      !url.includes('/projects')
+
+    if (!isApi) {
+      return nativeFetch.call(globalThis, input, init)
+    }
+
+    inflightApiRequests++
+    try {
+      const response = await nativeFetch.call(globalThis, input, init)
+      lastApiStatus = response.status
+      lastApiResponseTime = Date.now()
+      if (response.status >= 400 && firstApiErrorStatus === 0) {
+        firstApiErrorStatus = response.status
+      }
+      return response
+    } catch (err) {
+      lastApiResponseTime = Date.now()
+      throw err
+    } finally {
+      inflightApiRequests--
+    }
+  }
+}
+
+/**
+ * Wait for all API delivery activity to settle.
+ *
+ * The browser SDK's scheduleFlush uses a small random delay (100-600ms)
+ * between retry cycles, plus exponential backoff from pushWithBackoff.
+ * We wait until no API activity for a settling period.
+ */
+async function waitForDelivery(maxWaitMs = 60000): Promise<void> {
+  const start = Date.now()
+
+  // Wait for at least one API request
+  while (lastApiResponseTime === 0 && Date.now() - start < maxWaitMs) {
+    await sleep(100)
+  }
+
+  // Wait until no in-flight requests and enough quiet time
+  while (Date.now() - start < maxWaitMs) {
+    if (inflightApiRequests > 0) {
+      await sleep(100)
+      continue
+    }
+
+    const elapsed = Date.now() - lastApiResponseTime
+    // After success: brief settle for any remaining event dispatches.
+    // After error: longer settle to allow for retry scheduling + backoff.
+    // The fetch-dispatcher's core backoff reaches ~3200ms at attempt 5,
+    // plus schedule-flush jitter (~600ms), so we need >4s for error cases.
+    const settleMs = lastApiStatus < 400 ? 1500 : 5000
+
+    if (elapsed >= settleMs) {
+      return
+    }
+    await sleep(200)
+  }
+}
+
 // --- Helpers ---
 
 function parseArgs(): string | null {
@@ -63,7 +154,7 @@ function parseArgs(): string | null {
   return args[inputIndex + 1]
 }
 
-function delay(ms: number): Promise<void> {
+function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
@@ -80,6 +171,11 @@ async function main(): Promise<void> {
 
     const input: CLIInput = JSON.parse(inputJson)
 
+    // Install fetch monitor BEFORE importing the SDK
+    if (input.apiHost) {
+      installFetchMonitor(input.apiHost)
+    }
+
     // Create jsdom environment with the browser SDK
     const html = `
       <!DOCTYPE html>
@@ -112,7 +208,6 @@ async function main(): Promise<void> {
     ;(global as any).XMLHttpRequest = window.XMLHttpRequest
 
     // Import the browser SDK after setting up globals
-    // We need to dynamically import to ensure globals are set first
     const { AnalyticsBrowser } = await import('@segment/analytics-next')
 
     // Check if batching mode is enabled via environment variable
@@ -126,27 +221,40 @@ async function main(): Promise<void> {
       segmentConfig.protocol = protocol
 
       if (useBatching) {
-        // Batching mode: pass full URL (with scheme) since we patched batched-dispatcher
-        // to check for existing scheme
         segmentConfig.apiHost = input.apiHost
       } else {
-        // Standard mode: fetch-dispatcher uses the URL directly
         const apiHostStripped = input.apiHost.replace(/^https?:\/\//, '')
         segmentConfig.apiHost = apiHostStripped + '/v1'
       }
     }
 
+    // Wire maxRetries and backoff timing through httpConfig — this controls
+    // both the plugin's PriorityQueue (fetch-dispatcher path) and the
+    // batched-dispatcher's internal retry loop.
+    {
+      const backoffConfig: Record<string, unknown> = {
+        // Use a short base interval so batched-dispatcher backoff aligns with
+        // fetch-dispatcher's core backoff (100ms base). The default 500ms base
+        // produces gaps that exceed the CLI's settle-time detection.
+        baseBackoffInterval: 0.1,
+      }
+      if (input.config?.maxRetries != null) {
+        backoffConfig.maxRetryCount = input.config.maxRetries
+      }
+      segmentConfig.httpConfig = { backoffConfig }
+    }
+
     if (useBatching) {
       segmentConfig.deliveryStrategy = {
         strategy: 'batching',
         config: {
-          size: input.config?.flushAt ?? 1, // flush immediately for testing
+          size: input.config?.flushAt ?? 1,
           timeout: 1000,
         },
       }
     }
 
-    // Initialize analytics with the provided config
+    // Initialize analytics
     const [analytics] = await AnalyticsBrowser.load(
       {
         writeKey: input.writeKey,
@@ -160,21 +268,45 @@ async function main(): Promise<void> {
       }
     )
 
+    // Listen for delivery errors (now emitted by the Segment.io plugin)
+    const deliveryErrors: string[] = []
+    analytics.on('error', (err) => {
+      const reason = (err as any).reason
+      const msg =
+        reason instanceof Error
+          ? reason.message
+          : String(reason ?? (err as any).code)
+      deliveryErrors.push(msg)
+    })
+
     // Process event sequences
     for (const seq of input.sequences) {
       if (seq.delayMs > 0) {
-        await delay(seq.delayMs)
+        await sleep(seq.delayMs)
       }
 
       for (const event of seq.events) {
         await sendEvent(analytics, event)
       }
     }
 
-    // Wait for events to be sent (browser SDK auto-flushes)
-    await delay(3000)
-
-    output = { success: true, sentBatches: 1 }
+    // Wait for all delivery activity to settle
+    await waitForDelivery()
+
+    // Determine success/failure from delivery errors (emitted by the
+    // Segment.io plugin) and observed fetch responses as fallback.
+    if (deliveryErrors.length > 0) {
+      output = { success: false, error: deliveryErrors[0], sentBatches: 0 }
+    } else if (lastApiStatus >= 400) {
+      // Fetch monitor fallback: last response was an error
+      output = {
+        success: false,
+        error: `HTTP ${firstApiErrorStatus || lastApiStatus}`,
+        sentBatches: 0,
+      }
+    } else {
+      output = { success: true, sentBatches: 1 }
+    }
 
     // Cleanup
     dom.window.close()

packages/browser/jest.config.js

@@ -4,6 +4,9 @@ module.exports = createJestTSConfig(__dirname, {
   modulePathIgnorePatterns: ['<rootDir>/e2e-tests', '<rootDir>/qa'],
   setupFilesAfterEnv: ['./jest.setup.js'],
   testEnvironment: 'jsdom',
+  moduleNameMapper: {
+    '^@segment/analytics-page-tools$': '<rootDir>/../page-tools/src',
+  },
   coverageThreshold: {
     global: {
       branches: 0,

packages/browser/src/browser/__tests__/integration.test.ts

@@ -1603,9 +1603,11 @@ describe('setting headers', () => {
     const [call] = fetchCalls.filter((el) =>
       el.url.toString().includes('api.segment.io')
     )
-    expect(call.headers).toEqual({
-      'Content-Type': 'text/plain',
-      'X-Test': 'foo',
-    })
+    expect(call.headers).toEqual(
+      expect.objectContaining({
+        'Content-Type': 'text/plain',
+        'X-Test': 'foo',
+      })
+    )
   })
 })

packages/browser/src/browser/settings.ts

@@ -12,6 +12,7 @@ import { UserOptions } from '../core/user'
 import { HighEntropyHint } from '../lib/client-hints/interfaces'
 import { IntegrationsOptions } from '@segment/analytics-core'
 import { SegmentioSettings } from '../plugins/segmentio'
+import { HttpConfig } from '../plugins/segmentio/shared-dispatcher'
 
 interface VersionSettings {
   version?: string
@@ -74,6 +75,13 @@ export interface RemoteSegmentIOIntegrationSettings
   bundledConfigIds?: string[]
   unbundledConfigIds?: string[]
   maybeBundledConfigIds?: Record<string, string[]>
+
+  /**
+   * HTTP retry and backoff configuration.
+   * Controls rate-limit handling (429) and exponential backoff for transient errors.
+   * Fetched from CDN settings; can be overridden via init options.
+   */
+  httpConfig?: HttpConfig
 }
 
 /**
@@ -188,7 +196,7 @@ export interface AnalyticsSettings {
  */
 export type SegmentioIntegrationInitOptions = Pick<
   SegmentioSettings,
-  'apiHost' | 'protocol' | 'deliveryStrategy'
+  'apiHost' | 'protocol' | 'deliveryStrategy' | 'httpConfig'
 >
 
 /**

packages/browser/src/lib/priority-queue/__tests__/backoff.test.ts

@@ -2,14 +2,14 @@ import { backoff } from '../backoff'
 
 describe('backoff', () => {
   it('increases with the number of attempts', () => {
-    expect(backoff({ attempt: 1 })).toBeGreaterThan(1000)
-    expect(backoff({ attempt: 2 })).toBeGreaterThan(2000)
-    expect(backoff({ attempt: 3 })).toBeGreaterThan(3000)
-    expect(backoff({ attempt: 4 })).toBeGreaterThan(4000)
+    expect(backoff({ attempt: 1 })).toBeGreaterThan(200)
+    expect(backoff({ attempt: 2 })).toBeGreaterThan(400)
+    expect(backoff({ attempt: 3 })).toBeGreaterThan(800)
+    expect(backoff({ attempt: 4 })).toBeGreaterThan(1600)
   })
 
   it('accepts a max timeout', () => {
-    expect(backoff({ attempt: 1, maxTimeout: 3000 })).toBeGreaterThan(1000)
+    expect(backoff({ attempt: 1, maxTimeout: 3000 })).toBeGreaterThan(200)
     expect(backoff({ attempt: 3, maxTimeout: 3000 })).toBeLessThanOrEqual(3000)
     expect(backoff({ attempt: 4, maxTimeout: 3000 })).toBeLessThanOrEqual(3000)
   })

packages/browser/src/lib/priority-queue/__tests__/index.test.ts

@@ -121,7 +121,7 @@ describe('backoffs', () => {
     expect(spy).toHaveBeenCalled()
 
     const delay = spy.mock.calls[0][1]
-    expect(delay).toBeGreaterThan(1000)
+    expect(delay).toBeGreaterThan(200)
   })
 
   it('increases the delay as work gets requeued', () => {
@@ -147,12 +147,12 @@ describe('backoffs', () => {
     queue.pop()
 
     const firstDelay = spy.mock.calls[0][1]
-    expect(firstDelay).toBeGreaterThan(1000)
+    expect(firstDelay).toBeGreaterThan(200)
 
     const secondDelay = spy.mock.calls[1][1]
-    expect(secondDelay).toBeGreaterThan(2000)
+    expect(secondDelay).toBeGreaterThan(400)
 
     const thirdDelay = spy.mock.calls[2][1]
-    expect(thirdDelay).toBeGreaterThan(3000)
+    expect(thirdDelay).toBeGreaterThan(800)
   })
 })