fix(webapp): make native realtime change publishing fail-safe

Author: ericallam

.server-changes/harden-realtime-change-publishing.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Harden the native realtime backend's run-change publishing so a publish can never throw into a run lifecycle operation and never buffers commands in memory during a pub/sub Redis outage.

apps/webapp/app/redis.server.ts

@@ -11,6 +11,10 @@ export type RedisWithClusterOptions = {
   clusterMode?: boolean;
   clusterOptions?: Omit<ClusterOptions, "redisOptions">;
   keyPrefix?: string;
+  /** Reject commands immediately when the connection isn't ready instead of buffering them in memory (default: ioredis's `true`). */
+  enableOfflineQueue?: boolean;
+  /** Cap retries for an in-flight command before it rejects; `null` means unlimited (default: ioredis's default of 20). */
+  maxRetriesPerRequest?: number | null;
 };
 
 export type RedisClient = Redis | Cluster;
@@ -37,13 +41,19 @@ export function createRedisClient(
 
     redis = new Redis.Cluster(nodes, {
       ...options.clusterOptions,
+      ...(options.enableOfflineQueue !== undefined
+        ? { enableOfflineQueue: options.enableOfflineQueue }
+        : {}),
       redisOptions: {
         connectionName,
         keyPrefix: options.keyPrefix,
         username: options.username,
         password: options.password,
         enableAutoPipelining: true,
         reconnectOnError: defaultReconnectOnError,
+        ...(options.maxRetriesPerRequest !== undefined
+          ? { maxRetriesPerRequest: options.maxRetriesPerRequest }
+          : {}),
         ...(options.tlsDisabled
           ? {
               checkServerIdentity: () => {
@@ -72,6 +82,12 @@ export function createRedisClient(
       enableAutoPipelining: true,
       keyPrefix: options.keyPrefix,
       reconnectOnError: defaultReconnectOnError,
+      ...(options.enableOfflineQueue !== undefined
+        ? { enableOfflineQueue: options.enableOfflineQueue }
+        : {}),
+      ...(options.maxRetriesPerRequest !== undefined
+        ? { maxRetriesPerRequest: options.maxRetriesPerRequest }
+        : {}),
       ...(options.tlsDisabled ? {} : { tls: {} }),
     });
   }

apps/webapp/app/services/realtime/runChangeNotifier.server.ts

@@ -225,7 +225,13 @@ export class RunChangeNotifier {
 
   #ensurePublisher(): RedisClient {
     if (!this.#publisher) {
-      this.#publisher = createRedisClient(`${this.#connectionName}:pub`, this.options.redis);
+      // Publishes are fire-and-forget with a consumer-side backstop, so a dropped publish is
+      // latency-only. Fail fast rather than buffer commands in memory during a pub/sub outage.
+      this.#publisher = createRedisClient(`${this.#connectionName}:pub`, {
+        ...this.options.redis,
+        enableOfflineQueue: false,
+        maxRetriesPerRequest: 1,
+      });
     }
     return this.#publisher;
   }

apps/webapp/app/services/realtime/runChangeNotifierInstance.server.ts

@@ -1,6 +1,7 @@
 import { getMeter } from "@internal/tracing";
 import { env } from "~/env.server";
 import { singleton } from "~/utils/singleton";
+import { logger } from "../logger.server";
 import { RunChangeNotifier, type ChangeRecordInput } from "./runChangeNotifier.server";
 
 /**
@@ -74,12 +75,24 @@ export function publishChangeRecord(input: ChangeRecordInput): void {
   if (!nativeBackendEnabled) {
     return;
   }
-  getRunChangeNotifier().publish(input);
+  // Publish runs on the run-engine event bus / metadata flush loop; lazy init + encoding happen
+  // before the notifier's own try/catch, so guard the whole call — it must never throw at its caller.
+  try {
+    getRunChangeNotifier().publish(input);
+  } catch (error) {
+    logger.error("[runChangeNotifier] publishChangeRecord threw; dropping notification", { error });
+  }
 }
 
 export function publishManyChangeRecords(inputs: ChangeRecordInput[]): void {
   if (!nativeBackendEnabled) {
     return;
   }
-  getRunChangeNotifier().publishMany(inputs);
+  try {
+    getRunChangeNotifier().publishMany(inputs);
+  } catch (error) {
+    logger.error("[runChangeNotifier] publishManyChangeRecords threw; dropping notifications", {
+      error,
+    });
+  }
 }