fix: fenced lock release, wire retry processor, guard indexer metrics on RPC fail, yield tests

backend/src/index.ts

@@ -17,9 +17,9 @@ import {
   stopDefaultCheckerScheduler,
 } from "./services/defaultChecker.js";
 import {
-  startWebhookRetryScheduler,
-  stopWebhookRetryScheduler,
-} from "./services/webhookRetryScheduler.js";
+  startWebhookRetryProcessor,
+  stopWebhookRetryProcessor,
+} from "./services/webhookRetryProcessor.js";
 import { eventStreamService } from "./services/eventStreamService.js";
 import {
   startNotificationCleanupScheduler,
@@ -61,8 +61,8 @@ const server = app.listen(port, () => {
   // Start periodic on-chain default checks (if configured)
   startDefaultCheckerScheduler();
 
-  // Start webhook retry scheduler
-  startWebhookRetryScheduler();
+  // Start webhook retry processor (5m/15m/45m backoff via WebhookService.processRetries)
+  startWebhookRetryProcessor();
 
   // Start scheduled score reconciliation against on-chain state
   startScoreReconciliationScheduler();
@@ -87,7 +87,7 @@ const shutdown = async (signal: "SIGTERM" | "SIGINT") => {
   try {
     await stopIndexer();
     stopDefaultCheckerScheduler();
-    stopWebhookRetryScheduler();
+    stopWebhookRetryProcessor();
     stopScoreReconciliationScheduler();
     stopNotificationCleanupScheduler();
 

backend/src/services/__tests__/yieldHistoryService.test.ts

@@ -86,4 +86,124 @@ describe("yieldHistoryService", () => {
     expect(latest.currentValue).toBeGreaterThanOrEqual(1000);
     expect(latest.netYield).toBeGreaterThanOrEqual(0);
   });
+
+  // Issue #1171 — previously uncovered paths
+
+  it("Withdraw reduces cost basis proportionally", async () => {
+    const now = new Date();
+    const t1 = new Date(now);
+    t1.setUTCDate(t1.getUTCDate() - 2);
+    const t2 = new Date(now);
+    t2.setUTCDate(t2.getUTCDate() - 1);
+
+    // Pool events: Deposit 1000 then Withdraw 500 (value=null → shares=amount)
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        { event_type: "Deposit", amount: "1000", ledger_closed_at: t1, value: null },
+        { event_type: "Withdraw", amount: "500", ledger_closed_at: t2, value: null },
+      ],
+    });
+    // Depositor events: same
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        { event_type: "Deposit", amount: "1000", ledger_closed_at: t1, value: null },
+        { event_type: "Withdraw", amount: "500", ledger_closed_at: t2, value: null },
+      ],
+    });
+
+    const history = await buildDepositorYieldHistory("GDep", "GTok", 7, 500_000);
+    expect(history.length).toBeGreaterThan(0);
+    // After withdrawing half the shares the cost basis should have halved
+    const latest = history[history.length - 1]!;
+    // netYield = currentValue - costBasis; costBasis after withdraw ≈ 500
+    expect(latest.netYield).toBeGreaterThanOrEqual(-1); // may be slightly negative due to share price
+  });
+
+  it("EmergencyWithdraw follows the same cost-basis reduction path as Withdraw", async () => {
+    const now = new Date();
+    const t1 = new Date(now);
+    t1.setUTCDate(t1.getUTCDate() - 2);
+    const t2 = new Date(now);
+    t2.setUTCDate(t2.getUTCDate() - 1);
+
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        { event_type: "Deposit", amount: "1000", ledger_closed_at: t1, value: null },
+        { event_type: "EmergencyWithdraw", amount: "1000", ledger_closed_at: t2, value: null },
+      ],
+    });
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        { event_type: "Deposit", amount: "1000", ledger_closed_at: t1, value: null },
+        { event_type: "EmergencyWithdraw", amount: "1000", ledger_closed_at: t2, value: null },
+      ],
+    });
+
+    const history = await buildDepositorYieldHistory("GDep", "GTok", 7, 0);
+    // Full emergency withdraw → depositor has 0 shares → currentValue = 0
+    if (history.length > 0) {
+      const latest = history[history.length - 1]!;
+      expect(latest.currentValue).toBe(0);
+    }
+  });
+
+  it("decodes shares from base64 XDR value (BigInt conversion path)", async () => {
+    // XDR encodes [assetAmount=1000, shares=500] as a 2-element Vec of i128
+    // Generated with: nativeToScVal([BigInt(1000), BigInt(500)]).toXDR('base64')
+    const xdrDeposit = "AAAAEAAAAAEAAAACAAAABQAAAAAAAAPoAAAABQAAAAAAAAH0";
+    // [assetAmount=2000, shares=800]
+    const xdrWithdraw = "AAAAEAAAAAEAAAACAAAABQAAAAAAAAfQAAAABQAAAAAAAAMg";
+
+    const now = new Date();
+    const t1 = new Date(now);
+    t1.setUTCDate(t1.getUTCDate() - 2);
+    const t2 = new Date(now);
+    t2.setUTCDate(t2.getUTCDate() - 1);
+
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        { event_type: "Deposit", amount: "1000", ledger_closed_at: t1, value: xdrDeposit },
+        { event_type: "Withdraw", amount: "500", ledger_closed_at: t2, value: xdrWithdraw },
+      ],
+    });
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        { event_type: "Deposit", amount: "1000", ledger_closed_at: t1, value: xdrDeposit },
+        { event_type: "Withdraw", amount: "500", ledger_closed_at: t2, value: xdrWithdraw },
+      ],
+    });
+
+    // Should not throw even with non-null XDR values
+    const history = await buildDepositorYieldHistory("GDep", "GTok", 7, 1_000_000);
+    expect(history.length).toBeGreaterThan(0);
+  });
+
+  it("falls back gracefully when XDR value is malformed", async () => {
+    const now = new Date();
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        {
+          event_type: "Deposit",
+          amount: "1000",
+          ledger_closed_at: now,
+          value: "not-valid-base64-xdr!!!",
+        },
+      ],
+    });
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        {
+          event_type: "Deposit",
+          amount: "1000",
+          ledger_closed_at: now,
+          value: "not-valid-base64-xdr!!!",
+        },
+      ],
+    });
+
+    // Should not throw — falls back to assetAmount as shares
+    await expect(
+      buildDepositorYieldHistory("GDep", "GTok", 7),
+    ).resolves.toBeDefined();
+  });
 });

backend/src/services/cacheService.ts

@@ -145,6 +145,41 @@ class CacheService {
     }
   }
 
+  /**
+   * Delete a key only when its stored value matches `expectedValue` (fenced
+   * compare-and-delete).  Used by distributed locks so a run that outlives the
+   * TTL cannot delete a lock acquired by a different instance.
+   *
+   * @returns true if the key existed and the value matched (key deleted),
+   *          false if the key was absent or the value did not match.
+   */
+  async deleteIfMatch(key: string, expectedValue: string): Promise<boolean> {
+    try {
+      await this.ensureConnected();
+      const stored = await this.client!.get(key);
+      if (stored === null) return false;
+
+      let storedValue: unknown;
+      try {
+        storedValue = JSON.parse(stored);
+      } catch {
+        storedValue = stored;
+      }
+
+      if (storedValue !== expectedValue) return false;
+
+      await this.client!.del(key);
+      return true;
+    } catch (error) {
+      if (process.env.NODE_ENV !== "test") {
+        logger
+          .withContext()
+          .error(`Error in deleteIfMatch for key ${key}`, { error });
+      }
+      return false;
+    }
+  }
+
   /**
    * Invalidate multiple keys by a pattern (e.g. prefix)
    * Note: KEYS is generally not recommended in production, but suitable for exact or bounded patterns.

backend/src/services/defaultChecker.ts

@@ -94,6 +94,7 @@ export class DefaultChecker {
   private pollAttempts: number;
   private pollSleepMs: number;
   private concurrency: number;
+  private currentLockValue: string | null = null;
 
   constructor() {
     this.contractId = process.env.LOAN_MANAGER_CONTRACT_ID || "";
@@ -410,6 +411,9 @@ export class DefaultChecker {
         lockValue,
         LOCK_TTL_SECONDS,
       );
+      if (acquired) {
+        this.currentLockValue = lockValue;
+      }
       return acquired;
     } catch (error) {
       logger
@@ -420,11 +424,25 @@ export class DefaultChecker {
   }
 
   /**
-   * Releases the distributed lock.
+   * Releases the distributed lock only when the stored value matches the value
+   * this instance set at acquire time.  A run that outlives the TTL cannot
+   * delete a lock that now belongs to a different instance.
    */
   private async releaseLock(): Promise<void> {
+    const value = this.currentLockValue;
+    this.currentLockValue = null;
+
+    if (!value) return;
+
     try {
-      await cacheService.delete(LOCK_KEY);
+      const released = await cacheService.deleteIfMatch(LOCK_KEY, value);
+      if (!released) {
+        logger
+          .withContext()
+          .warn(
+            "default_checker: lock already expired or owned by another instance — skipping delete",
+          );
+      }
     } catch (error) {
       logger
         .withContext()

backend/src/services/eventIndexer.ts

@@ -270,6 +270,14 @@ export class EventIndexer {
     const lastIndexedLedger = await this.getLastIndexedLedger();
     const latestLedger = await this.getLatestLedgerSequence();
 
+    // latestLedger === 0 means getLatestLedgerSequence failed (RPC error or
+    // non-finite response).  Publishing lag=0 / chainTip=0 during an outage
+    // would silently defeat the behind-chain-tip alert, so skip the metric
+    // update and leave gauges at their last known values.
+    if (latestLedger === 0) {
+      return;
+    }
+
     if (latestLedger <= lastIndexedLedger) {
       recordIndexerLedgers(lastIndexedLedger, latestLedger);
       return;

backend/src/services/webhookRetryProcessor.ts

@@ -6,10 +6,19 @@ import { jobMetricsService } from "./jobMetricsService.js";
 let retryProcessorInterval: NodeJS.Timeout | null = null;
 
 /**
- * Starts the webhook retry processor that periodically checks for failed
- * webhook deliveries and retries them with exponential backoff.
+ * Starts the webhook retry processor.
  *
- * Runs every 10 seconds to process pending retries.
+ * Polls every 10 seconds and delegates to WebhookService.processRetries,
+ * which queries for deliveries whose next_retry_at <= now and applies the
+ * backoff schedule defined in webhookService.ts:
+ *   attempt 1 → retry after 5 min
+ *   attempt 2 → retry after 15 min
+ *   attempt 3 → retry after 45 min
+ *   attempt 4+ → permanently failed (MAX_RETRY_ATTEMPTS = 4)
+ *
+ * This is the single retry implementation wired in index.ts.
+ * webhookRetryScheduler.ts (which used a different backoff and ignored
+ * next_retry_at) has been removed.
  */
 export function startWebhookRetryProcessor(): void {
   if (retryProcessorInterval) {