Add seeded concurrency race harness for lease/reservation stores

src/credits/lease-store.ts

@@ -32,11 +32,18 @@ export interface ILeaseStore {
      * sharing the backend: if a live (unexpired) lease is already present —
      * even one with a different `leaseId`, e.g. acquired by a sibling pod that
      * raced this one — leave it untouched so its already-debited
-     * `localRemainingCredits` wins. Returns `true` if it wrote a fresh row,
-     * `false` if it kept an existing live lease (the caller should release the
-     * lease it acquired ONLY if its ID differs from the installed one — the
-     * server hands a racing acquire the same active lease back, and releasing
-     * that would pull the shared lease out from under every sibling).
+     * `localRemainingCredits` wins. If the slot holds an *expired* row with
+     * the SAME `leaseId`, implementations must NOT rewrite it either:
+     * rewriting resets `localRemainingCredits` to the full grant, erasing
+     * debits whose reservations are still open (the idempotent server hands a
+     * racing acquire the same active lease back, and a stale response can
+     * land after the local row's expiry — especially when a concurrent extend
+     * pushed the server-side expiry forward). Instead, reconcile it like an
+     * extend: granted-to-total, expiry only forward, balance untouched.
+     * Returns `true` if it wrote a fresh row, `false` if it kept (or
+     * reconciled) an existing lease (the caller should release the lease it
+     * acquired ONLY if its ID differs from the installed one — releasing the
+     * shared lease would pull it out from under every sibling).
      */
     replace(entry: Omit<LeaseEntry, "localRemainingCredits">): Promise<boolean>;
     /**
@@ -115,9 +122,11 @@ export class LeaseStore implements ILeaseStore {
      * Replace (or insert) the lease for the given (company, creditType).
      * Resets `localRemainingCredits` to `grantedAmount` for a new lease.
      * If a live lease already occupies the slot — regardless of `leaseId` — it
-     * is left alone so any already-debited `localRemainingCredits` wins.
-     * Returns `true` if a fresh row was written, `false` if an existing live
-     * lease was kept.
+     * is left alone so any already-debited `localRemainingCredits` wins. If
+     * the slot holds an expired row with the SAME `leaseId`, the entry is
+     * reconciled (granted-to-total, expiry forward) instead of rewritten —
+     * see `ILeaseStore.replace`. Returns `true` if a fresh row was written,
+     * `false` if an existing lease was kept.
      */
     async replace(entry: Omit<LeaseEntry, "localRemainingCredits">): Promise<boolean> {
         const key = leaseKey(entry.companyId, entry.creditTypeId);
@@ -128,6 +137,24 @@ export class LeaseStore implements ILeaseStore {
                 // already-debited localRemaining instead of clobbering it.
                 return false;
             }
+            if (existing && existing.leaseId === entry.leaseId) {
+                // The SAME lease coming back over its own expired row: a stale
+                // acquire response for a lease the idempotent server handed to
+                // a racing sibling too (possibly since extended, so still
+                // active server-side). Rewriting would reset
+                // localRemainingCredits to the full grant, erasing debits
+                // whose reservations are still open — reconcile like an
+                // extend instead (see ILeaseStore.replace).
+                const add = entry.grantedAmount - existing.grantedAmount;
+                if (add > 0) {
+                    existing.grantedAmount = entry.grantedAmount;
+                    existing.localRemainingCredits += add;
+                }
+                if (entry.expiresAt.getTime() > existing.expiresAt.getTime()) {
+                    existing.expiresAt = entry.expiresAt;
+                }
+                return false;
+            }
             this.leases.set(key, {
                 ...entry,
                 localRemainingCredits: entry.grantedAmount,

src/credits/redis-lease-store.ts

@@ -34,8 +34,13 @@ local now = (tonumber(t[1]) * 1000) + math.floor(tonumber(t[2]) / 1000)
  * installed by a sibling instance that raced this acquire. Keeping the existing
  * live lease preserves its already-debited `localRemainingCredits`; the caller
  * should release the lease it acquired only if its ID differs from the
- * installed one (see `CreditLeaseManager.acquire`). `now` is the Redis server
- * clock (see `LEASE_NOW_MS`).
+ * installed one (see `CreditLeaseManager.acquire`). An expired row with the
+ * SAME `leaseId` (a stale acquire response for a lease the idempotent server
+ * handed to a racing sibling, possibly since extended) is reconciled like an
+ * extend — granted-to-total, expiry only forward, balance untouched — instead
+ * of rewritten, which would reset the balance and erase debits whose
+ * reservations are still open. `now` is the Redis server clock (see
+ * `LEASE_NOW_MS`).
  */
 const REPLACE_SCRIPT =
     LEASE_NOW_MS +
@@ -51,6 +56,22 @@ if existing_id and existing_expiry > now then
     return 0
 end
 
+if existing_id == new_id then
+    local granted = tonumber(redis.call('HGET', KEYS[1], 'grantedAmount') or '0')
+    local add = tonumber(new_granted) - granted
+    if add > 0 then
+        local remaining = tonumber(redis.call('HGET', KEYS[1], 'localRemainingCredits') or '0')
+        redis.call('HSET', KEYS[1],
+            'grantedAmount', new_granted,
+            'localRemainingCredits', tostring(remaining + add))
+    end
+    if new_expiry > existing_expiry then
+        redis.call('HSET', KEYS[1], 'expiresAt', ARGV[3])
+        redis.call('PEXPIREAT', KEYS[1], new_expiry + grace)
+    end
+    return 0
+end
+
 redis.call('DEL', KEYS[1])
 redis.call('HSET', KEYS[1],
     'leaseId', new_id,
@@ -189,7 +210,7 @@ export class RedisLeaseStore implements ILeaseStore {
 
     async get(companyId: string, creditTypeId: string): Promise<LeaseEntry | undefined> {
         const raw = await this.client.hGetAll(this.hashKey(companyId, creditTypeId));
-        if (!raw || !raw.leaseId) return undefined;
+        if (!raw?.leaseId) return undefined;
         return decodeEntry(raw);
     }
 

tests/integration/redis-stores.test.ts

@@ -125,6 +125,35 @@ describeIf("Redis stores against a real redis-server", () => {
         expect(entry?.localRemainingCredits).toBe(200);
     });
 
+    it("replace reconciles an expired SAME-id lease instead of resetting its debited balance", async () => {
+        // A stale acquire response can hand back the lease already installed
+        // (the server is idempotent for an active slot) after the shared row
+        // expired — e.g. a concurrent extend pushed the server-side expiry
+        // forward. Rewriting would reset localRemainingCredits to the full
+        // grant, erasing debits whose reservations are still open.
+        const leaseStore = makeLeaseStore();
+        const creditTypeId = await installLease(leaseStore, { leaseId: "lse_a", grantedAmount: 100 });
+        expect(await leaseStore.tryReserve("co_1", creditTypeId, 40)).toBe(60);
+
+        await client.hSet(leaseStore.hashKey("co_1", creditTypeId), "expiresAt", String(Date.now() - 1_000));
+        const laterExpiry = new Date(Date.now() + 120_000);
+        const wrote = await leaseStore.replace({
+            leaseId: "lse_a",
+            companyId: "co_1",
+            creditTypeId,
+            grantedAmount: 150,
+            expiresAt: laterExpiry,
+        });
+        expect(wrote).toBe(false);
+        const entry = await leaseStore.get("co_1", creditTypeId);
+        expect(entry?.leaseId).toBe("lse_a");
+        // Granted reconciled to the server total; the 40-credit debit survives.
+        expect(entry?.grantedAmount).toBe(150);
+        expect(entry?.localRemainingCredits).toBe(110);
+        // Expiry moves forward with the response.
+        expect(entry?.expiresAt.getTime()).toBe(laterExpiry.getTime());
+    });
+
     it("tryReserve under contention admits exactly the available balance", async () => {
         const leaseStore = makeLeaseStore();
         const creditTypeId = await installLease(leaseStore, { grantedAmount: 100 });