diff --git a/lib/storage/providers/IDBKeyValProvider/createStore.ts b/lib/storage/providers/IDBKeyValProvider/createStore.ts index d7c6d0f8b..f6c64718d 100644 --- a/lib/storage/providers/IDBKeyValProvider/createStore.ts +++ b/lib/storage/providers/IDBKeyValProvider/createStore.ts @@ -2,12 +2,48 @@ import * as IDB from 'idb-keyval'; import type {UseStore} from 'idb-keyval'; import * as Logger from '../../../Logger'; +const HEAL_ATTEMPTS_MAX = 3; + +/** + * Detects the Chromium-specific IDB backing store corruption error. + * Fires when LevelDB files backing IndexedDB are corrupted and Chrome's + * internal recovery (RepairDB -> delete -> recreate) also fails. + */ +function isBackingStoreError(error: unknown): boolean { + return error instanceof Error && error.message.includes('Internal error opening backing store'); +} + +/** + * Detects Safari/WebKit IDB connection termination errors. + * Fires when Safari kills the IDB server process for backgrounded tabs. + * WebKit bugs: https://bugs.webkit.org/show_bug.cgi?id=197050, https://bugs.webkit.org/show_bug.cgi?id=201483 + */ +function isConnectionLostError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + const msg = error.message.toLowerCase(); + return msg.includes('connection to indexed database server lost') || msg.includes('connection is closing'); +} + +function isInvalidStateError(error: unknown): boolean { + return error instanceof Error && error.name === 'InvalidStateError'; +} + +/** Errors that trigger a budgeted heal-and-retry in store(). */ +function isBudgetedHealError(error: unknown): boolean { + return isBackingStoreError(error) || isConnectionLostError(error); +} + +function getBudgetedHealErrorLabel(error: unknown): 'backing store' | 'connection lost' { + return isBackingStoreError(error) ? 'backing store' : 'connection lost'; +} + // This is a copy of the createStore function from idb-keyval, we need a custom implementation // because we need to create the database manually in order to ensure that the store exists before we use it. // If the store does not exist, idb-keyval will throw an error // source: https://github.com/jakearchibald/idb-keyval/blob/9d19315b4a83897df1e0193dccdc29f78466a0f3/src/index.ts#L12 function createStore(dbName: string, storeName: string): UseStore { let dbp: Promise | undefined; + let healAttemptsRemaining = HEAL_ATTEMPTS_MAX; const attachHandlers = (db: IDBDatabase) => { // Browsers may close idle IDB connections at any time, especially Safari. @@ -30,18 +66,27 @@ function createStore(dbName: string, storeName: string): UseStore { }; }; + // Cache the open promise and attach handlers + rejection cleanup. + // On rejection, clears dbp so the next operation retries with a fresh indexedDB.open() + // instead of returning the same rejected promise. + // Guard: only clear if dbp hasn't been replaced by a concurrent heal/retry. + function cacheOpenPromise(openPromise: Promise) { + dbp = openPromise; + const currentPromise = openPromise; + openPromise.then(attachHandlers, () => { + if (dbp !== currentPromise) { + return; + } + dbp = undefined; + }); + return openPromise; + } + const getDB = () => { if (dbp) return dbp; const request = indexedDB.open(dbName); request.onupgradeneeded = () => request.result.createObjectStore(storeName); - dbp = IDB.promisifyRequest(request); - - dbp.then( - attachHandlers, - // eslint-disable-next-line @typescript-eslint/no-empty-function - () => {}, - ); - return dbp; + return cacheOpenPromise(IDB.promisifyRequest(request)); }; // Ensures the store exists in the DB. If missing, bumps the version to trigger @@ -66,10 +111,7 @@ function createStore(dbName: string, storeName: string): UseStore { updatedDatabase.createObjectStore(storeName); }; - dbp = IDB.promisifyRequest(request); - // eslint-disable-next-line @typescript-eslint/no-empty-function - dbp.then(attachHandlers, () => {}); - return dbp; + return cacheOpenPromise(IDB.promisifyRequest(request)); }; function executeTransaction(txMode: IDBTransactionMode, callback: (store: IDBObjectStore) => T | PromiseLike): Promise { @@ -78,23 +120,64 @@ function createStore(dbName: string, storeName: string): UseStore { .then((db) => callback(db.transaction(storeName, txMode).objectStore(storeName))); } - // If the connection was closed between getDB() resolving and db.transaction() executing, - // the transaction throws InvalidStateError. We catch it and retry once with a fresh connection. + function resetHealBudget(result: T): T { + healAttemptsRemaining = HEAL_ATTEMPTS_MAX; + return result; + } + + // Handles three recoverable error classes: + // 1. InvalidStateError — connection closed between getDB() resolving and db.transaction(). + // Retry once with a fresh connection. No budget limit (transient, always worth one reopen). + // 2. Backing store corruption (Chromium UnknownError) — drop cached connection and reopen. + // 3. Connection lost (Safari UnknownError) — IDB server terminated for backgrounded tabs. + // Both 2 and 3 share a heal budget (3 attempts, reset on success). + // Mirrors Dexie's PR1398_maxLoop pattern: https://github.com/dexie/Dexie.js/blob/master/src/functions/temp-transaction.ts + // Note: concurrent store() calls share the budget. Under overlapping failures each caller + // decrements independently, so the budget may drain faster than one-per-incident. This is + // acceptable — same as Dexie's approach — and the budget resets on any success. return (txMode, callback) => - executeTransaction(txMode, callback).catch((error) => { - if (error instanceof DOMException && error.name === 'InvalidStateError') { - Logger.logAlert('IDB InvalidStateError, retrying with fresh connection', { - dbName, - storeName, - txMode, - errorMessage: error.message, - }); - dbp = undefined; - // Retry only once — this call is not wrapped, so if it also fails the error propagates normally. - return executeTransaction(txMode, callback); - } - throw error; - }); + executeTransaction(txMode, callback) + .then(resetHealBudget) + .catch((error) => { + if (isInvalidStateError(error)) { + Logger.logAlert('IDB InvalidStateError — dropping cached connection and retrying', { + dbName, + storeName, + txMode, + errorMessage: error instanceof Error ? error.message : String(error), + }); + dbp = undefined; + return executeTransaction(txMode, callback).then(resetHealBudget); + } + + if (isBudgetedHealError(error) && healAttemptsRemaining > 0) { + healAttemptsRemaining--; + const label = getBudgetedHealErrorLabel(error); + Logger.logAlert(`IDB heal: ${label} error detected — dropping cached connection and reopening (${healAttemptsRemaining} attempts left)`, { + dbName, + storeName, + }); + dbp = undefined; + return executeTransaction(txMode, callback).then((result) => { + Logger.logInfo(`IDB heal: successfully recovered after ${label} error`, {dbName, storeName}); + return resetHealBudget(result); + }); + } + + if (isBudgetedHealError(error)) { + Logger.logAlert(`IDB heal: ${getBudgetedHealErrorLabel(error)} error — heal budget exhausted, giving up`, { + dbName, + storeName, + }); + } else { + Logger.logAlert('IDB error is not recoverable, giving up', { + dbName, + storeName, + errorMessage: error instanceof Error ? error.message : String(error), + }); + } + throw error; + }); } export default createStore; diff --git a/tests/unit/storage/providers/createStoreTest.ts b/tests/unit/storage/providers/createStoreTest.ts index 231244706..b9a71f57e 100644 --- a/tests/unit/storage/providers/createStoreTest.ts +++ b/tests/unit/storage/providers/createStoreTest.ts @@ -96,7 +96,8 @@ describe('createStore', () => { await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow(DOMException); expect(callCount).toBe(1); - expect(logAlertSpy).not.toHaveBeenCalled(); + expect(logAlertSpy).toHaveBeenCalledWith('IDB error is not recoverable, giving up', expect.objectContaining({errorMessage: 'Not found'})); + expect(logAlertSpy).not.toHaveBeenCalledWith(expect.stringContaining('dropping cached connection'), expect.anything()); }); it('should not retry on non-DOMException errors', async () => { @@ -115,7 +116,8 @@ describe('createStore', () => { await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow(TypeError); expect(callCount).toBe(1); - expect(logAlertSpy).not.toHaveBeenCalled(); + expect(logAlertSpy).toHaveBeenCalledWith('IDB error is not recoverable, giving up', expect.objectContaining({errorMessage: 'Something went wrong'})); + expect(logAlertSpy).not.toHaveBeenCalledWith(expect.stringContaining('dropping cached connection'), expect.anything()); }); it('should preserve data integrity after a successful retry', async () => { @@ -174,7 +176,7 @@ describe('createStore', () => { return IDB.promisifyRequest(s.transaction); }); - expect(logAlertSpy).toHaveBeenCalledWith('IDB InvalidStateError, retrying with fresh connection', { + expect(logAlertSpy).toHaveBeenCalledWith('IDB InvalidStateError — dropping cached connection and retrying', { dbName, storeName: STORE_NAME, txMode: 'readwrite', @@ -245,4 +247,310 @@ describe('createStore', () => { expect(result).toBe('value'); }); }); + + describe('backing store healing', () => { + /** + * Helper: creates a DOMException matching Chromium's backing store corruption error. + */ + function backingStoreError() { + return new DOMException('Internal error opening backing store for indexedDB.open.', 'UnknownError'); + } + + it('should heal mid-session by dropping cached connection and reopening', async () => { + const store = createStore(uniqueDBName(), STORE_NAME); + + await store('readwrite', (s) => { + s.put('value', 'key1'); + return IDB.promisifyRequest(s.transaction); + }); + + const original = IDBDatabase.prototype.transaction; + let callCount = 0; + jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(function (this: IDBDatabase, ...args) { + callCount++; + if (callCount === 1) { + throw backingStoreError(); + } + return original.apply(this, args); + }); + + const result = await store('readonly', (s) => IDB.promisifyRequest(s.get('key1'))); + expect(result).toBe('value'); + expect(callCount).toBe(2); + expect(logAlertSpy).toHaveBeenCalledWith('IDB heal: backing store error detected — dropping cached connection and reopening (2 attempts left)', expect.objectContaining({dbName: expect.any(String)})); + expect(logInfoSpy).toHaveBeenCalledWith('IDB heal: successfully recovered after backing store error', expect.objectContaining({dbName: expect.any(String)})); + }); + + it('should heal on init when indexedDB.open() rejects with UnknownError', async () => { + const dbName = uniqueDBName(); + + // Pre-create the DB with data + const setupStore = createStore(dbName, STORE_NAME); + await setupStore('readwrite', (s) => { + s.put('persisted', 'key1'); + return IDB.promisifyRequest(s.transaction); + }); + + // Fresh store instance (simulates app restart — dbp starts undefined) + const store = createStore(dbName, STORE_NAME); + + // Mock indexedDB.open to fail twice, then succeed. + // First failure: initial open in the first store() call. + // Second failure: the heal retry's open in that same call — propagates error. + // Third call (second store() invocation): succeeds, proving budget still has 1 left. + const realOpen = indexedDB.open.bind(indexedDB); + let openCallCount = 0; + jest.spyOn(indexedDB, 'open').mockImplementation((name: string, version?: number) => { + openCallCount++; + if (openCallCount <= 2) { + const req = {} as IDBOpenDBRequest; + Promise.resolve().then(() => { + Object.defineProperty(req, 'error', {value: backingStoreError(), configurable: true}); + req.onerror?.(new Event('error') as Event & {target: IDBOpenDBRequest}); + }); + return req; + } + return realOpen(name, version); + }); + + // First call: open fails, heal retry also fails — error propagates + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow('Internal error opening backing store'); + + // Second call: open succeeds (mock exhausted), heals + const result = await store('readonly', (s) => IDB.promisifyRequest(s.get('key1'))); + expect(result).toBe('persisted'); + expect(openCallCount).toBeGreaterThanOrEqual(3); + }); + + it('should stop healing after budget exhausts across multiple operations', async () => { + const store = createStore(uniqueDBName(), STORE_NAME); + + // All opens fail permanently + jest.spyOn(indexedDB, 'open').mockImplementation(() => { + const req = {} as IDBOpenDBRequest; + Promise.resolve().then(() => { + Object.defineProperty(req, 'error', {value: backingStoreError(), configurable: true}); + req.onerror?.(new Event('error') as Event & {target: IDBOpenDBRequest}); + }); + return req; + }); + + // Each call consumes 1 heal attempt (initial fails, heal retry also fails, propagates) + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store'); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store'); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store'); + + // Budget exhausted — 4th call should NOT attempt healing, but should log budget exhausted + logAlertSpy.mockClear(); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store'); + expect(logAlertSpy).toHaveBeenCalledWith(expect.stringContaining('heal budget exhausted'), expect.anything()); + expect(logAlertSpy).not.toHaveBeenCalledWith(expect.stringContaining('dropping cached connection and reopening'), expect.anything()); + }); + + it('should reset heal budget after a successful operation', async () => { + const dbName = uniqueDBName(); + const store = createStore(dbName, STORE_NAME); + + await store('readwrite', (s) => { + s.put('value', 'key1'); + return IDB.promisifyRequest(s.transaction); + }); + + const original = IDBDatabase.prototype.transaction; + + // Drain budget to 1 remaining: fail twice, each heals successfully + for (let i = 0; i < 2; i++) { + let callCount = 0; + const spy = jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(function (this: IDBDatabase, ...args) { + callCount++; + if (callCount === 1) { + throw backingStoreError(); + } + spy.mockRestore(); + return original.apply(this, args); + }); + await store('readonly', (s) => IDB.promisifyRequest(s.get('key1'))); + } + + // Clean success — resets budget to 3 + jest.restoreAllMocks(); + logInfoSpy = jest.spyOn(Logger, 'logInfo'); + await store('readonly', (s) => IDB.promisifyRequest(s.get('key1'))); + + // Now fail 3 more times — all should still heal (proving budget was reset) + for (let i = 0; i < 3; i++) { + let callCount = 0; + const spy = jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(function (this: IDBDatabase, ...args) { + callCount++; + if (callCount === 1) { + throw backingStoreError(); + } + spy.mockRestore(); + return original.apply(this, args); + }); + const result = await store('readonly', (s) => IDB.promisifyRequest(s.get('key1'))); + expect(result).toBe('value'); + } + }); + + it('should not attempt healing for non-backing-store errors', async () => { + const store = createStore(uniqueDBName(), STORE_NAME); + + await store('readwrite', (s) => { + s.put('value', 'key1'); + return IDB.promisifyRequest(s.transaction); + }); + + // UnknownError but different message + jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(() => { + throw new DOMException('Some other unknown error', 'UnknownError'); + }); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow('Some other unknown error'); + + jest.restoreAllMocks(); + logAlertSpy = jest.spyOn(Logger, 'logAlert'); + + // QuotaExceededError + jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(() => { + throw new DOMException('Quota exceeded', 'QuotaExceededError'); + }); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow('Quota exceeded'); + + expect(logAlertSpy).not.toHaveBeenCalledWith(expect.stringContaining('dropping cached connection and reopening'), expect.anything()); + expect(logAlertSpy).toHaveBeenCalledWith('IDB error is not recoverable, giving up', expect.objectContaining({errorMessage: 'Quota exceeded'})); + }); + }); + + describe('connection lost healing', () => { + function connectionLostError() { + return new DOMException('Connection to Indexed Database server lost. Refresh the page to try again', 'UnknownError'); + } + + it('should heal by dropping cached connection and reopening', async () => { + const store = createStore(uniqueDBName(), STORE_NAME); + + await store('readwrite', (s) => { + s.put('value', 'key1'); + return IDB.promisifyRequest(s.transaction); + }); + + const original = IDBDatabase.prototype.transaction; + let callCount = 0; + jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(function (this: IDBDatabase, ...args) { + callCount++; + if (callCount === 1) { + throw connectionLostError(); + } + return original.apply(this, args); + }); + + const result = await store('readonly', (s) => IDB.promisifyRequest(s.get('key1'))); + expect(result).toBe('value'); + expect(callCount).toBe(2); + expect(logAlertSpy).toHaveBeenCalledWith( + expect.stringContaining('connection lost error detected — dropping cached connection and reopening'), + expect.objectContaining({dbName: expect.any(String)}), + ); + expect(logInfoSpy).toHaveBeenCalledWith( + 'IDB heal: successfully recovered after connection lost error', + expect.objectContaining({dbName: expect.any(String)}), + ); + }); + + it('should stop healing after budget exhausts', async () => { + const store = createStore(uniqueDBName(), STORE_NAME); + + // All transaction calls fail permanently with connection lost. + // The heal path clears dbp and calls indexedDB.open() again — mock that to + // also fail so fake-indexeddb doesn't deadlock waiting for the old connection + // to close (same pattern as the backing store budget exhaustion test). + jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(() => { + throw connectionLostError(); + }); + jest.spyOn(indexedDB, 'open').mockImplementation(() => { + const req = {} as IDBOpenDBRequest; + Promise.resolve().then(() => { + Object.defineProperty(req, 'error', {value: connectionLostError(), configurable: true}); + req.onerror?.(new Event('error') as Event & {target: IDBOpenDBRequest}); + }); + return req; + }); + + // Each call drains 1 heal attempt (initial fails, heal retry also fails) + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Connection to Indexed Database server lost'); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Connection to Indexed Database server lost'); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Connection to Indexed Database server lost'); + + // Budget exhausted — 4th call should NOT attempt healing, but should log budget exhausted + logAlertSpy.mockClear(); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Connection to Indexed Database server lost'); + expect(logAlertSpy).toHaveBeenCalledWith(expect.stringContaining('heal budget exhausted'), expect.anything()); + expect(logAlertSpy).not.toHaveBeenCalledWith(expect.stringContaining('dropping cached connection and reopening'), expect.anything()); + }); + + it('should also heal "connection is closing" variant', async () => { + const store = createStore(uniqueDBName(), STORE_NAME); + + await store('readwrite', (s) => { + s.put('value', 'key1'); + return IDB.promisifyRequest(s.transaction); + }); + + const original = IDBDatabase.prototype.transaction; + let callCount = 0; + jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(function (this: IDBDatabase, ...args) { + callCount++; + if (callCount === 1) { + throw new DOMException('Connection is closing.', 'UnknownError'); + } + return original.apply(this, args); + }); + + const result = await store('readonly', (s) => IDB.promisifyRequest(s.get('key1'))); + expect(result).toBe('value'); + expect(callCount).toBe(2); + }); + + it('should share heal budget with backing store errors', async () => { + const store = createStore(uniqueDBName(), STORE_NAME); + + // All transaction calls fail permanently, alternating error types. + // The heal path clears dbp and calls indexedDB.open() again — mock that to + // also fail so fake-indexeddb doesn't deadlock waiting for the old connection + // to close. + const txErrors = [ + new DOMException('Internal error opening backing store for indexedDB.open.', 'UnknownError'), + connectionLostError(), + new DOMException('Internal error opening backing store for indexedDB.open.', 'UnknownError'), + ]; + let txErrorIndex = 0; + jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(() => { + const err = txErrors[Math.min(txErrorIndex, txErrors.length - 1)]; + txErrorIndex++; + throw err; + }); + jest.spyOn(indexedDB, 'open').mockImplementation(() => { + const req = {} as IDBOpenDBRequest; + Promise.resolve().then(() => { + Object.defineProperty(req, 'error', { + value: new DOMException('Internal error opening backing store for indexedDB.open.', 'UnknownError'), + configurable: true, + }); + req.onerror?.(new Event('error') as Event & {target: IDBOpenDBRequest}); + }); + return req; + }); + + // 3 calls drain the budget (each call: fail + heal retry fail = 1 budget) + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow(); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow(); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow(); + + // Budget exhausted — no more healing for either error type + logAlertSpy.mockClear(); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow(); + expect(logAlertSpy).toHaveBeenCalledWith(expect.stringContaining('heal budget exhausted'), expect.anything()); + expect(logAlertSpy).not.toHaveBeenCalledWith(expect.stringContaining('dropping cached connection and reopening'), expect.anything()); + }); + }); });