diff --git a/src/database/adapters/HybridStorageAdapter.ts b/src/database/adapters/HybridStorageAdapter.ts index 6bf42296f..d6c7283a9 100644 --- a/src/database/adapters/HybridStorageAdapter.ts +++ b/src/database/adapters/HybridStorageAdapter.ts @@ -232,22 +232,37 @@ export class HybridStorageAdapter implements IStorageAdapter { */ private async performInitialization(): Promise { try { + console.log('[HybridStorageAdapter] Starting initialization...'); + const migrator = new LegacyMigrator(this.app); const migrationNeeded = await migrator.isMigrationNeeded(); let actuallyMigrated = false; if (migrationNeeded) { + console.log('[HybridStorageAdapter] Legacy migration needed, running...'); const migrationResult = await migrator.migrate(); // Only count as "actually migrated" if something was migrated actuallyMigrated = migrationResult.needed && (migrationResult.stats.workspacesMigrated > 0 || migrationResult.stats.conversationsMigrated > 0); + console.log('[HybridStorageAdapter] Legacy migration complete:', { + workspaces: migrationResult.stats.workspacesMigrated, + conversations: migrationResult.stats.conversationsMigrated + }); } + console.log('[HybridStorageAdapter] Preparing storage plan...'); const storagePlan = await this.storageCoordinator.prepareStoragePlan(); this.applyStoragePlan(storagePlan); + console.log('[HybridStorageAdapter] Storage plan applied:', { + writePath: storagePlan.writeBasePath, + readPaths: storagePlan.readBasePaths, + migrationState: storagePlan.state.migration.state + }); // 1. Initialize SQLite cache + console.log('[HybridStorageAdapter] Initializing SQLite cache...'); await this.sqliteCache.initialize(); + console.log('[HybridStorageAdapter] SQLite cache initialized'); // 2. Ensure JSONL directories exist await this.jsonlWriter.ensureDirectory('workspaces'); @@ -266,19 +281,24 @@ export class HybridStorageAdapter implements IStorageAdapter { // The UI will show incrementally as data syncs in. const syncState = await this.sqliteCache.getSyncState(this.jsonlWriter.getDeviceId()); if (!syncState || actuallyMigrated) { + console.log('[HybridStorageAdapter] Running full rebuild...'); try { await this.syncCoordinator.fullRebuild(); + console.log('[HybridStorageAdapter] Full rebuild complete'); } catch (rebuildError) { console.error('[HybridStorageAdapter] Full rebuild failed:', rebuildError); } } else { + console.log('[HybridStorageAdapter] Running incremental sync...'); try { await this.syncCoordinator.sync(); + console.log('[HybridStorageAdapter] Incremental sync complete'); } catch (syncError) { console.error('[HybridStorageAdapter] Incremental sync failed:', syncError); } // 5. Reconcile JSONL workspaces missing from SQLite + console.log('[HybridStorageAdapter] Reconciling missing data...'); try { await this.reconcileMissingWorkspaces(); } catch (reconcileError) { @@ -298,8 +318,28 @@ export class HybridStorageAdapter implements IStorageAdapter { } catch (reconcileError) { console.error('[HybridStorageAdapter] Task reconciliation failed:', reconcileError); } + console.log('[HybridStorageAdapter] Reconciliation complete'); + } + + // Copy fully-populated cache.db to plugin-scoped storage after sync completes. + // Must happen AFTER rebuild/sync so the copy includes sync state. + if (this.storageCoordinator.backgroundMigration) { + try { + await this.storageCoordinator.backgroundMigration; + const dataRoot = this.storageCoordinator.roots.dataRoot; + const legacyCacheDb = `${this.basePath}/cache.db`; + const newCacheDb = `${dataRoot}/cache.db`; + if (await this.app.vault.adapter.exists(legacyCacheDb)) { + const content = await this.app.vault.adapter.readBinary(legacyCacheDb); + await this.app.vault.adapter.writeBinary(newCacheDb, content); + console.log('[HybridStorageAdapter] Copied cache.db to plugin-scoped storage'); + } + } catch (cacheError) { + console.warn('[HybridStorageAdapter] cache.db copy failed (will rebuild on next boot):', cacheError); + } } + console.log('[HybridStorageAdapter] Initialization complete'); } catch (error) { console.error('[HybridStorageAdapter] Initialization failed:', error); this.initError = error as Error; @@ -314,6 +354,7 @@ export class HybridStorageAdapter implements IStorageAdapter { this.basePath = plan.writeBasePath; this.jsonlWriter.setBasePath(plan.writeBasePath); this.jsonlWriter.setReadBasePaths(plan.readBasePaths); + this.sqliteCache.setDbPath(`${plan.writeBasePath}/cache.db`); } /** diff --git a/src/database/migration/PluginScopedStorageCoordinator.ts b/src/database/migration/PluginScopedStorageCoordinator.ts index 3e7db9b45..f73ceb0f2 100644 --- a/src/database/migration/PluginScopedStorageCoordinator.ts +++ b/src/database/migration/PluginScopedStorageCoordinator.ts @@ -1,4 +1,4 @@ -import { App, Plugin, normalizePath } from 'obsidian'; +import { App, Notice, Plugin, normalizePath } from 'obsidian'; import type { MCPSettings } from '../../types/plugin/PluginTypes'; import { resolvePluginStorageRoot, @@ -64,9 +64,24 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null; } +/** + * Two-stage migration coordinator for plugin-scoped storage. + * + * Stage 1 (first boot with legacy data): Returns legacy paths immediately, + * kicks off file copy + verify as fire-and-forget background work. + * When complete, persists state as 'verified'. No path changes this session. + * + * Stage 2 (subsequent boot after verified): Returns plugin-data paths instantly. + * No file I/O needed — the copy was completed in a prior session. + * + * Failed state: Stays on legacy paths. Background copy may retry on next boot. + */ export class PluginScopedStorageCoordinator { readonly roots: ResolvedPluginStorageRoot; + /** Exposed for testing — resolves when background migration finishes. */ + backgroundMigration: Promise | null = null; + constructor( private readonly app: App, private readonly plugin: Plugin, @@ -75,81 +90,110 @@ export class PluginScopedStorageCoordinator { this.roots = resolvePluginStorageRoot(app, plugin); } + /** + * Return a storage plan quickly. Never blocks on file copy I/O. + * + * - verified: instant cutover to plugin-data paths + * - not_started/copying/copied/failed with legacy files: legacy paths, + * background copy kicked off + * - no legacy files: plugin-data paths (nothing to migrate) + */ async prepareStoragePlan(): Promise { - await this.ensureDirectory(this.roots.dataRoot); - await this.ensureDirectory(this.roots.migrationRoot); + const state = await this.loadState(); - // Short-circuit: if migration already verified, skip all filesystem I/O - const persistedState = await this.loadState(); - if (persistedState.migration.state === 'verified') { - return { - writeBasePath: this.roots.dataRoot, - readBasePaths: [this.roots.dataRoot, this.legacyBasePath], - state: { - ...persistedState, - sourceOfTruthLocation: 'plugin-data', - migration: { - ...persistedState.migration, - activeDestination: this.roots.dataRoot - } - }, - roots: this.roots - }; + // Stage 2: migration already verified in a prior session — instant cutover + if (state.migration.state === 'verified') { + return this.buildPluginDataPlan(state); } + // Check whether legacy files exist (lightweight directory listing) const legacyFiles = await this.collectLegacyFiles(); - let state = persistedState; - state = { - ...state, - migration: { - ...state.migration, - activeDestination: this.roots.dataRoot, - legacySourcesDetected: legacyFiles.length > 0 ? [this.legacyBasePath] : [] - } - }; + // No legacy data — go straight to plugin-data paths if (legacyFiles.length === 0) { - const nextState: PluginScopedStorageState = { + const freshState: PluginScopedStorageState = { ...state, sourceOfTruthLocation: 'plugin-data', migration: { ...state.migration, + activeDestination: this.roots.dataRoot, + legacySourcesDetected: [], + // Preserve failed state if a prior attempt failed state: state.migration.state === 'failed' ? 'failed' : state.migration.state, lastError: state.migration.state === 'failed' ? state.migration.lastError : undefined } }; - await this.saveState(nextState); + await this.saveState(freshState); return { writeBasePath: this.roots.dataRoot, readBasePaths: [this.roots.dataRoot], - state: nextState, + state: freshState, roots: this.roots }; } + // Stage 1: legacy files exist — return legacy plan immediately, + // kick off copy+verify in the background + const legacyState: PluginScopedStorageState = { + ...state, + sourceOfTruthLocation: 'legacy-dotnexus', + migration: { + ...state.migration, + activeDestination: this.roots.dataRoot, + legacySourcesDetected: [this.legacyBasePath] + } + }; + + this.backgroundMigration = this.runBackgroundMigration(legacyState, legacyFiles); + + return { + writeBasePath: this.legacyBasePath, + readBasePaths: [this.legacyBasePath], + state: legacyState, + roots: this.roots + }; + } + + /** + * Fire-and-forget background migration. Copies legacy files to plugin-scoped + * storage, verifies them, and saves state as 'verified'. Errors are caught + * and persisted as 'failed' state — they don't propagate to the caller. + */ + private async runBackgroundMigration( + state: PluginScopedStorageState, + legacyFiles: string[] + ): Promise { try { - state = await this.runCopyOnlyMigration(state, legacyFiles); - if (state.migration.state === 'verified') { - const verifiedState: PluginScopedStorageState = { - ...state, - sourceOfTruthLocation: 'plugin-data' - }; - await this.saveState(verifiedState); - return { - writeBasePath: this.roots.dataRoot, - readBasePaths: [this.roots.dataRoot, this.legacyBasePath], - state: verifiedState, - roots: this.roots - }; + new Notice('Preparing your data for cross-device sync…'); + await this.ensureDirectory(this.roots.dataRoot); + await this.ensureDirectory(this.roots.migrationRoot); + + const finalState = await this.runCopyOnlyMigration(state, legacyFiles); + if (finalState.migration.state === 'verified') { + console.warn('[PluginScopedStorageCoordinator] Background migration verified — cutover will happen on next boot'); + new Notice('Data migration complete — changes take effect on next restart.'); } } catch (error) { - state = await this.saveFailureState(state, error instanceof Error ? error.message : String(error)); + console.error('[PluginScopedStorageCoordinator] Background migration failed:', error); + new Notice('Data migration encountered an issue — see console for details.'); + await this.saveFailureState(state, error instanceof Error ? error.message : String(error)).catch(() => { + // Best-effort — don't let state save failure mask the original error + }); } + } + private buildPluginDataPlan(state: PluginScopedStorageState): PluginScopedStoragePlan { return { - writeBasePath: this.legacyBasePath, - readBasePaths: [this.legacyBasePath], - state, + writeBasePath: this.roots.dataRoot, + readBasePaths: [this.roots.dataRoot, this.legacyBasePath], + state: { + ...state, + sourceOfTruthLocation: 'plugin-data', + migration: { + ...state.migration, + activeDestination: this.roots.dataRoot + } + }, roots: this.roots }; } @@ -383,4 +427,4 @@ export class PluginScopedStorageCoordinator { await this.app.vault.adapter.mkdir(path); } -} \ No newline at end of file +} diff --git a/src/database/storage/SQLiteCacheManager.ts b/src/database/storage/SQLiteCacheManager.ts index dc8f44262..f56826d27 100644 --- a/src/database/storage/SQLiteCacheManager.ts +++ b/src/database/storage/SQLiteCacheManager.ts @@ -118,6 +118,18 @@ export class SQLiteCacheManager implements IStorageBackend, ISQLiteCacheManager this.searchService = new SQLiteSearchService(this); } + /** + * Update the database path before initialization. + * Must be called before initialize() — has no effect after the DB is open. + */ + setDbPath(path: string): void { + if (this.isInitialized) { + console.warn('[SQLiteCacheManager] setDbPath called after initialization — ignoring'); + return; + } + this.dbPath = path; + } + private getSqlite3OrThrow(): SQLite3Module { if (!this.sqlite3) { throw new Error('SQLite module not initialized'); diff --git a/tests/unit/PluginScopedStorageCoordinator.test.ts b/tests/unit/PluginScopedStorageCoordinator.test.ts index a9707fb6d..1587adfc0 100644 --- a/tests/unit/PluginScopedStorageCoordinator.test.ts +++ b/tests/unit/PluginScopedStorageCoordinator.test.ts @@ -90,7 +90,7 @@ describe('PluginScopedStorageCoordinator', () => { expect(roots.dataRoot).toBe('.obsidian/plugins/claudesidian-mcp/data'); }); - it('copies verified legacy JSONL data into plugin-scoped storage and cuts over reads and writes', async () => { + it('returns legacy paths on first boot with legacy data, kicks off background copy', async () => { const adapter = createMockAdapter({ '.nexus/workspaces/ws_alpha.jsonl': '{"id":"evt-ws"}\n', '.nexus/conversations/conv_alpha.jsonl': '{"id":"evt-conv"}\n' @@ -113,13 +113,18 @@ describe('PluginScopedStorageCoordinator', () => { const plan = await coordinator.prepareStoragePlan(); - expect(plan.writeBasePath).toBe('.obsidian/plugins/claudesidian-mcp/data'); - expect(plan.readBasePaths).toEqual([ - '.obsidian/plugins/claudesidian-mcp/data', - '.nexus' - ]); - expect(plan.state.sourceOfTruthLocation).toBe('plugin-data'); - expect(plan.state.migration.state).toBe('verified'); + // Stage 1: returns legacy paths immediately + expect(plan.writeBasePath).toBe('.nexus'); + expect(plan.readBasePaths).toEqual(['.nexus']); + expect(plan.state.sourceOfTruthLocation).toBe('legacy-dotnexus'); + + // Background migration was started + expect(coordinator.backgroundMigration).not.toBeNull(); + + // Wait for background migration to complete + await coordinator.backgroundMigration; + + // Verify files were copied in the background expect(adapter.write).toHaveBeenCalledWith( '.obsidian/plugins/claudesidian-mcp/data/workspaces/ws_alpha.jsonl', '{"id":"evt-ws"}\n' @@ -128,11 +133,59 @@ describe('PluginScopedStorageCoordinator', () => { '.obsidian/plugins/claudesidian-mcp/data/conversations/conv_alpha.jsonl', '{"id":"evt-conv"}\n' ); - expect(adapter.exists).toHaveBeenCalledWith('.nexus/workspaces'); - expect(saveData).toHaveBeenCalled(); + + // State was persisted as verified + const lastSaveCall = saveData.mock.calls[saveData.mock.calls.length - 1][0]; + expect(lastSaveCall.pluginStorage.migration.state).toBe('verified'); }); - it('does not overwrite newer plugin-scoped data and falls back to legacy writes when migration detects conflicts', async () => { + it('returns plugin-data paths on second boot after verified migration', async () => { + const adapter = createMockAdapter({ + '.nexus/workspaces/ws_alpha.jsonl': '{"id":"evt-ws"}\n', + '.obsidian/plugins/claudesidian-mcp/data/workspaces/ws_alpha.jsonl': '{"id":"evt-ws"}\n' + }); + const coordinator = new PluginScopedStorageCoordinator( + { + vault: { adapter, configDir: '.obsidian' } + } as never, + { + manifest: { + id: 'nexus', + dir: '/mock/.obsidian/plugins/claudesidian-mcp' + }, + loadData: jest.fn(async () => ({ + pluginStorage: { + storageVersion: 1, + sourceOfTruthLocation: 'plugin-data', + migration: { + state: 'verified', + verifiedAt: Date.now(), + legacySourcesDetected: ['.nexus'], + activeDestination: '.obsidian/plugins/claudesidian-mcp/data' + } + } + })), + saveData: jest.fn(async () => undefined) + } as never, + '.nexus' + ); + + const plan = await coordinator.prepareStoragePlan(); + + // Stage 2: instant cutover to plugin-data paths + expect(plan.writeBasePath).toBe('.obsidian/plugins/claudesidian-mcp/data'); + expect(plan.readBasePaths).toEqual([ + '.obsidian/plugins/claudesidian-mcp/data', + '.nexus' + ]); + expect(plan.state.sourceOfTruthLocation).toBe('plugin-data'); + expect(plan.state.migration.state).toBe('verified'); + + // No background migration started + expect(coordinator.backgroundMigration).toBeNull(); + }); + + it('does not overwrite newer plugin-scoped data and records failure in background', async () => { const adapter = createMockAdapter({ '.nexus/workspaces/ws_alpha.jsonl': '{"id":"legacy-evt"}\n', '.obsidian/plugins/claudesidian-mcp/data/workspaces/ws_alpha.jsonl': '{"id":"plugin-evt","newer":true}\n' @@ -155,18 +208,49 @@ describe('PluginScopedStorageCoordinator', () => { const plan = await coordinator.prepareStoragePlan(); + // Returns legacy paths immediately (Stage 1) expect(plan.writeBasePath).toBe('.nexus'); expect(plan.readBasePaths).toEqual(['.nexus']); expect(plan.state.sourceOfTruthLocation).toBe('legacy-dotnexus'); - expect(plan.state.migration.state).toBe('failed'); - expect(plan.state.migration.lastError).toContain('destination already exists with different content'); - expect(adapter.write).not.toHaveBeenCalledWith( - '.obsidian/plugins/claudesidian-mcp/data/workspaces/ws_alpha.jsonl', - '{"id":"legacy-evt"}\n' - ); + + // Wait for background migration to complete (it will fail due to conflict) + await coordinator.backgroundMigration; + + // Plugin-scoped data was NOT overwritten await expect( adapter.read('.obsidian/plugins/claudesidian-mcp/data/workspaces/ws_alpha.jsonl') ).resolves.toBe('{"id":"plugin-evt","newer":true}\n'); - expect(saveData).toHaveBeenCalled(); + + // State was persisted as failed + const lastSaveCall = saveData.mock.calls[saveData.mock.calls.length - 1][0]; + expect(lastSaveCall.pluginStorage.migration.state).toBe('failed'); + expect(lastSaveCall.pluginStorage.migration.lastError).toContain( + 'destination already exists with different content' + ); + }); + + it('goes straight to plugin-data paths when no legacy files exist', async () => { + const adapter = createMockAdapter({}); + const saveData = jest.fn(async () => undefined); + const coordinator = new PluginScopedStorageCoordinator( + { + vault: { adapter, configDir: '.obsidian' } + } as never, + { + manifest: { + id: 'nexus', + dir: '/mock/.obsidian/plugins/claudesidian-mcp' + }, + loadData: jest.fn(async () => ({})), + saveData + } as never, + '.nexus' + ); + + const plan = await coordinator.prepareStoragePlan(); + + expect(plan.writeBasePath).toBe('.obsidian/plugins/claudesidian-mcp/data'); + expect(plan.readBasePaths).toEqual(['.obsidian/plugins/claudesidian-mcp/data']); + expect(coordinator.backgroundMigration).toBeNull(); }); -}); \ No newline at end of file +});