feat: plugin-scoped storage and mobile sync improvements

src/database/adapters/HybridStorageAdapter.ts

@@ -232,22 +232,37 @@ export class HybridStorageAdapter implements IStorageAdapter {
    */
   private async performInitialization(): Promise<void> {
     try {
+      console.log('[HybridStorageAdapter] Starting initialization...');
+
       const migrator = new LegacyMigrator(this.app);
       const migrationNeeded = await migrator.isMigrationNeeded();
       let actuallyMigrated = false;
 
       if (migrationNeeded) {
+        console.log('[HybridStorageAdapter] Legacy migration needed, running...');
         const migrationResult = await migrator.migrate();
         // Only count as "actually migrated" if something was migrated
         actuallyMigrated = migrationResult.needed &&
           (migrationResult.stats.workspacesMigrated > 0 || migrationResult.stats.conversationsMigrated > 0);
+        console.log('[HybridStorageAdapter] Legacy migration complete:', {
+          workspaces: migrationResult.stats.workspacesMigrated,
+          conversations: migrationResult.stats.conversationsMigrated
+        });
       }
 
+      console.log('[HybridStorageAdapter] Preparing storage plan...');
       const storagePlan = await this.storageCoordinator.prepareStoragePlan();
       this.applyStoragePlan(storagePlan);
+      console.log('[HybridStorageAdapter] Storage plan applied:', {
+        writePath: storagePlan.writeBasePath,
+        readPaths: storagePlan.readBasePaths,
+        migrationState: storagePlan.state.migration.state
+      });
 
       // 1. Initialize SQLite cache
+      console.log('[HybridStorageAdapter] Initializing SQLite cache...');
       await this.sqliteCache.initialize();
+      console.log('[HybridStorageAdapter] SQLite cache initialized');
 
       // 2. Ensure JSONL directories exist
       await this.jsonlWriter.ensureDirectory('workspaces');
@@ -266,19 +281,24 @@ export class HybridStorageAdapter implements IStorageAdapter {
       // The UI will show incrementally as data syncs in.
       const syncState = await this.sqliteCache.getSyncState(this.jsonlWriter.getDeviceId());
       if (!syncState || actuallyMigrated) {
+        console.log('[HybridStorageAdapter] Running full rebuild...');
         try {
           await this.syncCoordinator.fullRebuild();
+          console.log('[HybridStorageAdapter] Full rebuild complete');
         } catch (rebuildError) {
           console.error('[HybridStorageAdapter] Full rebuild failed:', rebuildError);
         }
       } else {
+        console.log('[HybridStorageAdapter] Running incremental sync...');
         try {
           await this.syncCoordinator.sync();
+          console.log('[HybridStorageAdapter] Incremental sync complete');
         } catch (syncError) {
           console.error('[HybridStorageAdapter] Incremental sync failed:', syncError);
         }
 
         // 5. Reconcile JSONL workspaces missing from SQLite
+        console.log('[HybridStorageAdapter] Reconciling missing data...');
         try {
           await this.reconcileMissingWorkspaces();
         } catch (reconcileError) {
@@ -298,8 +318,28 @@ export class HybridStorageAdapter implements IStorageAdapter {
         } catch (reconcileError) {
           console.error('[HybridStorageAdapter] Task reconciliation failed:', reconcileError);
         }
+        console.log('[HybridStorageAdapter] Reconciliation complete');
+      }
+
+      // Copy fully-populated cache.db to plugin-scoped storage after sync completes.
+      // Must happen AFTER rebuild/sync so the copy includes sync state.
+      if (this.storageCoordinator.backgroundMigration) {
+        try {
+          await this.storageCoordinator.backgroundMigration;
+          const dataRoot = this.storageCoordinator.roots.dataRoot;
+          const legacyCacheDb = `${this.basePath}/cache.db`;
+          const newCacheDb = `${dataRoot}/cache.db`;
+          if (await this.app.vault.adapter.exists(legacyCacheDb)) {
+            const content = await this.app.vault.adapter.readBinary(legacyCacheDb);
+            await this.app.vault.adapter.writeBinary(newCacheDb, content);
+            console.log('[HybridStorageAdapter] Copied cache.db to plugin-scoped storage');
+          }
+        } catch (cacheError) {
+          console.warn('[HybridStorageAdapter] cache.db copy failed (will rebuild on next boot):', cacheError);
+        }
       }
 
+      console.log('[HybridStorageAdapter] Initialization complete');
     } catch (error) {
       console.error('[HybridStorageAdapter] Initialization failed:', error);
       this.initError = error as Error;
@@ -314,6 +354,7 @@ export class HybridStorageAdapter implements IStorageAdapter {
     this.basePath = plan.writeBasePath;
     this.jsonlWriter.setBasePath(plan.writeBasePath);
     this.jsonlWriter.setReadBasePaths(plan.readBasePaths);
+    this.sqliteCache.setDbPath(`${plan.writeBasePath}/cache.db`);
   }
 
   /**

src/database/migration/PluginScopedStorageCoordinator.ts

@@ -1,4 +1,4 @@
-import { App, Plugin, normalizePath } from 'obsidian';
+import { App, Notice, Plugin, normalizePath } from 'obsidian';
 import type { MCPSettings } from '../../types/plugin/PluginTypes';
 import {
   resolvePluginStorageRoot,
@@ -64,9 +64,24 @@ function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null;
 }
 
+/**
+ * Two-stage migration coordinator for plugin-scoped storage.
+ *
+ * Stage 1 (first boot with legacy data): Returns legacy paths immediately,
+ *   kicks off file copy + verify as fire-and-forget background work.
+ *   When complete, persists state as 'verified'. No path changes this session.
+ *
+ * Stage 2 (subsequent boot after verified): Returns plugin-data paths instantly.
+ *   No file I/O needed — the copy was completed in a prior session.
+ *
+ * Failed state: Stays on legacy paths. Background copy may retry on next boot.
+ */
 export class PluginScopedStorageCoordinator {
   readonly roots: ResolvedPluginStorageRoot;
 
+  /** Exposed for testing — resolves when background migration finishes. */
+  backgroundMigration: Promise<void> | null = null;
+
   constructor(
     private readonly app: App,
     private readonly plugin: Plugin,
@@ -75,81 +90,110 @@ export class PluginScopedStorageCoordinator {
     this.roots = resolvePluginStorageRoot(app, plugin);
   }
 
+  /**
+   * Return a storage plan quickly. Never blocks on file copy I/O.
+   *
+   * - verified: instant cutover to plugin-data paths
+   * - not_started/copying/copied/failed with legacy files: legacy paths,
+   *   background copy kicked off
+   * - no legacy files: plugin-data paths (nothing to migrate)
+   */
   async prepareStoragePlan(): Promise<PluginScopedStoragePlan> {
-    await this.ensureDirectory(this.roots.dataRoot);
-    await this.ensureDirectory(this.roots.migrationRoot);
+    const state = await this.loadState();
 
-    // Short-circuit: if migration already verified, skip all filesystem I/O
-    const persistedState = await this.loadState();
-    if (persistedState.migration.state === 'verified') {
-      return {
-        writeBasePath: this.roots.dataRoot,
-        readBasePaths: [this.roots.dataRoot, this.legacyBasePath],
-        state: {
-          ...persistedState,
-          sourceOfTruthLocation: 'plugin-data',
-          migration: {
-            ...persistedState.migration,
-            activeDestination: this.roots.dataRoot
-          }
-        },
-        roots: this.roots
-      };
+    // Stage 2: migration already verified in a prior session — instant cutover
+    if (state.migration.state === 'verified') {
+      return this.buildPluginDataPlan(state);
     }
 
+    // Check whether legacy files exist (lightweight directory listing)
     const legacyFiles = await this.collectLegacyFiles();
-    let state = persistedState;
-    state = {
-      ...state,
-      migration: {
-        ...state.migration,
-        activeDestination: this.roots.dataRoot,
-        legacySourcesDetected: legacyFiles.length > 0 ? [this.legacyBasePath] : []
-      }
-    };
 
+    // No legacy data — go straight to plugin-data paths
     if (legacyFiles.length === 0) {
-      const nextState: PluginScopedStorageState = {
+      const freshState: PluginScopedStorageState = {
         ...state,
         sourceOfTruthLocation: 'plugin-data',
         migration: {
           ...state.migration,
+          activeDestination: this.roots.dataRoot,
+          legacySourcesDetected: [],
+          // Preserve failed state if a prior attempt failed
           state: state.migration.state === 'failed' ? 'failed' : state.migration.state,
           lastError: state.migration.state === 'failed' ? state.migration.lastError : undefined
         }
       };
-      await this.saveState(nextState);
+      await this.saveState(freshState);
       return {
         writeBasePath: this.roots.dataRoot,
         readBasePaths: [this.roots.dataRoot],
-        state: nextState,
+        state: freshState,
         roots: this.roots
       };
     }
 
+    // Stage 1: legacy files exist — return legacy plan immediately,
+    // kick off copy+verify in the background
+    const legacyState: PluginScopedStorageState = {
+      ...state,
+      sourceOfTruthLocation: 'legacy-dotnexus',
+      migration: {
+        ...state.migration,
+        activeDestination: this.roots.dataRoot,
+        legacySourcesDetected: [this.legacyBasePath]
+      }
+    };
+
+    this.backgroundMigration = this.runBackgroundMigration(legacyState, legacyFiles);
+
+    return {
+      writeBasePath: this.legacyBasePath,
+      readBasePaths: [this.legacyBasePath],
+      state: legacyState,
+      roots: this.roots
+    };
+  }
+
+  /**
+   * Fire-and-forget background migration. Copies legacy files to plugin-scoped
+   * storage, verifies them, and saves state as 'verified'. Errors are caught
+   * and persisted as 'failed' state — they don't propagate to the caller.
+   */
+  private async runBackgroundMigration(
+    state: PluginScopedStorageState,
+    legacyFiles: string[]
+  ): Promise<void> {
     try {
-      state = await this.runCopyOnlyMigration(state, legacyFiles);
-      if (state.migration.state === 'verified') {
-        const verifiedState: PluginScopedStorageState = {
-          ...state,
-          sourceOfTruthLocation: 'plugin-data'
-        };
-        await this.saveState(verifiedState);
-        return {
-          writeBasePath: this.roots.dataRoot,
-          readBasePaths: [this.roots.dataRoot, this.legacyBasePath],
-          state: verifiedState,
-          roots: this.roots
-        };
+      new Notice('Preparing your data for cross-device sync…');
+      await this.ensureDirectory(this.roots.dataRoot);
+      await this.ensureDirectory(this.roots.migrationRoot);
+
+      const finalState = await this.runCopyOnlyMigration(state, legacyFiles);
+      if (finalState.migration.state === 'verified') {
+        console.warn('[PluginScopedStorageCoordinator] Background migration verified — cutover will happen on next boot');
+        new Notice('Data migration complete — changes take effect on next restart.');
       }
     } catch (error) {
-      state = await this.saveFailureState(state, error instanceof Error ? error.message : String(error));
+      console.error('[PluginScopedStorageCoordinator] Background migration failed:', error);
+      new Notice('Data migration encountered an issue — see console for details.');
+      await this.saveFailureState(state, error instanceof Error ? error.message : String(error)).catch(() => {
+        // Best-effort — don't let state save failure mask the original error
+      });
     }
+  }
 
+  private buildPluginDataPlan(state: PluginScopedStorageState): PluginScopedStoragePlan {
     return {
-      writeBasePath: this.legacyBasePath,
-      readBasePaths: [this.legacyBasePath],
-      state,
+      writeBasePath: this.roots.dataRoot,
+      readBasePaths: [this.roots.dataRoot, this.legacyBasePath],
+      state: {
+        ...state,
+        sourceOfTruthLocation: 'plugin-data',
+        migration: {
+          ...state.migration,
+          activeDestination: this.roots.dataRoot
+        }
+      },
       roots: this.roots
     };
   }
@@ -383,4 +427,4 @@ export class PluginScopedStorageCoordinator {
 
     await this.app.vault.adapter.mkdir(path);
   }
-}
+}

src/database/storage/SQLiteCacheManager.ts

@@ -118,6 +118,18 @@ export class SQLiteCacheManager implements IStorageBackend, ISQLiteCacheManager
     this.searchService = new SQLiteSearchService(this);
   }
 
+  /**
+   * Update the database path before initialization.
+   * Must be called before initialize() — has no effect after the DB is open.
+   */
+  setDbPath(path: string): void {
+    if (this.isInitialized) {
+      console.warn('[SQLiteCacheManager] setDbPath called after initialization — ignoring');
+      return;
+    }
+    this.dbPath = path;
+  }
+
   private getSqlite3OrThrow(): SQLite3Module {
     if (!this.sqlite3) {
       throw new Error('SQLite module not initialized');