Improve vault performance and tab overflow

apps/desktop/electron.vite.config.ts

@@ -17,6 +17,17 @@ const MAIN_EXTERNALIZE_EXCLUSIONS = [
 ]
 
 function rendererManualChunk(id: string): string | undefined {
+  const normalizedId = id.split('\\').join('/')
+  if (normalizedId.endsWith('/packages/app-core/src/lib/wikilinks.ts')) {
+    return 'app-wikilinks'
+  }
+  if (normalizedId.endsWith('/packages/app-core/src/lib/local-assets.ts')) {
+    return 'app-local-assets'
+  }
+  if (normalizedId.endsWith('/packages/app-core/src/store.ts')) {
+    return 'app-store'
+  }
+
   if (!id.includes('node_modules')) return undefined
 
   if (id.includes('/react/') || id.includes('/react-dom/') || id.includes('/zustand/')) {
@@ -70,6 +81,31 @@ function rendererManualChunk(id: string): string | undefined {
   return undefined
 }
 
+function resolveRendererModulePreloads(
+  _filename: string,
+  deps: string[],
+  context: { hostType: 'html' | 'js' }
+): string[] {
+  if (context.hostType === 'html') {
+    return deps.filter((dep) => dep.includes('vendor-react'))
+  }
+  return deps.filter((dep) => !isDeferredRendererPreload(dep))
+}
+
+function isDeferredRendererPreload(dep: string): boolean {
+  return (
+    dep.includes('NoteHoverPreview-') ||
+    dep.includes('Preview-') ||
+    dep.includes('wardley-') ||
+    dep.includes('vendor-markdown') ||
+    dep.includes('vendor-highlight') ||
+    dep.includes('vendor-d3') ||
+    dep.includes('vendor-mermaid') ||
+    dep.includes('vendor-jsxgraph') ||
+    dep.includes('vendor-function-plot')
+  )
+}
+
 export default defineConfig({
   main: {
     plugins: [externalizeDepsPlugin({ exclude: MAIN_EXTERNALIZE_EXCLUSIONS })],
@@ -116,9 +152,13 @@ export default defineConfig({
     root: resolve(__dirname, 'src/renderer'),
     build: {
       outDir: 'out/renderer',
+      minify: 'esbuild',
       // This is a desktop app with multiple on-demand diagram stacks.
       // Some lazy chunks are intentionally larger than the web default.
       chunkSizeWarningLimit: 3500,
+      modulePreload: {
+        resolveDependencies: resolveRendererModulePreloads
+      },
       rollupOptions: {
         input: { index: resolve(__dirname, 'src/renderer/index.html') },
         output: {

apps/desktop/package.json

@@ -57,7 +57,6 @@
     "electron-updater": "^6.8.3",
     "font-list": "^2.0.2",
     "function-plot": "^1.25.3",
-    "fuse.js": "^7.0.0",
     "gray-matter": "^4.0.3",
     "highlight.js": "^11.10.0",
     "jsxgraph": "^1.12.2",

apps/desktop/src/main/index.ts

@@ -20,6 +20,7 @@ import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { IPC } from '@shared/ipc'
 import type {
+  NoteMeta,
   NoteCommentInput,
   NoteFolder,
   RemoteWorkspaceInfo,
@@ -50,6 +51,8 @@ import {
   getVaultSettings,
   hasAssetsDir,
   importFiles,
+  invalidateNoteMetaCache,
+  invalidateVaultTextSearchCache,
   listAssets,
   listFolders,
   listNotes,
@@ -161,6 +164,10 @@ const APP_DISCORD_URL = 'https://discord.gg/W4fWzapKS6'
 const APP_REPOSITORY_URL = 'https://github.com/ZenNotes/zennotes'
 const APP_RELEASES_URL = 'https://github.com/ZenNotes/zennotes/releases/latest'
 const APP_ISSUES_URL = 'https://github.com/ZenNotes/zennotes/issues'
+const userDataPathOverride = process.env['ZENNOTES_USER_DATA_PATH']?.trim()
+if (userDataPathOverride && (process.env['ZEN_PERF'] === '1' || !app.isPackaged)) {
+  app.setPath('userData', path.resolve(userDataPathOverride))
+}
 let currentZoomFactor = DEFAULT_ZOOM_FACTOR
 const pendingOpenNoteRequests: string[] = []
 const pendingFloatingNoteRequests: string[] = []
@@ -829,6 +836,8 @@ async function setVault(root: string): Promise<VaultInfo> {
     remoteWorkspaceProfileId: null
   }))
   watcher.start(root, (ev: VaultChangeEvent) => {
+    invalidateNoteMetaCache(root, ev.scope === 'vault-settings' ? undefined : ev.path)
+    invalidateVaultTextSearchCache(root)
     broadcastVaultChange(ev)
   })
   return currentVault
@@ -1315,6 +1324,40 @@ async function listFontFamilies(): Promise<string[]> {
   }
 }
 
+interface ListNotesStreamRequest {
+  requestId?: unknown
+  chunkSize?: unknown
+  offset?: unknown
+}
+
+interface ListNotesStreamState {
+  notes: NoteMeta[]
+  touchedAt: number
+}
+
+const DEFAULT_LIST_NOTES_STREAM_CHUNK_SIZE = 500
+const MAX_LIST_NOTES_STREAM_CHUNK_SIZE = 1000
+const LIST_NOTES_STREAM_STATE_TTL_MS = 60_000
+const listNotesStreamStates = new Map<string, ListNotesStreamState>()
+
+function listNotesStreamChunkSize(raw: unknown): number {
+  const parsed = Number.parseInt(String(raw ?? ''), 10)
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_LIST_NOTES_STREAM_CHUNK_SIZE
+  return Math.min(MAX_LIST_NOTES_STREAM_CHUNK_SIZE, parsed)
+}
+
+function listNotesStreamOffset(raw: unknown): number {
+  const parsed = Number.parseInt(String(raw ?? ''), 10)
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 0
+}
+
+function pruneListNotesStreamStates(): void {
+  const cutoff = Date.now() - LIST_NOTES_STREAM_STATE_TTL_MS
+  for (const [requestId, state] of listNotesStreamStates) {
+    if (state.touchedAt < cutoff) listNotesStreamStates.delete(requestId)
+  }
+}
+
 function registerIpc(): void {
   const handle = <Args extends unknown[], Result>(
     channel: string,
@@ -1447,6 +1490,38 @@ function registerIpc(): void {
     return await listNotes(v.root)
   })
 
+  handle(IPC.VAULT_LIST_NOTES_STREAM, async (_event, request: ListNotesStreamRequest) => {
+    if (typeof request?.requestId !== 'string' || request.requestId.length === 0) {
+      throw new Error('Missing list-notes stream request id')
+    }
+    const requestId = request.requestId
+    const chunkSize = listNotesStreamChunkSize(request.chunkSize)
+    const offset = listNotesStreamOffset(request.offset)
+    pruneListNotesStreamStates()
+
+    let state = listNotesStreamStates.get(requestId)
+    if (!state || offset === 0) {
+      const notes = isRemoteWorkspaceActive()
+        ? await requireRemoteWorkspaceClient().listNotes()
+        : await listNotes(requireVault().root)
+      state = { notes, touchedAt: Date.now() }
+      listNotesStreamStates.set(requestId, state)
+    } else {
+      state.touchedAt = Date.now()
+    }
+
+    const nextOffset = Math.min(state.notes.length, offset + chunkSize)
+    const done = nextOffset >= state.notes.length
+    const notes = state.notes.slice(offset, nextOffset)
+    if (done) listNotesStreamStates.delete(requestId)
+    return {
+      notes,
+      nextOffset,
+      done,
+      total: state.notes.length
+    }
+  })
+
   handle(IPC.VAULT_LIST_FOLDERS, async () => {
     if (isRemoteWorkspaceActive()) return await requireRemoteWorkspaceClient().listFolders()
     const v = requireVault()

apps/desktop/src/main/vault.perf.test.ts

@@ -0,0 +1,117 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { afterAll, describe, expect, it } from 'vitest'
+import {
+  ensureVaultLayout,
+  invalidateNoteMetaCache,
+  listNotes,
+  readNote,
+  searchVaultText,
+  writeNote
+} from './vault'
+
+const NOTE_COUNT = 5_000
+const FOLDER_COUNT = 50
+const WRITE_BATCH_SIZE = 250
+
+const tempDirs: string[] = []
+
+async function makeTempDir(prefix: string): Promise<string> {
+  const dir = await mkdtemp(path.join(os.tmpdir(), prefix))
+  tempDirs.push(dir)
+  return dir
+}
+
+afterAll(async () => {
+  await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })))
+})
+
+async function measure<T>(
+  label: string,
+  detail: Record<string, unknown>,
+  fn: () => Promise<T>
+): Promise<{ value: T; durationMs: number }> {
+  const startedAt = performance.now()
+  const value = await fn()
+  const durationMs = Math.round((performance.now() - startedAt) * 100) / 100
+  console.info(`[zen:bench] ${label} ${durationMs.toFixed(2)}ms`, detail)
+  return { value, durationMs }
+}
+
+async function seedLargeVault(root: string): Promise<void> {
+  await ensureVaultLayout(root)
+  await rm(path.join(root, 'inbox', 'Welcome.md'), { force: true })
+
+  const writes: Array<() => Promise<void>> = []
+  for (let index = 0; index < NOTE_COUNT; index += 1) {
+    const folder = `folder-${String(index % FOLDER_COUNT).padStart(2, '0')}`
+    const title = `Note ${String(index).padStart(5, '0')}`
+    const rel = path.join(root, 'inbox', folder, `${title}.md`)
+    const needle = `needle-${index}`
+    const body = [
+      `# ${title}`,
+      '',
+      `This synthetic benchmark note includes ${needle}.`,
+      'It has tags #perf #benchmark and enough body text to exercise metadata parsing.',
+      'The quick brown fox jumps over the lazy dog while ZenNotes indexes markdown.',
+      'A wikilink [[Synthetic Reference]] keeps wikilink extraction on the hot path.',
+      '',
+      '- [ ] task item for task extraction adjacent workloads',
+      '- [x] completed task item',
+      '',
+      'Final paragraph with repeated benchmark words for text search scoring.'
+    ].join('\n')
+    writes.push(async () => {
+      await mkdir(path.dirname(rel), { recursive: true })
+      await writeFile(rel, body, 'utf8')
+    })
+  }
+
+  for (let index = 0; index < writes.length; index += WRITE_BATCH_SIZE) {
+    await Promise.all(writes.slice(index, index + WRITE_BATCH_SIZE).map((write) => write()))
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+describe.skipIf(process.env['ZEN_PERF_BENCH'] !== '1')('large vault performance', () => {
+  it('measures note listing, note read, note write, and built-in text search', async () => {
+    const root = await makeTempDir('zennotes-large-vault-')
+    await measure('seedLargeVault', { notes: NOTE_COUNT }, () => seedLargeVault(root))
+
+    const coldList = await measure('listNotes.cold', { notes: NOTE_COUNT }, () => listNotes(root))
+    expect(coldList.value).toHaveLength(NOTE_COUNT)
+
+    const warmList = await measure('listNotes.warm', { notes: NOTE_COUNT }, () => listNotes(root))
+    expect(warmList.value).toHaveLength(NOTE_COUNT)
+
+    await sleep(1100)
+    invalidateNoteMetaCache(root)
+    const persistedWarmList = await measure('listNotes.persistedWarm', { notes: NOTE_COUNT }, () =>
+      listNotes(root)
+    )
+    expect(persistedWarmList.value).toHaveLength(NOTE_COUNT)
+
+    const target = 'inbox/folder-49/Note 04999.md'
+    const read = await measure('readNote', { path: target }, () => readNote(root, target))
+    expect(read.value.body).toContain('needle-4999')
+
+    const coldSearch = await measure('searchVaultText.builtin.cold', { query: 'needle-4999' }, () =>
+      searchVaultText(root, 'needle-4999', 'builtin', {}, 20)
+    )
+    expect(coldSearch.value.map((match) => match.path)).toContain(target)
+
+    const warmSearch = await measure('searchVaultText.builtin.warm', { query: 'needle-4999' }, () =>
+      searchVaultText(root, 'needle-4999', 'builtin', {}, 20)
+    )
+    expect(warmSearch.value.map((match) => match.path)).toContain(target)
+
+    const write = await measure('writeNote', { path: target }, () =>
+      writeNote(root, target, `${read.value.body}\n\nBenchmark write update.\n`)
+    )
+    expect(write.value.path).toBe(target)
+  }, 120_000)
+})