diff --git a/benchmarking/src/app/app.tsx b/benchmarking/src/app/app.tsx index 35f39d2f..4c862e3d 100644 --- a/benchmarking/src/app/app.tsx +++ b/benchmarking/src/app/app.tsx @@ -7,6 +7,7 @@ import { ParallelMemoryDBMProvider } from './dbm-context/parallel-memory-dbm-con import { RawDBMProvider } from './dbm-context/raw-dbm-context'; import { FileLoader } from './file-loader/file-loader'; import { NativeAppFileLoader } from './file-loader/native-app-file-loader'; +import { MemoryPoc } from './memory-poc/memory-poc'; import { QueryBenchmarking } from './query-benchmarking/query-benchmarking'; export function App() { @@ -32,6 +33,9 @@ export function App() {
  • Native Node DuckDB
    • +
  • + Memory POC +
    • @@ -114,6 +118,15 @@ export function App() { } /> + +

    Memory POC

    + + + } + />     ); diff --git a/benchmarking/src/app/memory-poc/memory-poc.tsx b/benchmarking/src/app/memory-poc/memory-poc.tsx new file mode 100644 index 00000000..4a4107c9 --- /dev/null +++ b/benchmarking/src/app/memory-poc/memory-poc.tsx @@ -0,0 +1,233 @@ +import { DBM, FileManagerType, MemoryDBFileManager } from '@devrev/meerkat-dbm'; +import axios from 'axios'; +import log from 'loglevel'; +import React, { useCallback, useRef, useState } from 'react'; +import { TAXI_FILE_URL } from '../file-loader/constants'; +import { useClassicEffect } from '../hooks/use-classic-effect'; +import { InstanceManager } from '../dbm-context/instance-manager'; +import { useAsyncDuckDB } from '../dbm-context/use-async-duckdb'; +import { generateViewQuery } from '../utils'; +import { + dropAllBuffers, + formatBytes, + MemorySnapshot, + POC_QUERIES, + resetEngine, + takeMemorySnapshot, + terminateEngine, +} from './memory-probe'; + +const TABLE_NAME = 'taxi'; +const FILE_NAME = 'taxi.parquet'; + +export const MemoryPoc = () => { + const instanceManagerRef = useRef(new InstanceManager()); + const fileManagerRef = useRef(null); + const dbmRef = useRef(null); + + const [status, setStatus] = useState('booting…'); + const [loaded, setLoaded] = useState(false); + const [busy, setBusy] = useState(false); + const [snapshots, setSnapshots] = useState([]); + const [queryMs, setQueryMs] = useState>({}); + const iterationRef = useRef(0); + const startMsRef = useRef(Date.now()); + const loadStartedRef = useRef(false); + + const dbState = useAsyncDuckDB(); + + const snap = useCallback(async (label: string) => { + if (!dbmRef.current) return; + iterationRef.current += 1; + const s = await takeMemorySnapshot(iterationRef.current, label, dbmRef.current, instanceManagerRef.current); + s.atMs = s.atMs - startMsRef.current; + setSnapshots((prev) => [...prev, s]); + }, []); + + // Build the DBM once DuckDB is ready, then auto-load the parquet + view. + useClassicEffect(() => { + if (!dbState || loadStartedRef.current) { + return; + } + loadStartedRef.current = true; + fileManagerRef.current = new MemoryDBFileManager({ + instanceManager: instanceManagerRef.current, + fetchTableFileBuffers: async () => [], + }); + const dbm = new DBM({ + instanceManager: instanceManagerRef.current, + fileManager: fileManagerRef.current, + logger: log, + onEvent: (event) => log.info(event), + }); + dbmRef.current = dbm; + + void (async () => { + try { + await snap('engine only (no data)'); + setStatus('downloading 452 MB parquet…'); + const res = await axios.get(TAXI_FILE_URL, { responseType: 'arraybuffer' }); + const buffer = new Uint8Array(res.data); + setStatus(`registering ${formatBytes(buffer.byteLength)}…`); + await fileManagerRef.current!.registerFileBuffer({ + tableName: TABLE_NAME, + fileName: FILE_NAME, + buffer, + }); + await dbm.query(generateViewQuery(TABLE_NAME, [FILE_NAME])); + const countRes = await dbm.query(`SELECT count(*)::BIGINT AS n FROM ${TABLE_NAME}`); + const n = countRes.toArray()[0]?.toJSON()?.['n']; + setStatus(`loaded — ${formatBytes(buffer.byteLength)}, ${n ?? '?'} rows`); + setLoaded(true); + await snap('after load'); + } catch (e) { + setStatus(`load failed: ${e instanceof Error ? e.message : String(e)}`); + } + })(); + }, [dbState]); + + const runQuery = useCallback( + async (q: { id: string; label: string; sql: string }) => { + if (!dbmRef.current) return; + setBusy(true); + try { + const start = performance.now(); + await dbmRef.current.query(q.sql); + setQueryMs((prev) => ({ ...prev, [q.id]: performance.now() - start })); + await snap(q.label); + } catch (e) { + setStatus(`query "${q.label}" failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setBusy(false); + } + }, + [snap], + ); + + const runAll = useCallback(async () => { + setBusy(true); + for (const q of POC_QUERIES) { + if (!dbmRef.current) break; + try { + const start = performance.now(); + await dbmRef.current.query(q.sql); + setQueryMs((prev) => ({ ...prev, [q.id]: performance.now() - start })); + await snap(q.label); + } catch { + // keep going + } + } + setBusy(false); + }, [snap]); + + const reclaim = useCallback( + async (mode: 'dropFiles' | 'reset' | 'terminate') => { + setBusy(true); + try { + if (mode === 'dropFiles') await dropAllBuffers(instanceManagerRef.current); + else if (mode === 'reset') await resetEngine(instanceManagerRef.current); + else await terminateEngine(instanceManagerRef.current); + setStatus(`${mode}() done`); + await snap(`after ${mode}()`); + } catch (e) { + setStatus(`${mode}() failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setBusy(false); + } + }, + [snap], + ); + + const last = snapshots[snapshots.length - 1]; + + return ( +
    +
    + Loads the 452 MB NYC-taxi parquet via registerFileBuffer, creates a view, runs varied SQL, and + measures DuckDB memory + measureUserAgentSpecificMemory() (true per-context incl. WASM worker). +
    +
    + status: {status} {busy ? '(busy…)' : ''} +
    + +
    + + {POC_QUERIES.map((q) => ( + + ))} +
    + +
    + + + + +
    + +
    + samples: {snapshots.length} + DuckDB: {formatBytes(last?.duckDbUsedBytes ?? null)} + + globFiles: {formatBytes(last?.duckDbFileBytes ?? null)} ({last?.duckDbFileCount ?? 0}) + + + UA total (incl. WASM): {formatBytes(last?.uaTotalBytes ?? null)} + + JS heap: {formatBytes(last?.jsHeapUsedBytes ?? null)} +
    + + {last && Object.keys(last.uaByType).length > 0 ? ( +
    + UA by type: + {Object.entries(last.uaByType) + .sort((a, b) => b[1] - a[1]) + .map(([t, b]) => ( + + {t}: {formatBytes(b)} + + ))} +
    + ) : null} + + + + + + + + + + + + + + + {snapshots.map((s) => ( + + + + + + + + + + ))} + +
    #t(s)eventDuckDBglobFilesUA totalJS heap
    {s.iteration}{(s.atMs / 1000).toFixed(1)}{s.label}{formatBytes(s.duckDbUsedBytes)}{formatBytes(s.duckDbFileBytes)}{formatBytes(s.uaTotalBytes)}{formatBytes(s.jsHeapUsedBytes)}
    +
    + ); +}; diff --git a/benchmarking/src/app/memory-poc/memory-probe.ts b/benchmarking/src/app/memory-poc/memory-probe.ts new file mode 100644 index 00000000..ce839b5c --- /dev/null +++ b/benchmarking/src/app/memory-poc/memory-probe.ts @@ -0,0 +1,183 @@ +import { DBM, InstanceManagerType } from '@devrev/meerkat-dbm'; + +/** + * One memory snapshot of the DuckDB WASM engine + the browser process. Because + * the benchmark app is cross-origin isolated (COOP/COEP set in vite.config), we + * can call measureUserAgentSpecificMemory() — which reports the TRUE per-context + * heap including the DuckDB WASM worker, the number performance.memory (main + * thread only) misses. + */ +export interface MemorySnapshot { + iteration: number; + atMs: number; + label: string; + /** DuckDB buffer-manager total, bytes — sum over duckdb_memory() tags. */ + duckDbUsedBytes: number | null; + /** Per-tag DuckDB memory (PARQUET_READER, HASH_TABLE, ORDER_BY, WINDOW, …). */ + duckDbByTag: Record; + /** Attached file bytes from DuckDB's WASM FS (globFiles). The resident parquet + * footprint the recycle/shutdown reclaims. */ + duckDbFileBytes: number; + duckDbFileCount: number; + /** measureUserAgentSpecificMemory() total across all contexts (incl. WASM + * worker) — the real browser-tab JS+WASM memory. null if unavailable. */ + uaTotalBytes: number | null; + /** Per-context breakdown from measureUserAgentSpecificMemory (type → bytes). */ + uaByType: Record; + /** performance.memory main-thread JS heap (Chromium) — for comparison. */ + jsHeapUsedBytes: number | null; +} + +interface GlobbableDuckDb { + globFiles?: (path: string) => Promise>; + dropFiles?: (names?: string[]) => Promise; + reset?: () => Promise; +} + +const toBytes = (value: unknown): number | null => { + if (value === undefined || value === null) return null; + if (typeof value === 'number' || typeof value === 'bigint') return Number(value); + const n = Number(String(value)); + return Number.isFinite(n) ? n : null; +}; + +/** Cross-origin-isolated memory measurement — the true per-context heap. */ +const measureUaMemory = async (): Promise<{ total: number | null; byType: Record }> => { + const measure = (performance as unknown as { + measureUserAgentSpecificMemory?: () => Promise<{ + bytes: number; + breakdown: Array<{ bytes: number; types: string[] }>; + }>; + }).measureUserAgentSpecificMemory; + + if (!measure || !(globalThis as unknown as { crossOriginIsolated?: boolean }).crossOriginIsolated) { + return { total: null, byType: {} }; + } + try { + const result = await measure(); + const byType: Record = {}; + for (const b of result.breakdown) { + const key = b.types.join('+') || 'unknown'; + byType[key] = (byType[key] ?? 0) + b.bytes; + } + return { total: result.bytes, byType }; + } catch { + return { total: null, byType: {} }; + } +}; + +export const takeMemorySnapshot = async ( + iteration: number, + label: string, + dbm: DBM, + instanceManager: InstanceManagerType, +): Promise => { + const perfMemory = (performance as unknown as { memory?: { usedJSHeapSize: number } }).memory; + + const snapshot: MemorySnapshot = { + iteration, + atMs: Date.now(), + label, + duckDbUsedBytes: null, + duckDbByTag: {}, + duckDbFileBytes: 0, + duckDbFileCount: 0, + uaTotalBytes: null, + uaByType: {}, + jsHeapUsedBytes: perfMemory?.usedJSHeapSize ?? null, + }; + + // duckdb_memory() per-tag breakdown via the DBM raw query. + try { + const res = await dbm.query( + 'SELECT tag, (memory_usage_bytes + temporary_storage_bytes)::BIGINT AS used FROM duckdb_memory()', + ); + const rows = res.toArray().map((r: { toJSON: () => Record }) => r.toJSON()); + let total = 0; + for (const row of rows) { + const tag = String(row['tag'] ?? ''); + const used = Number(row['used'] ?? 0); + if (Number.isFinite(used) && used > 0) { + snapshot.duckDbByTag[tag] = used; + total += used; + } + } + snapshot.duckDbUsedBytes = total; + } catch { + // duckdb_memory() unsupported — leave null + } + + // Attached files from the WASM FS (authoritative resident buffer set). + try { + const db = (await instanceManager.getDB()) as unknown as GlobbableDuckDb; + if (db.globFiles) { + const files = await db.globFiles('*'); + snapshot.duckDbFileCount = files.length; + snapshot.duckDbFileBytes = files.reduce((sum, f) => sum + (f.fileSize ?? 0), 0); + } + } catch { + // leave 0 + } + + const ua = await measureUaMemory(); + snapshot.uaTotalBytes = ua.total; + snapshot.uaByType = ua.byType; + + return snapshot; +}; + +/** dropFiles() — free registered parquet buffers, keep engine warm. */ +export const dropAllBuffers = async (instanceManager: InstanceManagerType): Promise => { + const db = (await instanceManager.getDB()) as unknown as GlobbableDuckDb; + await db.dropFiles?.(); +}; + +/** reset() — reset the DB (drop catalog + buffers + cache), keep worker warm. */ +export const resetEngine = async (instanceManager: InstanceManagerType): Promise => { + const db = (await instanceManager.getDB()) as unknown as GlobbableDuckDb; + await db.reset?.(); +}; + +/** terminate — kill the worker (full shutdown; next query pays cold boot). */ +export const terminateEngine = async (instanceManager: InstanceManagerType): Promise => { + await instanceManager.terminateDB(); +}; + +export const formatBytes = (bytes: number | null): string => { + if (bytes === null) return '—'; + if (bytes < 1024) return `${bytes} B`; + const units = ['KiB', 'MiB', 'GiB', 'TiB']; + let value = bytes / 1024; + let unitIndex = 0; + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024; + unitIndex += 1; + } + return `${value.toFixed(1)} ${units[unitIndex]}`; +}; + +/** Query variants over the taxi view — each stresses a different memory tag. */ +export const POC_QUERIES: Array<{ id: string; label: string; sql: string }> = [ + { id: 'scan', label: 'scan + limit', sql: 'SELECT * FROM taxi LIMIT 1000' }, + { id: 'count', label: 'count(*)', sql: 'SELECT count(*) AS n FROM taxi' }, + { + id: 'groupby', + label: 'group by license', + sql: 'SELECT hvfhs_license_num, count(*) AS trips, avg(base_passenger_fare) AS avg_fare FROM taxi GROUP BY hvfhs_license_num ORDER BY trips DESC', + }, + { + id: 'groupby-hi', + label: 'group by PU+DO (high card)', + sql: 'SELECT PULocationID, DOLocationID, count(*) AS n FROM taxi GROUP BY PULocationID, DOLocationID ORDER BY n DESC LIMIT 500', + }, + { + id: 'orderby', + label: 'order by fare (full sort)', + sql: 'SELECT hvfhs_license_num, base_passenger_fare FROM taxi ORDER BY base_passenger_fare DESC LIMIT 1000', + }, + { + id: 'window', + label: 'window: rank per license', + sql: 'SELECT hvfhs_license_num, base_passenger_fare, row_number() OVER (PARTITION BY hvfhs_license_num ORDER BY base_passenger_fare DESC) AS rnk FROM taxi QUALIFY rnk <= 100', + }, +];