fix: hydrate dashboard live updates incrementally

src/codex_usage_tracker/plugin_data/dashboard/dashboard.js

@@ -544,6 +544,7 @@
       loadedRowsDescription,
       refreshDashboardData,
       refreshDashboardIfStale,
+      refreshDashboardLive,
       rowHydrationTarget,
       rowsNeedHydration,
       scheduleAutoRefresh,
@@ -1253,6 +1254,15 @@
       activeView = view;
       resetVisibleRows();
       render();
+      if (!liveRefreshSupported) return;
+      scheduleAutoRefresh();
+      if (['calls', 'threads', 'insights'].includes(activeView)) {
+        if (rowsNeedHydration()) {
+          hydrateDashboardRows();
+        } else if (autoRefreshEl.checked) {
+          refreshDashboardLive();
+        }
+      }
     }
     async function routeBackToDashboard(view = 'calls') {
       activeView = ['calls', 'threads', 'insights'].includes(view) ? view : 'calls';
@@ -1286,11 +1296,17 @@
       const nextRows = payloadRows(nextPayload);
       if (applyOptions.appendRows) {
         data = mergedRows(data, nextRows);
+      } else if (applyOptions.prependRows) {
+        data = mergedRows(nextRows, data);
       } else if (applyOptions.preserveRows) {
         data = data.length ? data : nextRows;
       } else {
         data = nextRows;
       }
+      if (applyOptions.trimRowsToTarget) {
+        const target = rowHydrationTarget();
+        if (target > 0 && data.length > target) data = data.slice(0, target);
+      }
       summaryData = nextPayload.summary || summaryData;
       pricingConfigured = Boolean(nextPayload.pricing_configured);
       pricingSource = nextPayload.pricing_source || {};
@@ -1312,10 +1328,10 @@
       allHistoryAvailableRows = Number(nextPayload.all_history_available_rows || totalAvailableRows);
       archivedAvailableRows = Number(nextPayload.archived_available_rows || Math.max(allHistoryAvailableRows - activeAvailableRows, 0));
       includeArchived = Boolean(nextPayload.include_archived);
-      if (!applyOptions.appendRows) loadedLimit = payloadLimit(nextPayload);
-      if (!applyOptions.appendRows) supplementalRowsByRecordId = new Map();
+      if (!applyOptions.appendRows && !applyOptions.prependRows) loadedLimit = payloadLimit(nextPayload);
+      if (!applyOptions.appendRows && !applyOptions.prependRows && !applyOptions.preserveRows) supplementalRowsByRecordId = new Map();
       restoredAggregatePayloadFromCache = false;
-      if (!nextPayload.shell_boot && !applyOptions.appendRows) {
+      if (!nextPayload.shell_boot && !applyOptions.appendRows && !applyOptions.prependRows) {
         dashboardPayloadCache.writeAggregatePayloadCache({ ...nextPayload, api_token: apiToken });
       }
       rebuildDashboardIndexes();
@@ -1477,6 +1493,7 @@
       refreshDashboardData,
       refreshDashboardEl,
       refreshDashboardIfStale,
+      refreshDashboardLive,
       render,
       resetVisibleRows,
       routeBackToDashboard,

src/codex_usage_tracker/plugin_data/dashboard/dashboard_events.js

@@ -42,6 +42,7 @@
       refreshDashboardData,
       refreshDashboardEl,
       refreshDashboardIfStale,
+      refreshDashboardLive,
       render,
       resetVisibleRows,
       routeBackToDashboard,
@@ -100,10 +101,10 @@
     autoRefreshEl.addEventListener('change', () => {
       scheduleAutoRefresh();
       updateLiveStatus(autoRefreshEl.checked ? 'badge.live' : 'status.paused', `${autoRefreshEl.checked ? tf('live.every', { seconds: liveRefreshIntervalMs / 1000 }) : t('live.paused')}. ${loadedRowsDescription()}. ${historyRowsDescription()}`);
-      if (autoRefreshEl.checked) refreshDashboardIfStale();
+      if (autoRefreshEl.checked) refreshDashboardLive();
     });
     document.addEventListener('visibilitychange', () => {
-      if (document.visibilityState === 'visible' && autoRefreshEl.checked) refreshDashboardIfStale();
+      if (document.visibilityState === 'visible' && autoRefreshEl.checked) refreshDashboardLive();
     });
     document.addEventListener('keydown', event => {
       const target = event.target;

src/codex_usage_tracker/plugin_data/dashboard/dashboard_live.js

@@ -219,7 +219,7 @@
         const rowCountChanged = Number.isFinite(scopedRows) && scopedRows !== getTotalAvailableRows();
         const refreshChanged = statusRefreshAt && statusRefreshAt !== deps.latestRefreshAt();
         if (rowCountChanged || refreshChanged) {
-          refreshDashboardData(false, { refreshLogs: false, resetRows: true });
+          refreshDashboardLive();
         } else if (rowsNeedHydration()) {
           hydrateDashboardRows();
         }
@@ -228,6 +228,84 @@
       }
     }
 
+    async function refreshDashboardLive() {
+      if (!liveRefreshSupported || !apiToken() || activeView() === 'call') return;
+      if (refreshInFlight) return;
+      const previousTotal = Number(getTotalAvailableRows() || getData().length || 0);
+      refreshInFlight = true;
+      refreshDashboardEl.disabled = true;
+      updateLiveStatus('status.refreshing', t('live.refreshing_index'));
+      try {
+        const shellParams = new URLSearchParams({
+          limit: loadLimitEl.value,
+          include_archived: getIncludeArchived() ? '1' : '0',
+          lang: i18n.currentLanguage,
+          shell: '1',
+          _: String(Date.now()),
+          refresh: '1',
+        });
+        const shellResponse = await fetch(`/api/usage?${shellParams.toString()}`, {
+          headers: {
+            'Accept': 'application/json',
+            'X-Codex-Usage-Token': apiToken(),
+          },
+          cache: 'no-store',
+        });
+        if (!shellResponse.ok) throw new Error(`HTTP ${shellResponse.status}`);
+        const shellPayload = await shellResponse.json();
+        if (shellPayload.error) throw new Error(shellPayload.error);
+
+        const nextTotal = Number(shellPayload.total_available_rows || previousTotal);
+        const newRows = Math.max(0, nextTotal - previousTotal);
+        applyDashboardPayload(shellPayload, { preserveRows: true });
+
+        if (activeView() !== 'diagnostics' && newRows > 0) {
+          const loadedLimit = getLoadedLimit();
+          const visibleTarget = loadedLimit === null ? nextTotal : Math.min(nextTotal, Number(loadedLimit || nextTotal));
+          const rowsToFetch = Math.max(0, Math.min(newRows, visibleTarget || newRows));
+          if (rowsToFetch > 0) {
+            const rowParams = new URLSearchParams({
+              limit: String(rowsToFetch),
+              offset: '0',
+              include_archived: getIncludeArchived() ? '1' : '0',
+              lang: i18n.currentLanguage,
+              _: String(Date.now()),
+            });
+            const rowResponse = await fetch(`/api/usage?${rowParams.toString()}`, {
+              headers: {
+                'Accept': 'application/json',
+                'X-Codex-Usage-Token': apiToken(),
+              },
+              cache: 'no-store',
+            });
+            if (!rowResponse.ok) throw new Error(`HTTP ${rowResponse.status}`);
+            const rowPayload = await rowResponse.json();
+            if (rowPayload.error) throw new Error(rowPayload.error);
+            applyDashboardPayload(rowPayload, { prependRows: true, trimRowsToTarget: true });
+          }
+          rowHydrationComplete = getData().length >= rowHydrationTarget();
+          updateRowLoadProgress();
+        } else if (activeView() !== 'diagnostics' && rowsNeedHydration()) {
+          hydrateDashboardRows();
+        }
+
+        const result = shellPayload.refresh_result || {};
+        const indexed = result.inserted_or_updated_events === undefined
+          ? ''
+          : tf('live.indexed', { rows: number.format(result.inserted_or_updated_events), files: number.format(result.scanned_files || 0) });
+        const skipped = result.skipped_events
+          ? tf('live.skipped', { count: number.format(result.skipped_events) })
+          : '';
+        updateLiveStatus(autoRefreshEl.checked ? 'badge.live' : 'status.updated', tf('live.updated_detail', { time: formatTimestamp(shellPayload.refreshed_at), loaded: loadedRowsDescription(), history: historyRowsDescription(), indexed, skipped }));
+      } catch (error) {
+        const message = error.message || String(error);
+        updateLiveStatus('status.refresh_error', tf('live.refresh_unavailable', { message, suffix: '' }));
+      } finally {
+        refreshInFlight = false;
+        refreshDashboardEl.disabled = false;
+      }
+    }
+
     async function refreshDashboardData(manual = false, options = null) {
       if (!liveRefreshSupported) {
         updateLiveStatus('status.reloading', t('live.reloading_static'));
@@ -295,9 +373,9 @@
     function scheduleAutoRefresh() {
       if (autoRefreshTimer) window.clearInterval(autoRefreshTimer);
       autoRefreshTimer = null;
-      if (!autoRefreshEl.checked || !liveRefreshSupported || ['call', 'diagnostics'].includes(activeView())) return;
+      if (!autoRefreshEl.checked || !liveRefreshSupported || activeView() === 'call') return;
       autoRefreshTimer = window.setInterval(() => {
-        if (document.visibilityState === 'visible') refreshDashboardIfStale();
+        if (document.visibilityState === 'visible') refreshDashboardLive();
       }, liveRefreshIntervalMs);
     }
 
@@ -307,6 +385,7 @@
       loadedRowsDescription,
       refreshDashboardData,
       refreshDashboardIfStale,
+      refreshDashboardLive,
       rowHydrationTarget,
       rowsNeedHydration,
       scheduleAutoRefresh,

tests/test_dashboard_live.py

@@ -128,6 +128,109 @@ def test_dashboard_live_allows_diagnostics_bootstrap_refresh() -> None:
     assert payload["statusKeys"] == ["status.checking", "status.updated"]
 
 
+def test_dashboard_live_prepends_new_rows_after_cached_index_refresh() -> None:
+    payload = _run_dashboard_live_script(
+        """
+(async () => {
+  const calls = [];
+  const appliedPayloads = [];
+  let totalRows = 4;
+  let data = [
+    { record_id: 'old-1' },
+    { record_id: 'old-2' },
+    { record_id: 'old-3' },
+    { record_id: 'old-4' },
+  ];
+  globalThis.__fetch = async (url, options) => {
+    calls.push({ url, headers: options.headers });
+    const isRefresh = url.includes('refresh=1');
+    return {
+      ok: true,
+      json: async () => ({
+        rows: isRefresh ? [] : [{ record_id: 'new-1' }],
+        refreshed_at: '2026-06-19T00:00:00Z',
+        refresh_result: {
+          inserted_or_updated_events: 1,
+          scanned_files: 1,
+          skipped_events: 0,
+        },
+        total_available_rows: 5,
+      }),
+    };
+  };
+  const runtime = factory.create({
+    activeView: () => 'calls',
+    apiToken: () => 'test-token',
+    applyDashboardPayload: (payload, options = {}) => {
+      appliedPayloads.push({
+        rows: (payload.rows || []).map(row => row.record_id),
+        options,
+      });
+      totalRows = payload.total_available_rows || totalRows;
+      if (options.prependRows) {
+        const incoming = payload.rows || [];
+        const incomingIds = new Set(incoming.map(row => row.record_id));
+        data = [...incoming, ...data.filter(row => !incomingIds.has(row.record_id))];
+      }
+    },
+    autoRefreshEl: { checked: true },
+    backgroundHydrationChunkSize: 2000,
+    formatTimestamp: value => value,
+    getArchivedAvailableRows: () => 0,
+    getData: () => data,
+    getIncludeArchived: () => false,
+    getLoadedLimit: () => 5000,
+    getTotalAvailableRows: () => totalRows,
+    historyScopeEl: { value: 'active', parentElement: {} },
+    i18n: { currentLanguage: 'en' },
+    initialHydrationChunkSize: 500,
+    latestRefreshAt: () => '',
+    limitValue: value => value === null ? 'all' : String(value),
+    liveRefreshIntervalMs: 10000,
+    liveRefreshSupported: true,
+    loadLimitEl: { value: '5000', options: [], lastElementChild: null, insertBefore: () => {} },
+    number: new Intl.NumberFormat('en-US'),
+    payloadRows: payload => payload.rows || [],
+    rebuildDashboardIndexes: () => {},
+    rebuildFilterOptions: () => {},
+    refreshDashboardEl: { disabled: false },
+    render: () => {},
+    resetRowsForHydration: () => {},
+    rowLoadProgressBarEl: { style: {} },
+    rowLoadProgressCountEl: { textContent: '' },
+    rowLoadProgressEl: { hidden: true },
+    rowLoadProgressLabelEl: { textContent: '' },
+    setFastTooltip: () => {},
+    t: key => key,
+    tf: (key, values = {}) => `${key}:${JSON.stringify(values)}`,
+    updateLiveStatus: () => {},
+  });
+  await runtime.refreshDashboardLive();
+  console.log(JSON.stringify({
+    urls: calls.map(call => call.url.replace(/_=[0-9]+/, '_=<ts>')),
+    tokens: calls.map(call => call.headers['X-Codex-Usage-Token']),
+    appliedPayloads,
+    data: data.map(row => row.record_id),
+  }));
+})().catch(error => {
+  console.error(error);
+  process.exit(1);
+});
+"""
+    )
+
+    assert payload["urls"] == [
+        "/api/usage?limit=5000&include_archived=0&lang=en&shell=1&_=<ts>&refresh=1",
+        "/api/usage?limit=1&offset=0&include_archived=0&lang=en&_=<ts>",
+    ]
+    assert payload["tokens"] == ["test-token", "test-token"]
+    assert payload["appliedPayloads"] == [
+        {"rows": [], "options": {"preserveRows": True}},
+        {"rows": ["new-1"], "options": {"prependRows": True, "trimRowsToTarget": True}},
+    ]
+    assert payload["data"] == ["new-1", "old-1", "old-2", "old-3", "old-4"]
+
+
 def test_dashboard_bootstraps_direct_diagnostics_view() -> None:
     repo_root = Path(__file__).resolve().parents[1]
     dashboard_js = (

tests/test_dashboard_payload.py

@@ -305,6 +305,9 @@ def test_dashboard_and_csv_are_aggregate_only(tmp_path: Path) -> None:
     assert "direction: sortState.direction" in dashboard_diagnostics_js
     assert "diagnostics-expand-button" in dashboard_surface
     assert "selectedFactKey === key" in dashboard_diagnostics_js
+    assert "if (rowsNeedHydration())" in dashboard_js
+    assert "hydrateDashboardRows();" in dashboard_js
+    assert "refreshDashboardIfStale();" in dashboard_js
     assert "Needs Attention" in dashboard
     assert "Investigation Presets" in dashboard
     assert "presetDefinitions" in dashboard_insights_js