From 0238442b33c83dbcda53aeff5fc5510a2d4d4b8d Mon Sep 17 00:00:00 2001
From: nfebe <fenn25.fn@gmail.com>
Date: Sun, 21 Jun 2026 15:01:38 +0100
Subject: [PATCH] feat: Stream the transactions spreadsheet and speed up
 reports

Reports now load noticeably faster and no longer hide charts while data
is still arriving.

- Report transaction pages are fetched concurrently instead of one at a
  time, so wide date ranges render much sooner.
- Per-category spending trends are built in a single pass rather than
  rescanning every transaction once per category.
- The charts and other report tabs stay visible whenever chart data
  exists, with a loading state instead of a flash of the empty screen.
- A notice appears when a period has more transactions than can be
  charted at once, so partial figures are not mistaken for complete ones.
- Amounts that could not be converted to the default currency are now
  flagged, so cross-currency totals are no longer silently understated.

The transactions spreadsheet streams rows in chunks: the first rows and
running totals appear immediately and grow as the rest load. Columns can
be sorted, the current view can be exported to a CSV file, and search is
debounced so typing stays smooth on large sets.
---
 .../transactions/TransactionsSpreadsheet.vue  | 260 +++++++++++++++++-
 composables/useReportData.ts                  | 123 ++++++---
 pages/reports.vue                             |  53 +++-
 services/api/statsApi.ts                      |   4 +
 4 files changed, 392 insertions(+), 48 deletions(-)

diff --git a/components/transactions/TransactionsSpreadsheet.vue b/components/transactions/TransactionsSpreadsheet.vue
index 407a961..40b5ea5 100644
--- a/components/transactions/TransactionsSpreadsheet.vue
+++ b/components/transactions/TransactionsSpreadsheet.vue
@@ -7,10 +7,26 @@
           <h2>{{ t('Transactions spreadsheet') }}</h2>
           <span class="count">
             {{ t('{count} rows', { count: filteredRows.length }) }}
+            <span v-if="isStreaming" class="loading-more">· {{ t('loading…') }}</span>
           </span>
         </div>
         <div class="spreadsheet-actions">
-          <input v-model="search" type="search" class="search" :placeholder="t('Search rows...')" />
+          <input
+            v-model="searchInput"
+            type="search"
+            class="search"
+            :placeholder="t('Search rows...')"
+          />
+          <button
+            type="button"
+            class="action-btn"
+            :disabled="!filteredRows.length"
+            :aria-label="t('Export CSV')"
+            :title="t('Export CSV')"
+            @click="exportCsv"
+          >
+            <ArrowDownTrayIcon class="icon" />
+          </button>
           <button type="button" class="close-btn" :aria-label="t('Close')" @click="$emit('close')">
             <XMarkIcon class="icon" />
           </button>
@@ -31,18 +47,31 @@
           <thead>
             <tr>
               <th class="col-idx">#</th>
-              <th class="col-date">{{ t('Date') }}</th>
-              <th class="col-type">{{ t('Type') }}</th>
-              <th class="col-amount">{{ t('Amount') }}</th>
-              <th class="col-currency">{{ t('Currency') }}</th>
-              <th class="col-desc">{{ t('Description') }}</th>
-              <th class="col-cat">{{ t('Categories') }}</th>
-              <th class="col-wallet">{{ t('Wallet') }}</th>
-              <th class="col-party">{{ t('Party') }}</th>
+              <th
+                v-for="col in columns"
+                :key="col.key"
+                :class="[col.cls, 'sortable']"
+                :aria-sort="
+                  sortKey === col.key ? (sortDir === 'asc' ? 'ascending' : 'descending') : 'none'
+                "
+                @click="toggleSort(col.key)"
+              >
+                <span class="th-inner">
+                  {{ col.label }}
+                  <ChevronUpIcon
+                    v-if="sortKey === col.key && sortDir === 'asc'"
+                    class="sort-icon"
+                  />
+                  <ChevronDownIcon
+                    v-else-if="sortKey === col.key && sortDir === 'desc'"
+                    class="sort-icon"
+                  />
+                </span>
+              </th>
             </tr>
           </thead>
           <tbody>
-            <tr v-for="(row, i) in filteredRows" :key="row.id" :class="`row--${row.type}`">
+            <tr v-for="(row, i) in sortedRows" :key="row.id" :class="`row--${row.type}`">
               <td class="col-idx">{{ i + 1 }}</td>
               <td class="col-date">{{ formatDate(row.datetime) }}</td>
               <td class="col-type">
@@ -85,8 +114,14 @@
 </template>
 
 <script setup lang="ts">
-import { computed, onMounted, onUnmounted, ref } from 'vue';
-import { TableCellsIcon, XMarkIcon } from '@heroicons/vue/24/outline';
+import { computed, onMounted, onUnmounted, ref, watch } from 'vue';
+import {
+  TableCellsIcon,
+  XMarkIcon,
+  ArrowDownTrayIcon,
+  ChevronUpIcon,
+  ChevronDownIcon
+} from '@heroicons/vue/24/outline';
 import { api } from '@/services/api';
 
 const { t } = useI18n();
@@ -94,19 +129,52 @@ const emit = defineEmits<{ (e: 'close'): void }>();
 
 const rows = ref<any[]>([]);
 const isLoading = ref(true);
+const isStreaming = ref(false);
 const error = ref('');
+const searchInput = ref('');
 const search = ref('');
 
+let searchTimer: ReturnType<typeof setTimeout> | null = null;
+watch(searchInput, (value) => {
+  if (searchTimer) clearTimeout(searchTimer);
+  searchTimer = setTimeout(() => {
+    search.value = value;
+  }, 200);
+});
+
+const PAGE_SIZE = 100;
+const MAX_PAGES = 50;
+// Guards against an out-of-order load overwriting a newer one.
+let loadToken = 0;
+
 async function loadAll() {
+  const token = ++loadToken;
   isLoading.value = true;
+  isStreaming.value = true;
   error.value = '';
+  rows.value = [];
   try {
-    const response = await api.transactions.fetchAll({ limit: 500, page: 1 });
-    rows.value = Array.isArray(response?.data) ? response.data : [];
+    let page = 1;
+    while (page <= MAX_PAGES) {
+      const response = await api.transactions.fetchAll({ limit: PAGE_SIZE, page });
+      if (token !== loadToken) return;
+      const chunk = Array.isArray(response?.data) ? response.data : [];
+      // Append per page so the first chunk paints immediately and the totals
+      // climb as later pages stream in, rather than blocking on the full set.
+      rows.value = rows.value.concat(chunk);
+      isLoading.value = false;
+      const lastPage = Number(response?.last_page) || 1;
+      if (chunk.length < PAGE_SIZE || page >= lastPage) break;
+      page += 1;
+    }
   } catch (e) {
+    if (token !== loadToken) return;
     error.value = e instanceof Error ? e.message : t('Failed to load');
   } finally {
-    isLoading.value = false;
+    if (token === loadToken) {
+      isLoading.value = false;
+      isStreaming.value = false;
+    }
   }
 }
 
@@ -138,6 +206,117 @@ const totalExpense = computed(() =>
   filteredRows.value.filter((r) => r.type === 'expense').reduce((s, r) => s + Number(r.amount), 0)
 );
 
+type SortKey =
+  | 'datetime'
+  | 'type'
+  | 'amount'
+  | 'currency'
+  | 'description'
+  | 'categories'
+  | 'wallet'
+  | 'party';
+
+const sortKey = ref<SortKey | null>(null);
+const sortDir = ref<'asc' | 'desc'>('asc');
+
+const sortValue = (row: any, key: SortKey): string | number => {
+  switch (key) {
+    case 'datetime':
+      return row.datetime || '';
+    case 'type':
+      return row.type || '';
+    case 'amount':
+      return Number(row.amount) || 0;
+    case 'currency':
+      return row.wallet?.currency || '';
+    case 'description':
+      return (row.description || '').toLowerCase();
+    case 'categories':
+      return categoriesText(row).toLowerCase();
+    case 'wallet':
+      return (row.wallet?.name || '').toLowerCase();
+    case 'party':
+      return (row.party?.name || '').toLowerCase();
+    default:
+      return '';
+  }
+};
+
+const sortedRows = computed(() => {
+  const key = sortKey.value;
+  if (!key) return filteredRows.value;
+  const dir = sortDir.value === 'asc' ? 1 : -1;
+  return [...filteredRows.value].sort((a, b) => {
+    const av = sortValue(a, key);
+    const bv = sortValue(b, key);
+    if (av < bv) return -1 * dir;
+    if (av > bv) return 1 * dir;
+    return 0;
+  });
+});
+
+const columns = computed<{ key: SortKey; label: string; cls: string }[]>(() => [
+  { key: 'datetime', label: t('Date'), cls: 'col-date' },
+  { key: 'type', label: t('Type'), cls: 'col-type' },
+  { key: 'amount', label: t('Amount'), cls: 'col-amount' },
+  { key: 'currency', label: t('Currency'), cls: 'col-currency' },
+  { key: 'description', label: t('Description'), cls: 'col-desc' },
+  { key: 'categories', label: t('Categories'), cls: 'col-cat' },
+  { key: 'wallet', label: t('Wallet'), cls: 'col-wallet' },
+  { key: 'party', label: t('Party'), cls: 'col-party' }
+]);
+
+const toggleSort = (key: SortKey) => {
+  if (sortKey.value === key) {
+    sortDir.value = sortDir.value === 'asc' ? 'desc' : 'asc';
+  } else {
+    sortKey.value = key;
+    sortDir.value = 'asc';
+  }
+};
+
+const csvEscape = (value: unknown): string => {
+  const s = String(value ?? '');
+  return /["\n,]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
+};
+
+const exportCsv = () => {
+  const headers = [
+    t('Date'),
+    t('Type'),
+    t('Amount'),
+    t('Currency'),
+    t('Description'),
+    t('Categories'),
+    t('Wallet'),
+    t('Party')
+  ];
+  const lines = [headers.map(csvEscape).join(',')];
+  for (const r of sortedRows.value) {
+    lines.push(
+      [
+        formatDate(r.datetime),
+        r.type,
+        Number(r.amount) || 0,
+        r.wallet?.currency ?? '',
+        r.description ?? '',
+        categoriesText(r),
+        r.wallet?.name ?? '',
+        r.party?.name ?? ''
+      ]
+        .map(csvEscape)
+        .join(',')
+    );
+  }
+  const blob = new Blob(['' + lines.join('\n')], { type: 'text/csv;charset=utf-8;' });
+  const url = URL.createObjectURL(blob);
+  const link = document.createElement('a');
+  link.href = url;
+  link.download = 'transactions.csv';
+  link.click();
+  URL.revokeObjectURL(url);
+};
+
 const formatDate = (iso?: string) => {
   if (!iso) return '';
   const d = new Date(iso);
@@ -265,6 +444,36 @@ onUnmounted(() => {
   }
 }
 
+.action-btn {
+  background: transparent;
+  border: 1px solid $border-color;
+  border-radius: $radius-md;
+  padding: 6px;
+  cursor: pointer;
+  color: $text-muted;
+  display: inline-flex;
+
+  .icon {
+    width: 16px;
+    height: 16px;
+  }
+
+  &:hover:not(:disabled) {
+    color: $primary;
+    border-color: $primary-muted;
+  }
+
+  &:disabled {
+    opacity: 0.5;
+    cursor: not-allowed;
+  }
+}
+
+.loading-more {
+  color: $primary;
+  font-style: italic;
+}
+
 .state {
   display: flex;
   flex-direction: column;
@@ -319,6 +528,27 @@ onUnmounted(() => {
     letter-spacing: 0.05em;
     border-bottom: 1px solid $border-color;
     z-index: 2;
+
+    &.sortable {
+      cursor: pointer;
+      user-select: none;
+
+      &:hover {
+        color: $primary;
+      }
+    }
+
+    .th-inner {
+      display: inline-flex;
+      align-items: center;
+      gap: 4px;
+    }
+
+    .sort-icon {
+      width: 12px;
+      height: 12px;
+      color: $primary;
+    }
   }
 
   tbody td {
diff --git a/composables/useReportData.ts b/composables/useReportData.ts
index 20a9ee0..282c4f0 100644
--- a/composables/useReportData.ts
+++ b/composables/useReportData.ts
@@ -116,8 +116,11 @@ const apiStats = ref<StatsResponse['data'] | null>(null);
 const apiStatsPrev = ref<StatsResponse['data'] | null>(null);
 const periodTransactions = ref<FrontendTransaction[]>([]);
 const trailing12Transactions = ref<FrontendTransaction[]>([]);
-const isLoading = ref(false);
+// Starts true: the immediate watcher always kicks off a reload, so the first
+// paint should show loading rather than briefly flashing the empty state.
+const isLoading = ref(true);
 const error = ref<string | null>(null);
+const sweepTruncated = ref(false);
 
 const PALETTE_EXPENSE = [
   '#e11d48',
@@ -203,38 +206,68 @@ async function fetchAllTransactionsInRange(
     groups: readonly any[];
   }
 ): Promise<FrontendTransaction[]> {
-  const out: FrontendTransaction[] = [];
-  let page = 1;
   const limit = 200;
-  // Hard cap so we never spin forever; reports break degrade-gracefully past this.
+  // Hard cap so we never spin forever; reports degrade gracefully past this.
   const maxPages = 30;
-  while (true) {
-    const resp = await api.transactions.fetchAll({
+  // Fetch a bounded number of pages at once so a heavy account doesn't pay one
+  // serial round-trip per page before the report can render.
+  const concurrency = 6;
+
+  const fetchPage = (page: number) =>
+    api.transactions.fetchAll({
       page,
       limit,
       date_from: toISODate(start),
       date_to: toISODate(end)
     });
-    const mapped = (resp.data || []).map((t) =>
-      transactionMapper.toFrontend(
-        t,
-        lookups.parties as any[],
-        lookups.categories as any[],
-        lookups.wallets as any[],
-        lookups.groups as any[]
-      )
-    );
-    out.push(...mapped);
-    if (!resp.data || resp.data.length < limit || page >= resp.last_page || page >= maxPages) {
-      if (page >= maxPages && resp.last_page && page < resp.last_page) {
-        console.warn(
-          `[reports] transaction sweep truncated at ${maxPages} pages (${out.length} rows); some data omitted`
-        );
+
+  const first = await fetchPage(1);
+  const lastPage = Math.max(1, first.last_page || 1);
+  const targetPages = Math.min(lastPage, maxPages);
+
+  // Index pages by (page - 1) so the final concat preserves server order
+  // regardless of which request resolves first.
+  const pages: any[][] = new Array(targetPages);
+  pages[0] = first.data || [];
+
+  if (targetPages > 1) {
+    const queue: number[] = [];
+    for (let p = 2; p <= targetPages; p++) queue.push(p);
+
+    let cursor = 0;
+    const worker = async () => {
+      while (cursor < queue.length) {
+        const p = queue[cursor++];
+        const resp = await fetchPage(p);
+        pages[p - 1] = resp.data || [];
       }
-      break;
+    };
+    await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, () => worker()));
+  }
+
+  const out: FrontendTransaction[] = [];
+  for (const data of pages) {
+    if (!data) continue;
+    for (const t of data) {
+      out.push(
+        transactionMapper.toFrontend(
+          t,
+          lookups.parties as any[],
+          lookups.categories as any[],
+          lookups.wallets as any[],
+          lookups.groups as any[]
+        )
+      );
     }
-    page += 1;
   }
+
+  if (lastPage > maxPages) {
+    sweepTruncated.value = true;
+    console.warn(
+      `[reports] transaction sweep truncated at ${maxPages} pages (${out.length} rows); some data omitted`
+    );
+  }
+
   return out;
 }
 
@@ -264,6 +297,7 @@ async function reload(lookups: {
   const id = ++lastReloadId;
   isLoading.value = true;
   error.value = null;
+  sweepTruncated.value = false;
 
   const cur = getRange(selectedPeriod.value, customRange.value);
   const prev = previousRange(cur);
@@ -446,17 +480,35 @@ export const useReportData = () => {
     return Array.from(map.values());
   });
 
-  // Per-category 6-month trend computed from trailing transactions (API doesn't provide per-cat history)
-  const trendFor = (categoryName: string, type: 'INCOME' | 'EXPENSE'): number[] => {
+  // Per-category 6-month trend (API doesn't provide per-cat history). Built in a
+  // single pass over the trailing transactions and indexed by category, so the
+  // per-category lookups below stay O(1) instead of rescanning every transaction.
+  const trendIndex = computed(() => {
     const months = trailing6MonthBuckets.value.map((m) => m.month);
-    const acc = new Map(months.map((m) => [m, 0]));
+    const monthIdx = new Map(months.map((m, i) => [m, i]));
+    const income = new Map<string, number[]>();
+    const expense = new Map<string, number[]>();
     trailing12Transactions.value.forEach((t) => {
-      if (t.isTransfer || t.type !== type) return;
-      if ((t.category || 'Uncategorized') !== categoryName) return;
-      const key = t.date.slice(0, 7);
-      if (acc.has(key)) acc.set(key, (acc.get(key) || 0) + parseAmount(t.amount).value);
+      if (t.isTransfer) return;
+      const mi = monthIdx.get(t.date.slice(0, 7));
+      if (mi === undefined) return;
+      const bucket = t.type === 'INCOME' ? income : t.type === 'EXPENSE' ? expense : null;
+      if (!bucket) return;
+      const name = t.category || 'Uncategorized';
+      let series = bucket.get(name);
+      if (!series) {
+        series = new Array(months.length).fill(0);
+        bucket.set(name, series);
+      }
+      series[mi] += parseAmount(t.amount).value;
     });
-    return months.map((m) => acc.get(m) || 0);
+    return { months, income, expense };
+  });
+
+  const trendFor = (categoryName: string, type: 'INCOME' | 'EXPENSE'): number[] => {
+    const idx = trendIndex.value;
+    const series = (type === 'INCOME' ? idx.income : idx.expense).get(categoryName);
+    return series ? series.slice() : idx.months.map(() => 0);
   };
 
   const expenseCategories = computed<CategoryBucket[]>(() => {
@@ -703,6 +755,12 @@ export const useReportData = () => {
 
   const primaryCurrency = computed(() => getDefaultCurrency.value || 'USD');
 
+  // The backend excludes amounts it can't convert (no exchange rate for that
+  // currency) and flags the response partial. Surface it so party/category
+  // figures aren't silently understated.
+  const statsPartial = computed(() => apiStats.value?.partial === true);
+  const unconvertedCurrencies = computed(() => apiStats.value?.unconverted_currencies || []);
+
   const setPeriod = (p: ReportPeriodValue) => {
     selectedPeriod.value = p;
     if (p !== 'custom') customRange.value = null;
@@ -736,6 +794,9 @@ export const useReportData = () => {
     compareEnabled,
     isLoading,
     error,
+    sweepTruncated,
+    statsPartial,
+    unconvertedCurrencies,
     periods: REPORT_PERIODS,
     range,
     prevRange,
diff --git a/pages/reports.vue b/pages/reports.vue
index 7720ccc..29e122f 100644
--- a/pages/reports.vue
+++ b/pages/reports.vue
@@ -15,7 +15,32 @@
       @open-review="reviewOpen = true"
     />
 
-    <ReportsEmpty v-if="!hasData" />
+    <p v-if="statsPartial" class="reports-truncated" role="status">
+      {{
+        t(
+          'Some amounts could not be converted to your default currency and were left out, so figures may be understated. Affected: {currencies}.',
+          { currencies: unconvertedCurrencies.join(', ') }
+        )
+      }}
+    </p>
+
+    <p v-if="sweepTruncated" class="reports-truncated" role="status">
+      {{
+        t(
+          'This period has more transactions than we chart at once. Daily and trend figures show the most recent {count} and may be partial.',
+          { count: '6,000' }
+        )
+      }}
+    </p>
+
+    <LoadingSkeleton
+      v-if="isLoading && !hasData"
+      variant="card"
+      :count="3"
+      class="reports-loading"
+    />
+
+    <ReportsEmpty v-else-if="!hasData" />
 
     <Transition v-else name="tab" mode="out-in">
       <div :key="activeTab" class="tab-stage">
@@ -118,6 +143,10 @@ const {
   compareEnabled,
   periods,
   range,
+  isLoading,
+  sweepTruncated,
+  statsPartial,
+  unconvertedCurrencies,
   totals,
   dailyBuckets,
   monthlyBuckets,
@@ -156,7 +185,18 @@ const currency = computed(() => primaryCurrency.value);
 const format = (n, cur) =>
   formatShortAmount(`${Math.round((n || 0) * 100) / 100} ${cur || currency.value}`);
 
-const hasData = computed(() => totals.value.income > 0 || totals.value.expense > 0);
+// "Has data" must not hinge on the overview totals alone: charts read their own
+// buckets/category series, which can be present even when totals derive to zero
+// (e.g. amounts dropped for a missing exchange rate). Gate the empty state on any
+// real signal so charts aren't hidden while data exists. dailyBuckets is scaffolded
+// one-per-day, so check txCount rather than its length.
+const hasData = computed(() => {
+  if (totals.value.income > 0 || totals.value.expense > 0) return true;
+  if (expenseCategories.value.length > 0 || incomeCategories.value.length > 0) return true;
+  if (monthlyBuckets.value.some((m) => m.income || m.expense || m.net)) return true;
+  if (dailyBuckets.value.some((d) => d.txCount > 0)) return true;
+  return false;
+});
 
 const applyCustom = ({ start, end }) => {
   setCustomRange(start, end);
@@ -205,6 +245,15 @@ const reviewData = computed(() => reviewSnapshot.value || prevReviewSnapshot.val
   justify-content: flex-start;
 }
 
+.reports-truncated {
+  margin: 0;
+  padding: $spacing-3 $spacing-4;
+  border-radius: $radius-lg;
+  background: rgba(245, 158, 11, 0.12);
+  color: #92400e;
+  font-size: $font-size-sm;
+}
+
 .tab-stage {
   display: flex;
   flex-direction: column;
diff --git a/services/api/statsApi.ts b/services/api/statsApi.ts
index 5630632..60ffbc5 100644
--- a/services/api/statsApi.ts
+++ b/services/api/statsApi.ts
@@ -74,6 +74,10 @@ export interface StatsResponse {
       monthly_cash_flow: MonthlyCashFlow[];
       expense_by_wallet?: PartyData[];
     };
+    // Set when amounts in a currency with no available exchange rate were
+    // excluded from the totals, so figures are understated for those currencies.
+    partial?: boolean;
+    unconverted_currencies?: string[];
   };
 }