Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions packages/app/src/app/api/v1/benchmarks/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,30 @@ describe('GET /api/v1/benchmarks', () => {
expect(res.status).toBe(200);
const body = await res.json();
expect(body).toEqual(mockRows);
expect(mockGetLatestBenchmarks).toHaveBeenCalledWith('mock-sql', 'dsr1', undefined);
expect(mockGetLatestBenchmarks).toHaveBeenCalledWith('mock-sql', 'dsr1', undefined, undefined);
});

it('passes date param to query when provided', async () => {
mockGetLatestBenchmarks.mockResolvedValueOnce([]);

const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&date=2026-03-01'));
expect(res.status).toBe(200);
expect(mockGetLatestBenchmarks).toHaveBeenCalledWith('mock-sql', 'dsr1', '2026-03-01');
expect(mockGetLatestBenchmarks).toHaveBeenCalledWith(
'mock-sql',
'dsr1',
'2026-03-01',
undefined,
);
});

it('passes exact=true when query param set', async () => {
mockGetLatestBenchmarks.mockResolvedValueOnce([]);

const res = await GET(
req('/api/v1/benchmarks?model=DeepSeek-R1-0528&date=2026-03-01&exact=true'),
);
expect(res.status).toBe(200);
expect(mockGetLatestBenchmarks).toHaveBeenCalledWith('mock-sql', 'dsr1', '2026-03-01', true);
});

it('returns 500 when query throws', async () => {
Expand Down
7 changes: 4 additions & 3 deletions packages/app/src/app/api/v1/benchmarks/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ import { cachedJson, cachedQuery } from '@/lib/api-cache';
export const dynamic = 'force-dynamic';

const getCachedBenchmarks = cachedQuery(
async (dbModelKey: string, date?: string) => {
async (dbModelKey: string, date?: string, exact?: boolean) => {
const sql = getDb();
return getLatestBenchmarks(sql, dbModelKey, date);
return getLatestBenchmarks(sql, dbModelKey, date, exact);
},
'benchmarks',
{ blobOnly: true },
Expand All @@ -21,13 +21,14 @@ export async function GET(request: NextRequest) {
const params = request.nextUrl.searchParams;
const model = params.get('model') ?? '';
const date = params.get('date') ?? undefined;
const exact = params.get('exact') === 'true';
const dbModelKey = DISPLAY_MODEL_TO_DB[model];
if (!dbModelKey) {
return NextResponse.json({ error: 'Unknown model' }, { status: 400 });
}

try {
const rows = await getCachedBenchmarks(dbModelKey, date);
const rows = await getCachedBenchmarks(dbModelKey, date, exact || undefined);
return cachedJson(rows);
} catch (error) {
console.error('Error fetching benchmarks:', error);
Expand Down
4 changes: 3 additions & 1 deletion packages/app/src/components/inference/hooks/useChartData.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,9 @@ export function useChartData(
}, [selectedGPUs, selectedDates, selectedDateRange, selectedRunDate]);

const comparisonQueries = useQueries({
queries: comparisonDates.map((date) => benchmarkQueryOptions(selectedModel, date, enabled)),
queries: comparisonDates.map((date) =>
benchmarkQueryOptions(selectedModel, date, enabled, true),
),
});

const comparisonLoading = comparisonQueries.some((q) => q.isLoading);
Expand Down
36 changes: 35 additions & 1 deletion packages/app/src/components/inference/ui/ChartControls.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
} from '@/components/ui/chart-selectors';
import { DateRangePicker } from '@/components/ui/date-range-picker';
import { LabelWithTooltip } from '@/components/ui/label-with-tooltip';
import { MultiDatePicker } from '@/components/ui/multi-date-picker';
import { MultiSelect } from '@/components/ui/multi-select';
import {
Select,
Expand Down Expand Up @@ -68,9 +69,14 @@ const GROUPED_Y_AXIS_OPTIONS = (() => {
interface ChartControlsProps {
/** Hide GPU Config selector and related date pickers (used by Historical Trends tab) */
hideGpuComparison?: boolean;
/** Intermediate dates within the comparison range that have changelog entries */
intermediateDates?: string[];
}

export default function ChartControls({ hideGpuComparison = false }: ChartControlsProps) {
export default function ChartControls({
hideGpuComparison = false,
intermediateDates = [],
}: ChartControlsProps) {
const {
selectedModel,
setSelectedModel,
Expand All @@ -86,6 +92,8 @@ export default function ChartControls({ hideGpuComparison = false }: ChartContro
availableGPUs,
selectedDateRange,
setSelectedDateRange,
selectedDates,
setSelectedDates,
dateRangeAvailableDates,
isCheckingAvailableDates,
availablePrecisions,
Expand Down Expand Up @@ -327,6 +335,32 @@ export default function ChartControls({ hideGpuComparison = false }: ChartContro
/>
</div>
)}

{!hideGpuComparison &&
selectedGPUs.length > 0 &&
selectedDateRange.startDate &&
selectedDateRange.endDate &&
intermediateDates.length > 0 && (
<div className="flex flex-col space-y-1.5 lg:col-span-2">
<LabelWithTooltip
htmlFor="intermediate-dates-select"
label="Intermediate Dates"
tooltip="Select up to 2 additional intermediate dates with config changelog entries. These dates will be added to the chart for comparison alongside the start and end dates."
/>
<MultiDatePicker
dates={selectedDates}
onChange={(value) => {
setSelectedDates(value);
track('inference_intermediate_dates_selected', {
dates: value.join(','),
});
}}
availableDates={intermediateDates}
maxDates={2}
placeholder="Select intermediate dates"
/>
</div>
)}
</div>
</div>
</TooltipProvider>
Expand Down
71 changes: 13 additions & 58 deletions packages/app/src/components/inference/ui/ChartDisplay.tsx
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use client';
import { track } from '@/lib/analytics';
import Link from 'next/link';
import { useEffect, useMemo, useRef, useState } from 'react';
import { useMemo, useState } from 'react';
import { ChevronDown, X } from 'lucide-react';

import { useInference } from '@/components/inference/InferenceContext';
Expand Down Expand Up @@ -37,7 +37,6 @@ import {
Sequence,
} from '@/lib/data-mappings';
import { useComparisonChangelogs } from '@/hooks/api/use-comparison-changelogs';
import { configKeyMatchesHwKey } from '@/components/inference/utils/changelogFormatters';
import { useTrendData } from '@/components/inference/hooks/useTrendData';

import ChartControls from './ChartControls';
Expand Down Expand Up @@ -113,8 +112,6 @@ export default function ChartDisplay() {
selectedGPUs,
selectedPrecisions,
selectedDateRange,
setSelectedDates,
availableDates,
dateRangeAvailableDates,
selectedModel,
selectedSequence,
Expand All @@ -130,49 +127,10 @@ export default function ChartDisplay() {

const {
changelogs,
intermediateDates,
loading: changelogsLoading,
totalDatesQueried,
} = useComparisonChangelogs(selectedGPUs, selectedDateRange, availableDates);

// Auto-populate intermediate dates from changelog dates so they appear on the GPU graph
const prevChangelogDatesRef = useRef('');
useEffect(() => {
if (!selectedDateRange.startDate || !selectedDateRange.endDate || selectedGPUs.length === 0)
return;
const gpuDatesSet = new Set(dateRangeAvailableDates);
const precSet = new Set(selectedPrecisions);
const changelogDates = changelogs
.filter((c) =>
c.entries.some((entry) =>
entry.config_keys.some((key) => {
const precision = key.split('-')[1];
return (
precSet.has(precision) && selectedGPUs.some((gpu) => configKeyMatchesHwKey(key, gpu))
);
}),
),
)
.map((c) => c.date)
.filter(
(d) =>
d !== selectedDateRange.startDate &&
d !== selectedDateRange.endDate &&
gpuDatesSet.has(d),
)
.sort();
const key = changelogDates.join(',');
if (key !== prevChangelogDatesRef.current) {
prevChangelogDatesRef.current = key;
setSelectedDates(changelogDates);
}
}, [
changelogs,
selectedDateRange,
selectedGPUs,
selectedPrecisions,
setSelectedDates,
dateRangeAvailableDates,
]);
} = useComparisonChangelogs(selectedGPUs, selectedDateRange, dateRangeAvailableDates);

const { unofficialRunInfo, getOverlayData, isUnofficialRun } = useUnofficialRun();

Expand Down Expand Up @@ -705,21 +663,18 @@ export default function ChartDisplay() {
<SocialShareButtons compact className="hidden sm:flex" />
</div>
</div>
<ChartControls />
<ChartControls intermediateDates={intermediateDates} />
<ModelArchitectureDiagram model={selectedModel} />
{selectedGPUs.length === 0 && <WorkflowInfoDisplay workflowInfo={workflowInfo} />}
{selectedGPUs.length > 0 &&
selectedDateRange.startDate &&
selectedDateRange.endDate && (
<ComparisonChangelog
changelogs={changelogs}
selectedGPUs={selectedGPUs}
selectedPrecisions={selectedPrecisions}
selectedDateRange={selectedDateRange}
loading={changelogsLoading}
totalDatesQueried={totalDatesQueried}
/>
)}
{selectedGPUs.length > 0 && (
<ComparisonChangelog
changelogs={changelogs}
selectedGPUs={selectedGPUs}
selectedPrecisions={selectedPrecisions}
loading={changelogsLoading}
totalDatesQueried={totalDatesQueried}
/>
)}
</div>
</Card>
</section>
Expand Down
26 changes: 5 additions & 21 deletions packages/app/src/components/inference/ui/ComparisonChangelog.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ interface ComparisonChangelogProps {
changelogs: ComparisonChangelogType[];
selectedGPUs: string[];
selectedPrecisions: string[];
selectedDateRange: { startDate: string; endDate: string };
loading?: boolean;
totalDatesQueried: number;
}
Expand All @@ -26,11 +25,10 @@ export default function ComparisonChangelog({
changelogs,
selectedGPUs,
selectedPrecisions,
selectedDateRange,
loading,
totalDatesQueried,
}: ComparisonChangelogProps) {
const [isExpanded, setIsExpanded] = useState(false);
const [isExpanded, setIsExpanded] = useState(true);

// Filter changelog entries to only show those matching selected GPUs and precisions
const filteredChangelogs = useMemo(() => {
Expand All @@ -48,24 +46,10 @@ export default function ComparisonChangelog({
}),
),
}))
.filter((item) => item.entries.length > 0);
.filter((item) => item.entries.length > 0)
.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
}, [changelogs, selectedGPUs, selectedPrecisions]);

// Always include start/end dates, even if they have no changelog entries
const sortedChangelogs = useMemo(() => {
const byDate = new Map(filteredChangelogs.map((c) => [c.date, c]));
const { startDate, endDate } = selectedDateRange;
if (startDate && !byDate.has(startDate)) {
byDate.set(startDate, { date: startDate, entries: [] });
}
if (endDate && !byDate.has(endDate)) {
byDate.set(endDate, { date: endDate, entries: [] });
}
return [...byDate.values()].sort(
(a, b) => new Date(a.date).getTime() - new Date(b.date).getTime(),
);
}, [filteredChangelogs, selectedDateRange]);

const handleToggle = () => {
const newState = !isExpanded;
setIsExpanded(newState);
Expand Down Expand Up @@ -104,13 +88,13 @@ export default function ComparisonChangelog({
}`}
>
<div className="px-4 pt-2 pb-4 flex flex-col gap-3">
{sortedChangelogs.length === 0 ? (
{filteredChangelogs.length === 0 ? (
<p className="text-sm text-muted-foreground">
No config changelog data matching the selected GPUs and precisions for this date
range. Changelog tracking began Dec 30, 2025.
</p>
) : (
sortedChangelogs.map((item) => (
filteredChangelogs.map((item) => (
<div key={item.date} className="flex flex-col gap-1">
<div className="flex items-center gap-2 flex-wrap">
<span className="text-sm font-semibold">{item.date}</span>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ export const generateGPUGraphTooltipContent = (config: TooltipConfig): string =>
<div style="background: var(--popover); border: 1px solid var(--border); border-radius: 8px; padding: 12px; box-shadow: 0 4px 6px -1px rgb(0 0 0 / 0.1); user-select: ${isPinned ? 'text' : 'none'};">
${isPinned ? '<div style="color: var(--muted-foreground); font-size: 10px; margin-bottom: 6px; font-style: italic;">Click elsewhere to dismiss</div>' : ''}
<div style="color: var(--muted-foreground); font-size: 11px; margin-bottom: 4px;">
<strong>Date:</strong> ${d.actualDate ?? d.date}
<strong>Date:</strong> ${d.date}${d.actualDate && d.actualDate !== d.date ? ` <span style="opacity: 0.7">(data from ${d.actualDate})</span>` : ''}
</div>
<div style="color: var(--muted-foreground); font-size: 11px; margin-bottom: 4px;">
<strong>GPU Config:</strong> ${hardwareConfig[d.hwKey] ? getDisplayLabel(hardwareConfig[d.hwKey]) : d.hwKey}
Expand Down
7 changes: 6 additions & 1 deletion packages/app/src/hooks/api/use-benchmarks.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,12 @@ import { benchmarkQueryOptions } from '@/hooks/api/use-benchmarks';
describe('benchmarkQueryOptions', () => {
it('builds query key from model and date', () => {
const opts = benchmarkQueryOptions('DeepSeek-R1-0528', '2026-03-01');
expect(opts.queryKey).toEqual(['benchmarks', 'DeepSeek-R1-0528', '2026-03-01']);
expect(opts.queryKey).toEqual(['benchmarks', 'DeepSeek-R1-0528', '2026-03-01', 'latest']);
});

it('builds exact query key when exact=true', () => {
const opts = benchmarkQueryOptions('DeepSeek-R1-0528', '2026-03-01', true, true);
expect(opts.queryKey).toEqual(['benchmarks', 'DeepSeek-R1-0528', '2026-03-01', 'exact']);
});

it('produces distinct keys for different models', () => {
Expand Down
11 changes: 8 additions & 3 deletions packages/app/src/hooks/api/use-benchmarks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,15 @@ import { useQuery, keepPreviousData } from '@tanstack/react-query';
import { fetchBenchmarks } from '@/lib/api';

/** Shared query options — reused by useQueries for comparison dates. */
export function benchmarkQueryOptions(model: string, date: string, enabled: boolean = true) {
export function benchmarkQueryOptions(
model: string,
date: string,
enabled: boolean = true,
exact?: boolean,
) {
return {
queryKey: ['benchmarks', model, date] as const,
queryFn: () => fetchBenchmarks(model, date),
queryKey: ['benchmarks', model, date, exact ? 'exact' : 'latest'] as const,
queryFn: () => fetchBenchmarks(model, date, exact),
enabled: enabled && Boolean(model),
};
}
Expand Down
Loading