diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 074fa7c..49dd882 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,5 +129,6 @@ jobs: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - run: bun install --frozen-lockfile + - run: bun run web:test - run: bun run web:check - run: bun run web:smoke diff --git a/app/lib/export/leaderboard_exporter.dart b/app/lib/export/leaderboard_exporter.dart index b23e088..926a31a 100644 --- a/app/lib/export/leaderboard_exporter.dart +++ b/app/lib/export/leaderboard_exporter.dart @@ -15,6 +15,12 @@ import 'package:crypto/crypto.dart'; enum LeaderboardExportStrategy { aggregateCompatible, latestRun, bestObserved } +const supportedLeaderboardArtifactSchemaVersions = [1, 2]; +const acceptedLeaderboardDataPolicies = [ + 'aggregate-compatible', + 'latest-run', + 'best-observed', +]; const dartArenaBenchmarkVersion = '2026-05-31-master-spec'; const dartArenaEvaluatorSchemaVersion = 2; const _diagnosticOnlyScoringSchemaVersion = 2; diff --git a/app/test/export/leaderboard_exporter_test.dart b/app/test/export/leaderboard_exporter_test.dart index ae99005..2f91c8d 100644 --- a/app/test/export/leaderboard_exporter_test.dart +++ b/app/test/export/leaderboard_exporter_test.dart @@ -1,11 +1,16 @@ import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; import 'package:dart_arena/export/leaderboard_exporter.dart'; import 'package:dart_arena/storage/database.dart'; +import 'package:crypto/crypto.dart'; import 'package:drift/drift.dart'; import 'package:drift/native.dart'; import 'package:test/test.dart'; +import 'leaderboard_fixture_oracle.dart' as fixture_oracle; + void main() { late AppDatabase db; @@ -17,6 +22,257 @@ void main() { await db.close(); }); + test( + 'shared leaderboard compatibility corpus matches its manifest', + () async { + final manifest = await _loadLeaderboardFixtureManifest(); + final fixtureDirectory = Directory.fromUri(manifest.uri.resolve('.')); + final files = await fixtureDirectory + .list() + .where((entry) => entry is File && entry.path.endsWith('.json')) + .map((entry) => entry.uri.pathSegments.last) + .where((file) => file != 'manifest.v1.json') + .toList(); + files.sort(); + final listedFiles = [ + for (final entry in _fixtureEntries(manifest.decoded)) + entry['file']! as String, + ]..sort(); + expect(listedFiles, files); + expect(listedFiles.toSet(), hasLength(listedFiles.length)); + + for (final entry in _fixtureEntries(manifest.decoded)) { + final fixtureBytes = await File.fromUri( + manifest.uri.resolve(entry['file']! as String), + ).readAsBytes(); + expect( + sha256.convert(fixtureBytes).toString(), + entry['sha256'], + reason: entry['id']! as String, + ); + + final normalized = fixture_oracle.normalizeLeaderboardFixture( + utf8.decode(fixtureBytes), + ); + expect( + normalized == null ? 'reject' : 'accept', + entry['outcome'], + reason: entry['id']! as String, + ); + expect( + normalized, + entry['normalizedProjection'], + reason: entry['id']! as String, + ); + } + }, + ); + + test('fixed current export matches the current-v2 projection', () async { + await _seedRun( + db, + id: 'compat-run', + completedAt: DateTime.utc(2026, 5, 1), + provenanceJson: _compatibilityProvenanceJson(), + ); + await _seedTaskRun( + db, + id: 'deepseek-a', + runId: 'compat-run', + taskId: 'task.a', + providerId: 'deepseek', + modelId: 'deepseek-v4-pro', + completedAt: DateTime.utc(2026, 5, 1, 10), + latencyMs: 700, + promptTokens: 12, + completionTokens: 18, + ); + await _seedTaskRun( + db, + id: 'deepseek-b', + runId: 'compat-run', + taskId: 'task.b', + providerId: 'deepseek', + modelId: 'deepseek-v4-pro', + completedAt: DateTime.utc(2026, 5, 1, 11), + latencyMs: 900, + promptTokens: 14, + completionTokens: 22, + trialIndex: 1, + ); + await _seedTaskRun( + db, + id: 'openai-a', + runId: 'compat-run', + taskId: 'task.a', + completedAt: DateTime.utc(2026, 5, 1, 12), + latencyMs: 1000, + promptTokens: 10, + completionTokens: 20, + ); + await _seedTaskRun( + db, + id: 'openai-b', + runId: 'compat-run', + taskId: 'task.b', + completedAt: DateTime.utc(2026, 5, 1, 13), + primaryPass: false, + failureTag: 'public_tests_failed', + latencyMs: 3000, + promptTokens: 30, + completionTokens: 40, + aggregateScore: 0.4, + trialIndex: 1, + ); + await _seedEvaluation( + db, + taskRunId: 'deepseek-a', + evaluatorId: 'test', + passed: true, + ); + await _seedEvaluation( + db, + taskRunId: 'deepseek-a', + evaluatorId: 'agent_harness', + passed: true, + details: const {'step_count': 4, 'peak_context_tokens': 8000}, + ); + await _seedEvaluation( + db, + taskRunId: 'deepseek-a', + evaluatorId: 'hidden_test', + passed: true, + ); + await _seedEvaluation( + db, + taskRunId: 'deepseek-b', + evaluatorId: 'test', + passed: true, + ); + await _seedEvaluation( + db, + taskRunId: 'deepseek-b', + evaluatorId: 'agent_harness', + passed: true, + details: const {'step_count': 6, 'peak_context_tokens': 12000}, + ); + await _seedEvaluation( + db, + taskRunId: 'deepseek-b', + evaluatorId: 'hidden_test', + passed: true, + ); + await _seedEvaluation( + db, + taskRunId: 'openai-a', + evaluatorId: 'test', + passed: true, + ); + await _seedEvaluation( + db, + taskRunId: 'openai-a', + evaluatorId: 'agent_harness', + passed: true, + details: const {'step_count': 5, 'peak_context_tokens': 10000}, + ); + await _seedEvaluation( + db, + taskRunId: 'openai-a', + evaluatorId: 'hidden_test', + passed: true, + ); + await _seedEvaluation( + db, + taskRunId: 'openai-a', + evaluatorId: 'llm_judge', + passed: true, + details: const { + 'judge_overhead': { + 'prompt_tokens': 100, + 'completion_tokens': 20, + 'estimated_cost_micros': 325, + 'pricing_status': 'exact', + }, + }, + ); + await _seedEvaluation( + db, + taskRunId: 'openai-b', + evaluatorId: 'test', + passed: false, + ); + await _seedEvaluation( + db, + taskRunId: 'openai-b', + evaluatorId: 'agent_harness', + passed: true, + details: const {'step_count': 9, 'peak_context_tokens': 16000}, + ); + await _seedEvaluation( + db, + taskRunId: 'openai-b', + evaluatorId: 'hidden_test', + passed: false, + details: const {'blocked': true, 'blocked_by': 'test'}, + ); + + final export = await buildLeaderboardExport( + db, + options: const LeaderboardExportOptions(track: 'agentic'), + now: () => DateTime.utc(2026, 5, 31), + ); + final manifest = await _loadLeaderboardFixtureManifest(); + final current = _fixtureEntries( + manifest.decoded, + ).singleWhere((entry) => entry['id'] == 'current-v2'); + final fixtureBytes = await File.fromUri( + manifest.uri.resolve(current['file']! as String), + ).readAsBytes(); + + expect(jsonDecode(utf8.decode(fixtureBytes)), export); + expect( + fixture_oracle.normalizeLeaderboardFixture(jsonEncode(export)), + current['normalizedProjection'], + ); + expect( + fixture_oracle + .objectList(export['models']) + .map((row) => '${row['providerId']}:${row['modelId']}'), + ['deepseek:deepseek-v4-pro', 'openai:gpt-5'], + ); + expect( + fixture_oracle.objectList(export['tasks']).map((row) => row['taskId']), + ['task.a', 'task.b'], + ); + expect( + fixture_oracle + .objectList(export['taskModelCells']) + .map( + (row) => '${row['providerId']}:${row['modelId']}:${row['taskId']}', + ), + [ + 'deepseek:deepseek-v4-pro:task.a', + 'deepseek:deepseek-v4-pro:task.b', + 'openai:gpt-5:task.a', + 'openai:gpt-5:task.b', + ], + ); + expect( + fixture_oracle + .objectList(export['trialSummaries']) + .map( + (row) => + '${row['providerId']}:${row['modelId']}:${row['taskId']}:${row['trialIndex']}', + ), + [ + 'deepseek:deepseek-v4-pro:task.a:0', + 'deepseek:deepseek-v4-pro:task.b:1', + 'openai:gpt-5:task.a:0', + 'openai:gpt-5:task.b:1', + ], + ); + }); + test('aggregate-compatible aggregates compatible completed runs', () async { await _seedRun( db, @@ -2114,8 +2370,36 @@ String _environmentProvenanceJson({ }, }); -String _presetProvenanceJson({String corpusDigest = 'c'}) => jsonEncode({ +String _compatibilityProvenanceJson() => jsonEncode({ 'schemaVersion': 2, + 'providers': [ + { + 'id': 'deepseek', + 'selectedModelConfigs': [ + { + 'modelId': 'deepseek-v4-pro', + 'baseModelId': 'deepseek-v4-pro', + 'modelConfig': { + 'customModelDisplayName': 'DeepSeek Pro', + 'customModelProvider': 'DeepSeek', + }, + }, + ], + }, + { + 'id': 'openai', + 'selectedModelConfigs': [ + { + 'modelId': 'gpt-5', + 'baseModelId': 'gpt-5', + 'modelConfig': { + 'customModelDisplayName': 'GPT-5', + 'customModelProvider': 'OpenAI', + }, + }, + ], + }, + ], 'config': { 'scoringSchemaVersion': 2, 'evaluatorWeights': {'compile': 1.0}, @@ -2133,6 +2417,34 @@ String _presetProvenanceJson({String corpusDigest = 'c'}) => jsonEncode({ 'taskBundleDigest': List.filled(64, 'b').join(), }, ], + 'digestSha256': List.filled(64, 'c').join(), + }, + }, +}); + +String _presetProvenanceJson({ + String corpusDigest = 'c', + bool includeTaskB = true, +}) => jsonEncode({ + 'schemaVersion': 2, + 'config': { + 'scoringSchemaVersion': 2, + 'evaluatorWeights': {'compile': 1.0}, + 'corpusManifest': { + 'preset': 'mvp', + 'tasks': [ + { + 'taskId': 'task.a', + 'taskVersion': 1, + 'taskBundleDigest': List.filled(64, 'a').join(), + }, + if (includeTaskB) + { + 'taskId': 'task.b', + 'taskVersion': 1, + 'taskBundleDigest': List.filled(64, 'b').join(), + }, + ], 'digestSha256': List.filled(64, corpusDigest).join(), }, }, @@ -2207,3 +2519,33 @@ Future _seedEvaluation( ), ); } + +Future<({Uri uri, Map decoded})> +_loadLeaderboardFixtureManifest() async { + final exporterUri = await Isolate.resolvePackageUri( + Uri.parse('package:dart_arena/export/leaderboard_exporter.dart'), + ); + if (exporterUri == null) { + throw StateError('Could not resolve the dart_arena package location.'); + } + final manifestUri = exporterUri.resolve( + '../../../fixtures/leaderboard/compatibility/v1/manifest.v1.json', + ); + final bytes = await File.fromUri(manifestUri).readAsBytes(); + final decoded = fixture_oracle.objectMap(jsonDecode(utf8.decode(bytes))); + expect(decoded['fixtureManifestVersion'], 1); + expect(decoded['artifactFamily'], 'leaderboard.v1.json'); + expect( + decoded['supportedArtifactSchemaVersions'], + supportedLeaderboardArtifactSchemaVersions, + ); + expect(decoded['acceptedDataPolicies'], acceptedLeaderboardDataPolicies); + expect( + LeaderboardExportStrategy.values.map((strategy) => strategy.kebabName), + acceptedLeaderboardDataPolicies, + ); + return (uri: manifestUri, decoded: decoded); +} + +List> _fixtureEntries(Map manifest) => + fixture_oracle.objectList(manifest['entries']); diff --git a/app/test/export/leaderboard_fixture_oracle.dart b/app/test/export/leaderboard_fixture_oracle.dart new file mode 100644 index 0000000..888a6cd --- /dev/null +++ b/app/test/export/leaderboard_fixture_oracle.dart @@ -0,0 +1,624 @@ +import 'dart:convert'; + +import 'package:dart_arena/export/leaderboard_exporter.dart'; + +Map? normalizeLeaderboardFixture(String text) { + try { + final artifact = requiredMap(jsonDecode(text)); + final schemaVersion = requiredCount(artifact, 'schemaVersion'); + if (!supportedLeaderboardArtifactSchemaVersions.contains(schemaVersion)) { + throw const FormatException(); + } + final current = schemaVersion == 2; + final benchmark = requiredMap(artifact['benchmark']); + final source = requiredMap(artifact['source']); + final models = mapList(requiredList(artifact, 'models')); + final tasks = mapList(requiredList(artifact, 'tasks')); + final cells = mapList(listField(artifact, 'taskModelCells', current)); + final trials = mapList(listField(artifact, 'trialSummaries', current)); + final scoring = mapField(artifact, 'scoring', current); + final pricing = mapField(artifact, 'pricingRegistry', current); + final judgeOverhead = mapField(source, 'judgeOverhead', current); + final runProvenance = mapField(source, 'runProvenance', current); + + return { + 'schemaVersion': schemaVersion, + 'provisional': boolField(artifact, 'provisional', false), + 'generatedAt': nullableString(artifact, 'generatedAt'), + 'benchmark': { + 'title': requiredString(benchmark, 'title'), + 'version': nullableString(benchmark, 'version'), + 'taskSetId': nullableString(benchmark, 'taskSetId'), + 'evaluatorSchemaVersion': countField( + benchmark, + 'evaluatorSchemaVersion', + current, + 0, + ), + 'track': requiredString(benchmark, 'track'), + 'dataPolicy': dataPolicy(benchmark, 'dataPolicy'), + 'preset': nullableString(benchmark, 'preset'), + 'selectedTasks': [ + for (final task in mapList( + listField(benchmark, 'selectedTasks', false), + )) + { + 'taskId': requiredString(task, 'taskId'), + 'taskVersion': nullableVersion(task, 'taskVersion'), + 'taskBundleDigest': nullableString(task, 'taskBundleDigest'), + }, + ], + 'corpusManifestDigestSha256': nullableString( + benchmark, + 'corpusManifestDigestSha256', + ), + }, + 'source': { + 'anchorRunId': nullableString(source, 'anchorRunId'), + 'runIds': stringListField(source, 'runIds', current), + 'taskCount': requiredCount(source, 'taskCount'), + 'taskRunCount': requiredCount(source, 'taskRunCount'), + 'modelCount': countField(source, 'modelCount', current, models.length), + 'trialSummaryCount': countField( + source, + 'trialSummaryCount', + current, + trials.length, + ), + 'trialSummaryTotalCount': countField( + source, + 'trialSummaryTotalCount', + current, + requiredCount(source, 'taskRunCount'), + ), + 'trialSummaryTruncated': boolField( + source, + 'trialSummaryTruncated', + false, + current, + ), + 'trialSummaryLimit': countField( + source, + 'trialSummaryLimit', + current, + 0, + ), + 'warnings': stringListField(source, 'warnings', current), + 'judgeOverhead': projectJudgeOverhead(judgeOverhead, current), + 'runProvenance': projectRunProvenance(runProvenance, current), + }, + 'scoring': projectScoring(scoring, current), + 'pricingRegistry': { + 'version': nullableString(pricing, 'version'), + 'currency': nullableString(pricing, 'currency'), + 'modelCount': countField(pricing, 'modelCount', current, 0), + }, + 'models': [for (final model in models) projectModel(model, current)], + 'tasks': [for (final task in tasks) projectTask(task, current)], + 'taskModelCells': [ + for (final cell in cells) projectTaskModelCell(cell, current), + ], + 'trialSummaries': [ + for (final trial in trials) projectTrialSummary(trial, current), + ], + }; + } on Object { + return null; + } +} + +Map projectScoring(Map value, bool current) => + { + 'schemaVersion': countField(value, 'schemaVersion', current, 0), + 'primaryMetric': nullableString(value, 'primaryMetric'), + 'rankingMetric': nullableString(value, 'rankingMetric'), + 'confidenceInterval': nullableString(value, 'confidenceInterval'), + 'llmJudgePolicy': nullableString(value, 'llmJudgePolicy'), + 'objectiveEvaluatorIds': stringListField( + value, + 'objectiveEvaluatorIds', + current, + ), + 'secondaryEvaluatorIds': stringListField( + value, + 'secondaryEvaluatorIds', + current, + ), + 'hiddenVerifierPattern': nullableString(value, 'hiddenVerifierPattern'), + 'failureTags': stringListField(value, 'failureTags', current), + 'objectiveFailureCaps': numberMapField( + value, + 'objectiveFailureCaps', + current, + ), + 'defaultEvaluatorWeights': numberMapField( + value, + 'defaultEvaluatorWeights', + current, + ), + }; + +Map projectJudgeOverhead( + Map value, + bool current, +) => { + 'evaluationCount': countField(value, 'evaluationCount', current, 0), + 'promptTokens': countField(value, 'promptTokens', current, 0), + 'completionTokens': countField(value, 'completionTokens', current, 0), + 'knownEstimatedCostCount': countField( + value, + 'knownEstimatedCostCount', + current, + 0, + ), + 'unknownEstimatedCostCount': countField( + value, + 'unknownEstimatedCostCount', + current, + 0, + ), + 'totalEstimatedCostMicros': countField( + value, + 'totalEstimatedCostMicros', + current, + 0, + ), + 'pricingStatusCounts': countMapField(value, 'pricingStatusCounts', current), +}; + +Map projectRunProvenance( + Map value, + bool current, +) => { + 'runCount': countField(value, 'runCount', current, 0), + 'embeddedRunCount': countField(value, 'embeddedRunCount', current, 0), + 'sandboxEnforcedRunCount': countField( + value, + 'sandboxEnforcedRunCount', + current, + 0, + ), + 'taskExecutionPolicyRunCount': countField( + value, + 'taskExecutionPolicyRunCount', + current, + 0, + ), + 'networkDisabledTaskPolicyRunCount': countField( + value, + 'networkDisabledTaskPolicyRunCount', + current, + 0, + ), + 'taskResourceLimitRunCount': countField( + value, + 'taskResourceLimitRunCount', + current, + 0, + ), + 'sdkVersionRunCount': countField(value, 'sdkVersionRunCount', current, 0), + 'dependencySnapshotRunCount': countField( + value, + 'dependencySnapshotRunCount', + current, + 0, + ), + 'pricingRegistryRunCount': countField( + value, + 'pricingRegistryRunCount', + current, + 0, + ), + 'generatedCodeSandboxBackends': stringListField( + value, + 'generatedCodeSandboxBackends', + current, + ), + 'dartVersions': stringListField(value, 'dartVersions', current), + 'flutterVersions': stringListField(value, 'flutterVersions', current), + 'environmentIds': stringListField(value, 'environmentIds', current), + 'warnings': stringListField(value, 'warnings', current), +}; + +Map projectModel(Map value, bool current) { + final sampleCount = countField(value, 'sampleCount', current, 0); + return { + 'providerId': requiredString(value, 'providerId'), + 'modelId': requiredString(value, 'modelId'), + ...modelIdentity(value, current), + 'rank': nullableCount(value, 'rank'), + 'score': nullableNumber(value, 'score'), + 'passRate': nullableNumber(value, 'passRate'), + 'trialCount': countField(value, 'trialCount', current, sampleCount), + 'passCount': countField(value, 'passCount', current, 0), + 'sampleCount': sampleCount, + 'passAtK': projectPassAtK(mapField(value, 'passAtK', current), current), + 'medianStepCount': nullableCount(value, 'medianStepCount'), + 'medianPeakContextTokens': nullableCount(value, 'medianPeakContextTokens'), + 'publicPassCount': countField(value, 'publicPassCount', current, 0), + 'publicSampleCount': countField(value, 'publicSampleCount', current, 0), + 'publicPassRate': nullableNumber(value, 'publicPassRate'), + 'hiddenPassCount': countField(value, 'hiddenPassCount', current, 0), + 'hiddenSampleCount': countField(value, 'hiddenSampleCount', current, 0), + 'hiddenPassRate': nullableNumber(value, 'hiddenPassRate'), + 'confidenceInterval': projectConfidenceInterval( + mapField(value, 'confidenceInterval', current), + ), + 'lowSample': boolField(value, 'lowSample', false, current), + 'medianLatencyMs': nullableCount(value, 'medianLatencyMs'), + 'medianPromptTokens': nullableCount(value, 'medianPromptTokens'), + 'medianCompletionTokens': nullableCount(value, 'medianCompletionTokens'), + 'medianEstimatedCostMicros': nullableCount( + value, + 'medianEstimatedCostMicros', + ), + 'knownEstimatedCostCount': countField( + value, + 'knownEstimatedCostCount', + current, + 0, + ), + 'unknownEstimatedCostCount': countField( + value, + 'unknownEstimatedCostCount', + current, + 0, + ), + 'totalEstimatedCostMicros': nullableCount( + value, + 'totalEstimatedCostMicros', + ), + 'costPerSolvedTaskMicros': nullableCount(value, 'costPerSolvedTaskMicros'), + 'cheapestPassingEstimatedCostMicros': nullableCount( + value, + 'cheapestPassingEstimatedCostMicros', + ), + 'failureBreakdown': countMapField(value, 'failureBreakdown', current), + 'blockedEvaluationCount': countField( + value, + 'blockedEvaluationCount', + current, + 0, + ), + 'blockedTaskRunCount': countField(value, 'blockedTaskRunCount', current, 0), + }; +} + +Map projectTask(Map value, bool current) { + final sampleCount = countField(value, 'sampleCount', current, 0); + return { + 'taskId': requiredString(value, 'taskId'), + 'taskVersion': nullableVersion(value, 'taskVersion'), + 'taskBundleDigest': nullableString(value, 'taskBundleDigest'), + 'benchmarkTrack': nullableString(value, 'benchmarkTrack'), + 'trialCount': countField(value, 'trialCount', current, sampleCount), + 'sampleCount': sampleCount, + 'modelCount': countField(value, 'modelCount', current, 0), + 'passRate': nullableNumber(value, 'passRate'), + 'confidenceInterval': projectConfidenceInterval( + mapField(value, 'confidenceInterval', current), + ), + 'passAtK': projectPassAtK(mapField(value, 'passAtK', current), current), + 'medianStepCount': nullableCount(value, 'medianStepCount'), + 'medianPeakContextTokens': nullableCount(value, 'medianPeakContextTokens'), + 'publicPassCount': countField(value, 'publicPassCount', current, 0), + 'publicSampleCount': countField(value, 'publicSampleCount', current, 0), + 'publicPassRate': nullableNumber(value, 'publicPassRate'), + 'hiddenPassCount': countField(value, 'hiddenPassCount', current, 0), + 'hiddenSampleCount': countField(value, 'hiddenSampleCount', current, 0), + 'hiddenPassRate': nullableNumber(value, 'hiddenPassRate'), + 'blockedEvaluationCount': countField( + value, + 'blockedEvaluationCount', + current, + 0, + ), + 'blockedTaskRunCount': countField(value, 'blockedTaskRunCount', current, 0), + }; +} + +Map projectTaskModelCell( + Map value, + bool current, +) { + final sampleCount = countField(value, 'sampleCount', current, 0); + return { + 'providerId': requiredString(value, 'providerId'), + 'modelId': requiredString(value, 'modelId'), + ...modelIdentity(value, current), + 'taskId': requiredString(value, 'taskId'), + 'taskVersion': nullableVersion(value, 'taskVersion'), + 'benchmarkTrack': nullableString(value, 'benchmarkTrack'), + 'trialCount': countField(value, 'trialCount', current, sampleCount), + 'passCount': countField(value, 'passCount', current, 0), + 'sampleCount': sampleCount, + 'passRate': nullableNumber(value, 'passRate'), + 'confidenceInterval': projectConfidenceInterval( + mapField(value, 'confidenceInterval', current), + ), + 'errorCount': countField(value, 'errorCount', current, 0), + 'passAtK': projectPassAtK(mapField(value, 'passAtK', current), current), + 'medianStepCount': nullableCount(value, 'medianStepCount'), + 'medianPeakContextTokens': nullableCount(value, 'medianPeakContextTokens'), + 'publicPassCount': countField(value, 'publicPassCount', current, 0), + 'publicSampleCount': countField(value, 'publicSampleCount', current, 0), + 'publicPassRate': nullableNumber(value, 'publicPassRate'), + 'hiddenPassCount': countField(value, 'hiddenPassCount', current, 0), + 'hiddenSampleCount': countField(value, 'hiddenSampleCount', current, 0), + 'hiddenPassRate': nullableNumber(value, 'hiddenPassRate'), + 'blockedEvaluationCount': countField( + value, + 'blockedEvaluationCount', + current, + 0, + ), + 'blockedTaskRunCount': countField(value, 'blockedTaskRunCount', current, 0), + 'medianLatencyMs': nullableCount(value, 'medianLatencyMs'), + 'medianPromptTokens': nullableCount(value, 'medianPromptTokens'), + 'medianCompletionTokens': nullableCount(value, 'medianCompletionTokens'), + 'medianEstimatedCostMicros': nullableCount( + value, + 'medianEstimatedCostMicros', + ), + 'knownEstimatedCostCount': countField( + value, + 'knownEstimatedCostCount', + current, + 0, + ), + 'unknownEstimatedCostCount': countField( + value, + 'unknownEstimatedCostCount', + current, + 0, + ), + 'failureBreakdown': countMapField(value, 'failureBreakdown', current), + }; +} + +Map projectTrialSummary( + Map value, + bool current, +) => { + 'trialId': requiredString(value, 'trialId'), + 'runId': requiredString(value, 'runId'), + 'providerId': requiredString(value, 'providerId'), + 'modelId': requiredString(value, 'modelId'), + ...modelIdentity(value, current), + 'taskId': requiredString(value, 'taskId'), + 'taskVersion': nullableVersion(value, 'taskVersion'), + 'benchmarkTrack': nullableString(value, 'benchmarkTrack'), + 'trialIndex': requiredCount(value, 'trialIndex'), + 'completedAt': nullableString(value, 'completedAt'), + 'primaryPass': nullableBool(value, 'primaryPass'), + 'failureTag': requiredString(value, 'failureTag'), + 'aggregateScore': nullableNumber(value, 'aggregateScore'), + 'publicPassed': nullableBool(value, 'publicPassed'), + 'hiddenPassed': nullableBool(value, 'hiddenPassed'), + 'blockedEvaluationCount': countField( + value, + 'blockedEvaluationCount', + current, + 0, + ), + 'stepCount': nullableCount(value, 'stepCount'), + 'peakContextTokens': nullableCount(value, 'peakContextTokens'), + 'latencyMs': nullableCount(value, 'latencyMs'), + 'promptTokens': nullableCount(value, 'promptTokens'), + 'completionTokens': nullableCount(value, 'completionTokens'), + 'estimatedCostMicros': nullableCount(value, 'estimatedCostMicros'), +}; + +Map modelIdentity(Map value, bool current) { + final config = mapField(value, 'modelConfig', current); + return { + 'displayName': nullableNonEmptyString(config, 'customModelDisplayName'), + 'providerLabel': nullableNonEmptyString(config, 'customModelProvider'), + }; +} + +Map projectConfidenceInterval(Map value) => { + 'lower': nullableNumber(value, 'lower'), + 'upper': nullableNumber(value, 'upper'), +}; + +Map projectPassAtK(Map value, bool current) { + final result = {}; + for (final entry in value.entries) { + final item = requiredMap(entry.value); + final parsedKey = int.tryParse(entry.key); + final fallback = parsedKey != null && parsedKey >= 0 ? parsedKey : 0; + result[entry.key] = { + 'k': countField(item, 'k', current, fallback), + 'passCount': countField(item, 'passCount', current, 0), + 'sampleCount': countField(item, 'sampleCount', current, 0), + 'passRate': nullableNumber(item, 'passRate'), + }; + } + return result; +} + +String dataPolicy(Map value, String key) { + final entry = value[key]; + if (entry is! String || !acceptedLeaderboardDataPolicies.contains(entry)) { + throw const FormatException(); + } + return entry; +} + +Map objectMap(Object? value) { + if (value is! Map) return const {}; + return value.map((key, value) => MapEntry('$key', value)); +} + +List> objectList(Object? value) { + if (value is! List) return const []; + return [ + for (final entry in value) + if (entry is Map) objectMap(entry), + ]; +} + +Map requiredMap(Object? value) { + if (value is! Map) throw const FormatException(); + return objectMap(value); +} + +Map mapField( + Map value, + String key, + bool required, +) { + if (!value.containsKey(key)) { + if (required) throw const FormatException(); + return const {}; + } + return requiredMap(value[key]); +} + +List requiredList(Map value, String key) { + final entry = value[key]; + if (entry is! List) throw const FormatException(); + return entry; +} + +List listField(Map value, String key, bool required) { + if (!value.containsKey(key)) { + if (required) throw const FormatException(); + return const []; + } + return requiredList(value, key); +} + +List> mapList(List value) => [ + for (final entry in value) requiredMap(entry), +]; + +List stringListField( + Map value, + String key, + bool required, +) { + final entries = listField(value, key, required); + if (entries.any((entry) => entry is! String)) { + throw const FormatException(); + } + return entries.cast(); +} + +String requiredString(Map value, String key) { + final entry = value[key]; + if (entry is! String || entry.trim().isEmpty) { + throw const FormatException(); + } + return entry; +} + +String? nullableString(Map value, String key) { + if (!value.containsKey(key) || value[key] == null) return null; + final entry = value[key]; + if (entry is! String) throw const FormatException(); + return entry; +} + +String? nullableNonEmptyString(Map value, String key) { + final entry = nullableString(value, key); + if (entry != null && entry.trim().isEmpty) throw const FormatException(); + return entry; +} + +Object? nullableVersion(Map value, String key) { + if (!value.containsKey(key) || value[key] == null) return null; + final entry = value[key]; + if (entry is String) { + if (entry.trim().isEmpty) throw const FormatException(); + return entry; + } + if (entry is! num || !entry.isFinite) throw const FormatException(); + return entry; +} + +int requiredCount(Map value, String key) { + final entry = value[key]; + if (entry is! num || + !entry.isFinite || + entry < 0 || + entry != entry.truncate()) { + throw const FormatException(); + } + return entry.toInt(); +} + +int countField( + Map value, + String key, + bool required, + int fallback, +) { + if (!value.containsKey(key)) { + if (required) throw const FormatException(); + return fallback; + } + return requiredCount(value, key); +} + +int? nullableCount(Map value, String key) { + if (!value.containsKey(key) || value[key] == null) return null; + return requiredCount(value, key); +} + +num? nullableNumber(Map value, String key) { + if (!value.containsKey(key) || value[key] == null) return null; + final entry = value[key]; + if (entry is! num || !entry.isFinite) throw const FormatException(); + return entry; +} + +bool boolField( + Map value, + String key, + bool fallback, [ + bool required = false, +]) { + if (!value.containsKey(key)) { + if (required) throw const FormatException(); + return fallback; + } + final entry = value[key]; + if (entry is! bool) throw const FormatException(); + return entry; +} + +bool? nullableBool(Map value, String key) { + if (!value.containsKey(key) || value[key] == null) return null; + final entry = value[key]; + if (entry is! bool) throw const FormatException(); + return entry; +} + +Map numberMapField( + Map value, + String key, + bool required, +) { + final entries = mapField(value, key, required); + final result = {}; + for (final entry in entries.entries) { + final number = entry.value; + if (number is! num || !number.isFinite) throw const FormatException(); + result[entry.key] = number; + } + return result; +} + +Map countMapField( + Map value, + String key, + bool required, +) { + final entries = mapField(value, key, required); + return { + for (final entry in entries.entries) + entry.key: requiredCount(entries, entry.key), + }; +} diff --git a/fixtures/leaderboard/compatibility/v1/current-v2.json b/fixtures/leaderboard/compatibility/v1/current-v2.json new file mode 100644 index 0000000..ba06cc1 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/current-v2.json @@ -0,0 +1,815 @@ +{ + "schemaVersion": 2, + "generatedAt": "2026-05-31T00:00:00.000Z", + "benchmark": { + "name": "PickArena", + "brand": "Pickforge Studio", + "title": "PickArena by Pickforge Studio", + "version": "2026-05-31-master-spec", + "taskSetId": "taskset-de65ea47f84dc052", + "evaluatorSchemaVersion": 2, + "track": "agentic", + "dataPolicy": "aggregate-compatible", + "preset": "mvp", + "selectedTasks": [ + { + "taskId": "task.a", + "taskVersion": 1, + "taskBundleDigest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + { + "taskId": "task.b", + "taskVersion": 1, + "taskBundleDigest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + ], + "corpusManifestDigestSha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "source": { + "anchorRunId": "compat-run", + "runIds": [ + "compat-run" + ], + "taskCount": 2, + "taskRunCount": 4, + "modelCount": 2, + "trialSummaryCount": 4, + "trialSummaryTotalCount": 4, + "trialSummaryTruncated": false, + "trialSummaryLimit": 1000, + "warnings": [], + "judgeOverhead": { + "evaluationCount": 1, + "promptTokens": 100, + "completionTokens": 20, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": 325, + "pricingStatusCounts": { + "exact": 1 + } + }, + "runProvenance": { + "runCount": 1, + "embeddedRunCount": 1, + "sandboxEnforcedRunCount": 0, + "taskExecutionPolicyRunCount": 0, + "networkDisabledTaskPolicyRunCount": 0, + "taskResourceLimitRunCount": 0, + "sdkVersionRunCount": 0, + "dependencySnapshotRunCount": 0, + "pricingRegistryRunCount": 0, + "generatedCodeSandboxBackends": [], + "dartVersions": [], + "flutterVersions": [], + "environmentIds": [], + "warnings": [ + "Run compat-run does not record generated-code sandbox enforcement.", + "Run compat-run does not record network-disabled task execution policy.", + "Run compat-run has incomplete SDK version provenance.", + "Run compat-run has incomplete dependency lockfile provenance.", + "Run compat-run has incomplete or unenforced task resource limit provenance.", + "Run compat-run has incomplete pricing registry provenance.", + "Run compat-run has incomplete task execution policy provenance." + ] + } + }, + "scoring": { + "schemaVersion": 2, + "primaryMetric": "primary_pass", + "rankingMetric": "primary_pass_rate", + "confidenceInterval": "wilson_95", + "llmJudgePolicy": "diagnostic_only", + "diffSizePolicy": "diagnostic_only_full_patch", + "diagnosticOnlyEvaluatorIds": [ + "diff_size" + ], + "objectiveEvaluatorIds": [ + "analyze", + "compile", + "hidden_test", + "test", + "test_author", + "widget_tree" + ], + "secondaryEvaluatorIds": [ + "diff_size", + "llm_judge" + ], + "hiddenVerifierPattern": "*_hidden", + "failureTags": [ + "pass", + "hidden_verifier_failed", + "public_tests_failed", + "analysis_failed", + "compile_failed", + "harness_timeout", + "harness_error", + "no_patch", + "invalid_output", + "environment_error", + "unknown" + ], + "objectiveFailureCaps": { + "compile": 0.2, + "analyze": 0.35, + "public_test": 0.6, + "hidden_verifier": 0.6 + }, + "defaultEvaluatorWeights": { + "analyze": 0.5, + "compile": 0.5, + "hidden_test": 1.0, + "llm_judge": 0.7, + "test": 1.0, + "test_author": 4.0, + "widget_tree": 1.0 + } + }, + "pricingRegistry": { + "version": "2026-05-31", + "currency": "USD", + "modelCount": 7, + "models": { + "anthropic:claude-opus-4.7": { + "inputCostPerMToken": 15.0, + "outputCostPerMToken": 75.0, + "source": "manual", + "effectiveFrom": "2026-05-31" + }, + "anthropic:claude-sonnet-4.5": { + "inputCostPerMToken": 3.0, + "outputCostPerMToken": 15.0, + "source": "manual", + "effectiveFrom": "2026-05-31" + }, + "deepseek:deepseek-v4-flash": { + "inputCostPerMToken": 0.14, + "outputCostPerMToken": 0.28, + "source": "manual", + "effectiveFrom": "2026-05-31" + }, + "deepseek:deepseek-v4-pro": { + "inputCostPerMToken": 1.0, + "outputCostPerMToken": 3.0, + "source": "manual", + "effectiveFrom": "2026-05-31" + }, + "openai:gpt-5": { + "inputCostPerMToken": 1.25, + "outputCostPerMToken": 10.0, + "source": "manual", + "effectiveFrom": "2026-05-31" + }, + "openai:gpt-5.3-codex": { + "inputCostPerMToken": 1.25, + "outputCostPerMToken": 10.0, + "source": "manual", + "effectiveFrom": "2026-05-31" + }, + "openai:gpt-5.5": { + "inputCostPerMToken": 1.25, + "outputCostPerMToken": 10.0, + "source": "manual", + "effectiveFrom": "2026-05-31" + } + } + }, + "models": [ + { + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "baseModelId": "deepseek-v4-pro", + "modelConfig": { + "customModelDisplayName": "DeepSeek Pro", + "customModelProvider": "DeepSeek" + }, + "rank": 1, + "score": 1.0, + "passRate": 1.0, + "trialCount": 2, + "passCount": 2, + "sampleCount": 2, + "passAtK": { + "1": { + "k": 1, + "passCount": 2, + "sampleCount": 2, + "passRate": 1.0 + } + }, + "medianStepCount": 5, + "medianPeakContextTokens": 10000, + "traceMetricCoverage": { + "sampleCount": 2, + "stepCountKnownCount": 2, + "stepCountUnknownCount": 0, + "peakContextTokensKnownCount": 2, + "peakContextTokensUnknownCount": 0, + "completeTraceMetricCount": 2 + }, + "tokenUsageCoverage": { + "sampleCount": 2, + "promptTokensKnownCount": 2, + "promptTokensUnknownCount": 0, + "completionTokensKnownCount": 2, + "completionTokensUnknownCount": 0, + "completeTokenUsageCount": 2 + }, + "publicPassCount": 2, + "publicSampleCount": 2, + "publicPassRate": 1.0, + "hiddenPassCount": 2, + "hiddenSampleCount": 2, + "hiddenPassRate": 1.0, + "confidenceInterval": { + "lower": 0.3423811429771653, + "upper": 1.0 + }, + "lowSample": true, + "medianLatencyMs": 800, + "medianPromptTokens": 13, + "medianCompletionTokens": 20, + "medianEstimatedCostMicros": 73, + "knownEstimatedCostCount": 2, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": 146, + "costPerSolvedTaskMicros": 73, + "cheapestPassingEstimatedCostMicros": 66, + "failureBreakdown": { + "pass": 2 + }, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0 + }, + { + "providerId": "openai", + "modelId": "gpt-5", + "baseModelId": "gpt-5", + "modelConfig": { + "customModelDisplayName": "GPT-5", + "customModelProvider": "OpenAI" + }, + "rank": 2, + "score": 0.5, + "passRate": 0.5, + "trialCount": 2, + "passCount": 1, + "sampleCount": 2, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 2, + "passRate": 0.5 + } + }, + "medianStepCount": 7, + "medianPeakContextTokens": 13000, + "traceMetricCoverage": { + "sampleCount": 2, + "stepCountKnownCount": 2, + "stepCountUnknownCount": 0, + "peakContextTokensKnownCount": 2, + "peakContextTokensUnknownCount": 0, + "completeTraceMetricCount": 2 + }, + "tokenUsageCoverage": { + "sampleCount": 2, + "promptTokensKnownCount": 2, + "promptTokensUnknownCount": 0, + "completionTokensKnownCount": 2, + "completionTokensUnknownCount": 0, + "completeTokenUsageCount": 2 + }, + "publicPassCount": 1, + "publicSampleCount": 2, + "publicPassRate": 0.5, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1.0, + "confidenceInterval": { + "lower": 0.09453148796027494, + "upper": 0.905468512039725 + }, + "lowSample": true, + "medianLatencyMs": 2000, + "medianPromptTokens": 20, + "medianCompletionTokens": 30, + "medianEstimatedCostMicros": 326, + "knownEstimatedCostCount": 2, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": 651, + "costPerSolvedTaskMicros": 651, + "cheapestPassingEstimatedCostMicros": 213, + "failureBreakdown": { + "pass": 1, + "public_tests_failed": 1 + }, + "blockedEvaluationCount": 1, + "blockedTaskRunCount": 1 + } + ], + "tasks": [ + { + "taskId": "task.a", + "taskVersion": 1, + "taskBundleDigest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "benchmarkTrack": "agentic", + "trialCount": 2, + "sampleCount": 2, + "modelCount": 2, + "passRate": 1.0, + "confidenceInterval": { + "lower": 0.3423811429771653, + "upper": 1.0 + }, + "passAtK": { + "1": { + "k": 1, + "passCount": 2, + "sampleCount": 2, + "passRate": 1.0 + } + }, + "medianStepCount": 5, + "medianPeakContextTokens": 9000, + "traceMetricCoverage": { + "sampleCount": 2, + "stepCountKnownCount": 2, + "stepCountUnknownCount": 0, + "peakContextTokensKnownCount": 2, + "peakContextTokensUnknownCount": 0, + "completeTraceMetricCount": 2 + }, + "tokenUsageCoverage": { + "sampleCount": 2, + "promptTokensKnownCount": 2, + "promptTokensUnknownCount": 0, + "completionTokensKnownCount": 2, + "completionTokensUnknownCount": 0, + "completeTokenUsageCount": 2 + }, + "publicPassCount": 2, + "publicSampleCount": 2, + "publicPassRate": 1.0, + "hiddenPassCount": 2, + "hiddenSampleCount": 2, + "hiddenPassRate": 1.0, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0 + }, + { + "taskId": "task.b", + "taskVersion": 1, + "taskBundleDigest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "benchmarkTrack": "agentic", + "trialCount": 2, + "sampleCount": 2, + "modelCount": 2, + "passRate": 0.5, + "confidenceInterval": { + "lower": 0.09453148796027494, + "upper": 0.905468512039725 + }, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 2, + "passRate": 0.5 + } + }, + "medianStepCount": 8, + "medianPeakContextTokens": 14000, + "traceMetricCoverage": { + "sampleCount": 2, + "stepCountKnownCount": 2, + "stepCountUnknownCount": 0, + "peakContextTokensKnownCount": 2, + "peakContextTokensUnknownCount": 0, + "completeTraceMetricCount": 2 + }, + "tokenUsageCoverage": { + "sampleCount": 2, + "promptTokensKnownCount": 2, + "promptTokensUnknownCount": 0, + "completionTokensKnownCount": 2, + "completionTokensUnknownCount": 0, + "completeTokenUsageCount": 2 + }, + "publicPassCount": 1, + "publicSampleCount": 2, + "publicPassRate": 0.5, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1.0, + "blockedEvaluationCount": 1, + "blockedTaskRunCount": 1 + } + ], + "taskModelCells": [ + { + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "baseModelId": "deepseek-v4-pro", + "modelConfig": { + "customModelDisplayName": "DeepSeek Pro", + "customModelProvider": "DeepSeek" + }, + "taskId": "task.a", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialCount": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1.0, + "confidenceInterval": { + "lower": 0.20654998073085312, + "upper": 1.0 + }, + "errorCount": 0, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1.0 + } + }, + "medianStepCount": 4, + "medianPeakContextTokens": 8000, + "traceMetricCoverage": { + "sampleCount": 1, + "stepCountKnownCount": 1, + "stepCountUnknownCount": 0, + "peakContextTokensKnownCount": 1, + "peakContextTokensUnknownCount": 0, + "completeTraceMetricCount": 1 + }, + "tokenUsageCoverage": { + "sampleCount": 1, + "promptTokensKnownCount": 1, + "promptTokensUnknownCount": 0, + "completionTokensKnownCount": 1, + "completionTokensUnknownCount": 0, + "completeTokenUsageCount": 1 + }, + "publicPassCount": 1, + "publicSampleCount": 1, + "publicPassRate": 1.0, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1.0, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0, + "medianLatencyMs": 700, + "medianPromptTokens": 12, + "medianCompletionTokens": 18, + "medianEstimatedCostMicros": 66, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "failureBreakdown": { + "pass": 1 + } + }, + { + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "baseModelId": "deepseek-v4-pro", + "modelConfig": { + "customModelDisplayName": "DeepSeek Pro", + "customModelProvider": "DeepSeek" + }, + "taskId": "task.b", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialCount": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1.0, + "confidenceInterval": { + "lower": 0.20654998073085312, + "upper": 1.0 + }, + "errorCount": 0, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1.0 + } + }, + "medianStepCount": 6, + "medianPeakContextTokens": 12000, + "traceMetricCoverage": { + "sampleCount": 1, + "stepCountKnownCount": 1, + "stepCountUnknownCount": 0, + "peakContextTokensKnownCount": 1, + "peakContextTokensUnknownCount": 0, + "completeTraceMetricCount": 1 + }, + "tokenUsageCoverage": { + "sampleCount": 1, + "promptTokensKnownCount": 1, + "promptTokensUnknownCount": 0, + "completionTokensKnownCount": 1, + "completionTokensUnknownCount": 0, + "completeTokenUsageCount": 1 + }, + "publicPassCount": 1, + "publicSampleCount": 1, + "publicPassRate": 1.0, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1.0, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0, + "medianLatencyMs": 900, + "medianPromptTokens": 14, + "medianCompletionTokens": 22, + "medianEstimatedCostMicros": 80, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "failureBreakdown": { + "pass": 1 + } + }, + { + "providerId": "openai", + "modelId": "gpt-5", + "baseModelId": "gpt-5", + "modelConfig": { + "customModelDisplayName": "GPT-5", + "customModelProvider": "OpenAI" + }, + "taskId": "task.a", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialCount": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1.0, + "confidenceInterval": { + "lower": 0.20654998073085312, + "upper": 1.0 + }, + "errorCount": 0, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1.0 + } + }, + "medianStepCount": 5, + "medianPeakContextTokens": 10000, + "traceMetricCoverage": { + "sampleCount": 1, + "stepCountKnownCount": 1, + "stepCountUnknownCount": 0, + "peakContextTokensKnownCount": 1, + "peakContextTokensUnknownCount": 0, + "completeTraceMetricCount": 1 + }, + "tokenUsageCoverage": { + "sampleCount": 1, + "promptTokensKnownCount": 1, + "promptTokensUnknownCount": 0, + "completionTokensKnownCount": 1, + "completionTokensUnknownCount": 0, + "completeTokenUsageCount": 1 + }, + "publicPassCount": 1, + "publicSampleCount": 1, + "publicPassRate": 1.0, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1.0, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0, + "medianLatencyMs": 1000, + "medianPromptTokens": 10, + "medianCompletionTokens": 20, + "medianEstimatedCostMicros": 213, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "failureBreakdown": { + "pass": 1 + } + }, + { + "providerId": "openai", + "modelId": "gpt-5", + "baseModelId": "gpt-5", + "modelConfig": { + "customModelDisplayName": "GPT-5", + "customModelProvider": "OpenAI" + }, + "taskId": "task.b", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialCount": 1, + "passCount": 0, + "sampleCount": 1, + "passRate": 0.0, + "confidenceInterval": { + "lower": 0.0, + "upper": 0.7934500192691468 + }, + "errorCount": 1, + "passAtK": { + "1": { + "k": 1, + "passCount": 0, + "sampleCount": 1, + "passRate": 0.0 + } + }, + "medianStepCount": 9, + "medianPeakContextTokens": 16000, + "traceMetricCoverage": { + "sampleCount": 1, + "stepCountKnownCount": 1, + "stepCountUnknownCount": 0, + "peakContextTokensKnownCount": 1, + "peakContextTokensUnknownCount": 0, + "completeTraceMetricCount": 1 + }, + "tokenUsageCoverage": { + "sampleCount": 1, + "promptTokensKnownCount": 1, + "promptTokensUnknownCount": 0, + "completionTokensKnownCount": 1, + "completionTokensUnknownCount": 0, + "completeTokenUsageCount": 1 + }, + "publicPassCount": 0, + "publicSampleCount": 1, + "publicPassRate": 0.0, + "hiddenPassCount": 0, + "hiddenSampleCount": 0, + "hiddenPassRate": null, + "blockedEvaluationCount": 1, + "blockedTaskRunCount": 1, + "medianLatencyMs": 3000, + "medianPromptTokens": 30, + "medianCompletionTokens": 40, + "medianEstimatedCostMicros": 438, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "failureBreakdown": { + "public_tests_failed": 1 + } + } + ], + "trialSummaries": [ + { + "trialId": "ea26b7d144aa", + "runId": "compat-run", + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "baseModelId": "deepseek-v4-pro", + "modelConfig": { + "customModelDisplayName": "DeepSeek Pro", + "customModelProvider": "DeepSeek" + }, + "taskId": "task.a", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialIndex": 0, + "completedAt": "2026-05-01T10:00:00.000Z", + "primaryPass": true, + "failureTag": "pass", + "aggregateScore": 1.0, + "publicPassed": true, + "hiddenPassed": true, + "blockedEvaluationCount": 0, + "stepCount": 4, + "peakContextTokens": 8000, + "traceMetricStatus": { + "stepCount": "reported", + "peakContextTokens": "reported" + }, + "latencyMs": 700, + "promptTokens": 12, + "completionTokens": 18, + "tokenUsageStatus": { + "promptTokens": "reported", + "completionTokens": "reported" + }, + "estimatedCostMicros": 66 + }, + { + "trialId": "e8540dd0aba4", + "runId": "compat-run", + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "baseModelId": "deepseek-v4-pro", + "modelConfig": { + "customModelDisplayName": "DeepSeek Pro", + "customModelProvider": "DeepSeek" + }, + "taskId": "task.b", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialIndex": 1, + "completedAt": "2026-05-01T11:00:00.000Z", + "primaryPass": true, + "failureTag": "pass", + "aggregateScore": 1.0, + "publicPassed": true, + "hiddenPassed": true, + "blockedEvaluationCount": 0, + "stepCount": 6, + "peakContextTokens": 12000, + "traceMetricStatus": { + "stepCount": "reported", + "peakContextTokens": "reported" + }, + "latencyMs": 900, + "promptTokens": 14, + "completionTokens": 22, + "tokenUsageStatus": { + "promptTokens": "reported", + "completionTokens": "reported" + }, + "estimatedCostMicros": 80 + }, + { + "trialId": "0694cb931ace", + "runId": "compat-run", + "providerId": "openai", + "modelId": "gpt-5", + "baseModelId": "gpt-5", + "modelConfig": { + "customModelDisplayName": "GPT-5", + "customModelProvider": "OpenAI" + }, + "taskId": "task.a", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialIndex": 0, + "completedAt": "2026-05-01T12:00:00.000Z", + "primaryPass": true, + "failureTag": "pass", + "aggregateScore": 1.0, + "publicPassed": true, + "hiddenPassed": true, + "blockedEvaluationCount": 0, + "stepCount": 5, + "peakContextTokens": 10000, + "traceMetricStatus": { + "stepCount": "reported", + "peakContextTokens": "reported" + }, + "latencyMs": 1000, + "promptTokens": 10, + "completionTokens": 20, + "tokenUsageStatus": { + "promptTokens": "reported", + "completionTokens": "reported" + }, + "estimatedCostMicros": 213 + }, + { + "trialId": "1b0a30bef532", + "runId": "compat-run", + "providerId": "openai", + "modelId": "gpt-5", + "baseModelId": "gpt-5", + "modelConfig": { + "customModelDisplayName": "GPT-5", + "customModelProvider": "OpenAI" + }, + "taskId": "task.b", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialIndex": 1, + "completedAt": "2026-05-01T13:00:00.000Z", + "primaryPass": false, + "failureTag": "public_tests_failed", + "aggregateScore": 0.4, + "publicPassed": false, + "hiddenPassed": null, + "blockedEvaluationCount": 1, + "stepCount": 9, + "peakContextTokens": 16000, + "traceMetricStatus": { + "stepCount": "reported", + "peakContextTokens": "reported" + }, + "latencyMs": 3000, + "promptTokens": 30, + "completionTokens": 40, + "tokenUsageStatus": { + "promptTokens": "reported", + "completionTokens": "reported" + }, + "estimatedCostMicros": 438 + } + ] +} diff --git a/fixtures/leaderboard/compatibility/v1/empty-title.json b/fixtures/leaderboard/compatibility/v1/empty-title.json new file mode 100644 index 0000000..bf2e202 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/empty-title.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/empty-track.json b/fixtures/leaderboard/compatibility/v1/empty-track.json new file mode 100644 index 0000000..5147f20 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/empty-track.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":" ","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/fractional-count.json b/fixtures/leaderboard/compatibility/v1/fractional-count.json new file mode 100644 index 0000000..a0470ef --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/fractional-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0.5},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/fractional-model-count.json b/fixtures/leaderboard/compatibility/v1/fractional-model-count.json new file mode 100644 index 0000000..f6968a2 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/fractional-model-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1.5,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/fractional-pass-at-k-count.json b/fixtures/leaderboard/compatibility/v1/fractional-pass-at-k-count.json new file mode 100644 index 0000000..dc46ebf --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/fractional-pass-at-k-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2,"passAtK":{"1":{"k":1,"passCount":0.5,"sampleCount":2,"passRate":0.5}}}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/fractional-schema-version.json b/fixtures/leaderboard/compatibility/v1/fractional-schema-version.json new file mode 100644 index 0000000..61a365a --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/fractional-schema-version.json @@ -0,0 +1 @@ +{"schemaVersion":1.5,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/fractional-trial-index.json b/fixtures/leaderboard/compatibility/v1/fractional-trial-index.json new file mode 100644 index 0000000..9247f14 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/fractional-trial-index.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0.5,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/future-schema-version.json b/fixtures/leaderboard/compatibility/v1/future-schema-version.json new file mode 100644 index 0000000..f6a4823 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/future-schema-version.json @@ -0,0 +1 @@ +{"schemaVersion":3,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-benchmark.json b/fixtures/leaderboard/compatibility/v1/invalid-benchmark.json new file mode 100644 index 0000000..94c4267 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-benchmark.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":[],"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-data-policy.json b/fixtures/leaderboard/compatibility/v1/invalid-data-policy.json new file mode 100644 index 0000000..8859d7a --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-data-policy.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-models.json b/fixtures/leaderboard/compatibility/v1/invalid-models.json new file mode 100644 index 0000000..31f6cda --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-models.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":{},"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-pass-at-k-array.json b/fixtures/leaderboard/compatibility/v1/invalid-pass-at-k-array.json new file mode 100644 index 0000000..a313d37 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-pass-at-k-array.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2,"passAtK":[]}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-selected-tasks-array.json b/fixtures/leaderboard/compatibility/v1/invalid-selected-tasks-array.json new file mode 100644 index 0000000..47c202b --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-selected-tasks-array.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run","selectedTasks":{}},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-source.json b/fixtures/leaderboard/compatibility/v1/invalid-source.json new file mode 100644 index 0000000..e33546d --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-source.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":[],"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-task-model-cells-array.json b/fixtures/leaderboard/compatibility/v1/invalid-task-model-cells-array.json new file mode 100644 index 0000000..34f44d4 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-task-model-cells-array.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":{},"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-tasks.json b/fixtures/leaderboard/compatibility/v1/invalid-tasks.json new file mode 100644 index 0000000..8ed00e0 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-tasks.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":null} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-trial-summaries-array.json b/fixtures/leaderboard/compatibility/v1/invalid-trial-summaries-array.json new file mode 100644 index 0000000..d2aa00a --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-trial-summaries-array.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":"bad"} diff --git a/fixtures/leaderboard/compatibility/v1/invalid-warning-array-entry.json b/fixtures/leaderboard/compatibility/v1/invalid-warning-array-entry.json new file mode 100644 index 0000000..e25c5f4 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/invalid-warning-array-entry.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["ok",7]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/legacy-v1-minimal.json b/fixtures/leaderboard/compatibility/v1/legacy-v1-minimal.json new file mode 100644 index 0000000..aea46af --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/legacy-v1-minimal.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Legacy leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/legacy-v1-populated.json b/fixtures/leaderboard/compatibility/v1/legacy-v1-populated.json new file mode 100644 index 0000000..ba1bce6 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/legacy-v1-populated.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/malformed-json.json b/fixtures/leaderboard/compatibility/v1/malformed-json.json new file mode 100644 index 0000000..c9d7da6 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/malformed-json.json @@ -0,0 +1 @@ +{"schemaVersion":1, diff --git a/fixtures/leaderboard/compatibility/v1/manifest.v1.json b/fixtures/leaderboard/compatibility/v1/manifest.v1.json new file mode 100644 index 0000000..79c54f9 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/manifest.v1.json @@ -0,0 +1,1185 @@ +{ + "fixtureManifestVersion": 1, + "artifactFamily": "leaderboard.v1.json", + "supportedArtifactSchemaVersions": [ + 1, + 2 + ], + "acceptedDataPolicies": [ + "aggregate-compatible", + "latest-run", + "best-observed" + ], + "entries": [ + { + "id": "current-v2", + "file": "current-v2.json", + "sha256": "4e1e3e598f6773d38142df430aecc6e6d214ec63fde54262f3839721ddceca19", + "outcome": "accept", + "normalizedProjection": { + "schemaVersion": 2, + "provisional": false, + "generatedAt": "2026-05-31T00:00:00.000Z", + "benchmark": { + "title": "PickArena by Pickforge Studio", + "version": "2026-05-31-master-spec", + "taskSetId": "taskset-de65ea47f84dc052", + "evaluatorSchemaVersion": 2, + "track": "agentic", + "dataPolicy": "aggregate-compatible", + "preset": "mvp", + "selectedTasks": [ + { + "taskId": "task.a", + "taskVersion": 1, + "taskBundleDigest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + { + "taskId": "task.b", + "taskVersion": 1, + "taskBundleDigest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + ], + "corpusManifestDigestSha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "source": { + "anchorRunId": "compat-run", + "runIds": [ + "compat-run" + ], + "taskCount": 2, + "taskRunCount": 4, + "modelCount": 2, + "trialSummaryCount": 4, + "trialSummaryTotalCount": 4, + "trialSummaryTruncated": false, + "trialSummaryLimit": 1000, + "warnings": [], + "judgeOverhead": { + "evaluationCount": 1, + "promptTokens": 100, + "completionTokens": 20, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": 325, + "pricingStatusCounts": { + "exact": 1 + } + }, + "runProvenance": { + "runCount": 1, + "embeddedRunCount": 1, + "sandboxEnforcedRunCount": 0, + "taskExecutionPolicyRunCount": 0, + "networkDisabledTaskPolicyRunCount": 0, + "taskResourceLimitRunCount": 0, + "sdkVersionRunCount": 0, + "dependencySnapshotRunCount": 0, + "pricingRegistryRunCount": 0, + "generatedCodeSandboxBackends": [], + "dartVersions": [], + "flutterVersions": [], + "environmentIds": [], + "warnings": [ + "Run compat-run does not record generated-code sandbox enforcement.", + "Run compat-run does not record network-disabled task execution policy.", + "Run compat-run has incomplete SDK version provenance.", + "Run compat-run has incomplete dependency lockfile provenance.", + "Run compat-run has incomplete or unenforced task resource limit provenance.", + "Run compat-run has incomplete pricing registry provenance.", + "Run compat-run has incomplete task execution policy provenance." + ] + } + }, + "scoring": { + "schemaVersion": 2, + "primaryMetric": "primary_pass", + "rankingMetric": "primary_pass_rate", + "confidenceInterval": "wilson_95", + "llmJudgePolicy": "diagnostic_only", + "objectiveEvaluatorIds": [ + "analyze", + "compile", + "hidden_test", + "test", + "test_author", + "widget_tree" + ], + "secondaryEvaluatorIds": [ + "diff_size", + "llm_judge" + ], + "hiddenVerifierPattern": "*_hidden", + "failureTags": [ + "pass", + "hidden_verifier_failed", + "public_tests_failed", + "analysis_failed", + "compile_failed", + "harness_timeout", + "harness_error", + "no_patch", + "invalid_output", + "environment_error", + "unknown" + ], + "objectiveFailureCaps": { + "compile": 0.2, + "analyze": 0.35, + "public_test": 0.6, + "hidden_verifier": 0.6 + }, + "defaultEvaluatorWeights": { + "analyze": 0.5, + "compile": 0.5, + "hidden_test": 1, + "llm_judge": 0.7, + "test": 1, + "test_author": 4, + "widget_tree": 1 + } + }, + "pricingRegistry": { + "version": "2026-05-31", + "currency": "USD", + "modelCount": 7 + }, + "models": [ + { + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "displayName": "DeepSeek Pro", + "providerLabel": "DeepSeek", + "rank": 1, + "score": 1, + "passRate": 1, + "trialCount": 2, + "passCount": 2, + "sampleCount": 2, + "passAtK": { + "1": { + "k": 1, + "passCount": 2, + "sampleCount": 2, + "passRate": 1 + } + }, + "medianStepCount": 5, + "medianPeakContextTokens": 10000, + "publicPassCount": 2, + "publicSampleCount": 2, + "publicPassRate": 1, + "hiddenPassCount": 2, + "hiddenSampleCount": 2, + "hiddenPassRate": 1, + "confidenceInterval": { + "lower": 0.3423811429771653, + "upper": 1 + }, + "lowSample": true, + "medianLatencyMs": 800, + "medianPromptTokens": 13, + "medianCompletionTokens": 20, + "medianEstimatedCostMicros": 73, + "knownEstimatedCostCount": 2, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": 146, + "costPerSolvedTaskMicros": 73, + "cheapestPassingEstimatedCostMicros": 66, + "failureBreakdown": { + "pass": 2 + }, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0 + }, + { + "providerId": "openai", + "modelId": "gpt-5", + "displayName": "GPT-5", + "providerLabel": "OpenAI", + "rank": 2, + "score": 0.5, + "passRate": 0.5, + "trialCount": 2, + "passCount": 1, + "sampleCount": 2, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 2, + "passRate": 0.5 + } + }, + "medianStepCount": 7, + "medianPeakContextTokens": 13000, + "publicPassCount": 1, + "publicSampleCount": 2, + "publicPassRate": 0.5, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1, + "confidenceInterval": { + "lower": 0.09453148796027494, + "upper": 0.905468512039725 + }, + "lowSample": true, + "medianLatencyMs": 2000, + "medianPromptTokens": 20, + "medianCompletionTokens": 30, + "medianEstimatedCostMicros": 326, + "knownEstimatedCostCount": 2, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": 651, + "costPerSolvedTaskMicros": 651, + "cheapestPassingEstimatedCostMicros": 213, + "failureBreakdown": { + "pass": 1, + "public_tests_failed": 1 + }, + "blockedEvaluationCount": 1, + "blockedTaskRunCount": 1 + } + ], + "tasks": [ + { + "taskId": "task.a", + "taskVersion": 1, + "taskBundleDigest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "benchmarkTrack": "agentic", + "trialCount": 2, + "sampleCount": 2, + "modelCount": 2, + "passRate": 1, + "confidenceInterval": { + "lower": 0.3423811429771653, + "upper": 1 + }, + "passAtK": { + "1": { + "k": 1, + "passCount": 2, + "sampleCount": 2, + "passRate": 1 + } + }, + "medianStepCount": 5, + "medianPeakContextTokens": 9000, + "publicPassCount": 2, + "publicSampleCount": 2, + "publicPassRate": 1, + "hiddenPassCount": 2, + "hiddenSampleCount": 2, + "hiddenPassRate": 1, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0 + }, + { + "taskId": "task.b", + "taskVersion": 1, + "taskBundleDigest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "benchmarkTrack": "agentic", + "trialCount": 2, + "sampleCount": 2, + "modelCount": 2, + "passRate": 0.5, + "confidenceInterval": { + "lower": 0.09453148796027494, + "upper": 0.905468512039725 + }, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 2, + "passRate": 0.5 + } + }, + "medianStepCount": 8, + "medianPeakContextTokens": 14000, + "publicPassCount": 1, + "publicSampleCount": 2, + "publicPassRate": 0.5, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1, + "blockedEvaluationCount": 1, + "blockedTaskRunCount": 1 + } + ], + "taskModelCells": [ + { + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "displayName": "DeepSeek Pro", + "providerLabel": "DeepSeek", + "taskId": "task.a", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialCount": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1, + "confidenceInterval": { + "lower": 0.20654998073085312, + "upper": 1 + }, + "errorCount": 0, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1 + } + }, + "medianStepCount": 4, + "medianPeakContextTokens": 8000, + "publicPassCount": 1, + "publicSampleCount": 1, + "publicPassRate": 1, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0, + "medianLatencyMs": 700, + "medianPromptTokens": 12, + "medianCompletionTokens": 18, + "medianEstimatedCostMicros": 66, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "failureBreakdown": { + "pass": 1 + } + }, + { + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "displayName": "DeepSeek Pro", + "providerLabel": "DeepSeek", + "taskId": "task.b", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialCount": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1, + "confidenceInterval": { + "lower": 0.20654998073085312, + "upper": 1 + }, + "errorCount": 0, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1 + } + }, + "medianStepCount": 6, + "medianPeakContextTokens": 12000, + "publicPassCount": 1, + "publicSampleCount": 1, + "publicPassRate": 1, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0, + "medianLatencyMs": 900, + "medianPromptTokens": 14, + "medianCompletionTokens": 22, + "medianEstimatedCostMicros": 80, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "failureBreakdown": { + "pass": 1 + } + }, + { + "providerId": "openai", + "modelId": "gpt-5", + "displayName": "GPT-5", + "providerLabel": "OpenAI", + "taskId": "task.a", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialCount": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1, + "confidenceInterval": { + "lower": 0.20654998073085312, + "upper": 1 + }, + "errorCount": 0, + "passAtK": { + "1": { + "k": 1, + "passCount": 1, + "sampleCount": 1, + "passRate": 1 + } + }, + "medianStepCount": 5, + "medianPeakContextTokens": 10000, + "publicPassCount": 1, + "publicSampleCount": 1, + "publicPassRate": 1, + "hiddenPassCount": 1, + "hiddenSampleCount": 1, + "hiddenPassRate": 1, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0, + "medianLatencyMs": 1000, + "medianPromptTokens": 10, + "medianCompletionTokens": 20, + "medianEstimatedCostMicros": 213, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "failureBreakdown": { + "pass": 1 + } + }, + { + "providerId": "openai", + "modelId": "gpt-5", + "displayName": "GPT-5", + "providerLabel": "OpenAI", + "taskId": "task.b", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialCount": 1, + "passCount": 0, + "sampleCount": 1, + "passRate": 0, + "confidenceInterval": { + "lower": 0, + "upper": 0.7934500192691468 + }, + "errorCount": 1, + "passAtK": { + "1": { + "k": 1, + "passCount": 0, + "sampleCount": 1, + "passRate": 0 + } + }, + "medianStepCount": 9, + "medianPeakContextTokens": 16000, + "publicPassCount": 0, + "publicSampleCount": 1, + "publicPassRate": 0, + "hiddenPassCount": 0, + "hiddenSampleCount": 0, + "hiddenPassRate": null, + "blockedEvaluationCount": 1, + "blockedTaskRunCount": 1, + "medianLatencyMs": 3000, + "medianPromptTokens": 30, + "medianCompletionTokens": 40, + "medianEstimatedCostMicros": 438, + "knownEstimatedCostCount": 1, + "unknownEstimatedCostCount": 0, + "failureBreakdown": { + "public_tests_failed": 1 + } + } + ], + "trialSummaries": [ + { + "trialId": "ea26b7d144aa", + "runId": "compat-run", + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "displayName": "DeepSeek Pro", + "providerLabel": "DeepSeek", + "taskId": "task.a", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialIndex": 0, + "completedAt": "2026-05-01T10:00:00.000Z", + "primaryPass": true, + "failureTag": "pass", + "aggregateScore": 1, + "publicPassed": true, + "hiddenPassed": true, + "blockedEvaluationCount": 0, + "stepCount": 4, + "peakContextTokens": 8000, + "latencyMs": 700, + "promptTokens": 12, + "completionTokens": 18, + "estimatedCostMicros": 66 + }, + { + "trialId": "e8540dd0aba4", + "runId": "compat-run", + "providerId": "deepseek", + "modelId": "deepseek-v4-pro", + "displayName": "DeepSeek Pro", + "providerLabel": "DeepSeek", + "taskId": "task.b", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialIndex": 1, + "completedAt": "2026-05-01T11:00:00.000Z", + "primaryPass": true, + "failureTag": "pass", + "aggregateScore": 1, + "publicPassed": true, + "hiddenPassed": true, + "blockedEvaluationCount": 0, + "stepCount": 6, + "peakContextTokens": 12000, + "latencyMs": 900, + "promptTokens": 14, + "completionTokens": 22, + "estimatedCostMicros": 80 + }, + { + "trialId": "0694cb931ace", + "runId": "compat-run", + "providerId": "openai", + "modelId": "gpt-5", + "displayName": "GPT-5", + "providerLabel": "OpenAI", + "taskId": "task.a", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialIndex": 0, + "completedAt": "2026-05-01T12:00:00.000Z", + "primaryPass": true, + "failureTag": "pass", + "aggregateScore": 1, + "publicPassed": true, + "hiddenPassed": true, + "blockedEvaluationCount": 0, + "stepCount": 5, + "peakContextTokens": 10000, + "latencyMs": 1000, + "promptTokens": 10, + "completionTokens": 20, + "estimatedCostMicros": 213 + }, + { + "trialId": "1b0a30bef532", + "runId": "compat-run", + "providerId": "openai", + "modelId": "gpt-5", + "displayName": "GPT-5", + "providerLabel": "OpenAI", + "taskId": "task.b", + "taskVersion": 1, + "benchmarkTrack": "agentic", + "trialIndex": 1, + "completedAt": "2026-05-01T13:00:00.000Z", + "primaryPass": false, + "failureTag": "public_tests_failed", + "aggregateScore": 0.4, + "publicPassed": false, + "hiddenPassed": null, + "blockedEvaluationCount": 1, + "stepCount": 9, + "peakContextTokens": 16000, + "latencyMs": 3000, + "promptTokens": 30, + "completionTokens": 40, + "estimatedCostMicros": 438 + } + ] + } + }, + { + "id": "legacy-v1-minimal", + "file": "legacy-v1-minimal.json", + "sha256": "b19803b8631489fbc079f97f8e0451744d746bbe919ffa65960ce17fc5858408", + "outcome": "accept", + "normalizedProjection": { + "schemaVersion": 1, + "provisional": false, + "generatedAt": null, + "benchmark": { + "title": "Legacy leaderboard", + "version": null, + "taskSetId": null, + "evaluatorSchemaVersion": 0, + "track": "agentic", + "dataPolicy": "latest-run", + "preset": null, + "selectedTasks": [], + "corpusManifestDigestSha256": null + }, + "source": { + "anchorRunId": null, + "runIds": [], + "taskCount": 0, + "taskRunCount": 0, + "modelCount": 0, + "trialSummaryCount": 0, + "trialSummaryTotalCount": 0, + "trialSummaryTruncated": false, + "trialSummaryLimit": 0, + "warnings": [], + "judgeOverhead": { + "evaluationCount": 0, + "promptTokens": 0, + "completionTokens": 0, + "knownEstimatedCostCount": 0, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": 0, + "pricingStatusCounts": {} + }, + "runProvenance": { + "runCount": 0, + "embeddedRunCount": 0, + "sandboxEnforcedRunCount": 0, + "taskExecutionPolicyRunCount": 0, + "networkDisabledTaskPolicyRunCount": 0, + "taskResourceLimitRunCount": 0, + "sdkVersionRunCount": 0, + "dependencySnapshotRunCount": 0, + "pricingRegistryRunCount": 0, + "generatedCodeSandboxBackends": [], + "dartVersions": [], + "flutterVersions": [], + "environmentIds": [], + "warnings": [] + } + }, + "scoring": { + "schemaVersion": 0, + "primaryMetric": null, + "rankingMetric": null, + "confidenceInterval": null, + "llmJudgePolicy": null, + "objectiveEvaluatorIds": [], + "secondaryEvaluatorIds": [], + "hiddenVerifierPattern": null, + "failureTags": [], + "objectiveFailureCaps": {}, + "defaultEvaluatorWeights": {} + }, + "pricingRegistry": { + "version": null, + "currency": null, + "modelCount": 0 + }, + "models": [], + "tasks": [], + "taskModelCells": [], + "trialSummaries": [] + } + }, + { + "id": "legacy-v1-populated", + "file": "legacy-v1-populated.json", + "sha256": "7f7289ad2e167b1fb662a86df141c9513ed7532a06ab9bc85081cbbb4ae7b5ed", + "outcome": "accept", + "normalizedProjection": { + "schemaVersion": 1, + "provisional": true, + "generatedAt": "2025-12-01T00:00:00.000Z", + "benchmark": { + "title": "Historical PickArena leaderboard", + "version": "2025-12", + "taskSetId": null, + "evaluatorSchemaVersion": 0, + "track": "agentic", + "dataPolicy": "latest-run", + "preset": null, + "selectedTasks": [], + "corpusManifestDigestSha256": null + }, + "source": { + "anchorRunId": "legacy-run", + "runIds": [ + "legacy-run" + ], + "taskCount": 1, + "taskRunCount": 2, + "modelCount": 1, + "trialSummaryCount": 1, + "trialSummaryTotalCount": 2, + "trialSummaryTruncated": false, + "trialSummaryLimit": 0, + "warnings": [ + "Historical fixture" + ], + "judgeOverhead": { + "evaluationCount": 0, + "promptTokens": 0, + "completionTokens": 0, + "knownEstimatedCostCount": 0, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": 0, + "pricingStatusCounts": {} + }, + "runProvenance": { + "runCount": 0, + "embeddedRunCount": 0, + "sandboxEnforcedRunCount": 0, + "taskExecutionPolicyRunCount": 0, + "networkDisabledTaskPolicyRunCount": 0, + "taskResourceLimitRunCount": 0, + "sdkVersionRunCount": 0, + "dependencySnapshotRunCount": 0, + "pricingRegistryRunCount": 0, + "generatedCodeSandboxBackends": [], + "dartVersions": [], + "flutterVersions": [], + "environmentIds": [], + "warnings": [] + } + }, + "scoring": { + "schemaVersion": 0, + "primaryMetric": null, + "rankingMetric": null, + "confidenceInterval": null, + "llmJudgePolicy": null, + "objectiveEvaluatorIds": [], + "secondaryEvaluatorIds": [], + "hiddenVerifierPattern": null, + "failureTags": [], + "objectiveFailureCaps": {}, + "defaultEvaluatorWeights": {} + }, + "pricingRegistry": { + "version": null, + "currency": null, + "modelCount": 0 + }, + "models": [ + { + "providerId": "legacy-provider", + "modelId": "legacy-model", + "displayName": null, + "providerLabel": null, + "rank": 1, + "score": 0.5, + "passRate": 0.5, + "trialCount": 2, + "passCount": 1, + "sampleCount": 2, + "passAtK": {}, + "medianStepCount": null, + "medianPeakContextTokens": null, + "publicPassCount": 0, + "publicSampleCount": 0, + "publicPassRate": null, + "hiddenPassCount": 0, + "hiddenSampleCount": 0, + "hiddenPassRate": null, + "confidenceInterval": { + "lower": null, + "upper": null + }, + "lowSample": false, + "medianLatencyMs": null, + "medianPromptTokens": null, + "medianCompletionTokens": null, + "medianEstimatedCostMicros": null, + "knownEstimatedCostCount": 0, + "unknownEstimatedCostCount": 0, + "totalEstimatedCostMicros": null, + "costPerSolvedTaskMicros": null, + "cheapestPassingEstimatedCostMicros": null, + "failureBreakdown": {}, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0 + } + ], + "tasks": [ + { + "taskId": "legacy.task", + "taskVersion": "v1", + "taskBundleDigest": null, + "benchmarkTrack": "agentic", + "trialCount": 2, + "sampleCount": 2, + "modelCount": 1, + "passRate": 0.5, + "confidenceInterval": { + "lower": null, + "upper": null + }, + "passAtK": {}, + "medianStepCount": null, + "medianPeakContextTokens": null, + "publicPassCount": 0, + "publicSampleCount": 0, + "publicPassRate": null, + "hiddenPassCount": 0, + "hiddenSampleCount": 0, + "hiddenPassRate": null, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0 + } + ], + "taskModelCells": [ + { + "providerId": "legacy-provider", + "modelId": "legacy-model", + "displayName": null, + "providerLabel": null, + "taskId": "legacy.task", + "taskVersion": "v1", + "benchmarkTrack": "agentic", + "trialCount": 2, + "passCount": 1, + "sampleCount": 2, + "passRate": 0.5, + "confidenceInterval": { + "lower": null, + "upper": null + }, + "errorCount": 1, + "passAtK": {}, + "medianStepCount": null, + "medianPeakContextTokens": null, + "publicPassCount": 0, + "publicSampleCount": 0, + "publicPassRate": null, + "hiddenPassCount": 0, + "hiddenSampleCount": 0, + "hiddenPassRate": null, + "blockedEvaluationCount": 0, + "blockedTaskRunCount": 0, + "medianLatencyMs": null, + "medianPromptTokens": null, + "medianCompletionTokens": null, + "medianEstimatedCostMicros": null, + "knownEstimatedCostCount": 0, + "unknownEstimatedCostCount": 0, + "failureBreakdown": {} + } + ], + "trialSummaries": [ + { + "trialId": "legacy-trial-0", + "runId": "legacy-run", + "providerId": "legacy-provider", + "modelId": "legacy-model", + "displayName": null, + "providerLabel": null, + "taskId": "legacy.task", + "taskVersion": "v1", + "benchmarkTrack": "agentic", + "trialIndex": 0, + "completedAt": "2025-11-30T12:00:00.000Z", + "primaryPass": false, + "failureTag": "public_tests_failed", + "aggregateScore": 0.4, + "publicPassed": null, + "hiddenPassed": null, + "blockedEvaluationCount": 0, + "stepCount": null, + "peakContextTokens": null, + "latencyMs": 1500, + "promptTokens": 100, + "completionTokens": 50, + "estimatedCostMicros": 25 + } + ] + } + }, + { + "id": "empty-title", + "file": "empty-title.json", + "sha256": "00ab3c9e9517c1deb73609e67c59efb5668a1bd76c4b3e870b64c5b3b23dc635", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "empty-track", + "file": "empty-track.json", + "sha256": "c2005cb5239b976b657b4fb07ff7141ff37300c5161a95151fa53b6efd2dfd50", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "fractional-count", + "file": "fractional-count.json", + "sha256": "15a418c731c8bab719e06b7bd1891c790cdf402500bd4f5469b69c6452f67c67", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "fractional-model-count", + "file": "fractional-model-count.json", + "sha256": "8c239ef9cb6547689cbf83d6edad7f4237f15378913fcf0900d272fb5022b3c3", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "fractional-pass-at-k-count", + "file": "fractional-pass-at-k-count.json", + "sha256": "4f177d66ddd983c28b73b319aa3602c3acf242571845d916cd3685c4dcbc440b", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "fractional-schema-version", + "file": "fractional-schema-version.json", + "sha256": "8e0f81531341d44487cbbc5fcdfbe0e106aa82cebf508bbabccf1847ddea79a8", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "fractional-trial-index", + "file": "fractional-trial-index.json", + "sha256": "c8ac71301cb94251c854232c8bf7c0571b1a04dd69da0b69d58892c5361fbb7a", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "future-schema-version", + "file": "future-schema-version.json", + "sha256": "5177a9116a16c016945c2736e195127201c976fd68a76f7062c1643dfc8b387f", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "invalid-benchmark", + "file": "invalid-benchmark.json", + "sha256": "5fe66f791b08d76566546a1e5e9f1fc8efd4794cab01cd4ce731ce48dc6286eb", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "invalid-data-policy", + "file": "invalid-data-policy.json", + "sha256": "86bbe939bf55a2f240ac3e199786446276a19f28a25d126bdfb47c9278a5dc2c", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "invalid-models", + "file": "invalid-models.json", + "sha256": "778a201b337dd82e567fae8f52499965e66dca36d7a2a6106f45cc83a118a7ab", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "invalid-pass-at-k-array", + "file": "invalid-pass-at-k-array.json", + "sha256": "4881a2288a4552ca41b5d09cfef18341e7d47f9887ddec26b537816a9948e474", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "invalid-selected-tasks-array", + "file": "invalid-selected-tasks-array.json", + "sha256": "e36cad9a89afed70116baf1048f26be10b304744a2bf5a80e13bbffe24b23ce2", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "invalid-source", + "file": "invalid-source.json", + "sha256": "ffca0bc9e3242e4a5dbc9476a3a35a0b2693355e58446553ae8c4504d12db3e1", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "invalid-task-model-cells-array", + "outcome": "reject", + "normalizedProjection": null, + "file": "invalid-task-model-cells-array.json", + "sha256": "b7acbb2e938e00e4f1c9409e24fa74420ba3c1eb276135fb08e18f2ff3ac9e27" + }, + { + "id": "invalid-tasks", + "file": "invalid-tasks.json", + "sha256": "36ec4dcb76a84f48d74d051a84bbcc0c05735b59356fd11809aa9a249bcd4d41", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "invalid-trial-summaries-array", + "outcome": "reject", + "normalizedProjection": null, + "file": "invalid-trial-summaries-array.json", + "sha256": "59aa276fe9a4ed648b9e3b4bc7896d6f5a5b94093588cfcca81625ad3f570289" + }, + { + "id": "invalid-warning-array-entry", + "file": "invalid-warning-array-entry.json", + "sha256": "f33f851e0dfcd22f3e04ae900b94d36accddd844116d37f53ee25fc3b7b7d3ba", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "malformed-json", + "file": "malformed-json.json", + "sha256": "a8530a511bd06e418f4bc6562d9f5345ad3aa12916bdc1988c3b80dd62ef14f7", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "missing-benchmark", + "file": "missing-benchmark.json", + "sha256": "4f08cd697de544aecd8deb33fe80927250acf7d32f141e84cc6b9ac097c31283", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "missing-cell-identity", + "file": "missing-cell-identity.json", + "sha256": "0ec246a60d89ec3cfc2c02e02f9a67402fdb948ba487c3b7fde09796aa31dbb0", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "missing-count", + "file": "missing-count.json", + "sha256": "df9907b5e355997885caaecb061a7fa7095e9ed5f6d866277a10967bcfd3a08d", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "missing-model-identity", + "file": "missing-model-identity.json", + "sha256": "51a71879277732335fae4d4f53a8fd1a109749b5c550bf992d408d8ce7a9d587", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "missing-schema-version", + "file": "missing-schema-version.json", + "sha256": "15de34a293823e9630b777cd12a9f3a841263be812e027f1169e3102aa211c3c", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "missing-source", + "file": "missing-source.json", + "sha256": "c3d5d5656989d76f8027a66d0d8b1bc1fdb2a08352a5550fdecf90da69a45712", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "missing-task-identity", + "file": "missing-task-identity.json", + "sha256": "107477dffe07ed90ad408bdb7752edc1ed18708ba6bc64f4107d624f5dfd2d6d", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "missing-trial-identity", + "file": "missing-trial-identity.json", + "sha256": "d993a5675395424f4965401a34b27dc79c7809dfb5c3d78744cdaade9a977d25", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "negative-blocked-count", + "file": "negative-blocked-count.json", + "sha256": "74d6831bbfb40b23fefddbf9ddc833a4c23cbf5232321ed22025ebcfabb744ef", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "negative-count", + "file": "negative-count.json", + "sha256": "bcb1f83b96520a716cd0e2938de1133a1d7cb5920dc6aa07239fc4f06a5fe431", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "negative-sample-count", + "file": "negative-sample-count.json", + "sha256": "ee16963daa3ee2cd8ba83705e054c0c9e4de3cc35388f7319f11077b7eda6f47", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "negative-trial-index", + "file": "negative-trial-index.json", + "sha256": "52cd4a82c6a912ad717660fabc85dd912830bd6d9e1c5380d82596b2b917211b", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "negative-trial-summary-limit", + "file": "negative-trial-summary-limit.json", + "sha256": "b0e21d371879f497ab3be508e11b830c13131c8b23e47ac1eb15c767c52628d9", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "nested-fractional-count", + "file": "nested-fractional-count.json", + "sha256": "6247cd2878d589f2457abdac6fd8ed2da64a01aee27bf8d4bc2b89e7145d7b20", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "nested-negative-count", + "file": "nested-negative-count.json", + "sha256": "b572095de9a9a10913e7df86465e60fa896c5635d7a4d74899c721e72f2bac5f", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "non-finite-score", + "file": "non-finite-score.json", + "sha256": "529f91c27c11d16629e50799bd6a92979374aca59de1e4c28c7a83f3f9e36508", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "non-object-cell-entry", + "file": "non-object-cell-entry.json", + "sha256": "8c0cdaacb217355df6828edc921bf19c501f4387a0df8ce8d8acf78db173b5c4", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "non-object-model-entry", + "file": "non-object-model-entry.json", + "sha256": "825e8747f8f0261b9d63f7ba98dbfd42d9feb05d0c847fa55803be114cd90c14", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "non-object-root", + "file": "non-object-root.json", + "sha256": "1fb3ffcf3df89e31f56932d4a64a23eede8a2f6707898bf14d1fc42a78286868", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "non-object-task-entry", + "outcome": "reject", + "normalizedProjection": null, + "file": "non-object-task-entry.json", + "sha256": "1566a34a918e11116a37ece5c8cd36bfe91c8633fc1282f1372b35d2f648bb20" + }, + { + "id": "non-object-trial-entry", + "outcome": "reject", + "normalizedProjection": null, + "file": "non-object-trial-entry.json", + "sha256": "c991c8465e10c605b8baee013d4e849709ff52df9e7fb4ed8e564a0017a788ce" + }, + { + "id": "past-schema-version", + "file": "past-schema-version.json", + "sha256": "3cb1101a58c5be6ac3612695334a2326fc9dc804717969219299666b82ba1e04", + "outcome": "reject", + "normalizedProjection": null + }, + { + "id": "string-schema-version", + "file": "string-schema-version.json", + "sha256": "f7dcf2b2534a250c8582231d83702c6cacdbd364e99bc9ea609d955fecbc4b36", + "outcome": "reject", + "normalizedProjection": null + } + ] +} diff --git a/fixtures/leaderboard/compatibility/v1/missing-benchmark.json b/fixtures/leaderboard/compatibility/v1/missing-benchmark.json new file mode 100644 index 0000000..e6941f9 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/missing-benchmark.json @@ -0,0 +1 @@ +{"schemaVersion":1,"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/missing-cell-identity.json b/fixtures/leaderboard/compatibility/v1/missing-cell-identity.json new file mode 100644 index 0000000..5af8a60 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/missing-cell-identity.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/missing-count.json b/fixtures/leaderboard/compatibility/v1/missing-count.json new file mode 100644 index 0000000..671dda4 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/missing-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/missing-model-identity.json b/fixtures/leaderboard/compatibility/v1/missing-model-identity.json new file mode 100644 index 0000000..cb93542 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/missing-model-identity.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/missing-schema-version.json b/fixtures/leaderboard/compatibility/v1/missing-schema-version.json new file mode 100644 index 0000000..211a20b --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/missing-schema-version.json @@ -0,0 +1 @@ +{"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/missing-source.json b/fixtures/leaderboard/compatibility/v1/missing-source.json new file mode 100644 index 0000000..d271306 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/missing-source.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/missing-task-identity.json b/fixtures/leaderboard/compatibility/v1/missing-task-identity.json new file mode 100644 index 0000000..2249c55 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/missing-task-identity.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/missing-trial-identity.json b/fixtures/leaderboard/compatibility/v1/missing-trial-identity.json new file mode 100644 index 0000000..affcf16 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/missing-trial-identity.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/negative-blocked-count.json b/fixtures/leaderboard/compatibility/v1/negative-blocked-count.json new file mode 100644 index 0000000..97a3d49 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/negative-blocked-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2,"blockedTaskRunCount":-1}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/negative-count.json b/fixtures/leaderboard/compatibility/v1/negative-count.json new file mode 100644 index 0000000..2b5bfa3 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/negative-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":-1,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/negative-sample-count.json b/fixtures/leaderboard/compatibility/v1/negative-sample-count.json new file mode 100644 index 0000000..54014e9 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/negative-sample-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":-1,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/negative-trial-index.json b/fixtures/leaderboard/compatibility/v1/negative-trial-index.json new file mode 100644 index 0000000..7ba113d --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/negative-trial-index.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":-1,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/negative-trial-summary-limit.json b/fixtures/leaderboard/compatibility/v1/negative-trial-summary-limit.json new file mode 100644 index 0000000..8f9ee88 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/negative-trial-summary-limit.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"],"trialSummaryLimit":-1},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/nested-fractional-count.json b/fixtures/leaderboard/compatibility/v1/nested-fractional-count.json new file mode 100644 index 0000000..25c839f --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/nested-fractional-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1,"blockedEvaluationCount":0.5}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/nested-negative-count.json b/fixtures/leaderboard/compatibility/v1/nested-negative-count.json new file mode 100644 index 0000000..ed61c8e --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/nested-negative-count.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":-1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/non-finite-score.json b/fixtures/leaderboard/compatibility/v1/non-finite-score.json new file mode 100644 index 0000000..302e93a --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/non-finite-score.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":1e400,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/non-object-cell-entry.json b/fixtures/leaderboard/compatibility/v1/non-object-cell-entry.json new file mode 100644 index 0000000..a70b939 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/non-object-cell-entry.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1},7],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/non-object-model-entry.json b/fixtures/leaderboard/compatibility/v1/non-object-model-entry.json new file mode 100644 index 0000000..2a73efe --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/non-object-model-entry.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2},"bad-row"],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/non-object-root.json b/fixtures/leaderboard/compatibility/v1/non-object-root.json new file mode 100644 index 0000000..44e2ace --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/non-object-root.json @@ -0,0 +1 @@ +[1, 2] diff --git a/fixtures/leaderboard/compatibility/v1/non-object-task-entry.json b/fixtures/leaderboard/compatibility/v1/non-object-task-entry.json new file mode 100644 index 0000000..3c62cae --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/non-object-task-entry.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5},false],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25}]} diff --git a/fixtures/leaderboard/compatibility/v1/non-object-trial-entry.json b/fixtures/leaderboard/compatibility/v1/non-object-trial-entry.json new file mode 100644 index 0000000..8ac9f7d --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/non-object-trial-entry.json @@ -0,0 +1 @@ +{"schemaVersion":1,"provisional":true,"generatedAt":"2025-12-01T00:00:00.000Z","benchmark":{"title":"Historical PickArena leaderboard","version":"2025-12","track":"agentic","dataPolicy":"latest-run"},"source":{"anchorRunId":"legacy-run","runIds":["legacy-run"],"taskCount":1,"taskRunCount":2,"modelCount":1,"warnings":["Historical fixture"]},"models":[{"providerId":"legacy-provider","modelId":"legacy-model","rank":1,"score":0.5,"passRate":0.5,"passCount":1,"sampleCount":2}],"tasks":[{"taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","sampleCount":2,"modelCount":1,"passRate":0.5}],"taskModelCells":[{"providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","passCount":1,"sampleCount":2,"passRate":0.5,"errorCount":1}],"trialSummaries":[{"trialId":"legacy-trial-0","runId":"legacy-run","providerId":"legacy-provider","modelId":"legacy-model","taskId":"legacy.task","taskVersion":"v1","benchmarkTrack":"agentic","trialIndex":0,"completedAt":"2025-11-30T12:00:00.000Z","primaryPass":false,"failureTag":"public_tests_failed","aggregateScore":0.4,"latencyMs":1500,"promptTokens":100,"completionTokens":50,"estimatedCostMicros":25},[]]} diff --git a/fixtures/leaderboard/compatibility/v1/past-schema-version.json b/fixtures/leaderboard/compatibility/v1/past-schema-version.json new file mode 100644 index 0000000..7d89cf1 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/past-schema-version.json @@ -0,0 +1 @@ +{"schemaVersion":0,"benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/fixtures/leaderboard/compatibility/v1/string-schema-version.json b/fixtures/leaderboard/compatibility/v1/string-schema-version.json new file mode 100644 index 0000000..8ad5174 --- /dev/null +++ b/fixtures/leaderboard/compatibility/v1/string-schema-version.json @@ -0,0 +1 @@ +{"schemaVersion":"1","benchmark":{"title":"Leaderboard","track":"agentic","dataPolicy":"latest-run"},"source":{"taskCount":0,"taskRunCount":0},"models":[],"tasks":[]} diff --git a/package.json b/package.json index dde0a64..60dd6df 100644 --- a/package.json +++ b/package.json @@ -11,11 +11,13 @@ "web:install": "cd web && bun install", "web:dev": "cd web && bun run dev", "web:check": "cd web && bun run check", + "web:test": "cd web && bun run test", "web:build": "cd web && bun run build", "web:smoke": "cd web && bun run smoke", "bench:official:bubblewrap": "bash scripts/run-official-bubblewrap-benchmark.sh", "bench:publish:web": "bash scripts/publish-benchmark-to-web.sh", "leaderboard:export": "cd app && dart run --verbosity=error dart_arena:dart_arena_export_leaderboard --database .dart_arena/dart_arena.sqlite --out ../web/static/data/leaderboard.v1.json --track agentic", + "leaderboard:fixtures:hash": "bun run scripts/update-leaderboard-fixture-hashes.ts", "check": "bun run app:analyze && bun run app:test && bun run web:check", "build": "bun run web:build" } diff --git a/scripts/update-leaderboard-fixture-hashes.ts b/scripts/update-leaderboard-fixture-hashes.ts new file mode 100644 index 0000000..310397a --- /dev/null +++ b/scripts/update-leaderboard-fixture-hashes.ts @@ -0,0 +1,21 @@ +import { createHash } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const manifestPath = fileURLToPath( + new URL( + '../fixtures/leaderboard/compatibility/v1/manifest.v1.json', + import.meta.url + ) +); +const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) as { + entries: { file: string; sha256: string }[]; +}; + +for (const entry of manifest.entries) { + const bytes = await readFile(resolve(dirname(manifestPath), entry.file)); + entry.sha256 = createHash('sha256').update(bytes).digest('hex'); +} + +await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); diff --git a/web/package.json b/web/package.json index 979364f..549306d 100644 --- a/web/package.json +++ b/web/package.json @@ -8,6 +8,7 @@ "build": "vite build", "preview": "vite preview", "smoke": "bun run build && bun run scripts/smoke-static.ts", + "test": "bun test scripts/leaderboard-contract.test.ts", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json" }, "devDependencies": { diff --git a/web/scripts/leaderboard-contract.test.ts b/web/scripts/leaderboard-contract.test.ts new file mode 100644 index 0000000..654c219 --- /dev/null +++ b/web/scripts/leaderboard-contract.test.ts @@ -0,0 +1,136 @@ +import { createHash } from 'node:crypto'; +import { readFile, readdir } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, mock, test } from 'bun:test'; + +import { + acceptedDataPolicies, + parseLeaderboardArtifact, + supportedArtifactSchemaVersions, + type LeaderboardData +} from '../src/lib/data/leaderboard-contract'; + +mock.module('$app/paths', () => ({ base: '' })); +const { loadLeaderboard } = await import('../src/lib/data/leaderboard'); + +type FixtureEntry = { + id: string; + file: string; + sha256: string; + outcome: 'accept' | 'reject'; + normalizedProjection: unknown; +}; + +type FixtureManifest = { + fixtureManifestVersion: number; + artifactFamily: string; + supportedArtifactSchemaVersions: number[]; + acceptedDataPolicies: string[]; + entries: FixtureEntry[]; +}; + +const manifestPath = fileURLToPath( + new URL( + '../../fixtures/leaderboard/compatibility/v1/manifest.v1.json', + import.meta.url + ) +); +const fixtureDirectory = dirname(manifestPath); +const manifestBytes = await readFile(manifestPath); +const manifest = JSON.parse(manifestBytes.toString('utf8')) as FixtureManifest; + +describe('leaderboard compatibility corpus', () => { + test('uses the versioned manifest and runtime contract constants', () => { + expect(manifest.fixtureManifestVersion).toBe(1); + expect(manifest.artifactFamily).toBe('leaderboard.v1.json'); + expect(manifest.supportedArtifactSchemaVersions).toEqual([ + ...supportedArtifactSchemaVersions + ]); + expect(manifest.acceptedDataPolicies).toEqual([...acceptedDataPolicies]); + expect(manifest.entries.length).toBeGreaterThan(0); + }); + + test('lists every fixture exactly once and every listed fixture exists', async () => { + const files = (await readdir(fixtureDirectory)) + .filter((file) => file !== 'manifest.v1.json') + .sort(); + const listed = manifest.entries.map((entry) => entry.file).sort(); + expect(listed).toEqual(files); + expect(new Set(listed).size).toBe(listed.length); + await Promise.all( + listed.map((file) => readFile(resolve(fixtureDirectory, file))) + ); + }); + + for (const entry of manifest.entries) { + test(entry.id, async () => { + const fixturePath = resolve(fixtureDirectory, entry.file); + const bytes = await readFile(fixturePath); + expect(createHash('sha256').update(bytes).digest('hex')).toBe(entry.sha256); + + const parsed = parseLeaderboardArtifact(bytes.toString('utf8')); + expect(parsed === null ? 'reject' : 'accept').toBe(entry.outcome); + expect(parsed === null ? null : normalizedProjection(parsed)).toEqual( + entry.normalizedProjection + ); + }); + } + + test('preserves fixture ordering in every current collection', () => { + const current = manifest.entries.find((entry) => entry.id === 'current-v2'); + if (!current || current.outcome !== 'accept') throw new Error('missing current-v2'); + const projection = current.normalizedProjection as LeaderboardData; + expect(projection.models.map((row) => `${row.providerId}:${row.modelId}`)).toEqual([ + 'deepseek:deepseek-v4-pro', + 'openai:gpt-5' + ]); + expect(projection.tasks.map((row) => row.taskId)).toEqual(['task.a', 'task.b']); + expect( + projection.taskModelCells.map( + (row) => `${row.providerId}:${row.modelId}:${row.taskId}` + ) + ).toEqual([ + 'deepseek:deepseek-v4-pro:task.a', + 'deepseek:deepseek-v4-pro:task.b', + 'openai:gpt-5:task.a', + 'openai:gpt-5:task.b' + ]); + expect( + projection.trialSummaries.map( + (row) => `${row.providerId}:${row.modelId}:${row.taskId}:${row.trialIndex}` + ) + ).toEqual([ + 'deepseek:deepseek-v4-pro:task.a:0', + 'deepseek:deepseek-v4-pro:task.b:1', + 'openai:gpt-5:task.a:0', + 'openai:gpt-5:task.b:1' + ]); + }); +}); + +describe('leaderboard runtime loader warnings', () => { + test('preserves the load warning for malformed JSON syntax', async () => { + const result = await loadLeaderboard(async () => new Response('{', { status: 200 })); + expect(result.warning).toBe('Leaderboard data could not be loaded.'); + }); + + test('uses the malformed warning for a decoded schema error', async () => { + const result = await loadLeaderboard( + async () => new Response('{"schemaVersion":2}', { status: 200 }) + ); + expect(result.warning).toBe('Leaderboard data is malformed.'); + }); + + test('preserves the load warning for a fetch failure', async () => { + const result = await loadLeaderboard(async () => { + throw new Error('offline'); + }); + expect(result.warning).toBe('Leaderboard data could not be loaded.'); + }); +}); + +function normalizedProjection(value: LeaderboardData): LeaderboardData { + return value; +} diff --git a/web/src/lib/data/leaderboard-contract.ts b/web/src/lib/data/leaderboard-contract.ts new file mode 100644 index 0000000..46efc80 --- /dev/null +++ b/web/src/lib/data/leaderboard-contract.ts @@ -0,0 +1,1058 @@ +export const supportedArtifactSchemaVersions = [1, 2] as const; +export const acceptedDataPolicies = [ + 'aggregate-compatible', + 'latest-run', + 'best-observed' +] as const; + +export type LeaderboardBenchmark = { + title: string; + version: string | null; + taskSetId: string | null; + evaluatorSchemaVersion: number; + track: string; + dataPolicy: LeaderboardDataPolicy; + preset: string | null; + selectedTasks: LeaderboardSelectedTask[]; + corpusManifestDigestSha256: string | null; +}; + +export type LeaderboardDataPolicy = (typeof acceptedDataPolicies)[number]; + +export type LeaderboardSelectedTask = { + taskId: string; + taskVersion: number | string | null; + taskBundleDigest: string | null; +}; + +export type LeaderboardSource = { + anchorRunId: string | null; + runIds: string[]; + taskCount: number; + taskRunCount: number; + modelCount: number; + trialSummaryCount: number; + trialSummaryTotalCount: number; + trialSummaryTruncated: boolean; + trialSummaryLimit: number; + warnings: string[]; + judgeOverhead: LeaderboardJudgeOverhead; + runProvenance: LeaderboardRunProvenance; +}; + +export type LeaderboardJudgeOverhead = { + evaluationCount: number; + promptTokens: number; + completionTokens: number; + knownEstimatedCostCount: number; + unknownEstimatedCostCount: number; + totalEstimatedCostMicros: number; + pricingStatusCounts: Record; +}; + +export type LeaderboardRunProvenance = { + runCount: number; + embeddedRunCount: number; + sandboxEnforcedRunCount: number; + taskExecutionPolicyRunCount: number; + networkDisabledTaskPolicyRunCount: number; + taskResourceLimitRunCount: number; + sdkVersionRunCount: number; + dependencySnapshotRunCount: number; + pricingRegistryRunCount: number; + generatedCodeSandboxBackends: string[]; + dartVersions: string[]; + flutterVersions: string[]; + environmentIds: string[]; + warnings: string[]; +}; + +export type LeaderboardPricingRegistry = { + version: string | null; + currency: string | null; + modelCount: number; +}; + +export type LeaderboardScoring = { + schemaVersion: number; + primaryMetric: string | null; + rankingMetric: string | null; + confidenceInterval: string | null; + llmJudgePolicy: string | null; + objectiveEvaluatorIds: string[]; + secondaryEvaluatorIds: string[]; + hiddenVerifierPattern: string | null; + failureTags: string[]; + objectiveFailureCaps: Record; + defaultEvaluatorWeights: Record; +}; + +export type LeaderboardPassAtKEntry = { + k: number; + passCount: number; + sampleCount: number; + passRate: number | null; +}; + +export type LeaderboardPassAtK = Record; + +export type LeaderboardConfidenceInterval = { + lower: number | null; + upper: number | null; +}; + +export type LeaderboardModel = { + providerId: string; + modelId: string; + displayName: string | null; + providerLabel: string | null; + rank: number | null; + score: number | null; + passRate: number | null; + trialCount: number; + passCount: number; + sampleCount: number; + passAtK: LeaderboardPassAtK; + medianStepCount: number | null; + medianPeakContextTokens: number | null; + publicPassCount: number; + publicSampleCount: number; + publicPassRate: number | null; + hiddenPassCount: number; + hiddenSampleCount: number; + hiddenPassRate: number | null; + confidenceInterval: LeaderboardConfidenceInterval; + lowSample: boolean; + medianLatencyMs: number | null; + medianPromptTokens: number | null; + medianCompletionTokens: number | null; + medianEstimatedCostMicros: number | null; + knownEstimatedCostCount: number; + unknownEstimatedCostCount: number; + totalEstimatedCostMicros: number | null; + costPerSolvedTaskMicros: number | null; + cheapestPassingEstimatedCostMicros: number | null; + failureBreakdown: Record; + blockedEvaluationCount: number; + blockedTaskRunCount: number; +}; + +export type LeaderboardTask = { + taskId: string; + taskVersion: number | string | null; + taskBundleDigest: string | null; + benchmarkTrack: string | null; + trialCount: number; + sampleCount: number; + modelCount: number; + passRate: number | null; + confidenceInterval: LeaderboardConfidenceInterval; + passAtK: LeaderboardPassAtK; + medianStepCount: number | null; + medianPeakContextTokens: number | null; + publicPassCount: number; + publicSampleCount: number; + publicPassRate: number | null; + hiddenPassCount: number; + hiddenSampleCount: number; + hiddenPassRate: number | null; + blockedEvaluationCount: number; + blockedTaskRunCount: number; +}; + +export type LeaderboardTaskModelCell = { + providerId: string; + modelId: string; + displayName: string | null; + providerLabel: string | null; + taskId: string; + taskVersion: number | string | null; + benchmarkTrack: string | null; + trialCount: number; + passCount: number; + sampleCount: number; + passRate: number | null; + confidenceInterval: LeaderboardConfidenceInterval; + errorCount: number; + passAtK: LeaderboardPassAtK; + medianStepCount: number | null; + medianPeakContextTokens: number | null; + publicPassCount: number; + publicSampleCount: number; + publicPassRate: number | null; + hiddenPassCount: number; + hiddenSampleCount: number; + hiddenPassRate: number | null; + blockedEvaluationCount: number; + blockedTaskRunCount: number; + medianLatencyMs: number | null; + medianPromptTokens: number | null; + medianCompletionTokens: number | null; + medianEstimatedCostMicros: number | null; + knownEstimatedCostCount: number; + unknownEstimatedCostCount: number; + failureBreakdown: Record; +}; + +export type LeaderboardTrialSummary = { + trialId: string; + runId: string; + providerId: string; + modelId: string; + displayName: string | null; + providerLabel: string | null; + taskId: string; + taskVersion: number | string | null; + benchmarkTrack: string | null; + trialIndex: number; + completedAt: string | null; + primaryPass: boolean | null; + failureTag: string; + aggregateScore: number | null; + publicPassed: boolean | null; + hiddenPassed: boolean | null; + blockedEvaluationCount: number; + stepCount: number | null; + peakContextTokens: number | null; + latencyMs: number | null; + promptTokens: number | null; + completionTokens: number | null; + estimatedCostMicros: number | null; +}; + +export type LeaderboardData = { + schemaVersion: number; + provisional: boolean; + generatedAt: string | null; + benchmark: LeaderboardBenchmark; + source: LeaderboardSource; + scoring: LeaderboardScoring; + pricingRegistry: LeaderboardPricingRegistry; + models: LeaderboardModel[]; + tasks: LeaderboardTask[]; + taskModelCells: LeaderboardTaskModelCell[]; + trialSummaries: LeaderboardTrialSummary[]; +}; + +export type LeaderboardArtifactParseResult = + | { ok: true; value: LeaderboardData } + | { ok: false; error: 'syntax' | 'shape' }; + +export const emptyLeaderboard: LeaderboardData = { + schemaVersion: 1, + provisional: false, + generatedAt: null, + benchmark: { + title: 'PickArena by Pickforge Studio', + version: null, + taskSetId: null, + evaluatorSchemaVersion: 0, + track: 'agentic', + dataPolicy: 'aggregate-compatible', + preset: null, + selectedTasks: [], + corpusManifestDigestSha256: null + }, + source: { + anchorRunId: null, + runIds: [], + taskCount: 0, + taskRunCount: 0, + modelCount: 0, + trialSummaryCount: 0, + trialSummaryTotalCount: 0, + trialSummaryTruncated: false, + trialSummaryLimit: 0, + warnings: [], + judgeOverhead: emptyJudgeOverhead(), + runProvenance: emptyRunProvenance() + }, + scoring: emptyScoring(), + pricingRegistry: { version: null, currency: null, modelCount: 0 }, + models: [], + tasks: [], + taskModelCells: [], + trialSummaries: [] +}; + +export function parseLeaderboardArtifactResult( + text: string +): LeaderboardArtifactParseResult { + let decoded: unknown; + try { + decoded = JSON.parse(text); + } catch { + return { ok: false, error: 'syntax' }; + } + const value = parseLeaderboardValue(decoded); + return value === null + ? { ok: false, error: 'shape' } + : { ok: true, value }; +} + +export function parseLeaderboardArtifact(text: string): LeaderboardData | null { + const result = parseLeaderboardArtifactResult(text); + return result.ok ? result.value : null; +} + +export function parseLeaderboardValue(value: unknown): LeaderboardData | null { + try { + return parseLeaderboardRecord(requiredRecord(value)); + } catch { + return null; + } +} + +function parseLeaderboardRecord(value: Record): LeaderboardData { + const schemaVersion = requiredNonNegativeInteger(value, 'schemaVersion'); + if (!supportedArtifactSchemaVersions.includes(schemaVersion as 1 | 2)) { + invalid(); + } + const current = schemaVersion === 2; + const benchmark = requiredRecord(value.benchmark); + const source = requiredRecord(value.source); + const modelValues = requiredArray(value, 'models'); + const taskValues = requiredArray(value, 'tasks'); + const modelRecords = records(modelValues); + const taskRecords = records(taskValues); + const cellRecords = records(arrayField(value, 'taskModelCells', current)); + const trialRecords = records(arrayField(value, 'trialSummaries', current)); + const scoring = recordField(value, 'scoring', current); + const pricingRegistry = recordField(value, 'pricingRegistry', current); + const judgeOverhead = recordField(source, 'judgeOverhead', current); + const runProvenance = recordField(source, 'runProvenance', current); + + return { + schemaVersion, + provisional: booleanField(value, 'provisional', false), + generatedAt: nullableStringField(value, 'generatedAt'), + benchmark: { + title: requiredString(benchmark, 'title'), + version: nullableStringField(benchmark, 'version'), + taskSetId: nullableStringField(benchmark, 'taskSetId'), + evaluatorSchemaVersion: countField( + benchmark, + 'evaluatorSchemaVersion', + current, + 0 + ), + track: requiredString(benchmark, 'track'), + dataPolicy: dataPolicyField(benchmark, 'dataPolicy'), + preset: nullableStringField(benchmark, 'preset'), + selectedTasks: parseSelectedTasks(arrayField(benchmark, 'selectedTasks')), + corpusManifestDigestSha256: nullableStringField( + benchmark, + 'corpusManifestDigestSha256' + ) + }, + source: { + anchorRunId: nullableStringField(source, 'anchorRunId'), + runIds: stringArrayField(source, 'runIds', current), + taskCount: requiredNonNegativeInteger(source, 'taskCount'), + taskRunCount: requiredNonNegativeInteger(source, 'taskRunCount'), + modelCount: countField(source, 'modelCount', current, modelRecords.length), + trialSummaryCount: countField( + source, + 'trialSummaryCount', + current, + trialRecords.length + ), + trialSummaryTotalCount: countField( + source, + 'trialSummaryTotalCount', + current, + requiredNonNegativeInteger(source, 'taskRunCount') + ), + trialSummaryTruncated: booleanField( + source, + 'trialSummaryTruncated', + false, + current + ), + trialSummaryLimit: countField(source, 'trialSummaryLimit', current, 0), + warnings: stringArrayField(source, 'warnings', current), + judgeOverhead: parseJudgeOverhead(judgeOverhead, current), + runProvenance: parseRunProvenance(runProvenance, current) + }, + scoring: parseScoring(scoring, current), + pricingRegistry: { + version: nullableStringField(pricingRegistry, 'version'), + currency: nullableStringField(pricingRegistry, 'currency'), + modelCount: countField(pricingRegistry, 'modelCount', current, 0) + }, + models: modelRecords.map((entry) => parseModelRow(entry, current)), + tasks: taskRecords.map((entry) => parseTaskRow(entry, current)), + taskModelCells: cellRecords.map((entry) => + parseTaskModelCell(entry, current) + ), + trialSummaries: trialRecords.map((entry) => parseTrialSummary(entry, current)) + }; +} + +function emptyScoring(): LeaderboardScoring { + return { + schemaVersion: 0, + primaryMetric: null, + rankingMetric: null, + confidenceInterval: null, + llmJudgePolicy: null, + objectiveEvaluatorIds: [], + secondaryEvaluatorIds: [], + hiddenVerifierPattern: null, + failureTags: [], + objectiveFailureCaps: {}, + defaultEvaluatorWeights: {} + }; +} + +function emptyJudgeOverhead(): LeaderboardJudgeOverhead { + return { + evaluationCount: 0, + promptTokens: 0, + completionTokens: 0, + knownEstimatedCostCount: 0, + unknownEstimatedCostCount: 0, + totalEstimatedCostMicros: 0, + pricingStatusCounts: {} + }; +} + +function emptyRunProvenance(): LeaderboardRunProvenance { + return { + runCount: 0, + embeddedRunCount: 0, + sandboxEnforcedRunCount: 0, + taskExecutionPolicyRunCount: 0, + networkDisabledTaskPolicyRunCount: 0, + taskResourceLimitRunCount: 0, + sdkVersionRunCount: 0, + dependencySnapshotRunCount: 0, + pricingRegistryRunCount: 0, + generatedCodeSandboxBackends: [], + dartVersions: [], + flutterVersions: [], + environmentIds: [], + warnings: [] + }; +} + +function parseScoring( + value: Record, + current: boolean +): LeaderboardScoring { + return { + schemaVersion: countField(value, 'schemaVersion', current, 0), + primaryMetric: nullableStringField(value, 'primaryMetric'), + rankingMetric: nullableStringField(value, 'rankingMetric'), + confidenceInterval: nullableStringField(value, 'confidenceInterval'), + llmJudgePolicy: nullableStringField(value, 'llmJudgePolicy'), + objectiveEvaluatorIds: stringArrayField( + value, + 'objectiveEvaluatorIds', + current + ), + secondaryEvaluatorIds: stringArrayField( + value, + 'secondaryEvaluatorIds', + current + ), + hiddenVerifierPattern: nullableStringField(value, 'hiddenVerifierPattern'), + failureTags: stringArrayField(value, 'failureTags', current), + objectiveFailureCaps: numberRecordField( + value, + 'objectiveFailureCaps', + current + ), + defaultEvaluatorWeights: numberRecordField( + value, + 'defaultEvaluatorWeights', + current + ) + }; +} + +function parseJudgeOverhead( + value: Record, + current: boolean +): LeaderboardJudgeOverhead { + return { + evaluationCount: countField(value, 'evaluationCount', current, 0), + promptTokens: countField(value, 'promptTokens', current, 0), + completionTokens: countField(value, 'completionTokens', current, 0), + knownEstimatedCostCount: countField( + value, + 'knownEstimatedCostCount', + current, + 0 + ), + unknownEstimatedCostCount: countField( + value, + 'unknownEstimatedCostCount', + current, + 0 + ), + totalEstimatedCostMicros: countField( + value, + 'totalEstimatedCostMicros', + current, + 0 + ), + pricingStatusCounts: countRecordField( + value, + 'pricingStatusCounts', + current + ) + }; +} + +function parseRunProvenance( + value: Record, + current: boolean +): LeaderboardRunProvenance { + return { + runCount: countField(value, 'runCount', current, 0), + embeddedRunCount: countField(value, 'embeddedRunCount', current, 0), + sandboxEnforcedRunCount: countField( + value, + 'sandboxEnforcedRunCount', + current, + 0 + ), + taskExecutionPolicyRunCount: countField( + value, + 'taskExecutionPolicyRunCount', + current, + 0 + ), + networkDisabledTaskPolicyRunCount: countField( + value, + 'networkDisabledTaskPolicyRunCount', + current, + 0 + ), + taskResourceLimitRunCount: countField( + value, + 'taskResourceLimitRunCount', + current, + 0 + ), + sdkVersionRunCount: countField(value, 'sdkVersionRunCount', current, 0), + dependencySnapshotRunCount: countField( + value, + 'dependencySnapshotRunCount', + current, + 0 + ), + pricingRegistryRunCount: countField( + value, + 'pricingRegistryRunCount', + current, + 0 + ), + generatedCodeSandboxBackends: stringArrayField( + value, + 'generatedCodeSandboxBackends', + current + ), + dartVersions: stringArrayField(value, 'dartVersions', current), + flutterVersions: stringArrayField(value, 'flutterVersions', current), + environmentIds: stringArrayField(value, 'environmentIds', current), + warnings: stringArrayField(value, 'warnings', current) + }; +} + +function parseModelIdentity( + value: Record, + current: boolean +): { displayName: string | null; providerLabel: string | null } { + const config = recordField(value, 'modelConfig', current); + return { + displayName: nullableNonEmptyStringField( + config, + 'customModelDisplayName' + ), + providerLabel: nullableNonEmptyStringField(config, 'customModelProvider') + }; +} + +function parseModelRow( + value: Record, + current: boolean +): LeaderboardModel { + const sampleCount = countField(value, 'sampleCount', current, 0); + return { + providerId: requiredString(value, 'providerId'), + modelId: requiredString(value, 'modelId'), + ...parseModelIdentity(value, current), + rank: nullableNonNegativeIntegerField(value, 'rank'), + score: nullableFiniteNumberField(value, 'score'), + passRate: nullableFiniteNumberField(value, 'passRate'), + trialCount: countField(value, 'trialCount', current, sampleCount), + passCount: countField(value, 'passCount', current, 0), + sampleCount, + passAtK: parsePassAtK(recordField(value, 'passAtK', current), current), + medianStepCount: nullableNonNegativeIntegerField(value, 'medianStepCount'), + medianPeakContextTokens: nullableNonNegativeIntegerField( + value, + 'medianPeakContextTokens' + ), + publicPassCount: countField(value, 'publicPassCount', current, 0), + publicSampleCount: countField(value, 'publicSampleCount', current, 0), + publicPassRate: nullableFiniteNumberField(value, 'publicPassRate'), + hiddenPassCount: countField(value, 'hiddenPassCount', current, 0), + hiddenSampleCount: countField(value, 'hiddenSampleCount', current, 0), + hiddenPassRate: nullableFiniteNumberField(value, 'hiddenPassRate'), + confidenceInterval: parseConfidenceInterval( + recordField(value, 'confidenceInterval', current) + ), + lowSample: booleanField(value, 'lowSample', false, current), + medianLatencyMs: nullableNonNegativeIntegerField(value, 'medianLatencyMs'), + medianPromptTokens: nullableNonNegativeIntegerField( + value, + 'medianPromptTokens' + ), + medianCompletionTokens: nullableNonNegativeIntegerField( + value, + 'medianCompletionTokens' + ), + medianEstimatedCostMicros: nullableNonNegativeIntegerField( + value, + 'medianEstimatedCostMicros' + ), + knownEstimatedCostCount: countField( + value, + 'knownEstimatedCostCount', + current, + 0 + ), + unknownEstimatedCostCount: countField( + value, + 'unknownEstimatedCostCount', + current, + 0 + ), + totalEstimatedCostMicros: nullableNonNegativeIntegerField( + value, + 'totalEstimatedCostMicros' + ), + costPerSolvedTaskMicros: nullableNonNegativeIntegerField( + value, + 'costPerSolvedTaskMicros' + ), + cheapestPassingEstimatedCostMicros: nullableNonNegativeIntegerField( + value, + 'cheapestPassingEstimatedCostMicros' + ), + failureBreakdown: countRecordField(value, 'failureBreakdown', current), + blockedEvaluationCount: countField( + value, + 'blockedEvaluationCount', + current, + 0 + ), + blockedTaskRunCount: countField( + value, + 'blockedTaskRunCount', + current, + 0 + ) + }; +} + +function parseTaskRow( + value: Record, + current: boolean +): LeaderboardTask { + const sampleCount = countField(value, 'sampleCount', current, 0); + return { + taskId: requiredString(value, 'taskId'), + taskVersion: nullableVersionField(value, 'taskVersion'), + taskBundleDigest: nullableStringField(value, 'taskBundleDigest'), + benchmarkTrack: nullableStringField(value, 'benchmarkTrack'), + trialCount: countField(value, 'trialCount', current, sampleCount), + sampleCount, + modelCount: countField(value, 'modelCount', current, 0), + passRate: nullableFiniteNumberField(value, 'passRate'), + confidenceInterval: parseConfidenceInterval( + recordField(value, 'confidenceInterval', current) + ), + passAtK: parsePassAtK(recordField(value, 'passAtK', current), current), + medianStepCount: nullableNonNegativeIntegerField(value, 'medianStepCount'), + medianPeakContextTokens: nullableNonNegativeIntegerField( + value, + 'medianPeakContextTokens' + ), + publicPassCount: countField(value, 'publicPassCount', current, 0), + publicSampleCount: countField(value, 'publicSampleCount', current, 0), + publicPassRate: nullableFiniteNumberField(value, 'publicPassRate'), + hiddenPassCount: countField(value, 'hiddenPassCount', current, 0), + hiddenSampleCount: countField(value, 'hiddenSampleCount', current, 0), + hiddenPassRate: nullableFiniteNumberField(value, 'hiddenPassRate'), + blockedEvaluationCount: countField( + value, + 'blockedEvaluationCount', + current, + 0 + ), + blockedTaskRunCount: countField( + value, + 'blockedTaskRunCount', + current, + 0 + ) + }; +} + +function parseTaskModelCell( + value: Record, + current: boolean +): LeaderboardTaskModelCell { + const sampleCount = countField(value, 'sampleCount', current, 0); + return { + providerId: requiredString(value, 'providerId'), + modelId: requiredString(value, 'modelId'), + ...parseModelIdentity(value, current), + taskId: requiredString(value, 'taskId'), + taskVersion: nullableVersionField(value, 'taskVersion'), + benchmarkTrack: nullableStringField(value, 'benchmarkTrack'), + trialCount: countField(value, 'trialCount', current, sampleCount), + passCount: countField(value, 'passCount', current, 0), + sampleCount, + passRate: nullableFiniteNumberField(value, 'passRate'), + confidenceInterval: parseConfidenceInterval( + recordField(value, 'confidenceInterval', current) + ), + errorCount: countField(value, 'errorCount', current, 0), + passAtK: parsePassAtK(recordField(value, 'passAtK', current), current), + medianStepCount: nullableNonNegativeIntegerField(value, 'medianStepCount'), + medianPeakContextTokens: nullableNonNegativeIntegerField( + value, + 'medianPeakContextTokens' + ), + publicPassCount: countField(value, 'publicPassCount', current, 0), + publicSampleCount: countField(value, 'publicSampleCount', current, 0), + publicPassRate: nullableFiniteNumberField(value, 'publicPassRate'), + hiddenPassCount: countField(value, 'hiddenPassCount', current, 0), + hiddenSampleCount: countField(value, 'hiddenSampleCount', current, 0), + hiddenPassRate: nullableFiniteNumberField(value, 'hiddenPassRate'), + blockedEvaluationCount: countField( + value, + 'blockedEvaluationCount', + current, + 0 + ), + blockedTaskRunCount: countField( + value, + 'blockedTaskRunCount', + current, + 0 + ), + medianLatencyMs: nullableNonNegativeIntegerField(value, 'medianLatencyMs'), + medianPromptTokens: nullableNonNegativeIntegerField( + value, + 'medianPromptTokens' + ), + medianCompletionTokens: nullableNonNegativeIntegerField( + value, + 'medianCompletionTokens' + ), + medianEstimatedCostMicros: nullableNonNegativeIntegerField( + value, + 'medianEstimatedCostMicros' + ), + knownEstimatedCostCount: countField( + value, + 'knownEstimatedCostCount', + current, + 0 + ), + unknownEstimatedCostCount: countField( + value, + 'unknownEstimatedCostCount', + current, + 0 + ), + failureBreakdown: countRecordField(value, 'failureBreakdown', current) + }; +} + +function parseTrialSummary( + value: Record, + current: boolean +): LeaderboardTrialSummary { + return { + trialId: requiredString(value, 'trialId'), + runId: requiredString(value, 'runId'), + providerId: requiredString(value, 'providerId'), + modelId: requiredString(value, 'modelId'), + ...parseModelIdentity(value, current), + taskId: requiredString(value, 'taskId'), + taskVersion: nullableVersionField(value, 'taskVersion'), + benchmarkTrack: nullableStringField(value, 'benchmarkTrack'), + trialIndex: requiredNonNegativeInteger(value, 'trialIndex'), + completedAt: nullableStringField(value, 'completedAt'), + primaryPass: nullableBooleanField(value, 'primaryPass'), + failureTag: requiredString(value, 'failureTag'), + aggregateScore: nullableFiniteNumberField(value, 'aggregateScore'), + publicPassed: nullableBooleanField(value, 'publicPassed'), + hiddenPassed: nullableBooleanField(value, 'hiddenPassed'), + blockedEvaluationCount: countField( + value, + 'blockedEvaluationCount', + current, + 0 + ), + stepCount: nullableNonNegativeIntegerField(value, 'stepCount'), + peakContextTokens: nullableNonNegativeIntegerField( + value, + 'peakContextTokens' + ), + latencyMs: nullableNonNegativeIntegerField(value, 'latencyMs'), + promptTokens: nullableNonNegativeIntegerField(value, 'promptTokens'), + completionTokens: nullableNonNegativeIntegerField( + value, + 'completionTokens' + ), + estimatedCostMicros: nullableNonNegativeIntegerField( + value, + 'estimatedCostMicros' + ) + }; +} + +function parseConfidenceInterval( + value: Record +): LeaderboardConfidenceInterval { + return { + lower: nullableFiniteNumberField(value, 'lower'), + upper: nullableFiniteNumberField(value, 'upper') + }; +} + +function parsePassAtK( + value: Record, + current: boolean +): LeaderboardPassAtK { + const result: LeaderboardPassAtK = {}; + for (const [key, rawEntry] of Object.entries(value)) { + const entry = requiredRecord(rawEntry); + const parsedKey = Number.parseInt(key, 10); + const fallback = Number.isInteger(parsedKey) && parsedKey >= 0 ? parsedKey : 0; + result[key] = { + k: countField(entry, 'k', current, fallback), + passCount: countField(entry, 'passCount', current, 0), + sampleCount: countField(entry, 'sampleCount', current, 0), + passRate: nullableFiniteNumberField(entry, 'passRate') + }; + } + return result; +} + +function parseSelectedTasks(value: unknown[]): LeaderboardSelectedTask[] { + return records(value).map((entry) => ({ + taskId: requiredString(entry, 'taskId'), + taskVersion: nullableVersionField(entry, 'taskVersion'), + taskBundleDigest: nullableStringField(entry, 'taskBundleDigest') + })); +} + +function requiredRecord(value: unknown): Record { + if (!isRecord(value)) invalid(); + return value; +} + +function recordField( + value: Record, + key: string, + required: boolean +): Record { + if (!has(value, key)) { + if (required) invalid(); + return {}; + } + return requiredRecord(value[key]); +} + +function records(values: unknown[]): Record[] { + return values.map(requiredRecord); +} + +function requiredArray(value: Record, key: string): unknown[] { + if (!Array.isArray(value[key])) invalid(); + return value[key]; +} + +function arrayField( + value: Record, + key: string, + required = false +): unknown[] { + if (!has(value, key)) { + if (required) invalid(); + return []; + } + return requiredArray(value, key); +} + +function stringArrayField( + value: Record, + key: string, + required = false +): string[] { + const entries = arrayField(value, key, required); + if (!entries.every((entry) => typeof entry === 'string')) invalid(); + return entries as string[]; +} + +function requiredString(value: Record, key: string): string { + const entry = value[key]; + if (typeof entry !== 'string' || entry.trim().length === 0) invalid(); + return entry; +} + +function nullableStringField( + value: Record, + key: string +): string | null { + if (!has(value, key) || value[key] === null) return null; + if (typeof value[key] !== 'string') invalid(); + return value[key]; +} + +function nullableNonEmptyStringField( + value: Record, + key: string +): string | null { + const entry = nullableStringField(value, key); + if (entry !== null && entry.trim().length === 0) invalid(); + return entry; +} + +function nullableVersionField( + value: Record, + key: string +): number | string | null { + if (!has(value, key) || value[key] === null) return null; + const entry = value[key]; + if (typeof entry === 'string') { + if (entry.trim().length === 0) invalid(); + return entry; + } + if (typeof entry !== 'number' || !Number.isFinite(entry)) invalid(); + return entry; +} + +function requiredNonNegativeInteger( + value: Record, + key: string +): number { + const entry = value[key]; + if ( + typeof entry !== 'number' || + !Number.isInteger(entry) || + entry < 0 || + !Number.isFinite(entry) + ) { + invalid(); + } + return entry; +} + +function countField( + value: Record, + key: string, + required: boolean, + fallback: number +): number { + if (!has(value, key)) { + if (required) invalid(); + return fallback; + } + return requiredNonNegativeInteger(value, key); +} + +function nullableNonNegativeIntegerField( + value: Record, + key: string +): number | null { + if (!has(value, key) || value[key] === null) return null; + return requiredNonNegativeInteger(value, key); +} + +function nullableFiniteNumberField( + value: Record, + key: string +): number | null { + if (!has(value, key) || value[key] === null) return null; + const entry = value[key]; + if (typeof entry !== 'number' || !Number.isFinite(entry)) invalid(); + return entry; +} + +function booleanField( + value: Record, + key: string, + fallback: boolean, + required = false +): boolean { + if (!has(value, key)) { + if (required) invalid(); + return fallback; + } + if (typeof value[key] !== 'boolean') invalid(); + return value[key]; +} + +function nullableBooleanField( + value: Record, + key: string +): boolean | null { + if (!has(value, key) || value[key] === null) return null; + if (typeof value[key] !== 'boolean') invalid(); + return value[key]; +} + +function numberRecordField( + value: Record, + key: string, + required: boolean +): Record { + const record = recordField(value, key, required); + for (const entry of Object.values(record)) { + if (typeof entry !== 'number' || !Number.isFinite(entry)) invalid(); + } + return record as Record; +} + +function countRecordField( + value: Record, + key: string, + required: boolean +): Record { + const record = recordField(value, key, required); + for (const entry of Object.values(record)) { + if (!Number.isInteger(entry) || (entry as number) < 0) invalid(); + } + return record as Record; +} + +function dataPolicyField( + value: Record, + key: string +): LeaderboardDataPolicy { + const entry = value[key]; + if (!acceptedDataPolicies.includes(entry as LeaderboardDataPolicy)) invalid(); + return entry as LeaderboardDataPolicy; +} + +function has(value: Record, key: string): boolean { + return Object.prototype.hasOwnProperty.call(value, key); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function invalid(): never { + throw new TypeError('Malformed leaderboard artifact'); +} diff --git a/web/src/lib/data/leaderboard.ts b/web/src/lib/data/leaderboard.ts index 452383e..4cf4498 100644 --- a/web/src/lib/data/leaderboard.ts +++ b/web/src/lib/data/leaderboard.ts @@ -1,221 +1,12 @@ import { base } from '$app/paths'; -export type LeaderboardBenchmark = { - title: string; - version: string | null; - taskSetId: string | null; - evaluatorSchemaVersion: number; - track: string; - dataPolicy: string; -}; - -export type LeaderboardSource = { - anchorRunId: string | null; - runIds: string[]; - taskCount: number; - taskRunCount: number; - modelCount: number; - trialSummaryCount: number; - trialSummaryTotalCount: number; - trialSummaryTruncated: boolean; - trialSummaryLimit: number; - warnings: string[]; - judgeOverhead: LeaderboardJudgeOverhead; - runProvenance: LeaderboardRunProvenance; -}; - -export type LeaderboardJudgeOverhead = { - evaluationCount: number; - promptTokens: number; - completionTokens: number; - knownEstimatedCostCount: number; - unknownEstimatedCostCount: number; - totalEstimatedCostMicros: number; - pricingStatusCounts: Record; -}; - -export type LeaderboardRunProvenance = { - runCount: number; - embeddedRunCount: number; - sandboxEnforcedRunCount: number; - taskExecutionPolicyRunCount: number; - networkDisabledTaskPolicyRunCount: number; - taskResourceLimitRunCount: number; - sdkVersionRunCount: number; - dependencySnapshotRunCount: number; - pricingRegistryRunCount: number; - generatedCodeSandboxBackends: string[]; - dartVersions: string[]; - flutterVersions: string[]; - environmentIds: string[]; - warnings: string[]; -}; - -export type LeaderboardPricingRegistry = { - version: string | null; - currency: string | null; - modelCount: number; -}; - -export type LeaderboardScoring = { - schemaVersion: number; - primaryMetric: string | null; - rankingMetric: string | null; - confidenceInterval: string | null; - llmJudgePolicy: string | null; - objectiveEvaluatorIds: string[]; - secondaryEvaluatorIds: string[]; - hiddenVerifierPattern: string | null; - failureTags: string[]; - objectiveFailureCaps: Record; - defaultEvaluatorWeights: Record; -}; - -export type LeaderboardPassAtKEntry = { - k: number; - passCount: number; - sampleCount: number; - passRate: number | null; -}; - -export type LeaderboardPassAtK = Record; - -export type LeaderboardConfidenceInterval = { - lower: number | null; - upper: number | null; -}; +import { + emptyLeaderboard, + parseLeaderboardArtifactResult, + type LeaderboardData +} from './leaderboard-contract'; -export type LeaderboardModel = { - providerId: string; - modelId: string; - displayName: string | null; - providerLabel: string | null; - rank: number | null; - score: number | null; - passRate: number | null; - trialCount: number; - passCount: number; - sampleCount: number; - passAtK: LeaderboardPassAtK; - medianStepCount: number | null; - medianPeakContextTokens: number | null; - publicPassCount: number; - publicSampleCount: number; - publicPassRate: number | null; - hiddenPassCount: number; - hiddenSampleCount: number; - hiddenPassRate: number | null; - confidenceInterval: LeaderboardConfidenceInterval; - lowSample: boolean; - medianLatencyMs: number | null; - medianPromptTokens: number | null; - medianCompletionTokens: number | null; - medianEstimatedCostMicros: number | null; - knownEstimatedCostCount: number; - unknownEstimatedCostCount: number; - totalEstimatedCostMicros: number | null; - costPerSolvedTaskMicros: number | null; - cheapestPassingEstimatedCostMicros: number | null; - failureBreakdown: Record; - blockedEvaluationCount: number; - blockedTaskRunCount: number; -}; - -export type LeaderboardTask = { - taskId: string; - taskVersion: number | string | null; - benchmarkTrack: string | null; - trialCount: number; - sampleCount: number; - modelCount: number; - passRate: number | null; - confidenceInterval: LeaderboardConfidenceInterval; - passAtK: LeaderboardPassAtK; - medianStepCount: number | null; - medianPeakContextTokens: number | null; - publicPassCount: number; - publicSampleCount: number; - publicPassRate: number | null; - hiddenPassCount: number; - hiddenSampleCount: number; - hiddenPassRate: number | null; - blockedEvaluationCount: number; - blockedTaskRunCount: number; -}; - -export type LeaderboardTaskModelCell = { - providerId: string; - modelId: string; - displayName: string | null; - providerLabel: string | null; - taskId: string; - taskVersion: number | string | null; - benchmarkTrack: string | null; - trialCount: number; - passCount: number; - sampleCount: number; - passRate: number | null; - confidenceInterval: LeaderboardConfidenceInterval; - errorCount: number; - passAtK: LeaderboardPassAtK; - medianStepCount: number | null; - medianPeakContextTokens: number | null; - publicPassCount: number; - publicSampleCount: number; - publicPassRate: number | null; - hiddenPassCount: number; - hiddenSampleCount: number; - hiddenPassRate: number | null; - blockedEvaluationCount: number; - blockedTaskRunCount: number; - medianLatencyMs: number | null; - medianPromptTokens: number | null; - medianCompletionTokens: number | null; - medianEstimatedCostMicros: number | null; - knownEstimatedCostCount: number; - unknownEstimatedCostCount: number; - failureBreakdown: Record; -}; - -export type LeaderboardTrialSummary = { - trialId: string; - runId: string; - providerId: string; - modelId: string; - displayName: string | null; - providerLabel: string | null; - taskId: string; - taskVersion: number | string | null; - benchmarkTrack: string | null; - trialIndex: number; - completedAt: string | null; - primaryPass: boolean | null; - failureTag: string; - aggregateScore: number | null; - publicPassed: boolean | null; - hiddenPassed: boolean | null; - blockedEvaluationCount: number; - stepCount: number | null; - peakContextTokens: number | null; - latencyMs: number | null; - promptTokens: number | null; - completionTokens: number | null; - estimatedCostMicros: number | null; -}; - -export type LeaderboardData = { - schemaVersion: number; - provisional: boolean; - generatedAt: string | null; - benchmark: LeaderboardBenchmark; - source: LeaderboardSource; - scoring: LeaderboardScoring; - pricingRegistry: LeaderboardPricingRegistry; - models: LeaderboardModel[]; - tasks: LeaderboardTask[]; - taskModelCells: LeaderboardTaskModelCell[]; - trialSummaries: LeaderboardTrialSummary[]; -}; +export type * from './leaderboard-contract'; export type LeaderboardLoadResult = { leaderboard: LeaderboardData; @@ -227,44 +18,6 @@ type LeaderboardFetch = ( init?: RequestInit ) => Promise; -const emptyLeaderboard: LeaderboardData = { - schemaVersion: 1, - provisional: false, - generatedAt: null, - benchmark: { - title: 'PickArena by Pickforge Studio', - version: null, - taskSetId: null, - evaluatorSchemaVersion: 0, - track: 'agentic', - dataPolicy: 'aggregate-compatible' - }, - source: { - anchorRunId: null, - runIds: [], - taskCount: 0, - taskRunCount: 0, - modelCount: 0, - trialSummaryCount: 0, - trialSummaryTotalCount: 0, - trialSummaryTruncated: false, - trialSummaryLimit: 0, - warnings: [], - judgeOverhead: emptyJudgeOverhead(), - runProvenance: emptyRunProvenance() - }, - scoring: emptyScoring(), - pricingRegistry: { - version: null, - currency: null, - modelCount: 0 - }, - models: [], - tasks: [], - taskModelCells: [], - trialSummaries: [] -}; - export async function loadLeaderboard( fetcher: LeaderboardFetch = fetch ): Promise { @@ -274,461 +27,21 @@ export async function loadLeaderboard( return withWarning(`Leaderboard data is unavailable (${response.status}).`); } - const decoded: unknown = await response.json(); - const leaderboard = parseLeaderboard(decoded); - if (!leaderboard) { - return withWarning('Leaderboard data is malformed.'); + const parsed = parseLeaderboardArtifactResult(await response.text()); + if (!parsed.ok) { + return withWarning( + parsed.error === 'syntax' + ? 'Leaderboard data could not be loaded.' + : 'Leaderboard data is malformed.' + ); } - return { - leaderboard, - warning: null - }; + return { leaderboard: parsed.value, warning: null }; } catch { return withWarning('Leaderboard data could not be loaded.'); } } function withWarning(warning: string): LeaderboardLoadResult { - return { - leaderboard: emptyLeaderboard, - warning - }; -} - -function parseLeaderboard(value: unknown): LeaderboardData | null { - if (!isRecord(value)) return null; - const schemaVersion = parseNumber(value.schemaVersion); - if (schemaVersion == null || schemaVersion < 1 || schemaVersion > 2) return null; - if (!isRecord(value.benchmark)) return null; - if (!isRecord(value.source)) return null; - if (!Array.isArray(value.models)) return null; - if (!Array.isArray(value.tasks)) return null; - - const title = value.benchmark.title; - const track = value.benchmark.track; - const dataPolicy = value.benchmark.dataPolicy; - const taskCount = value.source.taskCount; - const taskRunCount = value.source.taskRunCount; - const pricingRegistry = isRecord(value.pricingRegistry) - ? value.pricingRegistry - : {}; - const scoring = isRecord(value.scoring) ? value.scoring : {}; - const judgeOverhead = isRecord(value.source.judgeOverhead) - ? value.source.judgeOverhead - : {}; - const runProvenance = isRecord(value.source.runProvenance) - ? value.source.runProvenance - : {}; - - if ( - typeof title !== 'string' || - typeof track !== 'string' || - typeof dataPolicy !== 'string' || - typeof taskCount !== 'number' || - typeof taskRunCount !== 'number' - ) { - return null; - } - - const models = value.models.filter(isRecord).map(parseModelRow); - const tasks = value.tasks.filter(isRecord).map(parseTaskRow); - const taskModelCells = Array.isArray(value.taskModelCells) - ? value.taskModelCells.filter(isRecord).map(parseTaskModelCell) - : []; - const trialSummaries = Array.isArray(value.trialSummaries) - ? value.trialSummaries.filter(isRecord).map(parseTrialSummary) - : []; - - return { - schemaVersion, - provisional: value.provisional === true, - generatedAt: typeof value.generatedAt === 'string' ? value.generatedAt : null, - benchmark: { - title, - version: - typeof value.benchmark.version === 'string' - ? value.benchmark.version - : null, - taskSetId: - typeof value.benchmark.taskSetId === 'string' - ? value.benchmark.taskSetId - : null, - evaluatorSchemaVersion: - parseNumber(value.benchmark.evaluatorSchemaVersion) ?? 0, - track, - dataPolicy - }, - source: { - anchorRunId: - typeof value.source.anchorRunId === 'string' ? value.source.anchorRunId : null, - runIds: parseStringArray(value.source.runIds), - taskCount, - taskRunCount, - modelCount: parseNumber(value.source.modelCount) ?? models.length, - trialSummaryCount: - parseNumber(value.source.trialSummaryCount) ?? trialSummaries.length, - trialSummaryTotalCount: - parseNumber(value.source.trialSummaryTotalCount) ?? taskRunCount, - trialSummaryTruncated: - typeof value.source.trialSummaryTruncated === 'boolean' - ? value.source.trialSummaryTruncated - : false, - trialSummaryLimit: parseNumber(value.source.trialSummaryLimit) ?? 0, - warnings: parseWarnings(value.source.warnings), - judgeOverhead: parseJudgeOverhead(judgeOverhead), - runProvenance: parseRunProvenance(runProvenance) - }, - scoring: parseScoring(scoring), - pricingRegistry: { - version: - typeof pricingRegistry.version === 'string' ? pricingRegistry.version : null, - currency: - typeof pricingRegistry.currency === 'string' ? pricingRegistry.currency : null, - modelCount: parseNumber(pricingRegistry.modelCount) ?? 0 - }, - models, - tasks, - taskModelCells, - trialSummaries - }; -} - -function emptyScoring(): LeaderboardScoring { - return { - schemaVersion: 0, - primaryMetric: null, - rankingMetric: null, - confidenceInterval: null, - llmJudgePolicy: null, - objectiveEvaluatorIds: [], - secondaryEvaluatorIds: [], - hiddenVerifierPattern: null, - failureTags: [], - objectiveFailureCaps: {}, - defaultEvaluatorWeights: {} - }; -} - -function emptyJudgeOverhead(): LeaderboardJudgeOverhead { - return { - evaluationCount: 0, - promptTokens: 0, - completionTokens: 0, - knownEstimatedCostCount: 0, - unknownEstimatedCostCount: 0, - totalEstimatedCostMicros: 0, - pricingStatusCounts: {} - }; -} - -function emptyRunProvenance(): LeaderboardRunProvenance { - return { - runCount: 0, - embeddedRunCount: 0, - sandboxEnforcedRunCount: 0, - taskExecutionPolicyRunCount: 0, - networkDisabledTaskPolicyRunCount: 0, - taskResourceLimitRunCount: 0, - sdkVersionRunCount: 0, - dependencySnapshotRunCount: 0, - pricingRegistryRunCount: 0, - generatedCodeSandboxBackends: [], - dartVersions: [], - flutterVersions: [], - environmentIds: [], - warnings: [] - }; -} - -function parseScoring(value: Record): LeaderboardScoring { - return { - schemaVersion: parseNumber(value.schemaVersion) ?? 0, - primaryMetric: - typeof value.primaryMetric === 'string' ? value.primaryMetric : null, - rankingMetric: - typeof value.rankingMetric === 'string' ? value.rankingMetric : null, - confidenceInterval: - typeof value.confidenceInterval === 'string' - ? value.confidenceInterval - : null, - llmJudgePolicy: - typeof value.llmJudgePolicy === 'string' ? value.llmJudgePolicy : null, - objectiveEvaluatorIds: parseStringArray(value.objectiveEvaluatorIds), - secondaryEvaluatorIds: parseStringArray(value.secondaryEvaluatorIds), - hiddenVerifierPattern: - typeof value.hiddenVerifierPattern === 'string' - ? value.hiddenVerifierPattern - : null, - failureTags: parseStringArray(value.failureTags), - objectiveFailureCaps: parseNumberRecord(value.objectiveFailureCaps), - defaultEvaluatorWeights: parseNumberRecord(value.defaultEvaluatorWeights) - }; -} - -function parseJudgeOverhead( - value: Record -): LeaderboardJudgeOverhead { - return { - evaluationCount: parseNumber(value.evaluationCount) ?? 0, - promptTokens: parseNumber(value.promptTokens) ?? 0, - completionTokens: parseNumber(value.completionTokens) ?? 0, - knownEstimatedCostCount: parseNumber(value.knownEstimatedCostCount) ?? 0, - unknownEstimatedCostCount: parseNumber(value.unknownEstimatedCostCount) ?? 0, - totalEstimatedCostMicros: parseNumber(value.totalEstimatedCostMicros) ?? 0, - pricingStatusCounts: parseNumberRecord(value.pricingStatusCounts) - }; -} - -function parseRunProvenance( - value: Record -): LeaderboardRunProvenance { - return { - runCount: parseNumber(value.runCount) ?? 0, - embeddedRunCount: parseNumber(value.embeddedRunCount) ?? 0, - sandboxEnforcedRunCount: parseNumber(value.sandboxEnforcedRunCount) ?? 0, - taskExecutionPolicyRunCount: - parseNumber(value.taskExecutionPolicyRunCount) ?? 0, - networkDisabledTaskPolicyRunCount: - parseNumber(value.networkDisabledTaskPolicyRunCount) ?? 0, - taskResourceLimitRunCount: - parseNumber(value.taskResourceLimitRunCount) ?? 0, - sdkVersionRunCount: parseNumber(value.sdkVersionRunCount) ?? 0, - dependencySnapshotRunCount: - parseNumber(value.dependencySnapshotRunCount) ?? 0, - pricingRegistryRunCount: parseNumber(value.pricingRegistryRunCount) ?? 0, - generatedCodeSandboxBackends: parseStringArray( - value.generatedCodeSandboxBackends - ), - dartVersions: parseStringArray(value.dartVersions), - flutterVersions: parseStringArray(value.flutterVersions), - environmentIds: parseStringArray(value.environmentIds), - warnings: parseWarnings(value.warnings) - }; -} - -function parseModelIdentity(value: Record): { - displayName: string | null; - providerLabel: string | null; -} { - const config = isRecord(value.modelConfig) ? value.modelConfig : {}; - const displayName = - typeof config.customModelDisplayName === 'string' && - config.customModelDisplayName.length > 0 - ? config.customModelDisplayName - : null; - const providerLabel = - typeof config.customModelProvider === 'string' && - config.customModelProvider.length > 0 - ? config.customModelProvider - : null; - return { displayName, providerLabel }; -} - -function parseModelRow(value: Record): LeaderboardModel { - return { - providerId: parseString(value.providerId, 'unknown-provider'), - modelId: parseString(value.modelId, 'unknown-model'), - ...parseModelIdentity(value), - rank: parseNumber(value.rank), - score: parseNumber(value.score), - passRate: parseNumber(value.passRate), - trialCount: parseNumber(value.trialCount) ?? parseNumber(value.sampleCount) ?? 0, - passCount: parseNumber(value.passCount) ?? 0, - sampleCount: parseNumber(value.sampleCount) ?? 0, - passAtK: parsePassAtK(value.passAtK), - medianStepCount: parseNumber(value.medianStepCount), - medianPeakContextTokens: parseNumber(value.medianPeakContextTokens), - publicPassCount: parseNumber(value.publicPassCount) ?? 0, - publicSampleCount: parseNumber(value.publicSampleCount) ?? 0, - publicPassRate: parseNumber(value.publicPassRate), - hiddenPassCount: parseNumber(value.hiddenPassCount) ?? 0, - hiddenSampleCount: parseNumber(value.hiddenSampleCount) ?? 0, - hiddenPassRate: parseNumber(value.hiddenPassRate), - confidenceInterval: parseConfidenceInterval(value.confidenceInterval), - lowSample: typeof value.lowSample === 'boolean' ? value.lowSample : false, - medianLatencyMs: parseNumber(value.medianLatencyMs), - medianPromptTokens: parseNumber(value.medianPromptTokens), - medianCompletionTokens: parseNumber(value.medianCompletionTokens), - medianEstimatedCostMicros: parseNumber(value.medianEstimatedCostMicros), - knownEstimatedCostCount: parseNumber(value.knownEstimatedCostCount) ?? 0, - unknownEstimatedCostCount: parseNumber(value.unknownEstimatedCostCount) ?? 0, - totalEstimatedCostMicros: parseNumber(value.totalEstimatedCostMicros), - costPerSolvedTaskMicros: parseNumber(value.costPerSolvedTaskMicros), - cheapestPassingEstimatedCostMicros: parseNumber( - value.cheapestPassingEstimatedCostMicros - ), - failureBreakdown: parseNumberRecord(value.failureBreakdown), - blockedEvaluationCount: parseNumber(value.blockedEvaluationCount) ?? 0, - blockedTaskRunCount: parseNumber(value.blockedTaskRunCount) ?? 0 - }; -} - -function parseTaskRow(value: Record): LeaderboardTask { - return { - taskId: parseString(value.taskId, 'unknown-task'), - taskVersion: - typeof value.taskVersion === 'string' || typeof value.taskVersion === 'number' - ? value.taskVersion - : null, - benchmarkTrack: - typeof value.benchmarkTrack === 'string' ? value.benchmarkTrack : null, - trialCount: parseNumber(value.trialCount) ?? parseNumber(value.sampleCount) ?? 0, - sampleCount: parseNumber(value.sampleCount) ?? 0, - modelCount: parseNumber(value.modelCount) ?? 0, - passRate: parseNumber(value.passRate), - confidenceInterval: parseConfidenceInterval(value.confidenceInterval), - passAtK: parsePassAtK(value.passAtK), - medianStepCount: parseNumber(value.medianStepCount), - medianPeakContextTokens: parseNumber(value.medianPeakContextTokens), - publicPassCount: parseNumber(value.publicPassCount) ?? 0, - publicSampleCount: parseNumber(value.publicSampleCount) ?? 0, - publicPassRate: parseNumber(value.publicPassRate), - hiddenPassCount: parseNumber(value.hiddenPassCount) ?? 0, - hiddenSampleCount: parseNumber(value.hiddenSampleCount) ?? 0, - hiddenPassRate: parseNumber(value.hiddenPassRate), - blockedEvaluationCount: parseNumber(value.blockedEvaluationCount) ?? 0, - blockedTaskRunCount: parseNumber(value.blockedTaskRunCount) ?? 0 - }; -} - -function parseTaskModelCell( - value: Record -): LeaderboardTaskModelCell { - return { - providerId: parseString(value.providerId, 'unknown-provider'), - modelId: parseString(value.modelId, 'unknown-model'), - ...parseModelIdentity(value), - taskId: parseString(value.taskId, 'unknown-task'), - taskVersion: - typeof value.taskVersion === 'string' || typeof value.taskVersion === 'number' - ? value.taskVersion - : null, - benchmarkTrack: - typeof value.benchmarkTrack === 'string' ? value.benchmarkTrack : null, - trialCount: parseNumber(value.trialCount) ?? parseNumber(value.sampleCount) ?? 0, - passCount: parseNumber(value.passCount) ?? 0, - sampleCount: parseNumber(value.sampleCount) ?? 0, - passRate: parseNumber(value.passRate), - confidenceInterval: parseConfidenceInterval(value.confidenceInterval), - errorCount: parseNumber(value.errorCount) ?? 0, - passAtK: parsePassAtK(value.passAtK), - medianStepCount: parseNumber(value.medianStepCount), - medianPeakContextTokens: parseNumber(value.medianPeakContextTokens), - publicPassCount: parseNumber(value.publicPassCount) ?? 0, - publicSampleCount: parseNumber(value.publicSampleCount) ?? 0, - publicPassRate: parseNumber(value.publicPassRate), - hiddenPassCount: parseNumber(value.hiddenPassCount) ?? 0, - hiddenSampleCount: parseNumber(value.hiddenSampleCount) ?? 0, - hiddenPassRate: parseNumber(value.hiddenPassRate), - blockedEvaluationCount: parseNumber(value.blockedEvaluationCount) ?? 0, - blockedTaskRunCount: parseNumber(value.blockedTaskRunCount) ?? 0, - medianLatencyMs: parseNumber(value.medianLatencyMs), - medianPromptTokens: parseNumber(value.medianPromptTokens), - medianCompletionTokens: parseNumber(value.medianCompletionTokens), - medianEstimatedCostMicros: parseNumber(value.medianEstimatedCostMicros), - knownEstimatedCostCount: parseNumber(value.knownEstimatedCostCount) ?? 0, - unknownEstimatedCostCount: parseNumber(value.unknownEstimatedCostCount) ?? 0, - failureBreakdown: parseNumberRecord(value.failureBreakdown) - }; -} - -function parseConfidenceInterval(value: unknown): LeaderboardConfidenceInterval { - const confidenceInterval = isRecord(value) ? value : {}; - return { - lower: parseNumber(confidenceInterval.lower), - upper: parseNumber(confidenceInterval.upper) - }; -} - -function parseTrialSummary( - value: Record -): LeaderboardTrialSummary { - return { - trialId: parseString(value.trialId, 'unknown-trial'), - runId: parseString(value.runId, 'unknown-run'), - providerId: parseString(value.providerId, 'unknown-provider'), - modelId: parseString(value.modelId, 'unknown-model'), - ...parseModelIdentity(value), - taskId: parseString(value.taskId, 'unknown-task'), - taskVersion: - typeof value.taskVersion === 'string' || typeof value.taskVersion === 'number' - ? value.taskVersion - : null, - benchmarkTrack: - typeof value.benchmarkTrack === 'string' ? value.benchmarkTrack : null, - trialIndex: parseNumber(value.trialIndex) ?? 0, - completedAt: typeof value.completedAt === 'string' ? value.completedAt : null, - primaryPass: - typeof value.primaryPass === 'boolean' ? value.primaryPass : null, - failureTag: parseString(value.failureTag, 'unknown'), - aggregateScore: parseNumber(value.aggregateScore), - publicPassed: - typeof value.publicPassed === 'boolean' ? value.publicPassed : null, - hiddenPassed: - typeof value.hiddenPassed === 'boolean' ? value.hiddenPassed : null, - blockedEvaluationCount: parseNumber(value.blockedEvaluationCount) ?? 0, - stepCount: parseNumber(value.stepCount), - peakContextTokens: parseNumber(value.peakContextTokens), - latencyMs: parseNumber(value.latencyMs), - promptTokens: parseNumber(value.promptTokens), - completionTokens: parseNumber(value.completionTokens), - estimatedCostMicros: parseNumber(value.estimatedCostMicros) - }; -} - -function parsePassAtK(value: unknown): LeaderboardPassAtK { - if (!isRecord(value)) return {}; - - const parsed: [string, LeaderboardPassAtKEntry][] = []; - for (const [key, rawEntry] of Object.entries(value)) { - if (!isRecord(rawEntry)) continue; - - const k = parseNumber(rawEntry.k) ?? Number.parseInt(key, 10); - if (!Number.isFinite(k)) continue; - - parsed.push([ - key, - { - k, - passCount: parseNumber(rawEntry.passCount) ?? 0, - sampleCount: parseNumber(rawEntry.sampleCount) ?? 0, - passRate: parseNumber(rawEntry.passRate) - } - ]); - } - - return Object.fromEntries(parsed); -} - -function parseString(value: unknown, fallback: string): string { - return typeof value === 'string' && value.length > 0 ? value : fallback; -} - -function parseNumber(value: unknown): number | null { - return typeof value === 'number' && Number.isFinite(value) ? value : null; -} - -function parseWarnings(value: unknown): string[] { - return parseStringArray(value); -} - -function parseStringArray(value: unknown): string[] { - if (!Array.isArray(value)) return []; - return value.filter((entry): entry is string => typeof entry === 'string'); -} - -function parseNumberRecord(value: unknown): Record { - if (!isRecord(value)) return {}; - - return Object.fromEntries( - Object.entries(value).filter( - (entry): entry is [string, number] => - typeof entry[1] === 'number' && Number.isFinite(entry[1]) - ) - ); -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); + return { leaderboard: emptyLeaderboard, warning }; }