From d2de497e019553233aff4b699994248c37313187 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Wed, 15 Jul 2026 22:00:43 +0100 Subject: [PATCH 1/5] Add adaptive stopping replay experiment --- experiments/adaptive-stopping/README.md | 273 ++ experiments/adaptive-stopping/elo-deltas.csv | 113 + experiments/adaptive-stopping/replay.py | 602 +++ .../results-sentinels-excluded.json | 1735 +++++++ experiments/adaptive-stopping/results.json | 3996 +++++++++++++++++ .../adaptive-stopping/round-history.csv | 71 + .../adaptive-stopping/strategy-summary.csv | 9 + tests/test_adaptive_replay_experiment.py | 81 + 8 files changed, 6880 insertions(+) create mode 100644 experiments/adaptive-stopping/README.md create mode 100644 experiments/adaptive-stopping/elo-deltas.csv create mode 100644 experiments/adaptive-stopping/replay.py create mode 100644 experiments/adaptive-stopping/results-sentinels-excluded.json create mode 100644 experiments/adaptive-stopping/results.json create mode 100644 experiments/adaptive-stopping/round-history.csv create mode 100644 experiments/adaptive-stopping/strategy-summary.csv create mode 100644 tests/test_adaptive_replay_experiment.py diff --git a/experiments/adaptive-stopping/README.md b/experiments/adaptive-stopping/README.md new file mode 100644 index 0000000..071d2e2 --- /dev/null +++ b/experiments/adaptive-stopping/README.md @@ -0,0 +1,273 @@ +# Adaptive stopping counterfactual replay + +**Recommendation: revise the targeted strategy and keep it experimental.** Do not change the +balanced default yet, and do not proceed directly to a live trial from this one replay. + +The opt-in targeted allocator delivers the expected call reduction, but not by stopping early: +it uses all 10 five-sample rounds while selecting fewer pairs per round. Against the full stored +board, plain targeted allocation saves **79.7%** of comparisons, while the 3× size rule saves +**82.9%**. Both preserve top-3 membership, but both swap ranks 2 and 3 and move individual ELOs +substantially. The size rule adds only 3.1 percentage points of savings over plain targeted while +increasing median/max ELO drift. + +No production bug was found. This is therefore an experiment/documentation-only change. + +## Scope + +This experiment follows [issue #35](https://github.com/davanstrien/ocr-bench/issues/35) and +[PR #53](https://github.com/davanstrien/ocr-bench/pull/53), using main at `ec802fa`. + +Data is pinned to: + +- repo: `davanstrien/ocr-bench-britannica-results` +- config: `comparisons` +- revision: `48a0f42de26009892d2784a3a97d6d61525f4040` +- 4,293 stored outcomes, 14 models, 50 samples, and all 91 possible model pairs + +The source metadata records 4,299 attempted comparisons and 4,293 valid comparisons. Only the +4,293 published verdicts are observable, so “used” and “saved” below count stored valid outcomes, +not reconstructed API attempts. + +This is a **counterfactual replay**, not independent validation. It asks which already-stored +outcomes each policy would have selected. It makes no judge API calls and writes nothing to the +Hub. + +## Method + +The replay: + +1. Processes sample indices in production batches of five: 0–4, 5–9, ..., 45–49. +2. Starts targeted runs with balanced evidence. +3. First checks after production's `max(3 * n_pairs, 20)` threshold: 273 outcomes here. +4. Uses production `compute_elo` with 1,000 bootstrap replicates and seed 42. +5. Uses production `classify_adjacent_pairs`, `unresolved_pairs`, pair counting, model-size + parsing, and practical-preference helpers. +6. Recomputes adjacency after every batch, just as the CLI does. + +The primary replay intentionally treats all 4,293 published rows as outcomes because that is the +requested reference board and reproduces the historical leaderboard. The data includes 628 rows +containing OCR error sentinels, judged before the current sentinel exclusion policy. A separate +robustness replay applies the current exclusion rule below. + +### Metrics + +- Rank correlations compare complete 14-model orders with the full stored board. +- ELO deltas use the same 14-model Bradley–Terry centering. +- Pair coverage is observed undirected edges out of 91; min/median/max are direct outcomes per + observed edge. +- `stat/practical/unresolved` counts the final 13 adjacent pairs. “Practical” means the opt-in + smaller-model deployment preference, not statistical resolution. + +## Primary results + +All adaptive policies reached round 10 and exhausted the available sample batches. **None met its +stopping criteria early.** + +| Strategy | Used | Saved | Stop | Kendall τ | Spearman ρ | Top-3 members | Top-3 order | Median abs(ΔELO) | Max abs(ΔELO) | Pair coverage (min/med/max) | Stat/practical/unresolved | +|---|---:|---:|---|---:|---:|---:|---|---:|---:|---|---| +| Full stored board | 4,293 | 0.0% | reference, round 10 | 1.000 | 1.000 | 3/3 | yes | 0.0 | 0.0 | 91/91 (43/46/50) | 4/0/9 | +| Balanced adaptive | 4,293 | 0.0% | samples exhausted, round 10 | 1.000 | 1.000 | 3/3 | yes | 0.0 | 0.0 | 91/91 (43/46/50) | 4/0/9 | +| Targeted | 870 | 79.7% | samples exhausted, round 10 | 0.934 | 0.982 | 3/3 | **no** | 36.3 | 85.5 | 91/91 (2/5/46) | 1/0/12 | +| Targeted + 3×, min 10 | 736 | 82.9% | samples exhausted, round 10 | 0.912 | 0.978 | 3/3 | **no** | 59.0 | 105.9 | 91/91 (2/5/41) | 1/6/6 | + +The rough 870 targeted estimate is reproduced exactly. The rigorous current-helper result for the +3× rule is 736 rather than the rough ~715. + +### Final rank order + +| Rank | Full / balanced | Targeted | Targeted + 3× | +|---:|---|---|---| +| 1 | dots.mocr | dots.mocr | dots.mocr | +| 2 | LightOnOCR-2-1B | GLM-OCR | GLM-OCR | +| 3 | GLM-OCR | LightOnOCR-2-1B | LightOnOCR-2-1B | +| 4 | olmOCR-2-7B-1025-FP8 | olmOCR-2-7B-1025-FP8 | olmOCR-2-7B-1025-FP8 | +| 5 | NuExtract3 | FireRed-OCR | FireRed-OCR | +| 6 | Qianfan-OCR | NuExtract3 | NuExtract3 | +| 7 | FireRed-OCR | Qianfan-OCR | Qianfan-OCR | +| 8 | Unlimited-OCR | Unlimited-OCR | PaddleOCR-VL-1.6 | +| 9 | PaddleOCR-VL-1.6 | PaddleOCR-VL-1.6 | Unlimited-OCR | +| 10 | DeepSeek-OCR | DeepSeek-OCR | DeepSeek-OCR | +| 11 | PP-OCRv6_medium | PP-OCRv6_medium | PP-OCRv6_medium | +| 12 | DeepSeek-OCR-2 | DeepSeek-OCR-2 | DeepSeek-OCR-2 | +| 13 | tesseract-5 | tesseract-5 | tesseract-5 | +| 14 | dots.ocr | dots.ocr | dots.ocr | + +Top-3 **membership** is stable, but its ordering is not: GLM-OCR and LightOnOCR-2-1B swap. +Targeting also changes ranks 5–7; the size rule additionally swaps ranks 8–9. + +### Per-model ELO change + +Balanced adaptive consumes the complete board, so every balanced delta is zero. Parentheses show +change from the full stored board. + +| Model | Full ELO | Targeted ELO (Δ) | Targeted + 3× ELO (Δ) | +|---|---:|---:|---:| +| rednote-hilab/dots.mocr | 1745.4 | 1792.9 (+47.5) | 1810.3 (+64.9) | +| lightonai/LightOnOCR-2-1B | 1741.3 | 1749.4 (+8.1) | 1737.3 (-3.9) | +| zai-org/GLM-OCR | 1738.2 | 1771.9 (+33.8) | 1773.3 (+35.2) | +| allenai/olmOCR-2-7B-1025-FP8 | 1719.2 | 1642.8 (-76.4) | 1635.9 (-83.3) | +| numind/NuExtract3 | 1689.6 | 1605.7 (-83.9) | 1585.8 (-103.8) | +| baidu/Qianfan-OCR | 1568.1 | 1539.9 (-28.2) | 1530.4 (-37.7) | +| FireRedTeam/FireRed-OCR | 1557.4 | 1608.7 (+51.2) | 1624.9 (+67.5) | +| baidu/Unlimited-OCR | 1545.2 | 1506.3 (-38.9) | 1480.1 (-65.1) | +| PaddlePaddle/PaddleOCR-VL-1.6 | 1463.0 | 1454.6 (-8.4) | 1491.0 (+28.0) | +| deepseek-ai/DeepSeek-OCR | 1452.0 | 1426.5 (-25.4) | 1398.9 (-53.1) | +| PaddlePaddle/PP-OCRv6_medium | 1397.1 | 1377.7 (-19.4) | 1376.9 (-20.1) | +| deepseek-ai/DeepSeek-OCR-2 | 1380.2 | 1377.3 (-2.9) | 1357.2 (-23.0) | +| tesseract-5 | 1112.6 | 1198.1 (+85.5) | 1218.5 (+105.9) | +| rednote-hilab/dots.ocr | 890.7 | 948.2 (+57.5) | 979.3 (+88.7) | + +The targeted graph remains connected and retains direct evidence for every pair because of the +balanced warm-up. Connectivity alone is not enough: targeted median pair evidence falls from 46 +to 5, with some pairs frozen at 2 outcomes. The nonuniform edge weights are reflected in the ELO +movement. + +## Statistical resolution versus practical preference + +At the end of the full board, only 4 of 13 adjacent pairs are statistically resolved; 9 still have +overlapping marginal CIs. Plain targeted ends with only 1 resolved and 12 unresolved. + +With the 3× rule, the final board has: + +- 1 statistically resolved adjacent pair; +- 6 overlapping pairs annotated as practical smaller-model preferences; and +- 6 still unresolved pairs. + +The practical annotations produced are: + +| Prefer smaller | Compared with | Parameter ratio | +|---|---|---:| +| FireRedTeam/FireRed-OCR | allenai/olmOCR-2-7B-1025-FP8 | 4.0× | +| PaddlePaddle/PP-OCRv6_medium | deepseek-ai/DeepSeek-OCR | 115.9× | +| PaddlePaddle/PP-OCRv6_medium | deepseek-ai/DeepSeek-OCR-2 | 98.6× | +| PaddlePaddle/PaddleOCR-VL-1.6 | baidu/Qianfan-OCR | 5.2× | +| PaddlePaddle/PaddleOCR-VL-1.6 | baidu/Unlimited-OCR | 3.7× | +| zai-org/GLM-OCR | rednote-hilab/dots.mocr | 3.3× | + +These do **not** establish equivalence or alter ELO/rank. CI overlap is a failure to statistically +separate two marginal estimates, not evidence that model quality is equal. The annotation says +only: given overlap, enough direct pair evidence, and the configured parameter proxy, prefer the +smaller deployment. Parameter count is also not a direct latency, memory, price, or throughput +measurement. + +## Sensitivity + +All sensitivity runs also exhausted round 10 rather than meeting all stopping criteria. + +| Size ratio | Min evidence | Used | Saved | Kendall τ | Spearman ρ | Top-3 members | Top-3 order | Median abs(ΔELO) | Max abs(ΔELO) | Unresolved | +|---:|---:|---:|---:|---:|---:|---:|---|---:|---:|---:| +| 2× | 10 | 629 | 85.3% | 0.890 | 0.952 | **2/3** | no | 79.2 | 167.4 | 3 | +| 3× | 5 | 735 | 82.9% | 0.824 | 0.912 | 3/3 | no | 79.0 | 155.3 | 8 | +| 3× | 10 | 736 | 82.9% | 0.912 | 0.978 | 3/3 | no | 59.0 | 105.9 | 6 | +| 3× | 15 | 786 | 81.7% | 0.934 | 0.982 | 3/3 | no | 46.0 | 98.4 | 7 | +| 5× | 10 | 810 | 81.1% | 0.934 | 0.982 | 3/3 | no | 35.8 | 85.1 | 9 | + +The response is not monotonic because each interim board changes adjacency and therefore later +allocation. A permissive 2× threshold loses one full-board top-3 member. Lowering minimum evidence +from 10 to 5 barely saves another comparison but materially worsens rank correlation and ELO +error. A 15-sample minimum or 5× ratio is less aggressive and tracks the full board better, but +neither restores top-3 ordering or achieves early stopping. + +## Current sentinel-policy robustness check + +The primary result must use the requested 4,293-row stored board. However, current production +would exclude the 628 historical comparisons containing an OCR error sentinel and mark dots.ocr +as a fully failed model. Replaying the remaining 3,665 rows over 13 models gives the same broad +conclusion: + +| Strategy | Used | Saved vs 3,665 | Kendall τ | Spearman ρ | Top-3 members/order | Median/max abs(ΔELO) | +|---|---:|---:|---:|---:|---|---:| +| Balanced | 3,665 | 0.0% | 1.000 | 1.000 | 3/3, yes | 0.0 / 0.0 | +| Targeted | 753 | 79.5% | 0.923 | 0.984 | 3/3, no | 27.0 / 69.0 | +| Targeted + 3×, min 10 | 624 | 83.0% | 0.897 | 0.973 | 3/3, no | 48.6 / 86.0 | + +All three adaptive runs still exhaust the sample batches. See +[`results-sentinels-excluded.json`](results-sentinels-excluded.json) for full details. + +## Determinism + +Each of the four primary strategies was replayed twice. Selected comparison keys, interim rank +orders, ELOs, bootstrap CIs, adjacent-pair decisions, final annotations, stop round, and stop +reason matched exactly. This confirms deterministic execution in the tested environment, helped +by the fixed bootstrap seed and deterministic equal-ELO tie-break. + +It does not establish reproducibility across different SciPy/NumPy versions or statistical +validity under repeated data collection. + +## Limitations + +1. **Counterfactual, not independent.** The selected outcomes are a subset of the same run used as + the reference. This does not test a new judge run, judge drift, API nondeterminism, or a new + sample of pages. +2. **Outcome-conditioned sampling.** Later targeted pairs depend on interim outcomes and ranks. + Ordinary percentile bootstrap CIs computed after selection do not account for that adaptive, + optional-stopping process and may be optimistic. +3. **The bootstrap does not replay the policy.** Production resamples selected comparisons as if + the selected set were fixed. It does not re-run allocation inside each bootstrap replicate. +4. **Comparison-level resampling ignores page clustering.** Many pair outcomes share one page and + are correlated. A page/round-clustered bootstrap would better match the sampling unit. +5. **Bradley–Terry fit under sparse, nonuniform allocation.** The graph is connected, so a fit is + identifiable, but adjacency-driven edge counts can amplify model misspecification and shift the + global ELO scale relative to a balanced grid. +6. **One collection and one stored judge run.** Britannica cannot establish general behavior for + manuscripts, tables, noisy scans, or other judges. +7. **Unobserved failed judge calls.** Six attempted comparisons have no published valid verdict. + The replay cannot know how a different allocation would have distributed those failures, so + savings are relative to 4,293 valid stored outcomes rather than the 4,299 attempted calls. +8. **Historical sentinel rows.** The primary board predates the current exclusion policy; the + robustness replay is closer to current input integrity semantics but has only 13 rankable + models. + +## Recommendation and next experiment + +**Revise strategy; keep `targeted` opt-in and `balanced` as the default.** Specifically, evaluate a +revision that: + +1. uses a per-pair balanced warm-up floor rather than only the aggregate 273-outcome threshold; +2. injects periodic balanced/exploration batches so early adjacency does not permanently starve + other edges; +3. monitors rank/top-k stability and ELO drift across rounds, not only overlapping adjacent + marginal CIs; and +4. evaluates page-clustered or policy-aware uncertainty for adaptive runs. + +Replay that revision on several existing result boards before one explicitly budgeted live trial. +The 3× practical rule can remain as an annotation/stopping preference, but this board does not +support making it a default or treating it as statistical resolution. + +## Reproduce + +From the repository root: + +```bash +uv sync --dev +uv run python experiments/adaptive-stopping/replay.py +``` + +The full run takes about five minutes on the machine used for this report. The pinned dataset +revision is downloaded read-only. The script contains no judge backend construction and no Hub +push/upload call. + +Optional current-sentinel-policy replay: + +```bash +uv run python experiments/adaptive-stopping/replay.py \ + --exclude-sentinel-comparisons \ + --skip-sensitivity \ + --repeats 1 +``` + +Generated artifacts: + +- [`results.json`](results.json): complete metrics, ranks, CIs, annotations, and round history +- [`strategy-summary.csv`](strategy-summary.csv): strategy-level metrics +- [`elo-deltas.csv`](elo-deltas.csv): per-model ELO and deltas +- [`round-history.csv`](round-history.csv): batch-by-batch allocation and decisions +- [`results-sentinels-excluded.json`](results-sentinels-excluded.json): robustness replay + +Validation commands: + +```bash +uv run pytest tests/ -q +uv run ruff check src/ tests/ experiments/adaptive-stopping/replay.py +uv run ty check src/ experiments/adaptive-stopping/replay.py +``` diff --git a/experiments/adaptive-stopping/elo-deltas.csv b/experiments/adaptive-stopping/elo-deltas.csv new file mode 100644 index 0000000..8e3fe79 --- /dev/null +++ b/experiments/adaptive-stopping/elo-deltas.csv @@ -0,0 +1,113 @@ +strategy,model,elo,full_elo,delta,absolute_delta +full_stored_board,FireRedTeam/FireRed-OCR,1557.449386684769,1557.449386684769,0.0,0.0 +full_stored_board,PaddlePaddle/PP-OCRv6_medium,1397.0965599286017,1397.0965599286017,0.0,0.0 +full_stored_board,PaddlePaddle/PaddleOCR-VL-1.6,1462.9745524973894,1462.9745524973894,0.0,0.0 +full_stored_board,allenai/olmOCR-2-7B-1025-FP8,1719.231138029067,1719.231138029067,0.0,0.0 +full_stored_board,baidu/Qianfan-OCR,1568.0944503021342,1568.0944503021342,0.0,0.0 +full_stored_board,baidu/Unlimited-OCR,1545.2060026385532,1545.2060026385532,0.0,0.0 +full_stored_board,deepseek-ai/DeepSeek-OCR,1451.9970235866688,1451.9970235866688,0.0,0.0 +full_stored_board,deepseek-ai/DeepSeek-OCR-2,1380.2125479868414,1380.2125479868414,0.0,0.0 +full_stored_board,lightonai/LightOnOCR-2-1B,1741.2525107847546,1741.2525107847546,0.0,0.0 +full_stored_board,numind/NuExtract3,1689.6190357899554,1689.6190357899554,0.0,0.0 +full_stored_board,rednote-hilab/dots.mocr,1745.4252593866306,1745.4252593866306,0.0,0.0 +full_stored_board,rednote-hilab/dots.ocr,890.6755485577996,890.6755485577996,0.0,0.0 +full_stored_board,tesseract-5,1112.5865083646895,1112.5865083646895,0.0,0.0 +full_stored_board,zai-org/GLM-OCR,1738.1794754621453,1738.1794754621453,0.0,0.0 +balanced_adaptive,FireRedTeam/FireRed-OCR,1557.449386684769,1557.449386684769,0.0,0.0 +balanced_adaptive,PaddlePaddle/PP-OCRv6_medium,1397.0965599286017,1397.0965599286017,0.0,0.0 +balanced_adaptive,PaddlePaddle/PaddleOCR-VL-1.6,1462.9745524973894,1462.9745524973894,0.0,0.0 +balanced_adaptive,allenai/olmOCR-2-7B-1025-FP8,1719.231138029067,1719.231138029067,0.0,0.0 +balanced_adaptive,baidu/Qianfan-OCR,1568.0944503021342,1568.0944503021342,0.0,0.0 +balanced_adaptive,baidu/Unlimited-OCR,1545.2060026385532,1545.2060026385532,0.0,0.0 +balanced_adaptive,deepseek-ai/DeepSeek-OCR,1451.9970235866688,1451.9970235866688,0.0,0.0 +balanced_adaptive,deepseek-ai/DeepSeek-OCR-2,1380.2125479868414,1380.2125479868414,0.0,0.0 +balanced_adaptive,lightonai/LightOnOCR-2-1B,1741.2525107847546,1741.2525107847546,0.0,0.0 +balanced_adaptive,numind/NuExtract3,1689.6190357899554,1689.6190357899554,0.0,0.0 +balanced_adaptive,rednote-hilab/dots.mocr,1745.4252593866306,1745.4252593866306,0.0,0.0 +balanced_adaptive,rednote-hilab/dots.ocr,890.6755485577996,890.6755485577996,0.0,0.0 +balanced_adaptive,tesseract-5,1112.5865083646895,1112.5865083646895,0.0,0.0 +balanced_adaptive,zai-org/GLM-OCR,1738.1794754621453,1738.1794754621453,0.0,0.0 +targeted,FireRedTeam/FireRed-OCR,1608.6660112077338,1557.449386684769,51.21662452296482,51.21662452296482 +targeted,PaddlePaddle/PP-OCRv6_medium,1377.6646857166104,1397.0965599286017,-19.43187421199127,19.43187421199127 +targeted,PaddlePaddle/PaddleOCR-VL-1.6,1454.615998571742,1462.9745524973894,-8.358553925647357,8.358553925647357 +targeted,allenai/olmOCR-2-7B-1025-FP8,1642.818285019612,1719.231138029067,-76.41285300945515,76.41285300945515 +targeted,baidu/Qianfan-OCR,1539.9111178560452,1568.0944503021342,-28.183332446089025,28.183332446089025 +targeted,baidu/Unlimited-OCR,1506.3234161401497,1545.2060026385532,-38.8825864984035,38.8825864984035 +targeted,deepseek-ai/DeepSeek-OCR,1426.5476771700012,1451.9970235866688,-25.449346416667595,25.449346416667595 +targeted,deepseek-ai/DeepSeek-OCR-2,1377.2800066566067,1380.2125479868414,-2.9325413302346988,2.9325413302346988 +targeted,lightonai/LightOnOCR-2-1B,1749.396477632365,1741.2525107847546,8.1439668476105,8.1439668476105 +targeted,numind/NuExtract3,1605.6969627860044,1689.6190357899554,-83.92207300395103,83.92207300395103 +targeted,rednote-hilab/dots.mocr,1792.8899530394683,1745.4252593866306,47.464693652837695,47.464693652837695 +targeted,rednote-hilab/dots.ocr,948.1699172928749,890.6755485577996,57.49436873507523,57.49436873507523 +targeted,tesseract-5,1198.0745767135008,1112.5865083646895,85.48806834881134,85.48806834881134 +targeted,zai-org/GLM-OCR,1771.9449141972862,1738.1794754621453,33.76543873514083,33.76543873514083 +targeted_size_3_min_10,FireRedTeam/FireRed-OCR,1624.9227279420727,1557.449386684769,67.47334125730367,67.47334125730367 +targeted_size_3_min_10,PaddlePaddle/PP-OCRv6_medium,1376.9484337859622,1397.0965599286017,-20.148126142639512,20.148126142639512 +targeted_size_3_min_10,PaddlePaddle/PaddleOCR-VL-1.6,1490.9995257145788,1462.9745524973894,28.024973217189427,28.024973217189427 +targeted_size_3_min_10,allenai/olmOCR-2-7B-1025-FP8,1635.9035087399477,1719.231138029067,-83.32762928911939,83.32762928911939 +targeted_size_3_min_10,baidu/Qianfan-OCR,1530.4176737122787,1568.0944503021342,-37.676776589855535,37.676776589855535 +targeted_size_3_min_10,baidu/Unlimited-OCR,1480.0702592753057,1545.2060026385532,-65.13574336324746,65.13574336324746 +targeted_size_3_min_10,deepseek-ai/DeepSeek-OCR,1398.8879735474673,1451.9970235866688,-53.10905003920152,53.10905003920152 +targeted_size_3_min_10,deepseek-ai/DeepSeek-OCR-2,1357.233351240866,1380.2125479868414,-22.979196745975287,22.979196745975287 +targeted_size_3_min_10,lightonai/LightOnOCR-2-1B,1737.341289759878,1741.2525107847546,-3.9112210248765678,3.9112210248765678 +targeted_size_3_min_10,numind/NuExtract3,1585.7734560830231,1689.6190357899554,-103.84557970693231,103.84557970693231 +targeted_size_3_min_10,rednote-hilab/dots.mocr,1810.2964669395187,1745.4252593866306,64.87120755288811,64.87120755288811 +targeted_size_3_min_10,rednote-hilab/dots.ocr,979.3393152695397,890.6755485577996,88.6637667117401,88.6637667117401 +targeted_size_3_min_10,tesseract-5,1218.526808949212,1112.5865083646895,105.94030058452245,105.94030058452245 +targeted_size_3_min_10,zai-org/GLM-OCR,1773.3392090403493,1738.1794754621453,35.15973357820394,35.15973357820394 +targeted_size_2_min_10,FireRedTeam/FireRed-OCR,1724.8282863076872,1557.449386684769,167.3788996229182,167.3788996229182 +targeted_size_2_min_10,PaddlePaddle/PP-OCRv6_medium,1367.8142585858143,1397.0965599286017,-29.282301342787378,29.282301342787378 +targeted_size_2_min_10,PaddlePaddle/PaddleOCR-VL-1.6,1471.007968958292,1462.9745524973894,8.033416460902572,8.033416460902572 +targeted_size_2_min_10,allenai/olmOCR-2-7B-1025-FP8,1613.8096859733894,1719.231138029067,-105.42145205567772,105.42145205567772 +targeted_size_2_min_10,baidu/Qianfan-OCR,1512.007840306965,1568.0944503021342,-56.08660999516928,56.08660999516928 +targeted_size_2_min_10,baidu/Unlimited-OCR,1469.7309981503452,1545.2060026385532,-75.47500448820801,75.47500448820801 +targeted_size_2_min_10,deepseek-ai/DeepSeek-OCR,1389.8648193014194,1451.9970235866688,-62.13220428524937,62.13220428524937 +targeted_size_2_min_10,deepseek-ai/DeepSeek-OCR-2,1347.8889020867468,1380.2125479868414,-32.32364590009456,32.32364590009456 +targeted_size_2_min_10,lightonai/LightOnOCR-2-1B,1832.5442474811484,1741.2525107847546,91.29173669639385,91.29173669639385 +targeted_size_2_min_10,numind/NuExtract3,1571.3398967045782,1689.6190357899554,-118.27913908537721,118.27913908537721 +targeted_size_2_min_10,rednote-hilab/dots.mocr,1866.7924427944317,1745.4252593866306,121.36718340780112,121.36718340780112 +targeted_size_2_min_10,rednote-hilab/dots.ocr,956.3624072814648,890.6755485577996,65.68685872366518,65.68685872366518 +targeted_size_2_min_10,tesseract-5,1220.7009515760715,1112.5865083646895,108.11444321138197,108.11444321138197 +targeted_size_2_min_10,zai-org/GLM-OCR,1655.3072944916464,1738.1794754621453,-82.8721809704989,82.8721809704989 +targeted_size_3_min_5,FireRedTeam/FireRed-OCR,1643.480912656966,1557.449386684769,86.03152597219696,86.03152597219696 +targeted_size_3_min_5,PaddlePaddle/PP-OCRv6_medium,1379.6772153920847,1397.0965599286017,-17.419344536516974,17.419344536516974 +targeted_size_3_min_5,PaddlePaddle/PaddleOCR-VL-1.6,1618.2259346774688,1462.9745524973894,155.25138218007942,155.25138218007942 +targeted_size_3_min_5,allenai/olmOCR-2-7B-1025-FP8,1597.4359505773557,1719.231138029067,-121.7951874517114,121.7951874517114 +targeted_size_3_min_5,baidu/Qianfan-OCR,1502.0187746993581,1568.0944503021342,-66.07567560277607,66.07567560277607 +targeted_size_3_min_5,baidu/Unlimited-OCR,1440.5804020769444,1545.2060026385532,-104.62560056160874,104.62560056160874 +targeted_size_3_min_5,deepseek-ai/DeepSeek-OCR,1380.0392388821308,1451.9970235866688,-71.95778470453797,71.95778470453797 +targeted_size_3_min_5,deepseek-ai/DeepSeek-OCR-2,1352.7679837448773,1380.2125479868414,-27.444564241964144,27.444564241964144 +targeted_size_3_min_5,lightonai/LightOnOCR-2-1B,1736.1743812456357,1741.2525107847546,-5.0781295391188905,5.0781295391188905 +targeted_size_3_min_5,numind/NuExtract3,1558.0228780431187,1689.6190357899554,-131.59615774683675,131.59615774683675 +targeted_size_3_min_5,rednote-hilab/dots.mocr,1848.054900386524,1745.4252593866306,102.62964099989335,102.62964099989335 +targeted_size_3_min_5,rednote-hilab/dots.ocr,952.5729095569341,890.6755485577996,61.897360999134435,61.897360999134435 +targeted_size_3_min_5,tesseract-5,1209.2828405221953,1112.5865083646895,96.69633215750582,96.69633215750582 +targeted_size_3_min_5,zai-org/GLM-OCR,1781.6656775384063,1738.1794754621453,43.48620207626095,43.48620207626095 +targeted_size_3_min_15,FireRedTeam/FireRed-OCR,1613.4265061457595,1557.449386684769,55.97711946099048,55.97711946099048 +targeted_size_3_min_15,PaddlePaddle/PP-OCRv6_medium,1387.404191581624,1397.0965599286017,-9.692368346977673,9.692368346977673 +targeted_size_3_min_15,PaddlePaddle/PaddleOCR-VL-1.6,1462.4611566414483,1462.9745524973894,-0.5133958559410985,0.5133958559410985 +targeted_size_3_min_15,allenai/olmOCR-2-7B-1025-FP8,1628.6862596842361,1719.231138029067,-90.54487834483098,90.54487834483098 +targeted_size_3_min_15,baidu/Qianfan-OCR,1541.1377796548968,1568.0944503021342,-26.956670647237388,26.956670647237388 +targeted_size_3_min_15,baidu/Unlimited-OCR,1477.7399853405952,1545.2060026385532,-67.46601729795793,67.46601729795793 +targeted_size_3_min_15,deepseek-ai/DeepSeek-OCR,1416.0457043006472,1451.9970235866688,-35.951319286021544,35.951319286021544 +targeted_size_3_min_15,deepseek-ai/DeepSeek-OCR-2,1381.398389655434,1380.2125479868414,1.1858416685925022,1.1858416685925022 +targeted_size_3_min_15,lightonai/LightOnOCR-2-1B,1738.5779937997568,1741.2525107847546,-2.6745169849978083,2.6745169849978083 +targeted_size_3_min_15,numind/NuExtract3,1597.707417047832,1689.6190357899554,-91.91161874212344,91.91161874212344 +targeted_size_3_min_15,rednote-hilab/dots.mocr,1843.8304414058673,1745.4252593866306,98.40518201923669,98.40518201923669 +targeted_size_3_min_15,rednote-hilab/dots.ocr,947.0957655919705,890.6755485577996,56.42021703417083,56.42021703417083 +targeted_size_3_min_15,tesseract-5,1197.3374054968394,1112.5865083646895,84.75089713214993,84.75089713214993 +targeted_size_3_min_15,zai-org/GLM-OCR,1767.151003653093,1738.1794754621453,28.971528190947765,28.971528190947765 +targeted_size_5_min_10,FireRedTeam/FireRed-OCR,1607.869184480983,1557.449386684769,50.41979779621397,50.41979779621397 +targeted_size_5_min_10,PaddlePaddle/PP-OCRv6_medium,1389.0671908582858,1397.0965599286017,-8.029369070315852,8.029369070315852 +targeted_size_5_min_10,PaddlePaddle/PaddleOCR-VL-1.6,1459.276992666841,1462.9745524973894,-3.697559830548471,3.697559830548471 +targeted_size_5_min_10,allenai/olmOCR-2-7B-1025-FP8,1641.6028731708125,1719.231138029067,-77.62826485825462,77.62826485825462 +targeted_size_5_min_10,baidu/Qianfan-OCR,1534.835864168096,1568.0944503021342,-33.258586134038296,33.258586134038296 +targeted_size_5_min_10,baidu/Unlimited-OCR,1507.8178621579427,1545.2060026385532,-37.388140480610446,37.388140480610446 +targeted_size_5_min_10,deepseek-ai/DeepSeek-OCR,1417.789766415991,1451.9970235866688,-34.20725717067785,34.20725717067785 +targeted_size_5_min_10,deepseek-ai/DeepSeek-OCR-2,1378.4484194360455,1380.2125479868414,-1.764128550795931,1.764128550795931 +targeted_size_5_min_10,lightonai/LightOnOCR-2-1B,1748.8083433448191,1741.2525107847546,7.555832560064573,7.555832560064573 +targeted_size_5_min_10,numind/NuExtract3,1605.3996229247402,1689.6190357899554,-84.21941286521519,84.21941286521519 +targeted_size_5_min_10,rednote-hilab/dots.mocr,1792.358569623159,1745.4252593866306,46.933310236528314,46.933310236528314 +targeted_size_5_min_10,rednote-hilab/dots.ocr,947.6817327503769,890.6755485577996,57.00618419257728,57.00618419257728 +targeted_size_5_min_10,tesseract-5,1197.6390075573959,1112.5865083646895,85.05249919270636,85.05249919270636 +targeted_size_5_min_10,zai-org/GLM-OCR,1771.4045704445111,1738.1794754621453,33.22509498236582,33.22509498236582 diff --git a/experiments/adaptive-stopping/replay.py b/experiments/adaptive-stopping/replay.py new file mode 100644 index 0000000..e0f9ae7 --- /dev/null +++ b/experiments/adaptive-stopping/replay.py @@ -0,0 +1,602 @@ +#!/usr/bin/env python3 +"""Replay adaptive allocation against stored Britannica comparison outcomes. + +This experiment is deliberately read-only: it downloads the pinned ``comparisons`` +config, selects already-published outcomes in production sample-batch order, and +never constructs a judge backend or writes to the Hugging Face Hub. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import statistics +from collections import Counter, defaultdict +from collections.abc import Iterable, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Literal, cast + +from datasets import load_dataset +from scipy.stats import kendalltau, spearmanr + +from ocr_bench.adaptive import ( + AdjacentPairDecision, + classify_adjacent_pairs, + comparison_pair_counts, + model_parameter_counts, + normalize_model_pair, + practical_preferences, + unresolved_pairs, +) +from ocr_bench.elo import ComparisonResult, Leaderboard, Winner, compute_elo +from ocr_bench.judge import is_sentinel + +DEFAULT_REPO = "davanstrien/ocr-bench-britannica-results" +# Pin the board used for the report so later Hub updates cannot change the replay. +DEFAULT_REVISION = "48a0f42de26009892d2784a3a97d6d61525f4040" +BATCH_SAMPLES = 5 + +StrategyKind = Literal["full", "balanced", "targeted"] + + +@dataclass(frozen=True) +class StrategyConfig: + """One replay configuration.""" + + name: str + kind: StrategyKind + size_tie_ratio: float | None = None + size_tie_min_samples: int = 10 + + +@dataclass +class ReplayResult: + """Internal replay result, including objects needed for later metrics.""" + + config: StrategyConfig + comparisons: list[ComparisonResult] + board: Leaderboard + decisions: list[AdjacentPairDecision] + stopping_round: int + stopping_reason: str + round_history: list[dict[str, Any]] + + +PRIMARY_CONFIGS = ( + StrategyConfig("full_stored_board", "full"), + StrategyConfig("balanced_adaptive", "balanced"), + StrategyConfig("targeted", "targeted"), + StrategyConfig("targeted_size_3_min_10", "targeted", 3.0, 10), +) + +SENSITIVITY_CONFIGS = ( + StrategyConfig("targeted_size_2_min_10", "targeted", 2.0, 10), + StrategyConfig("targeted_size_3_min_5", "targeted", 3.0, 5), + StrategyConfig("targeted_size_3_min_15", "targeted", 3.0, 15), + StrategyConfig("targeted_size_5_min_10", "targeted", 5.0, 10), +) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", default=DEFAULT_REPO) + parser.add_argument("--revision", default=DEFAULT_REVISION) + parser.add_argument( + "--bootstrap", + type=int, + default=1000, + help="Bootstrap replicates per interim board (production default: 1000).", + ) + parser.add_argument( + "--repeats", + type=int, + default=2, + help="Repeated primary replays used for the determinism check.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(__file__).resolve().parent, + ) + parser.add_argument( + "--skip-sensitivity", + action="store_true", + help="Run only the four primary configurations.", + ) + parser.add_argument( + "--exclude-sentinel-comparisons", + action="store_true", + help=( + "Robustness mode: remove historical rows containing current error " + "sentinels. The committed primary report intentionally uses all 4,293 " + "stored outcomes as requested." + ), + ) + return parser.parse_args() + + +def load_stored_comparisons( + repo: str, + revision: str, + *, + exclude_sentinels: bool = False, +) -> tuple[list[ComparisonResult], int, int]: + """Load canonical stored verdicts at a pinned, read-only Hub revision.""" + dataset = load_dataset(repo, "comparisons", split="train", revision=revision) + source_rows = len(dataset) + results: list[ComparisonResult] = [] + sentinel_rows = 0 + for row in dataset: + has_sentinel = is_sentinel(row.get("text_a")) or is_sentinel(row.get("text_b")) + sentinel_rows += int(has_sentinel) + if exclude_sentinels and has_sentinel: + continue + winner = row["winner"] + if winner not in {"A", "B", "tie"}: + raise ValueError(f"Unexpected stored winner: {winner!r}") + results.append( + ComparisonResult( + sample_idx=row["sample_idx"], + model_a=row["model_a"], + model_b=row["model_b"], + winner=cast(Winner, winner), + reason=row.get("reason", ""), + agreement=row.get("agreement", "1/1"), + # Published winners are already canonical/unswapped. + swapped=False, + text_a=row.get("text_a", ""), + text_b=row.get("text_b", ""), + col_a=row.get("col_a", ""), + col_b=row.get("col_b", ""), + truncated_a=row.get("truncated_a", False), + truncated_b=row.get("truncated_b", False), + ) + ) + return results, source_rows, sentinel_rows + + +def _group_by_sample( + comparisons: Iterable[ComparisonResult], +) -> dict[int, list[ComparisonResult]]: + grouped: dict[int, list[ComparisonResult]] = defaultdict(list) + for comparison in comparisons: + grouped[comparison.sample_idx].append(comparison) + return dict(grouped) + + +def _classify( + board: Leaderboard, + comparisons: Sequence[ComparisonResult], + config: StrategyConfig, +) -> list[AdjacentPairDecision]: + """Call the production classifier with the production registry sizes.""" + return classify_adjacent_pairs( + board, + comparison_pair_counts(comparisons), + size_tie_ratio=config.size_tie_ratio, + size_tie_min_samples=config.size_tie_min_samples, + parameter_counts=model_parameter_counts(), + ) + + +def _status_counts(decisions: Sequence[AdjacentPairDecision]) -> dict[str, int]: + counts = Counter(decision.status for decision in decisions) + return { + "resolved": counts["resolved"], + "prefer-smaller": counts["prefer-smaller"], + "unresolved": counts["unresolved"], + } + + +def replay_strategy( + stored: Sequence[ComparisonResult], + model_names: Sequence[str], + config: StrategyConfig, + *, + n_bootstrap: int = 1000, + batch_samples: int = BATCH_SAMPLES, +) -> ReplayResult: + """Replay one strategy with the production batching and decision rules. + + The first targeted allocation is balanced. As in ``cmd_judge``, decisions + are first checked after ``max(3 * n_pairs, 20)`` accumulated outcomes and + targeted allocation then follows the currently unresolved adjacent pairs. + """ + if not stored: + raise ValueError("Cannot replay an empty comparison set") + grouped = _group_by_sample(stored) + sample_order = list(range(min(grouped), max(grouped) + 1)) + n_pairs = len(model_names) * (len(model_names) - 1) // 2 + min_before_check = max(3 * n_pairs, 20) + + if config.kind == "full": + comparisons = list(stored) + board = compute_elo(comparisons, list(model_names), n_bootstrap=n_bootstrap) + decisions = _classify(board, comparisons, config) + return ReplayResult( + config=config, + comparisons=comparisons, + board=board, + decisions=decisions, + stopping_round=math.ceil(len(sample_order) / batch_samples), + stopping_reason="reference_full_stored_board", + round_history=[], + ) + + accumulated: list[ComparisonResult] = [] + active_pairs: set[tuple[str, str]] | None = None + round_history: list[dict[str, Any]] = [] + board: Leaderboard | None = None + decisions: list[AdjacentPairDecision] = [] + stopping_round = math.ceil(len(sample_order) / batch_samples) + stopping_reason = "sample_batches_exhausted" + + for round_number, start in enumerate(range(0, len(sample_order), batch_samples), start=1): + batch_indices = sample_order[start : start + batch_samples] + batch = [ + comparison + for sample_idx in batch_indices + for comparison in grouped.get(sample_idx, []) + if active_pairs is None + or normalize_model_pair(comparison.model_a, comparison.model_b) in active_pairs + ] + # Production skips the decision check when build_comparisons returns no rows. + if not batch: + continue + accumulated.extend(batch) + if len(accumulated) < min_before_check: + round_history.append( + { + "round": round_number, + "sample_indices": batch_indices, + "batch_comparisons": len(batch), + "cumulative_comparisons": len(accumulated), + "active_pairs": None if active_pairs is None else len(active_pairs), + "decision": "warm-up", + } + ) + continue + + board = compute_elo(accumulated, list(model_names), n_bootstrap=n_bootstrap) + decisions = _classify(board, accumulated, config) + next_pairs = unresolved_pairs(decisions) + round_history.append( + { + "round": round_number, + "sample_indices": batch_indices, + "batch_comparisons": len(batch), + "cumulative_comparisons": len(accumulated), + "active_pairs": None if active_pairs is None else len(active_pairs), + "next_active_pairs": len(next_pairs), + "decision": _status_counts(decisions), + "rank_order": [model for model, _ in board.ranked], + } + ) + if not next_pairs: + stopping_round = round_number + stopping_reason = "adaptive_criteria_met" + break + if config.kind == "targeted": + active_pairs = next_pairs + + # cmd_judge refits once more for final publication, even after the last check. + board = compute_elo(accumulated, list(model_names), n_bootstrap=n_bootstrap) + decisions = _classify(board, accumulated, config) + return ReplayResult( + config=config, + comparisons=accumulated, + board=board, + decisions=decisions, + stopping_round=stopping_round, + stopping_reason=stopping_reason, + round_history=round_history, + ) + + +def graph_metrics( + comparisons: Sequence[ComparisonResult], model_names: Sequence[str] +) -> dict[str, Any]: + """Summarize direct-pair coverage and undirected comparison connectivity.""" + pair_counts = comparison_pair_counts(comparisons) + possible_pairs = len(model_names) * (len(model_names) - 1) // 2 + adjacency = {model: set() for model in model_names} + for model_a, model_b in pair_counts: + adjacency[model_a].add(model_b) + adjacency[model_b].add(model_a) + + components: list[list[str]] = [] + unseen = set(model_names) + while unseen: + root = min(unseen) + stack = [root] + component: set[str] = set() + while stack: + model = stack.pop() + if model in component: + continue + component.add(model) + stack.extend(adjacency[model] - component) + unseen -= component + components.append(sorted(component)) + + direct_counts = list(pair_counts.values()) + return { + "observed_pairs": len(pair_counts), + "possible_pairs": possible_pairs, + "pair_coverage_pct": 100.0 * len(pair_counts) / possible_pairs, + "min_direct_comparisons": min(direct_counts, default=0), + "median_direct_comparisons": statistics.median(direct_counts) if direct_counts else 0, + "max_direct_comparisons": max(direct_counts, default=0), + "connected": len(components) == 1, + "connected_components": components, + } + + +def _preference_rows(decisions: Sequence[AdjacentPairDecision]) -> list[dict[str, Any]]: + preferences = practical_preferences(decisions) + return [ + { + "smaller_model": smaller, + "larger_model": decision.larger_model, + "size_ratio": decision.size_ratio, + "higher_ranked_model": decision.higher_model, + } + for smaller, grouped in sorted(preferences.items()) + for decision in grouped + ] + + +def _determinism_signature(result: ReplayResult) -> dict[str, Any]: + return { + "comparison_keys": [ + (r.sample_idx, normalize_model_pair(r.model_a, r.model_b), r.winner) + for r in result.comparisons + ], + "elo": result.board.elo, + "elo_ci": result.board.elo_ci, + "ranked": result.board.ranked, + "decisions": [asdict(decision) for decision in result.decisions], + "stopping_round": result.stopping_round, + "stopping_reason": result.stopping_reason, + "round_history": result.round_history, + } + + +def strategy_metrics( + result: ReplayResult, + reference: ReplayResult, + model_names: Sequence[str], + *, + deterministic: bool | None, +) -> dict[str, Any]: + """Compute all requested comparison-to-reference metrics.""" + reference_rank = [model for model, _ in reference.board.ranked] + strategy_rank = [model for model, _ in result.board.ranked] + ref_position = {model: rank for rank, model in enumerate(reference_rank, start=1)} + strategy_position = {model: rank for rank, model in enumerate(strategy_rank, start=1)} + ordered_models = sorted(model_names) + ref_positions = [ref_position[model] for model in ordered_models] + strategy_positions = [strategy_position[model] for model in ordered_models] + kendall = kendalltau(ref_positions, strategy_positions).statistic + spearman = spearmanr(ref_positions, strategy_positions).statistic + + elo_deltas = { + model: result.board.elo[model] - reference.board.elo[model] for model in ordered_models + } + absolute_deltas = [abs(delta) for delta in elo_deltas.values()] + statuses = _status_counts(result.decisions) + if result.stopping_reason == "adaptive_criteria_met": + resolution = ( + f"criteria met: {statuses['resolved']} statistically resolved, " + f"{statuses['prefer-smaller']} practical preferences" + ) + elif result.config.kind == "full": + resolution = "reference only; no adaptive stop" + else: + resolution = f"sample batches exhausted: {statuses['unresolved']} adjacent pairs unresolved" + + full_count = len(reference.comparisons) + top3_reference = reference_rank[:3] + top3_strategy = strategy_rank[:3] + return { + "name": result.config.name, + "kind": result.config.kind, + "size_tie_ratio": result.config.size_tie_ratio, + "size_tie_min_samples": result.config.size_tie_min_samples, + "comparisons_consumed": len(result.comparisons), + "comparisons_saved": full_count - len(result.comparisons), + "percentage_saved": 100.0 * (full_count - len(result.comparisons)) / full_count, + "stopping_round": result.stopping_round, + "stopping_reason": result.stopping_reason, + "resolution_summary": resolution, + "final_rank_order": strategy_rank, + "kendall_tau": float(kendall), + "spearman_rho": float(spearman), + "top3": top3_strategy, + "top3_membership_matches": set(top3_strategy) == set(top3_reference), + "top3_membership_overlap": len(set(top3_strategy) & set(top3_reference)), + "top3_order_matches": top3_strategy == top3_reference, + "elo": result.board.elo, + "elo_ci": result.board.elo_ci, + "elo_delta_from_full": elo_deltas, + "median_absolute_elo_delta": statistics.median(absolute_deltas), + "max_absolute_elo_delta": max(absolute_deltas), + "max_absolute_elo_delta_model": max(elo_deltas, key=lambda model: abs(elo_deltas[model])), + "decision_statuses": statuses, + "practical_preferences": _preference_rows(result.decisions), + "graph": graph_metrics(result.comparisons, model_names), + "deterministic_across_repeats": deterministic, + "round_history": result.round_history, + } + + +def _write_csvs(output_dir: Path, metrics: Sequence[dict[str, Any]]) -> None: + summary_fields = [ + "name", + "comparisons_consumed", + "comparisons_saved", + "percentage_saved", + "stopping_round", + "stopping_reason", + "kendall_tau", + "spearman_rho", + "top3_membership_matches", + "top3_order_matches", + "median_absolute_elo_delta", + "max_absolute_elo_delta", + "deterministic_across_repeats", + ] + with (output_dir / "strategy-summary.csv").open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=summary_fields, lineterminator="\n") + writer.writeheader() + for metric in metrics: + writer.writerow({field: metric[field] for field in summary_fields}) + + with (output_dir / "elo-deltas.csv").open("w", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=["strategy", "model", "elo", "full_elo", "delta", "absolute_delta"], + lineterminator="\n", + ) + writer.writeheader() + reference = metrics[0] + for metric in metrics: + for model, delta in metric["elo_delta_from_full"].items(): + writer.writerow( + { + "strategy": metric["name"], + "model": model, + "elo": metric["elo"][model], + "full_elo": reference["elo"][model], + "delta": delta, + "absolute_delta": abs(delta), + } + ) + + with (output_dir / "round-history.csv").open("w", newline="") as handle: + fields = [ + "strategy", + "round", + "sample_indices", + "batch_comparisons", + "cumulative_comparisons", + "active_pairs", + "next_active_pairs", + "resolved", + "prefer_smaller", + "unresolved", + ] + writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") + writer.writeheader() + for metric in metrics: + for row in metric["round_history"]: + decision = row.get("decision", {}) + if not isinstance(decision, dict): + decision = {} + writer.writerow( + { + "strategy": metric["name"], + "round": row["round"], + "sample_indices": json.dumps(row["sample_indices"]), + "batch_comparisons": row["batch_comparisons"], + "cumulative_comparisons": row["cumulative_comparisons"], + "active_pairs": row.get("active_pairs"), + "next_active_pairs": row.get("next_active_pairs"), + "resolved": decision.get("resolved"), + "prefer_smaller": decision.get("prefer-smaller"), + "unresolved": decision.get("unresolved"), + } + ) + + +def main() -> None: + args = _parse_args() + if args.bootstrap < 1: + raise ValueError("--bootstrap must be at least 1 because adaptive decisions require CIs") + if args.repeats < 1: + raise ValueError("--repeats must be at least 1") + + stored, source_rows, sentinel_rows = load_stored_comparisons( + args.repo, + args.revision, + exclude_sentinels=args.exclude_sentinel_comparisons, + ) + model_names = sorted({model for result in stored for model in (result.model_a, result.model_b)}) + configs = list(PRIMARY_CONFIGS) + if not args.skip_sensitivity: + configs.extend(SENSITIVITY_CONFIGS) + + results: dict[str, ReplayResult] = {} + deterministic: dict[str, bool | None] = {} + primary_names = {config.name for config in PRIMARY_CONFIGS} + for config in configs: + print(f"Replaying {config.name} ...", flush=True) + first = replay_strategy(stored, model_names, config, n_bootstrap=args.bootstrap) + results[config.name] = first + signatures = [_determinism_signature(first)] + repeats = args.repeats if config.name in primary_names else 1 + for _ in range(1, repeats): + repeated = replay_strategy( + stored, + model_names, + config, + n_bootstrap=args.bootstrap, + ) + signatures.append(_determinism_signature(repeated)) + deterministic[config.name] = ( + all(signature == signatures[0] for signature in signatures[1:]) if repeats > 1 else None + ) + + reference = results[PRIMARY_CONFIGS[0].name] + metrics = [ + strategy_metrics( + results[config.name], + reference, + model_names, + deterministic=deterministic[config.name], + ) + for config in configs + ] + payload = { + "experiment": { + "source_repo": args.repo, + "source_revision": args.revision, + "stored_comparisons_at_revision": source_rows, + "comparisons_replayed": len(stored), + "historical_sentinel_comparisons": sentinel_rows, + "sentinel_comparisons_excluded": args.exclude_sentinel_comparisons, + "models": model_names, + "model_count": len(model_names), + "sample_count": len({result.sample_idx for result in stored}), + "batch_samples": BATCH_SAMPLES, + "min_before_check": max(3 * (len(model_names) * (len(model_names) - 1) // 2), 20), + "bootstrap_replicates": args.bootstrap, + "bootstrap_seed": 42, + "primary_repeats": args.repeats, + "production_helpers": [ + "compute_elo", + "classify_adjacent_pairs", + "comparison_pair_counts", + "model_parameter_counts", + "normalize_model_pair", + "practical_preferences", + "unresolved_pairs", + ], + }, + "strategies": metrics, + } + + args.output_dir.mkdir(parents=True, exist_ok=True) + suffix = "-sentinels-excluded" if args.exclude_sentinel_comparisons else "" + json_path = args.output_dir / f"results{suffix}.json" + json_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + if not args.exclude_sentinel_comparisons: + _write_csvs(args.output_dir, metrics) + print(f"Wrote {json_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/adaptive-stopping/results-sentinels-excluded.json b/experiments/adaptive-stopping/results-sentinels-excluded.json new file mode 100644 index 0000000..5897cb4 --- /dev/null +++ b/experiments/adaptive-stopping/results-sentinels-excluded.json @@ -0,0 +1,1735 @@ +{ + "experiment": { + "batch_samples": 5, + "bootstrap_replicates": 1000, + "bootstrap_seed": 42, + "comparisons_replayed": 3665, + "historical_sentinel_comparisons": 628, + "min_before_check": 234, + "model_count": 13, + "models": [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ], + "primary_repeats": 1, + "production_helpers": [ + "compute_elo", + "classify_adjacent_pairs", + "comparison_pair_counts", + "model_parameter_counts", + "normalize_model_pair", + "practical_preferences", + "unresolved_pairs" + ], + "sample_count": 50, + "sentinel_comparisons_excluded": true, + "source_repo": "davanstrien/ocr-bench-britannica-results", + "source_revision": "48a0f42de26009892d2784a3a97d6d61525f4040", + "stored_comparisons_at_revision": 4293 + }, + "strategies": [ + { + "comparisons_consumed": 3665, + "comparisons_saved": 0, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 8 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1511.854751011696, + "PaddlePaddle/PP-OCRv6_medium": 1346.3573650492863, + "PaddlePaddle/PaddleOCR-VL-1.6": 1414.2755806608957, + "allenai/olmOCR-2-7B-1025-FP8": 1673.7529792732425, + "baidu/Qianfan-OCR": 1524.0060491781535, + "baidu/Unlimited-OCR": 1498.0434434911501, + "deepseek-ai/DeepSeek-OCR": 1408.8292451079392, + "deepseek-ai/DeepSeek-OCR-2": 1336.865396676597, + "lightonai/LightOnOCR-2-1B": 1697.6032576835216, + "numind/NuExtract3": 1649.7536414287124, + "rednote-hilab/dots.mocr": 1700.0913003009869, + "tesseract-5": 1042.6333111257677, + "zai-org/GLM-OCR": 1695.933679012051 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1482.3991211797122, + 1545.2857414618684 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1313.1038770351304, + 1376.9109033878265 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1385.5157023047452, + 1444.0549433565216 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1643.3912352609393, + 1708.3579193771732 + ], + "baidu/Qianfan-OCR": [ + 1493.9433681270984, + 1553.7467893652977 + ], + "baidu/Unlimited-OCR": [ + 1466.0632813223278, + 1526.7470192965795 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1379.6929975698667, + 1438.107003207418 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1302.4628329656473, + 1368.2201737628789 + ], + "lightonai/LightOnOCR-2-1B": [ + 1666.3946389097268, + 1731.8860003005097 + ], + "numind/NuExtract3": [ + 1617.210500089359, + 1683.2562861656986 + ], + "rednote-hilab/dots.mocr": [ + 1671.8726911951383, + 1733.7848900535528 + ], + "tesseract-5": [ + 986.2557407473818, + 1089.0385516713936 + ], + "zai-org/GLM-OCR": [ + 1663.9401260430195, + 1726.382015223743 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 0.0, + "PaddlePaddle/PP-OCRv6_medium": 0.0, + "PaddlePaddle/PaddleOCR-VL-1.6": 0.0, + "allenai/olmOCR-2-7B-1025-FP8": 0.0, + "baidu/Qianfan-OCR": 0.0, + "baidu/Unlimited-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR-2": 0.0, + "lightonai/LightOnOCR-2-1B": 0.0, + "numind/NuExtract3": 0.0, + "rednote-hilab/dots.mocr": 0.0, + "tesseract-5": 0.0, + "zai-org/GLM-OCR": 0.0 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 46.0, + "min_direct_comparisons": 43, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9999999999999998, + "kind": "full", + "max_absolute_elo_delta": 0.0, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 0.0, + "name": "full_stored_board", + "percentage_saved": 0.0, + "practical_preferences": [], + "resolution_summary": "reference only; no adaptive stop", + "round_history": [], + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 1.0, + "stopping_reason": "reference_full_stored_board", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true + }, + { + "comparisons_consumed": 3665, + "comparisons_saved": 0, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 8 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1511.854751011696, + "PaddlePaddle/PP-OCRv6_medium": 1346.3573650492863, + "PaddlePaddle/PaddleOCR-VL-1.6": 1414.2755806608957, + "allenai/olmOCR-2-7B-1025-FP8": 1673.7529792732425, + "baidu/Qianfan-OCR": 1524.0060491781535, + "baidu/Unlimited-OCR": 1498.0434434911501, + "deepseek-ai/DeepSeek-OCR": 1408.8292451079392, + "deepseek-ai/DeepSeek-OCR-2": 1336.865396676597, + "lightonai/LightOnOCR-2-1B": 1697.6032576835216, + "numind/NuExtract3": 1649.7536414287124, + "rednote-hilab/dots.mocr": 1700.0913003009869, + "tesseract-5": 1042.6333111257677, + "zai-org/GLM-OCR": 1695.933679012051 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1482.3991211797122, + 1545.2857414618684 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1313.1038770351304, + 1376.9109033878265 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1385.5157023047452, + 1444.0549433565216 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1643.3912352609393, + 1708.3579193771732 + ], + "baidu/Qianfan-OCR": [ + 1493.9433681270984, + 1553.7467893652977 + ], + "baidu/Unlimited-OCR": [ + 1466.0632813223278, + 1526.7470192965795 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1379.6929975698667, + 1438.107003207418 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1302.4628329656473, + 1368.2201737628789 + ], + "lightonai/LightOnOCR-2-1B": [ + 1666.3946389097268, + 1731.8860003005097 + ], + "numind/NuExtract3": [ + 1617.210500089359, + 1683.2562861656986 + ], + "rednote-hilab/dots.mocr": [ + 1671.8726911951383, + 1733.7848900535528 + ], + "tesseract-5": [ + 986.2557407473818, + 1089.0385516713936 + ], + "zai-org/GLM-OCR": [ + 1663.9401260430195, + 1726.382015223743 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 0.0, + "PaddlePaddle/PP-OCRv6_medium": 0.0, + "PaddlePaddle/PaddleOCR-VL-1.6": 0.0, + "allenai/olmOCR-2-7B-1025-FP8": 0.0, + "baidu/Qianfan-OCR": 0.0, + "baidu/Unlimited-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR-2": 0.0, + "lightonai/LightOnOCR-2-1B": 0.0, + "numind/NuExtract3": 0.0, + "rednote-hilab/dots.mocr": 0.0, + "tesseract-5": 0.0, + "zai-org/GLM-OCR": 0.0 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 46.0, + "min_direct_comparisons": 43, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9999999999999998, + "kind": "balanced", + "max_absolute_elo_delta": 0.0, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 0.0, + "name": "balanced_adaptive", + "percentage_saved": 0.0, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 8 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 315, + "cumulative_comparisons": 582, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 388, + "cumulative_comparisons": 970, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 390, + "cumulative_comparisons": 1360, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 390, + "cumulative_comparisons": 1750, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 389, + "cumulative_comparisons": 2139, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 390, + "cumulative_comparisons": 2529, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 390, + "cumulative_comparisons": 2919, + "decision": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 357, + "cumulative_comparisons": 3276, + "decision": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 389, + "cumulative_comparisons": 3665, + "decision": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 1.0, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true + }, + { + "comparisons_consumed": 753, + "comparisons_saved": 2912, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1580.874213718402, + "PaddlePaddle/PP-OCRv6_medium": 1319.3227775512091, + "PaddlePaddle/PaddleOCR-VL-1.6": 1434.472485469595, + "allenai/olmOCR-2-7B-1025-FP8": 1617.2454829500064, + "baidu/Qianfan-OCR": 1519.6355377277357, + "baidu/Unlimited-OCR": 1491.3076994756007, + "deepseek-ai/DeepSeek-OCR": 1383.2381845896082, + "deepseek-ai/DeepSeek-OCR-2": 1325.6550319273726, + "lightonai/LightOnOCR-2-1B": 1720.9108108450519, + "numind/NuExtract3": 1589.384134889466, + "rednote-hilab/dots.mocr": 1764.1282436141787, + "tesseract-5": 1009.8122740342296, + "zai-org/GLM-OCR": 1744.0131232075441 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1518.8482710241274, + 1799.5736483326505 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1227.5197049885348, + 1546.8649875392728 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1372.9588770106732, + 1648.8416158407233 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1559.7323671654024, + 1835.1450101673204 + ], + "baidu/Qianfan-OCR": [ + 1463.2331948150375, + 1737.2951454405734 + ], + "baidu/Unlimited-OCR": [ + 1430.374762133354, + 1708.5227832419662 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1312.7016427558865, + 1604.6879783887803 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1246.2221390762734, + 1554.5203599578194 + ], + "lightonai/LightOnOCR-2-1B": [ + 1645.9553791446951, + 1946.4268424489621 + ], + "numind/NuExtract3": [ + 1527.6676569852266, + 1812.0294250840504 + ], + "rednote-hilab/dots.mocr": [ + 1686.3512346677337, + 1997.258728625838 + ], + "tesseract-5": [ + -1389.1652340162436, + 1182.6057601764962 + ], + "zai-org/GLM-OCR": [ + 1667.5246143830188, + 1973.4203023685607 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 69.01946270670601, + "PaddlePaddle/PP-OCRv6_medium": -27.034587498077144, + "PaddlePaddle/PaddleOCR-VL-1.6": 20.1969048086994, + "allenai/olmOCR-2-7B-1025-FP8": -56.50749632323618, + "baidu/Qianfan-OCR": -4.37051145041778, + "baidu/Unlimited-OCR": -6.735744015549471, + "deepseek-ai/DeepSeek-OCR": -25.591060518331005, + "deepseek-ai/DeepSeek-OCR-2": -11.210364749224254, + "lightonai/LightOnOCR-2-1B": 23.307553161530222, + "numind/NuExtract3": -60.36950653924646, + "rednote-hilab/dots.mocr": 64.03694331319184, + "tesseract-5": -32.821037091538074, + "zai-org/GLM-OCR": 48.079444195493124 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 44, + "median_direct_comparisons": 5.0, + "min_direct_comparisons": 2, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "targeted", + "max_absolute_elo_delta": 69.01946270670601, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 27.034587498077144, + "name": "targeted", + "percentage_saved": 79.45429740791269, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "batch_comparisons": 52, + "cumulative_comparisons": 319, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 54, + "cumulative_comparisons": 373, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 428, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 483, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 538, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 593, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 648, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 50, + "cumulative_comparisons": 698, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 753, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9835164835164836, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false + }, + { + "comparisons_consumed": 624, + "comparisons_saved": 3041, + "decision_statuses": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 5 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1597.893609965803, + "PaddlePaddle/PP-OCRv6_medium": 1345.4472271187867, + "PaddlePaddle/PaddleOCR-VL-1.6": 1470.0510725134177, + "allenai/olmOCR-2-7B-1025-FP8": 1611.7199041946376, + "baidu/Qianfan-OCR": 1510.4522825767392, + "baidu/Unlimited-OCR": 1465.4308202827929, + "deepseek-ai/DeepSeek-OCR": 1360.2597673138464, + "deepseek-ai/DeepSeek-OCR-2": 1323.006845943473, + "lightonai/LightOnOCR-2-1B": 1710.249592559738, + "numind/NuExtract3": 1566.0218597228734, + "rednote-hilab/dots.mocr": 1783.0603618591579, + "tesseract-5": 1009.7009312662467, + "zai-org/GLM-OCR": 1746.7057246824875 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1535.3303123052528, + 1826.4654515488583 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1251.756585138917, + 1567.1082831571525 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1401.2369362577963, + 1706.871796238772 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1548.640412637941, + 1839.7784021033285 + ], + "baidu/Qianfan-OCR": [ + 1443.4083364855344, + 1738.4392343811203 + ], + "baidu/Unlimited-OCR": [ + 1389.765429723917, + 1685.889473322771 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1272.2432991713881, + 1585.155292331488 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1228.9764980486973, + 1554.902813555286 + ], + "lightonai/LightOnOCR-2-1B": [ + 1637.5134077539665, + 1949.588134399969 + ], + "numind/NuExtract3": [ + 1505.7333283439796, + 1786.920876812792 + ], + "rednote-hilab/dots.mocr": [ + 1690.8923428361034, + 2029.8173266221638 + ], + "tesseract-5": [ + -1436.2798009269263, + 1166.5970573519855 + ], + "zai-org/GLM-OCR": [ + 1674.5389270417318, + 1982.2259241547752 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 86.03885895410713, + "PaddlePaddle/PP-OCRv6_medium": -0.9101379304995589, + "PaddlePaddle/PaddleOCR-VL-1.6": 55.77549185252201, + "allenai/olmOCR-2-7B-1025-FP8": -62.03307507860495, + "baidu/Qianfan-OCR": -13.553766601414281, + "baidu/Unlimited-OCR": -32.61262320835726, + "deepseek-ai/DeepSeek-OCR": -48.56947779409279, + "deepseek-ai/DeepSeek-OCR-2": -13.85855073312382, + "lightonai/LightOnOCR-2-1B": 12.646334876216315, + "numind/NuExtract3": -83.73178170583901, + "rednote-hilab/dots.mocr": 82.969061558171, + "tesseract-5": -32.932379859521006, + "zai-org/GLM-OCR": 50.772045670436455 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 35, + "median_direct_comparisons": 5.0, + "min_direct_comparisons": 2, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "targeted", + "max_absolute_elo_delta": 86.03885895410713, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 48.56947779409279, + "name": "targeted_size_3_min_10", + "percentage_saved": 82.97407912687585, + "practical_preferences": [ + { + "higher_ranked_model": "allenai/olmOCR-2-7B-1025-FP8", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 3.9523809523809526, + "smaller_model": "FireRedTeam/FireRed-OCR" + }, + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 115.94202898550725, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "PaddlePaddle/PP-OCRv6_medium", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Qianfan-OCR", + "larger_model": "baidu/Qianfan-OCR", + "size_ratio": 5.222222222222222, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 5 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "batch_comparisons": 52, + "cumulative_comparisons": 319, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 54, + "cumulative_comparisons": 373, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 9, + "batch_comparisons": 45, + "cumulative_comparisons": 418, + "decision": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 40, + "cumulative_comparisons": 458, + "decision": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 40, + "cumulative_comparisons": 498, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 9, + "batch_comparisons": 45, + "cumulative_comparisons": 543, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 6, + "batch_comparisons": 30, + "cumulative_comparisons": 573, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 6, + "batch_comparisons": 26, + "cumulative_comparisons": 599, + "decision": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 5, + "batch_comparisons": 25, + "cumulative_comparisons": 624, + "decision": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9725274725274725, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false + } + ] +} diff --git a/experiments/adaptive-stopping/results.json b/experiments/adaptive-stopping/results.json new file mode 100644 index 0000000..11680bc --- /dev/null +++ b/experiments/adaptive-stopping/results.json @@ -0,0 +1,3996 @@ +{ + "experiment": { + "batch_samples": 5, + "bootstrap_replicates": 1000, + "bootstrap_seed": 42, + "comparisons_replayed": 4293, + "historical_sentinel_comparisons": 628, + "min_before_check": 273, + "model_count": 14, + "models": [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ], + "primary_repeats": 2, + "production_helpers": [ + "compute_elo", + "classify_adjacent_pairs", + "comparison_pair_counts", + "model_parameter_counts", + "normalize_model_pair", + "practical_preferences", + "unresolved_pairs" + ], + "sample_count": 50, + "sentinel_comparisons_excluded": false, + "source_repo": "davanstrien/ocr-bench-britannica-results", + "source_revision": "48a0f42de26009892d2784a3a97d6d61525f4040", + "stored_comparisons_at_revision": 4293 + }, + "strategies": [ + { + "comparisons_consumed": 4293, + "comparisons_saved": 0, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1557.449386684769, + "PaddlePaddle/PP-OCRv6_medium": 1397.0965599286017, + "PaddlePaddle/PaddleOCR-VL-1.6": 1462.9745524973894, + "allenai/olmOCR-2-7B-1025-FP8": 1719.231138029067, + "baidu/Qianfan-OCR": 1568.0944503021342, + "baidu/Unlimited-OCR": 1545.2060026385532, + "deepseek-ai/DeepSeek-OCR": 1451.9970235866688, + "deepseek-ai/DeepSeek-OCR-2": 1380.2125479868414, + "lightonai/LightOnOCR-2-1B": 1741.2525107847546, + "numind/NuExtract3": 1689.6190357899554, + "rednote-hilab/dots.mocr": 1745.4252593866306, + "rednote-hilab/dots.ocr": 890.6755485577996, + "tesseract-5": 1112.5865083646895, + "zai-org/GLM-OCR": 1738.1794754621453 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1527.5808934337085, + 1587.576236691464 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1366.8263296608684, + 1427.6765862005336 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1434.6496521153135, + 1492.5649214642008 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1687.5478854387866, + 1753.337400988653 + ], + "baidu/Qianfan-OCR": [ + 1538.904110460085, + 1599.6559183608647 + ], + "baidu/Unlimited-OCR": [ + 1518.1663034474218, + 1575.0208526743218 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1421.5890820195257, + 1482.1437322978022 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1345.4486303621293, + 1415.1325861820442 + ], + "lightonai/LightOnOCR-2-1B": [ + 1708.946380889712, + 1778.5039641086157 + ], + "numind/NuExtract3": [ + 1657.11563443145, + 1724.8134296653323 + ], + "rednote-hilab/dots.mocr": [ + 1714.2617786978212, + 1781.8822281056453 + ], + "rednote-hilab/dots.ocr": [ + 796.8367480482991, + 963.3312313749263 + ], + "tesseract-5": [ + 1061.9245755673062, + 1153.1614650034103 + ], + "zai-org/GLM-OCR": [ + 1707.1033483530432, + 1772.6200954256956 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 0.0, + "PaddlePaddle/PP-OCRv6_medium": 0.0, + "PaddlePaddle/PaddleOCR-VL-1.6": 0.0, + "allenai/olmOCR-2-7B-1025-FP8": 0.0, + "baidu/Qianfan-OCR": 0.0, + "baidu/Unlimited-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR-2": 0.0, + "lightonai/LightOnOCR-2-1B": 0.0, + "numind/NuExtract3": 0.0, + "rednote-hilab/dots.mocr": 0.0, + "rednote-hilab/dots.ocr": 0.0, + "tesseract-5": 0.0, + "zai-org/GLM-OCR": 0.0 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 46, + "min_direct_comparisons": 43, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 1.0, + "kind": "full", + "max_absolute_elo_delta": 0.0, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 0.0, + "name": "full_stored_board", + "percentage_saved": 0.0, + "practical_preferences": [], + "resolution_summary": "reference only; no adaptive stop", + "round_history": [], + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 1.0, + "stopping_reason": "reference_full_stored_board", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true + }, + { + "comparisons_consumed": 4293, + "comparisons_saved": 0, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1557.449386684769, + "PaddlePaddle/PP-OCRv6_medium": 1397.0965599286017, + "PaddlePaddle/PaddleOCR-VL-1.6": 1462.9745524973894, + "allenai/olmOCR-2-7B-1025-FP8": 1719.231138029067, + "baidu/Qianfan-OCR": 1568.0944503021342, + "baidu/Unlimited-OCR": 1545.2060026385532, + "deepseek-ai/DeepSeek-OCR": 1451.9970235866688, + "deepseek-ai/DeepSeek-OCR-2": 1380.2125479868414, + "lightonai/LightOnOCR-2-1B": 1741.2525107847546, + "numind/NuExtract3": 1689.6190357899554, + "rednote-hilab/dots.mocr": 1745.4252593866306, + "rednote-hilab/dots.ocr": 890.6755485577996, + "tesseract-5": 1112.5865083646895, + "zai-org/GLM-OCR": 1738.1794754621453 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1527.5808934337085, + 1587.576236691464 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1366.8263296608684, + 1427.6765862005336 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1434.6496521153135, + 1492.5649214642008 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1687.5478854387866, + 1753.337400988653 + ], + "baidu/Qianfan-OCR": [ + 1538.904110460085, + 1599.6559183608647 + ], + "baidu/Unlimited-OCR": [ + 1518.1663034474218, + 1575.0208526743218 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1421.5890820195257, + 1482.1437322978022 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1345.4486303621293, + 1415.1325861820442 + ], + "lightonai/LightOnOCR-2-1B": [ + 1708.946380889712, + 1778.5039641086157 + ], + "numind/NuExtract3": [ + 1657.11563443145, + 1724.8134296653323 + ], + "rednote-hilab/dots.mocr": [ + 1714.2617786978212, + 1781.8822281056453 + ], + "rednote-hilab/dots.ocr": [ + 796.8367480482991, + 963.3312313749263 + ], + "tesseract-5": [ + 1061.9245755673062, + 1153.1614650034103 + ], + "zai-org/GLM-OCR": [ + 1707.1033483530432, + 1772.6200954256956 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 0.0, + "PaddlePaddle/PP-OCRv6_medium": 0.0, + "PaddlePaddle/PaddleOCR-VL-1.6": 0.0, + "allenai/olmOCR-2-7B-1025-FP8": 0.0, + "baidu/Qianfan-OCR": 0.0, + "baidu/Unlimited-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR-2": 0.0, + "lightonai/LightOnOCR-2-1B": 0.0, + "numind/NuExtract3": 0.0, + "rednote-hilab/dots.mocr": 0.0, + "rednote-hilab/dots.ocr": 0.0, + "tesseract-5": 0.0, + "zai-org/GLM-OCR": 0.0 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 46, + "min_direct_comparisons": 43, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 1.0, + "kind": "balanced", + "max_absolute_elo_delta": 0.0, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 0.0, + "name": "balanced_adaptive", + "percentage_saved": 0.0, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 9 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 373, + "cumulative_comparisons": 693, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 453, + "cumulative_comparisons": 1146, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 455, + "cumulative_comparisons": 1601, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 455, + "cumulative_comparisons": 2056, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 454, + "cumulative_comparisons": 2510, + "decision": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 455, + "cumulative_comparisons": 2965, + "decision": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 455, + "cumulative_comparisons": 3420, + "decision": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 419, + "cumulative_comparisons": 3839, + "decision": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": null, + "batch_comparisons": 454, + "cumulative_comparisons": 4293, + "decision": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 1.0, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true + }, + { + "comparisons_consumed": 870, + "comparisons_saved": 3423, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1608.6660112077338, + "PaddlePaddle/PP-OCRv6_medium": 1377.6646857166104, + "PaddlePaddle/PaddleOCR-VL-1.6": 1454.615998571742, + "allenai/olmOCR-2-7B-1025-FP8": 1642.818285019612, + "baidu/Qianfan-OCR": 1539.9111178560452, + "baidu/Unlimited-OCR": 1506.3234161401497, + "deepseek-ai/DeepSeek-OCR": 1426.5476771700012, + "deepseek-ai/DeepSeek-OCR-2": 1377.2800066566067, + "lightonai/LightOnOCR-2-1B": 1749.396477632365, + "numind/NuExtract3": 1605.6969627860044, + "rednote-hilab/dots.mocr": 1792.8899530394683, + "rednote-hilab/dots.ocr": 948.1699172928749, + "tesseract-5": 1198.0745767135008, + "zai-org/GLM-OCR": 1771.9449141972862 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1545.5819668204665, + 1677.4763048231573 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1303.657715609379, + 1450.9921931671456 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1378.7134004987934, + 1520.5255221452223 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1581.9013344122984, + 1717.4354346981218 + ], + "baidu/Qianfan-OCR": [ + 1475.1980495620346, + 1608.4182326690727 + ], + "baidu/Unlimited-OCR": [ + 1444.3127099702376, + 1576.1270775334935 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1356.073318299339, + 1491.574653831446 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1304.0099952704252, + 1451.005101864267 + ], + "lightonai/LightOnOCR-2-1B": [ + 1675.5077363065138, + 1841.5118490455209 + ], + "numind/NuExtract3": [ + 1543.861045393346, + 1673.428582800026 + ], + "rednote-hilab/dots.mocr": [ + 1713.8552497726523, + 1886.8502541195887 + ], + "rednote-hilab/dots.ocr": [ + 739.7565195610962, + 1080.618204193135 + ], + "tesseract-5": [ + 1061.6697888992635, + 1292.2920086620372 + ], + "zai-org/GLM-OCR": [ + 1697.6078252056416, + 1863.20408460815 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 51.21662452296482, + "PaddlePaddle/PP-OCRv6_medium": -19.43187421199127, + "PaddlePaddle/PaddleOCR-VL-1.6": -8.358553925647357, + "allenai/olmOCR-2-7B-1025-FP8": -76.41285300945515, + "baidu/Qianfan-OCR": -28.183332446089025, + "baidu/Unlimited-OCR": -38.8825864984035, + "deepseek-ai/DeepSeek-OCR": -25.449346416667595, + "deepseek-ai/DeepSeek-OCR-2": -2.9325413302346988, + "lightonai/LightOnOCR-2-1B": 8.1439668476105, + "numind/NuExtract3": -83.92207300395103, + "rednote-hilab/dots.mocr": 47.464693652837695, + "rednote-hilab/dots.ocr": 57.49436873507523, + "tesseract-5": 85.48806834881134, + "zai-org/GLM-OCR": 33.76543873514083 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 46, + "median_direct_comparisons": 5, + "min_direct_comparisons": 2, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "targeted", + "max_absolute_elo_delta": 85.48806834881134, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 36.324012616772166, + "name": "targeted", + "percentage_saved": 79.73445143256464, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 12 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 56, + "cumulative_comparisons": 376, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 64, + "cumulative_comparisons": 440, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 65, + "cumulative_comparisons": 505, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "batch_comparisons": 60, + "cumulative_comparisons": 565, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 65, + "cumulative_comparisons": 630, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 65, + "cumulative_comparisons": 695, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 12, + "batch_comparisons": 60, + "cumulative_comparisons": 755, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 12, + "batch_comparisons": 55, + "cumulative_comparisons": 810, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 12, + "batch_comparisons": 60, + "cumulative_comparisons": 870, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9824175824175824, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false + }, + { + "comparisons_consumed": 736, + "comparisons_saved": 3557, + "decision_statuses": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 6 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1624.9227279420727, + "PaddlePaddle/PP-OCRv6_medium": 1376.9484337859622, + "PaddlePaddle/PaddleOCR-VL-1.6": 1490.9995257145788, + "allenai/olmOCR-2-7B-1025-FP8": 1635.9035087399477, + "baidu/Qianfan-OCR": 1530.4176737122787, + "baidu/Unlimited-OCR": 1480.0702592753057, + "deepseek-ai/DeepSeek-OCR": 1398.8879735474673, + "deepseek-ai/DeepSeek-OCR-2": 1357.233351240866, + "lightonai/LightOnOCR-2-1B": 1737.341289759878, + "numind/NuExtract3": 1585.7734560830231, + "rednote-hilab/dots.mocr": 1810.2964669395187, + "rednote-hilab/dots.ocr": 979.3393152695397, + "tesseract-5": 1218.526808949212, + "zai-org/GLM-OCR": 1773.3392090403493 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1562.7837220710585, + 1698.634927722427 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1281.5837669618547, + 1467.5216694596666 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1409.1279647868157, + 1573.0049328566743 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1565.5036501586656, + 1717.30802096474 + ], + "baidu/Qianfan-OCR": [ + 1460.9440554985438, + 1603.4599066781986 + ], + "baidu/Unlimited-OCR": [ + 1399.4241584564875, + 1556.5219637334098 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1324.033498095611, + 1465.7100096756717 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1271.7404372036422, + 1432.668694116761 + ], + "lightonai/LightOnOCR-2-1B": [ + 1656.336243098424, + 1829.6919119009408 + ], + "numind/NuExtract3": [ + 1524.0794962793475, + 1659.3205185195495 + ], + "rednote-hilab/dots.mocr": [ + 1717.3897359508037, + 1933.8652844563953 + ], + "rednote-hilab/dots.ocr": [ + 782.4363335134382, + 1094.2262348116394 + ], + "tesseract-5": [ + 1103.2763286226277, + 1297.4078303068397 + ], + "zai-org/GLM-OCR": [ + 1703.6783067423682, + 1863.5991590878095 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 67.47334125730367, + "PaddlePaddle/PP-OCRv6_medium": -20.148126142639512, + "PaddlePaddle/PaddleOCR-VL-1.6": 28.024973217189427, + "allenai/olmOCR-2-7B-1025-FP8": -83.32762928911939, + "baidu/Qianfan-OCR": -37.676776589855535, + "baidu/Unlimited-OCR": -65.13574336324746, + "deepseek-ai/DeepSeek-OCR": -53.10905003920152, + "deepseek-ai/DeepSeek-OCR-2": -22.979196745975287, + "lightonai/LightOnOCR-2-1B": -3.9112210248765678, + "numind/NuExtract3": -103.84557970693231, + "rednote-hilab/dots.mocr": 64.87120755288811, + "rednote-hilab/dots.ocr": 88.6637667117401, + "tesseract-5": 105.94030058452245, + "zai-org/GLM-OCR": 35.15973357820394 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 41, + "median_direct_comparisons": 5, + "min_direct_comparisons": 2, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9120879120879122, + "kind": "targeted", + "max_absolute_elo_delta": 105.94030058452245, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 58.990128796044814, + "name": "targeted_size_3_min_10", + "percentage_saved": 82.8558117866294, + "practical_preferences": [ + { + "higher_ranked_model": "allenai/olmOCR-2-7B-1025-FP8", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 3.9523809523809526, + "smaller_model": "FireRedTeam/FireRed-OCR" + }, + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 115.94202898550725, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "PaddlePaddle/PP-OCRv6_medium", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Qianfan-OCR", + "larger_model": "baidu/Qianfan-OCR", + "size_ratio": 5.222222222222222, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 6 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 56, + "cumulative_comparisons": 376, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 64, + "cumulative_comparisons": 440, + "decision": { + "prefer-smaller": 3, + "resolved": 0, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 10, + "batch_comparisons": 50, + "cumulative_comparisons": 490, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 10, + "batch_comparisons": 50, + "cumulative_comparisons": 540, + "decision": { + "prefer-smaller": 2, + "resolved": 0, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 595, + "decision": { + "prefer-smaller": 5, + "resolved": 0, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 40, + "cumulative_comparisons": 635, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 7, + "batch_comparisons": 35, + "cumulative_comparisons": 670, + "decision": { + "prefer-smaller": 5, + "resolved": 0, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 36, + "cumulative_comparisons": 706, + "decision": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 6, + "batch_comparisons": 30, + "cumulative_comparisons": 736, + "decision": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9780219780219781, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false + }, + { + "comparisons_consumed": 629, + "comparisons_saved": 3664, + "decision_statuses": { + "prefer-smaller": 9, + "resolved": 1, + "unresolved": 3 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1724.8282863076872, + "PaddlePaddle/PP-OCRv6_medium": 1367.8142585858143, + "PaddlePaddle/PaddleOCR-VL-1.6": 1471.007968958292, + "allenai/olmOCR-2-7B-1025-FP8": 1613.8096859733894, + "baidu/Qianfan-OCR": 1512.007840306965, + "baidu/Unlimited-OCR": 1469.7309981503452, + "deepseek-ai/DeepSeek-OCR": 1389.8648193014194, + "deepseek-ai/DeepSeek-OCR-2": 1347.8889020867468, + "lightonai/LightOnOCR-2-1B": 1832.5442474811484, + "numind/NuExtract3": 1571.3398967045782, + "rednote-hilab/dots.mocr": 1866.7924427944317, + "rednote-hilab/dots.ocr": 956.3624072814648, + "tesseract-5": 1220.7009515760715, + "zai-org/GLM-OCR": 1655.3072944916464 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1631.9633127308584, + 1835.5512927735508 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1275.9934622181388, + 1453.9953822269133 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1386.0291700691264, + 1548.968512978189 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1534.2669924301463, + 1706.5739155813255 + ], + "baidu/Qianfan-OCR": [ + 1438.6632446967662, + 1581.4262039345153 + ], + "baidu/Unlimited-OCR": [ + 1391.4364775398828, + 1545.3622161274357 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1310.7885900380925, + 1459.0026506383879 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1258.4560876905875, + 1421.4082099568373 + ], + "lightonai/LightOnOCR-2-1B": [ + 1716.6493566657398, + 2017.2337201801101 + ], + "numind/NuExtract3": [ + 1502.7743650389643, + 1650.1436018416384 + ], + "rednote-hilab/dots.mocr": [ + 1765.7636667685692, + 2043.3260855171757 + ], + "rednote-hilab/dots.ocr": [ + 746.7472343775213, + 1073.6940532730353 + ], + "tesseract-5": [ + 1108.033202710418, + 1302.7717661367615 + ], + "zai-org/GLM-OCR": [ + 1559.5608629579697, + 1755.386958673927 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 167.3788996229182, + "PaddlePaddle/PP-OCRv6_medium": -29.282301342787378, + "PaddlePaddle/PaddleOCR-VL-1.6": 8.033416460902572, + "allenai/olmOCR-2-7B-1025-FP8": -105.42145205567772, + "baidu/Qianfan-OCR": -56.08660999516928, + "baidu/Unlimited-OCR": -75.47500448820801, + "deepseek-ai/DeepSeek-OCR": -62.13220428524937, + "deepseek-ai/DeepSeek-OCR-2": -32.32364590009456, + "lightonai/LightOnOCR-2-1B": 91.29173669639385, + "numind/NuExtract3": -118.27913908537721, + "rednote-hilab/dots.mocr": 121.36718340780112, + "rednote-hilab/dots.ocr": 65.68685872366518, + "tesseract-5": 108.11444321138197, + "zai-org/GLM-OCR": -82.8721809704989 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 36, + "median_direct_comparisons": 4, + "min_direct_comparisons": 2, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8901098901098902, + "kind": "targeted", + "max_absolute_elo_delta": 167.3788996229182, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 79.17359272935346, + "name": "targeted_size_2_min_10", + "percentage_saved": 85.3482413230841, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 115.94202898550725, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "PaddlePaddle/PP-OCRv6_medium", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Qianfan-OCR", + "larger_model": "baidu/Qianfan-OCR", + "size_ratio": 5.222222222222222, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.0, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "FireRedTeam/FireRed-OCR", + "size_ratio": 2.1, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "allenai/olmOCR-2-7B-1025-FP8", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 2.075, + "smaller_model": "numind/NuExtract3" + }, + { + "higher_ranked_model": "FireRedTeam/FireRed-OCR", + "larger_model": "FireRedTeam/FireRed-OCR", + "size_ratio": 2.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 3 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 56, + "cumulative_comparisons": 376, + "decision": { + "prefer-smaller": 2, + "resolved": 0, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 54, + "cumulative_comparisons": 430, + "decision": { + "prefer-smaller": 6, + "resolved": 0, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 7, + "batch_comparisons": 35, + "cumulative_comparisons": 465, + "decision": { + "prefer-smaller": 5, + "resolved": 0, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 40, + "cumulative_comparisons": 505, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 40, + "cumulative_comparisons": 545, + "decision": { + "prefer-smaller": 7, + "resolved": 0, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 6, + "batch_comparisons": 30, + "cumulative_comparisons": 575, + "decision": { + "prefer-smaller": 8, + "resolved": 0, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 5, + "batch_comparisons": 25, + "cumulative_comparisons": 600, + "decision": { + "prefer-smaller": 9, + "resolved": 1, + "unresolved": 3 + }, + "next_active_pairs": 3, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 3, + "batch_comparisons": 14, + "cumulative_comparisons": 614, + "decision": { + "prefer-smaller": 9, + "resolved": 1, + "unresolved": 3 + }, + "next_active_pairs": 3, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 3, + "batch_comparisons": 15, + "cumulative_comparisons": 629, + "decision": { + "prefer-smaller": 9, + "resolved": 1, + "unresolved": 3 + }, + "next_active_pairs": 3, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 10, + "size_tie_ratio": 2.0, + "spearman_rho": 0.9516483516483516, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false + }, + { + "comparisons_consumed": 735, + "comparisons_saved": 3558, + "decision_statuses": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1643.480912656966, + "PaddlePaddle/PP-OCRv6_medium": 1379.6772153920847, + "PaddlePaddle/PaddleOCR-VL-1.6": 1618.2259346774688, + "allenai/olmOCR-2-7B-1025-FP8": 1597.4359505773557, + "baidu/Qianfan-OCR": 1502.0187746993581, + "baidu/Unlimited-OCR": 1440.5804020769444, + "deepseek-ai/DeepSeek-OCR": 1380.0392388821308, + "deepseek-ai/DeepSeek-OCR-2": 1352.7679837448773, + "lightonai/LightOnOCR-2-1B": 1736.1743812456357, + "numind/NuExtract3": 1558.0228780431187, + "rednote-hilab/dots.mocr": 1848.054900386524, + "rednote-hilab/dots.ocr": 952.5729095569341, + "tesseract-5": 1209.2828405221953, + "zai-org/GLM-OCR": 1781.6656775384063 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1582.7272168095594, + 1715.7344049666015 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1268.325671762669, + 1475.5864768854833 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1538.4561165731207, + 1711.8707558513315 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1525.1036278487147, + 1674.3030249540375 + ], + "baidu/Qianfan-OCR": [ + 1428.5685390280723, + 1571.16126625356 + ], + "baidu/Unlimited-OCR": [ + 1360.8518319201428, + 1516.6676072548219 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1296.6815140899125, + 1453.5933121622338 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1260.155270219713, + 1427.9778336663262 + ], + "lightonai/LightOnOCR-2-1B": [ + 1658.1497040299382, + 1829.2478414701275 + ], + "numind/NuExtract3": [ + 1492.8410780500449, + 1632.4994317119447 + ], + "rednote-hilab/dots.mocr": [ + 1742.3816293279253, + 2016.4275336668138 + ], + "rednote-hilab/dots.ocr": [ + 751.2937734790955, + 1068.6715784480825 + ], + "tesseract-5": [ + 1086.5399312023187, + 1295.683537830649 + ], + "zai-org/GLM-OCR": [ + 1706.0396541303815, + 1883.0289738899974 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 86.03152597219696, + "PaddlePaddle/PP-OCRv6_medium": -17.419344536516974, + "PaddlePaddle/PaddleOCR-VL-1.6": 155.25138218007942, + "allenai/olmOCR-2-7B-1025-FP8": -121.7951874517114, + "baidu/Qianfan-OCR": -66.07567560277607, + "baidu/Unlimited-OCR": -104.62560056160874, + "deepseek-ai/DeepSeek-OCR": -71.95778470453797, + "deepseek-ai/DeepSeek-OCR-2": -27.444564241964144, + "lightonai/LightOnOCR-2-1B": -5.0781295391188905, + "numind/NuExtract3": -131.59615774683675, + "rednote-hilab/dots.mocr": 102.62964099989335, + "rednote-hilab/dots.ocr": 61.897360999134435, + "tesseract-5": 96.69633215750582, + "zai-org/GLM-OCR": 43.48620207626095 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 46, + "median_direct_comparisons": 5, + "min_direct_comparisons": 2, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "targeted", + "max_absolute_elo_delta": 155.25138218007942, + "max_absolute_elo_delta_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "median_absolute_elo_delta": 78.99465533836747, + "name": "targeted_size_3_min_5", + "percentage_saved": 82.87910552061496, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 115.94202898550725, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "PaddlePaddle/PP-OCRv6_medium", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 8 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 56, + "cumulative_comparisons": 376, + "decision": { + "prefer-smaller": 3, + "resolved": 0, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 10, + "batch_comparisons": 49, + "cumulative_comparisons": 425, + "decision": { + "prefer-smaller": 4, + "resolved": 0, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 9, + "batch_comparisons": 45, + "cumulative_comparisons": 470, + "decision": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 9, + "batch_comparisons": 45, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 2, + "resolved": 0, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 570, + "decision": { + "prefer-smaller": 4, + "resolved": 0, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 9, + "batch_comparisons": 45, + "cumulative_comparisons": 615, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 40, + "cumulative_comparisons": 655, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 36, + "cumulative_comparisons": 691, + "decision": { + "prefer-smaller": 4, + "resolved": 0, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 9, + "batch_comparisons": 44, + "cumulative_comparisons": 735, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 5, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9120879120879121, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false + }, + { + "comparisons_consumed": 786, + "comparisons_saved": 3507, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1613.4265061457595, + "PaddlePaddle/PP-OCRv6_medium": 1387.404191581624, + "PaddlePaddle/PaddleOCR-VL-1.6": 1462.4611566414483, + "allenai/olmOCR-2-7B-1025-FP8": 1628.6862596842361, + "baidu/Qianfan-OCR": 1541.1377796548968, + "baidu/Unlimited-OCR": 1477.7399853405952, + "deepseek-ai/DeepSeek-OCR": 1416.0457043006472, + "deepseek-ai/DeepSeek-OCR-2": 1381.398389655434, + "lightonai/LightOnOCR-2-1B": 1738.5779937997568, + "numind/NuExtract3": 1597.707417047832, + "rednote-hilab/dots.mocr": 1843.8304414058673, + "rednote-hilab/dots.ocr": 947.0957655919705, + "tesseract-5": 1197.3374054968394, + "zai-org/GLM-OCR": 1767.151003653093 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1543.5243309932873, + 1688.395945721398 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1300.8791377951052, + 1471.2457811197476 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1387.1966244410883, + 1535.1619969198853 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1564.306710374943, + 1704.4702672741244 + ], + "baidu/Qianfan-OCR": [ + 1485.6797750880469, + 1609.9797875882643 + ], + "baidu/Unlimited-OCR": [ + 1402.8844039168926, + 1551.2751298427988 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1344.1704638635304, + 1485.5716414491087 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1299.4958911109131, + 1461.6330904865256 + ], + "lightonai/LightOnOCR-2-1B": [ + 1665.5053309925438, + 1830.6998748653925 + ], + "numind/NuExtract3": [ + 1536.985021838651, + 1664.3516059543645 + ], + "rednote-hilab/dots.mocr": [ + 1756.93715701477, + 1968.734915931455 + ], + "rednote-hilab/dots.ocr": [ + 718.266120352759, + 1068.52386099673 + ], + "tesseract-5": [ + 1065.6767253171126, + 1291.62845459734 + ], + "zai-org/GLM-OCR": [ + 1698.5180973251058, + 1853.4069611201721 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 55.97711946099048, + "PaddlePaddle/PP-OCRv6_medium": -9.692368346977673, + "PaddlePaddle/PaddleOCR-VL-1.6": -0.5133958559410985, + "allenai/olmOCR-2-7B-1025-FP8": -90.54487834483098, + "baidu/Qianfan-OCR": -26.956670647237388, + "baidu/Unlimited-OCR": -67.46601729795793, + "deepseek-ai/DeepSeek-OCR": -35.951319286021544, + "deepseek-ai/DeepSeek-OCR-2": 1.1858416685925022, + "lightonai/LightOnOCR-2-1B": -2.6745169849978083, + "numind/NuExtract3": -91.91161874212344, + "rednote-hilab/dots.mocr": 98.40518201923669, + "rednote-hilab/dots.ocr": 56.42021703417083, + "tesseract-5": 84.75089713214993, + "zai-org/GLM-OCR": 28.971528190947765 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 46, + "median_direct_comparisons": 5, + "min_direct_comparisons": 2, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "targeted", + "max_absolute_elo_delta": 98.40518201923669, + "max_absolute_elo_delta_model": "rednote-hilab/dots.mocr", + "median_absolute_elo_delta": 45.96421937350601, + "name": "targeted_size_3_min_15", + "percentage_saved": 81.69112508735151, + "practical_preferences": [ + { + "higher_ranked_model": "allenai/olmOCR-2-7B-1025-FP8", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 3.9523809523809526, + "smaller_model": "FireRedTeam/FireRed-OCR" + }, + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 115.94202898550725, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "PaddlePaddle/PP-OCRv6_medium", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 7 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 56, + "cumulative_comparisons": 376, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 64, + "cumulative_comparisons": 440, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 65, + "cumulative_comparisons": 505, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 10, + "batch_comparisons": 50, + "cumulative_comparisons": 555, + "decision": { + "prefer-smaller": 2, + "resolved": 0, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 610, + "decision": { + "prefer-smaller": 1, + "resolved": 0, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "batch_comparisons": 60, + "cumulative_comparisons": 670, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 40, + "cumulative_comparisons": 710, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 36, + "cumulative_comparisons": 746, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 8, + "batch_comparisons": 40, + "cumulative_comparisons": 786, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 15, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9824175824175824, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false + }, + { + "comparisons_consumed": 810, + "comparisons_saved": 3483, + "decision_statuses": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1607.869184480983, + "PaddlePaddle/PP-OCRv6_medium": 1389.0671908582858, + "PaddlePaddle/PaddleOCR-VL-1.6": 1459.276992666841, + "allenai/olmOCR-2-7B-1025-FP8": 1641.6028731708125, + "baidu/Qianfan-OCR": 1534.835864168096, + "baidu/Unlimited-OCR": 1507.8178621579427, + "deepseek-ai/DeepSeek-OCR": 1417.789766415991, + "deepseek-ai/DeepSeek-OCR-2": 1378.4484194360455, + "lightonai/LightOnOCR-2-1B": 1748.8083433448191, + "numind/NuExtract3": 1605.3996229247402, + "rednote-hilab/dots.mocr": 1792.358569623159, + "rednote-hilab/dots.ocr": 947.6817327503769, + "tesseract-5": 1197.6390075573959, + "zai-org/GLM-OCR": 1771.4045704445111 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1546.7379528040656, + 1681.8277669015442 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1297.1150354802521, + 1478.799320575555 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1384.9487384779895, + 1530.8548793145546 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1577.1836511725007, + 1713.8314446286734 + ], + "baidu/Qianfan-OCR": [ + 1469.195183145721, + 1607.3072574750356 + ], + "baidu/Unlimited-OCR": [ + 1440.006171898046, + 1581.1878081967768 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1344.1199433971506, + 1491.0980051560161 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1293.4277890746114, + 1456.9191870387003 + ], + "lightonai/LightOnOCR-2-1B": [ + 1676.830217215075, + 1839.6471344134725 + ], + "numind/NuExtract3": [ + 1538.4106926645422, + 1675.4561894264639 + ], + "rednote-hilab/dots.mocr": [ + 1711.081227136105, + 1897.8510878321426 + ], + "rednote-hilab/dots.ocr": [ + 719.1641945568995, + 1088.585708795392 + ], + "tesseract-5": [ + 1067.4005082692627, + 1291.4384291451663 + ], + "zai-org/GLM-OCR": [ + 1702.1938165606, + 1855.603613648787 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 50.41979779621397, + "PaddlePaddle/PP-OCRv6_medium": -8.029369070315852, + "PaddlePaddle/PaddleOCR-VL-1.6": -3.697559830548471, + "allenai/olmOCR-2-7B-1025-FP8": -77.62826485825462, + "baidu/Qianfan-OCR": -33.258586134038296, + "baidu/Unlimited-OCR": -37.388140480610446, + "deepseek-ai/DeepSeek-OCR": -34.20725717067785, + "deepseek-ai/DeepSeek-OCR-2": -1.764128550795931, + "lightonai/LightOnOCR-2-1B": 7.555832560064573, + "numind/NuExtract3": -84.21941286521519, + "rednote-hilab/dots.mocr": 46.933310236528314, + "rednote-hilab/dots.ocr": 57.00618419257728, + "tesseract-5": 85.05249919270636, + "zai-org/GLM-OCR": 33.22509498236582 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 46, + "median_direct_comparisons": 5, + "min_direct_comparisons": 2, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "targeted", + "max_absolute_elo_delta": 85.05249919270636, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 35.79769882564415, + "name": "targeted_size_5_min_10", + "percentage_saved": 81.13207547169812, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 115.94202898550725, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "PaddlePaddle/PP-OCRv6_medium", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 8.3, + "smaller_model": "lightonai/LightOnOCR-2-1B" + } + ], + "resolution_summary": "sample batches exhausted: 9 adjacent pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 56, + "cumulative_comparisons": 376, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 13, + "batch_comparisons": 64, + "cumulative_comparisons": 440, + "decision": { + "prefer-smaller": 2, + "resolved": 0, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 495, + "decision": { + "prefer-smaller": 1, + "resolved": 0, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "batch_comparisons": 60, + "cumulative_comparisons": 555, + "decision": { + "prefer-smaller": 1, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 610, + "decision": { + "prefer-smaller": 2, + "resolved": 0, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "batch_comparisons": 55, + "cumulative_comparisons": 665, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 10, + "batch_comparisons": 50, + "cumulative_comparisons": 715, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 10, + "batch_comparisons": 45, + "cumulative_comparisons": 760, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 10, + "batch_comparisons": 50, + "cumulative_comparisons": 810, + "decision": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_tie_min_samples": 10, + "size_tie_ratio": 5.0, + "spearman_rho": 0.9824175824175824, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false + } + ] +} diff --git a/experiments/adaptive-stopping/round-history.csv b/experiments/adaptive-stopping/round-history.csv new file mode 100644 index 0000000..e5df111 --- /dev/null +++ b/experiments/adaptive-stopping/round-history.csv @@ -0,0 +1,71 @@ +strategy,round,sample_indices,batch_comparisons,cumulative_comparisons,active_pairs,next_active_pairs,resolved,prefer_smaller,unresolved +balanced_adaptive,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 +balanced_adaptive,2,"[5, 6, 7, 8, 9]",373,693,,12,1,0,12 +balanced_adaptive,3,"[10, 11, 12, 13, 14]",453,1146,,12,1,0,12 +balanced_adaptive,4,"[15, 16, 17, 18, 19]",455,1601,,12,1,0,12 +balanced_adaptive,5,"[20, 21, 22, 23, 24]",455,2056,,11,2,0,11 +balanced_adaptive,6,"[25, 26, 27, 28, 29]",454,2510,,10,3,0,10 +balanced_adaptive,7,"[30, 31, 32, 33, 34]",455,2965,,9,4,0,9 +balanced_adaptive,8,"[35, 36, 37, 38, 39]",455,3420,,9,4,0,9 +balanced_adaptive,9,"[40, 41, 42, 43, 44]",419,3839,,9,4,0,9 +balanced_adaptive,10,"[45, 46, 47, 48, 49]",454,4293,,9,4,0,9 +targeted,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 +targeted,2,"[5, 6, 7, 8, 9]",56,376,13,13,0,0,13 +targeted,3,"[10, 11, 12, 13, 14]",64,440,13,13,0,0,13 +targeted,4,"[15, 16, 17, 18, 19]",65,505,13,12,1,0,12 +targeted,5,"[20, 21, 22, 23, 24]",60,565,12,13,0,0,13 +targeted,6,"[25, 26, 27, 28, 29]",65,630,13,13,0,0,13 +targeted,7,"[30, 31, 32, 33, 34]",65,695,13,12,1,0,12 +targeted,8,"[35, 36, 37, 38, 39]",60,755,12,12,1,0,12 +targeted,9,"[40, 41, 42, 43, 44]",55,810,12,12,1,0,12 +targeted,10,"[45, 46, 47, 48, 49]",60,870,12,12,1,0,12 +targeted_size_3_min_10,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 +targeted_size_3_min_10,2,"[5, 6, 7, 8, 9]",56,376,13,13,0,0,13 +targeted_size_3_min_10,3,"[10, 11, 12, 13, 14]",64,440,13,10,0,3,10 +targeted_size_3_min_10,4,"[15, 16, 17, 18, 19]",50,490,10,10,1,2,10 +targeted_size_3_min_10,5,"[20, 21, 22, 23, 24]",50,540,10,11,0,2,11 +targeted_size_3_min_10,6,"[25, 26, 27, 28, 29]",55,595,11,8,0,5,8 +targeted_size_3_min_10,7,"[30, 31, 32, 33, 34]",40,635,8,7,1,5,7 +targeted_size_3_min_10,8,"[35, 36, 37, 38, 39]",35,670,7,8,0,5,8 +targeted_size_3_min_10,9,"[40, 41, 42, 43, 44]",36,706,8,6,1,6,6 +targeted_size_3_min_10,10,"[45, 46, 47, 48, 49]",30,736,6,6,1,6,6 +targeted_size_2_min_10,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 +targeted_size_2_min_10,2,"[5, 6, 7, 8, 9]",56,376,13,11,0,2,11 +targeted_size_2_min_10,3,"[10, 11, 12, 13, 14]",54,430,11,7,0,6,7 +targeted_size_2_min_10,4,"[15, 16, 17, 18, 19]",35,465,7,8,0,5,8 +targeted_size_2_min_10,5,"[20, 21, 22, 23, 24]",40,505,8,8,1,4,8 +targeted_size_2_min_10,6,"[25, 26, 27, 28, 29]",40,545,8,6,0,7,6 +targeted_size_2_min_10,7,"[30, 31, 32, 33, 34]",30,575,6,5,0,8,5 +targeted_size_2_min_10,8,"[35, 36, 37, 38, 39]",25,600,5,3,1,9,3 +targeted_size_2_min_10,9,"[40, 41, 42, 43, 44]",14,614,3,3,1,9,3 +targeted_size_2_min_10,10,"[45, 46, 47, 48, 49]",15,629,3,3,1,9,3 +targeted_size_3_min_5,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 +targeted_size_3_min_5,2,"[5, 6, 7, 8, 9]",56,376,13,10,0,3,10 +targeted_size_3_min_5,3,"[10, 11, 12, 13, 14]",49,425,10,9,0,4,9 +targeted_size_3_min_5,4,"[15, 16, 17, 18, 19]",45,470,9,9,1,3,9 +targeted_size_3_min_5,5,"[20, 21, 22, 23, 24]",45,515,9,11,0,2,11 +targeted_size_3_min_5,6,"[25, 26, 27, 28, 29]",55,570,11,9,0,4,9 +targeted_size_3_min_5,7,"[30, 31, 32, 33, 34]",45,615,9,8,1,4,8 +targeted_size_3_min_5,8,"[35, 36, 37, 38, 39]",40,655,8,8,1,4,8 +targeted_size_3_min_5,9,"[40, 41, 42, 43, 44]",36,691,8,9,0,4,9 +targeted_size_3_min_5,10,"[45, 46, 47, 48, 49]",44,735,9,8,1,4,8 +targeted_size_3_min_15,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 +targeted_size_3_min_15,2,"[5, 6, 7, 8, 9]",56,376,13,13,0,0,13 +targeted_size_3_min_15,3,"[10, 11, 12, 13, 14]",64,440,13,13,0,0,13 +targeted_size_3_min_15,4,"[15, 16, 17, 18, 19]",65,505,13,10,1,2,10 +targeted_size_3_min_15,5,"[20, 21, 22, 23, 24]",50,555,10,11,0,2,11 +targeted_size_3_min_15,6,"[25, 26, 27, 28, 29]",55,610,11,12,0,1,12 +targeted_size_3_min_15,7,"[30, 31, 32, 33, 34]",60,670,12,8,1,4,8 +targeted_size_3_min_15,8,"[35, 36, 37, 38, 39]",40,710,8,8,1,4,8 +targeted_size_3_min_15,9,"[40, 41, 42, 43, 44]",36,746,8,8,1,4,8 +targeted_size_3_min_15,10,"[45, 46, 47, 48, 49]",40,786,8,7,1,5,7 +targeted_size_5_min_10,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 +targeted_size_5_min_10,2,"[5, 6, 7, 8, 9]",56,376,13,13,0,0,13 +targeted_size_5_min_10,3,"[10, 11, 12, 13, 14]",64,440,13,11,0,2,11 +targeted_size_5_min_10,4,"[15, 16, 17, 18, 19]",55,495,11,12,0,1,12 +targeted_size_5_min_10,5,"[20, 21, 22, 23, 24]",60,555,12,11,1,1,11 +targeted_size_5_min_10,6,"[25, 26, 27, 28, 29]",55,610,11,11,0,2,11 +targeted_size_5_min_10,7,"[30, 31, 32, 33, 34]",55,665,11,10,1,2,10 +targeted_size_5_min_10,8,"[35, 36, 37, 38, 39]",50,715,10,10,1,2,10 +targeted_size_5_min_10,9,"[40, 41, 42, 43, 44]",45,760,10,10,1,2,10 +targeted_size_5_min_10,10,"[45, 46, 47, 48, 49]",50,810,10,9,1,3,9 diff --git a/experiments/adaptive-stopping/strategy-summary.csv b/experiments/adaptive-stopping/strategy-summary.csv new file mode 100644 index 0000000..ed2c71b --- /dev/null +++ b/experiments/adaptive-stopping/strategy-summary.csv @@ -0,0 +1,9 @@ +name,comparisons_consumed,comparisons_saved,percentage_saved,stopping_round,stopping_reason,kendall_tau,spearman_rho,top3_membership_matches,top3_order_matches,median_absolute_elo_delta,max_absolute_elo_delta,deterministic_across_repeats +full_stored_board,4293,0,0.0,10,reference_full_stored_board,1.0,1.0,True,True,0.0,0.0,True +balanced_adaptive,4293,0,0.0,10,sample_batches_exhausted,1.0,1.0,True,True,0.0,0.0,True +targeted,870,3423,79.73445143256464,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,36.324012616772166,85.48806834881134,True +targeted_size_3_min_10,736,3557,82.8558117866294,10,sample_batches_exhausted,0.9120879120879122,0.9780219780219781,True,False,58.990128796044814,105.94030058452245,True +targeted_size_2_min_10,629,3664,85.3482413230841,10,sample_batches_exhausted,0.8901098901098902,0.9516483516483516,False,False,79.17359272935346,167.3788996229182, +targeted_size_3_min_5,735,3558,82.87910552061496,10,sample_batches_exhausted,0.8241758241758242,0.9120879120879121,True,False,78.99465533836747,155.25138218007942, +targeted_size_3_min_15,786,3507,81.69112508735151,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,45.96421937350601,98.40518201923669, +targeted_size_5_min_10,810,3483,81.13207547169812,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,35.79769882564415,85.05249919270636, diff --git a/tests/test_adaptive_replay_experiment.py b/tests/test_adaptive_replay_experiment.py new file mode 100644 index 0000000..2bad9f0 --- /dev/null +++ b/tests/test_adaptive_replay_experiment.py @@ -0,0 +1,81 @@ +"""Regression tests for the read-only adaptive replay experiment.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +from ocr_bench.elo import ComparisonResult, Leaderboard + +_REPLAY_PATH = ( + Path(__file__).resolve().parents[1] / "experiments" / "adaptive-stopping" / "replay.py" +) +_SPEC = importlib.util.spec_from_file_location("adaptive_stopping_replay", _REPLAY_PATH) +assert _SPEC is not None and _SPEC.loader is not None +replay = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = replay +_SPEC.loader.exec_module(replay) + + +def _stored_grid(n_samples: int = 8) -> list[ComparisonResult]: + return [ + ComparisonResult(sample_idx, model_a, model_b, "A") + for sample_idx in range(n_samples) + for model_a, model_b in (("a", "b"), ("a", "c"), ("b", "c")) + ] + + +def _fixed_board() -> Leaderboard: + return Leaderboard( + elo={"a": 1600.0, "b": 1500.0, "c": 1400.0}, + elo_ci={ + "a": (1580.0, 1620.0), + "b": (1450.0, 1550.0), + "c": (1380.0, 1520.0), + }, + ) + + +def test_targeted_replay_warms_up_balanced_then_filters_pairs(monkeypatch): + stored = _stored_grid() + monkeypatch.setattr(replay, "compute_elo", lambda *args, **kwargs: _fixed_board()) + + targeted = replay.replay_strategy( + stored, + ["a", "b", "c"], + replay.StrategyConfig("targeted", "targeted"), + n_bootstrap=1, + batch_samples=1, + ) + balanced = replay.replay_strategy( + stored, + ["a", "b", "c"], + replay.StrategyConfig("balanced", "balanced"), + n_bootstrap=1, + batch_samples=1, + ) + + # Production waits for max(3 * 3 pairs, 20) outcomes: seven complete + # samples (21 rows), then only the unresolved b/c pair is selected. + assert len(targeted.comparisons) == 22 + assert len(balanced.comparisons) == 24 + assert targeted.round_history[-1]["active_pairs"] == 1 + assert targeted.comparisons[-1].model_a == "b" + assert targeted.comparisons[-1].model_b == "c" + assert targeted.stopping_reason == "sample_batches_exhausted" + + +def test_graph_metrics_report_coverage_and_connectivity(): + complete = replay.graph_metrics(_stored_grid(1), ["a", "b", "c"]) + disconnected = replay.graph_metrics( + [ComparisonResult(0, "a", "b", "A")], + ["a", "b", "c"], + ) + + assert complete["observed_pairs"] == 3 + assert complete["pair_coverage_pct"] == 100.0 + assert complete["connected"] is True + assert disconnected["observed_pairs"] == 1 + assert disconnected["connected"] is False + assert disconnected["connected_components"] == [["a", "b"], ["c"]] From 6b852f249f1fe507e0781b550bf042d2e35d1a21 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Wed, 15 Jul 2026 22:28:52 +0100 Subject: [PATCH 2/5] Evaluate targeted v2 allocation variants --- experiments/adaptive-stopping/README.md | 90 +- experiments/adaptive-stopping/elo-deltas.csv | 84 + experiments/adaptive-stopping/replay.py | 174 +- .../results-sentinels-excluded.json | 3236 +++++++++++++++- experiments/adaptive-stopping/results.json | 3447 ++++++++++++++++- .../adaptive-stopping/round-history.csv | 202 +- .../adaptive-stopping/strategy-summary.csv | 24 +- tests/test_adaptive_replay_experiment.py | 68 + 8 files changed, 7180 insertions(+), 145 deletions(-) diff --git a/experiments/adaptive-stopping/README.md b/experiments/adaptive-stopping/README.md index 071d2e2..0c60b09 100644 --- a/experiments/adaptive-stopping/README.md +++ b/experiments/adaptive-stopping/README.md @@ -1,7 +1,8 @@ # Adaptive stopping counterfactual replay -**Recommendation: revise the targeted strategy and keep it experimental.** Do not change the -balanced default yet, and do not proceed directly to a live trial from this one replay. +**Recommendation: keep targeted allocation experimental and do not run a live trial yet.** A +follow-up replay tested stronger per-pair warm-ups, periodic balanced exploration, and separating +size annotations from sampling. None passed the predeclared fidelity/savings gate. The opt-in targeted allocator delivers the expected call reduction, but not by stopping early: it uses all 10 five-sample rounds while selecting fewer pairs per round. Against the full stored @@ -10,7 +11,7 @@ board, plain targeted allocation saves **79.7%** of comparisons, while the 3× s substantially. The size rule adds only 3.1 percentage points of savings over plain targeted while increasing median/max ELO drift. -No production bug was found. This is therefore an experiment/documentation-only change. +No production bug was found. This remains an experiment/documentation-only change. ## Scope @@ -168,6 +169,45 @@ from 10 to 5 barely saves another comparison but materially worsens rank correla error. A 15-sample minimum or 5× ratio is less aggressive and tracks the full board better, but neither restores top-3 ordering or achieves early stopping. +## Follow-up targeted-v2 replay + +A second replay tested the proposed next step without changing production code. The gate was +fixed before running: + +- exact full-board top-3 order; +- Kendall τ ≥ 0.95; +- maximum absolute ELO delta ≤ 50; and +- at least 60% of stored outcomes saved. + +“Explore every 3” means every third post-warm-up allocation batch is balanced; the other two are +targeted. “Annotate 3×” computes practical smaller-model annotations on the final board but does +not let that rule remove pairs from sampling. “Size-stop 3×” does let it control sampling, matching +the production option. + +| Follow-up strategy | Used | Saved | Kendall τ | Spearman ρ | Exact top-3 order | Median abs(ΔELO) | Max abs(ΔELO) | Min pair evidence | Gate | +|---|---:|---:|---:|---:|---|---:|---:|---:|---| +| Per-pair warm-up 5 | 1,153 | 73.1% | 0.912 | 0.969 | yes | 74.0 | 157.5 | 5 | **fail** | +| Per-pair warm-up 10 | 1,561 | 63.6% | 0.934 | 0.982 | yes | 50.0 | 99.1 | 10 | **fail** | +| Balanced exploration every 3 | 2,023 | 52.9% | 0.956 | 0.991 | no | 31.8 | 98.6 | 17 | **fail** | +| Warm-up 5 + explore 3 + annotate 3× | 1,946 | 54.7% | 0.934 | 0.987 | no | 29.8 | 88.2 | 15 | **fail** | +| Warm-up 10 + explore 3 + annotate 3× | 2,313 | 46.1% | 0.956 | 0.991 | yes | 37.1 | 99.6 | 18 | **fail** | +| Warm-up 5 + explore 3 + size-stop 3× | 1,820 | 57.6% | 0.934 | 0.987 | no | 31.8 | 90.8 | 15 | **fail** | + +The candidate suggested after the first replay—warm-up 5 plus every-third-batch exploration—did +not pass. Periodic balanced batches improved global rank correlation and reduced median ELO drift, +but consumed enough outcomes to miss the savings gate and still left a large worst-model delta. +Warm-up floors alone restored the full-board top-3 order, but did not stabilize the rest of the +board. No follow-up strategy met its stopping criteria before round 10. + +Separating annotation from allocation was directionally preferable: with warm-up 5 and exploration, +annotation-only used 1,946 outcomes and had max drift 88.2, while size-controlled stopping used +1,820 and had max drift 90.8. The additional 126-outcome saving is not enough evidence to let a +practical deployment preference influence statistical allocation. + +The exact order of the full board's top three is itself weakly identified—their CIs overlap—so it +should not become a universal product criterion. It remains useful here as a deliberately strict +counterfactual fidelity check, alongside full-order correlation and ELO drift. + ## Current sentinel-policy robustness check The primary result must use the requested 4,293-row stored board. However, current production @@ -181,15 +221,19 @@ conclusion: | Targeted | 753 | 79.5% | 0.923 | 0.984 | 3/3, no | 27.0 / 69.0 | | Targeted + 3×, min 10 | 624 | 83.0% | 0.897 | 0.973 | 3/3, no | 48.6 / 86.0 | -All three adaptive runs still exhaust the sample batches. See -[`results-sentinels-excluded.json`](results-sentinels-excluded.json) for full details. +All three adaptive runs still exhaust the sample batches. The targeted-v2 variants also fail the +predeclared gate under current sentinel handling. The closest on ELO drift—warm-up 5 plus +exploration 3, annotation-only—uses 1,684 outcomes (54.1% saved), has median/max absolute ELO +deltas of 18.1/57.0, and still swaps the statistically unresolved models in ranks 2 and 3. +See [`results-sentinels-excluded.json`](results-sentinels-excluded.json) for full details. ## Determinism -Each of the four primary strategies was replayed twice. Selected comparison keys, interim rank -orders, ELOs, bootstrap CIs, adjacent-pair decisions, final annotations, stop round, and stop -reason matched exactly. This confirms deterministic execution in the tested environment, helped -by the fixed bootstrap seed and deterministic equal-ELO tie-break. +Each of the four primary strategies and the warm-up-5/exploration-3 annotation-only candidate was +replayed twice. Selected comparison keys, interim rank orders, ELOs, bootstrap CIs, adjacent-pair +decisions, final annotations, stop round, and stop reason matched exactly. This confirms +deterministic execution in the tested environment, helped by the fixed bootstrap seed and +deterministic equal-ELO tie-break. It does not establish reproducibility across different SciPy/NumPy versions or statistical validity under repeated data collection. @@ -220,19 +264,22 @@ validity under repeated data collection. ## Recommendation and next experiment -**Revise strategy; keep `targeted` opt-in and `balanced` as the default.** Specifically, evaluate a -revision that: +**Keep `targeted` opt-in, retain `balanced` as the default, and do not implement the tested v2 +variants in production.** The follow-up falsified the simple “more warm-up plus periodic balanced +batches” revision under the predeclared gate. + +The next useful direction is a different estimator/design rather than more threshold tuning: -1. uses a per-pair balanced warm-up floor rather than only the aggregate 273-outcome threshold; -2. injects periodic balanced/exploration batches so early adjacency does not permanently starve - other edges; -3. monitors rank/top-k stability and ELO drift across rounds, not only overlapping adjacent - marginal CIs; and -4. evaluates page-clustered or policy-aware uncertainty for adaptive runs. +1. compare fixed balanced subsampling at matched budgets (for example 700, 1,200, and 2,000 + outcomes) against outcome-conditioned targeting; +2. test a predeclared mixed allocation with random exploration independent of interim winners; +3. evaluate top-k membership and pairwise decisions separately from exact ordering among + statistically unresolved models; and +4. use page-clustered, policy-aware uncertainty before considering a live adaptive trial. -Replay that revision on several existing result boards before one explicitly budgeted live trial. -The 3× practical rule can remain as an annotation/stopping preference, but this board does not -support making it a default or treating it as statistical resolution. +Keep the 3× rule as a post-hoc deployment annotation only during that work. This board does not +support letting it influence sampling, making it a default, or treating it as statistical +resolution. ## Reproduce @@ -243,7 +290,8 @@ uv sync --dev uv run python experiments/adaptive-stopping/replay.py ``` -The full run takes about five minutes on the machine used for this report. The pinned dataset +The full primary, sensitivity, and targeted-v2 run takes about eleven minutes on the machine used +for this report. The pinned dataset revision is downloaded read-only. The script contains no judge backend construction and no Hub push/upload call. diff --git a/experiments/adaptive-stopping/elo-deltas.csv b/experiments/adaptive-stopping/elo-deltas.csv index 8e3fe79..85e0b39 100644 --- a/experiments/adaptive-stopping/elo-deltas.csv +++ b/experiments/adaptive-stopping/elo-deltas.csv @@ -111,3 +111,87 @@ targeted_size_5_min_10,rednote-hilab/dots.mocr,1792.358569623159,1745.4252593866 targeted_size_5_min_10,rednote-hilab/dots.ocr,947.6817327503769,890.6755485577996,57.00618419257728,57.00618419257728 targeted_size_5_min_10,tesseract-5,1197.6390075573959,1112.5865083646895,85.05249919270636,85.05249919270636 targeted_size_5_min_10,zai-org/GLM-OCR,1771.4045704445111,1738.1794754621453,33.22509498236582,33.22509498236582 +targeted_v2_warmup_5,FireRedTeam/FireRed-OCR,1633.0661587781437,1557.449386684769,75.61677209337472,75.61677209337472 +targeted_v2_warmup_5,PaddlePaddle/PP-OCRv6_medium,1378.5509421275347,1397.0965599286017,-18.54561780106701,18.54561780106701 +targeted_v2_warmup_5,PaddlePaddle/PaddleOCR-VL-1.6,1489.5851365342287,1462.9745524973894,26.610584036839327,26.610584036839327 +targeted_v2_warmup_5,allenai/olmOCR-2-7B-1025-FP8,1617.7060474973778,1719.231138029067,-101.52509053168933,101.52509053168933 +targeted_v2_warmup_5,baidu/Qianfan-OCR,1539.3832484613636,1568.0944503021342,-28.711201840770627,28.711201840770627 +targeted_v2_warmup_5,baidu/Unlimited-OCR,1520.2549659248068,1545.2060026385532,-24.951036713746362,24.951036713746362 +targeted_v2_warmup_5,deepseek-ai/DeepSeek-OCR,1437.8082997396057,1451.9970235866688,-14.188723847063102,14.188723847063102 +targeted_v2_warmup_5,deepseek-ai/DeepSeek-OCR-2,1393.7406984861343,1380.2125479868414,13.528150499292906,13.528150499292906 +targeted_v2_warmup_5,lightonai/LightOnOCR-2-1B,1663.2040351738883,1741.2525107847546,-78.04847561086626,78.04847561086626 +targeted_v2_warmup_5,numind/NuExtract3,1583.5634855810165,1689.6190357899554,-106.05555020893894,106.05555020893894 +targeted_v2_warmup_5,rednote-hilab/dots.mocr,1817.811063950996,1745.4252593866306,72.38580456436534,72.38580456436534 +targeted_v2_warmup_5,rednote-hilab/dots.ocr,1048.1461002900596,890.6755485577996,157.47055173226,157.47055173226 +targeted_v2_warmup_5,tesseract-5,1219.1549769230849,1112.5865083646895,106.56846855839535,106.56846855839535 +targeted_v2_warmup_5,zai-org/GLM-OCR,1658.024840531759,1738.1794754621453,-80.15463493038624,80.15463493038624 +targeted_v2_warmup_10,FireRedTeam/FireRed-OCR,1609.5530540751233,1557.449386684769,52.1036673903543,52.1036673903543 +targeted_v2_warmup_10,PaddlePaddle/PP-OCRv6_medium,1405.5694788204771,1397.0965599286017,8.472918891875452,8.472918891875452 +targeted_v2_warmup_10,PaddlePaddle/PaddleOCR-VL-1.6,1504.841469006784,1462.9745524973894,41.86691650939474,41.86691650939474 +targeted_v2_warmup_10,allenai/olmOCR-2-7B-1025-FP8,1638.649188052229,1719.231138029067,-80.58194997683813,80.58194997683813 +targeted_v2_warmup_10,baidu/Qianfan-OCR,1561.8338533353342,1568.0944503021342,-6.260596966799994,6.260596966799994 +targeted_v2_warmup_10,baidu/Unlimited-OCR,1550.500239335426,1545.2060026385532,5.294236696872758,5.294236696872758 +targeted_v2_warmup_10,deepseek-ai/DeepSeek-OCR,1473.6932550160866,1451.9970235866688,21.69623142941782,21.69623142941782 +targeted_v2_warmup_10,deepseek-ai/DeepSeek-OCR-2,1450.3162027749963,1380.2125479868414,70.10365478815493,70.10365478815493 +targeted_v2_warmup_10,lightonai/LightOnOCR-2-1B,1678.0896999616534,1741.2525107847546,-63.162810823101154,63.162810823101154 +targeted_v2_warmup_10,numind/NuExtract3,1603.1064302580833,1689.6190357899554,-86.51260553187217,86.51260553187217 +targeted_v2_warmup_10,rednote-hilab/dots.mocr,1703.374304097687,1745.4252593866306,-42.05095528894367,42.05095528894367 +targeted_v2_warmup_10,rednote-hilab/dots.ocr,989.7414932324835,890.6755485577996,99.06594467468392,99.06594467468392 +targeted_v2_warmup_10,tesseract-5,1160.5043899819825,1112.5865083646895,47.917881617293006,47.917881617293006 +targeted_v2_warmup_10,zai-org/GLM-OCR,1670.2269420516536,1738.1794754621453,-67.9525334104917,67.9525334104917 +targeted_v2_explore_every_3,FireRedTeam/FireRed-OCR,1630.6920706317392,1557.449386684769,73.24268394697015,73.24268394697015 +targeted_v2_explore_every_3,PaddlePaddle/PP-OCRv6_medium,1399.2162696618263,1397.0965599286017,2.1197097332246813,2.1197097332246813 +targeted_v2_explore_every_3,PaddlePaddle/PaddleOCR-VL-1.6,1482.4525331108623,1462.9745524973894,19.477980613472937,19.477980613472937 +targeted_v2_explore_every_3,allenai/olmOCR-2-7B-1025-FP8,1734.9551475708581,1719.231138029067,15.72400954179102,15.72400954179102 +targeted_v2_explore_every_3,baidu/Qianfan-OCR,1599.5590661571557,1568.0944503021342,31.46461585502152,31.46461585502152 +targeted_v2_explore_every_3,baidu/Unlimited-OCR,1545.0083820431003,1545.2060026385532,-0.19762059545291777,0.19762059545291777 +targeted_v2_explore_every_3,deepseek-ai/DeepSeek-OCR,1413.1052520605142,1451.9970235866688,-38.89177152615457,38.89177152615457 +targeted_v2_explore_every_3,deepseek-ai/DeepSeek-OCR-2,1305.1771911858696,1380.2125479868414,-75.03535680097184,75.03535680097184 +targeted_v2_explore_every_3,lightonai/LightOnOCR-2-1B,1777.5430173968446,1741.2525107847546,36.290506612089985,36.290506612089985 +targeted_v2_explore_every_3,numind/NuExtract3,1717.252180625907,1689.6190357899554,27.633144835951498,27.633144835951498 +targeted_v2_explore_every_3,rednote-hilab/dots.mocr,1772.3700975287675,1745.4252593866306,26.94483814213686,26.94483814213686 +targeted_v2_explore_every_3,rednote-hilab/dots.ocr,792.1181443689077,890.6755485577996,-98.55740418889195,98.55740418889195 +targeted_v2_explore_every_3,tesseract-5,1060.215191730965,1112.5865083646895,-52.37131663372452,52.37131663372452 +targeted_v2_explore_every_3,zai-org/GLM-OCR,1770.335455926683,1738.1794754621453,32.155980464537606,32.155980464537606 +targeted_v2_warmup_5_explore_3_annotate_3x,FireRedTeam/FireRed-OCR,1563.6216723985356,1557.449386684769,6.1722857137665414,6.1722857137665414 +targeted_v2_warmup_5_explore_3_annotate_3x,PaddlePaddle/PP-OCRv6_medium,1359.1283790504326,1397.0965599286017,-37.96818087816905,37.96818087816905 +targeted_v2_warmup_5_explore_3_annotate_3x,PaddlePaddle/PaddleOCR-VL-1.6,1462.1563493606204,1462.9745524973894,-0.8182031367689433,0.8182031367689433 +targeted_v2_warmup_5_explore_3_annotate_3x,allenai/olmOCR-2-7B-1025-FP8,1678.5702750473943,1719.231138029067,-40.660862981672835,40.660862981672835 +targeted_v2_warmup_5_explore_3_annotate_3x,baidu/Qianfan-OCR,1563.059300514666,1568.0944503021342,-5.035149787468299,5.035149787468299 +targeted_v2_warmup_5_explore_3_annotate_3x,baidu/Unlimited-OCR,1523.620796710941,1545.2060026385532,-21.58520592761215,21.58520592761215 +targeted_v2_warmup_5_explore_3_annotate_3x,deepseek-ai/DeepSeek-OCR,1434.8844874590588,1451.9970235866688,-17.11253612760993,17.11253612760993 +targeted_v2_warmup_5_explore_3_annotate_3x,deepseek-ai/DeepSeek-OCR-2,1389.3976050877043,1380.2125479868414,9.185057100862878,9.185057100862878 +targeted_v2_warmup_5_explore_3_annotate_3x,lightonai/LightOnOCR-2-1B,1695.4889972978024,1741.2525107847546,-45.763513486952206,45.763513486952206 +targeted_v2_warmup_5_explore_3_annotate_3x,numind/NuExtract3,1636.9966241204509,1689.6190357899554,-52.62241166950457,52.62241166950457 +targeted_v2_warmup_5_explore_3_annotate_3x,rednote-hilab/dots.mocr,1792.726987698778,1745.4252593866306,47.30172831214736,47.30172831214736 +targeted_v2_warmup_5_explore_3_annotate_3x,rednote-hilab/dots.ocr,978.8284730418867,890.6755485577996,88.15292448408707,88.15292448408707 +targeted_v2_warmup_5_explore_3_annotate_3x,tesseract-5,1181.7368334994678,1112.5865083646895,69.15032513477831,69.15032513477831 +targeted_v2_warmup_5_explore_3_annotate_3x,zai-org/GLM-OCR,1739.7832187122617,1738.1794754621453,1.6037432501163948,1.6037432501163948 +targeted_v2_warmup_10_explore_3_annotate_3x,FireRedTeam/FireRed-OCR,1566.505914167425,1557.449386684769,9.056527482656065,9.056527482656065 +targeted_v2_warmup_10_explore_3_annotate_3x,PaddlePaddle/PP-OCRv6_medium,1399.588726777184,1397.0965599286017,2.4921668485824284,2.4921668485824284 +targeted_v2_warmup_10_explore_3_annotate_3x,PaddlePaddle/PaddleOCR-VL-1.6,1493.208336041072,1462.9745524973894,30.233783543682648,30.233783543682648 +targeted_v2_warmup_10_explore_3_annotate_3x,allenai/olmOCR-2-7B-1025-FP8,1660.6962647727582,1719.231138029067,-58.534873256308856,58.534873256308856 +targeted_v2_warmup_10_explore_3_annotate_3x,baidu/Qianfan-OCR,1560.8388644628626,1568.0944503021342,-7.255585839271589,7.255585839271589 +targeted_v2_warmup_10_explore_3_annotate_3x,baidu/Unlimited-OCR,1552.4905896931555,1545.2060026385532,7.284587054602298,7.284587054602298 +targeted_v2_warmup_10_explore_3_annotate_3x,deepseek-ai/DeepSeek-OCR,1481.1312209253567,1451.9970235866688,29.134197338687954,29.134197338687954 +targeted_v2_warmup_10_explore_3_annotate_3x,deepseek-ai/DeepSeek-OCR-2,1435.0430425115603,1380.2125479868414,54.830494524718915,54.830494524718915 +targeted_v2_warmup_10_explore_3_annotate_3x,lightonai/LightOnOCR-2-1B,1703.4194733050372,1741.2525107847546,-37.833037479717404,37.833037479717404 +targeted_v2_warmup_10_explore_3_annotate_3x,numind/NuExtract3,1625.1807493413353,1689.6190357899554,-64.43828644862015,64.43828644862015 +targeted_v2_warmup_10_explore_3_annotate_3x,rednote-hilab/dots.mocr,1704.1059067393662,1745.4252593866306,-41.319352647264395,41.319352647264395 +targeted_v2_warmup_10_explore_3_annotate_3x,rednote-hilab/dots.ocr,990.3151628354453,890.6755485577996,99.6396142776457,99.6396142776457 +targeted_v2_warmup_10_explore_3_annotate_3x,tesseract-5,1148.9315733513142,1112.5865083646895,36.34506498662472,36.34506498662472 +targeted_v2_warmup_10_explore_3_annotate_3x,zai-org/GLM-OCR,1678.5441750761272,1738.1794754621453,-59.635300386018116,59.635300386018116 +targeted_v2_warmup_5_explore_3_size_stop_3x,FireRedTeam/FireRed-OCR,1568.1460930084588,1557.449386684769,10.696706323689796,10.696706323689796 +targeted_v2_warmup_5_explore_3_size_stop_3x,PaddlePaddle/PP-OCRv6_medium,1357.4682136865583,1397.0965599286017,-39.628346242043335,39.628346242043335 +targeted_v2_warmup_5_explore_3_size_stop_3x,PaddlePaddle/PaddleOCR-VL-1.6,1466.904200168557,1462.9745524973894,3.929647671167686,3.929647671167686 +targeted_v2_warmup_5_explore_3_size_stop_3x,allenai/olmOCR-2-7B-1025-FP8,1674.0928720139548,1719.231138029067,-45.13826601511232,45.13826601511232 +targeted_v2_warmup_5_explore_3_size_stop_3x,baidu/Qianfan-OCR,1564.7900946582236,1568.0944503021342,-3.3043556439106396,3.3043556439106396 +targeted_v2_warmup_5_explore_3_size_stop_3x,baidu/Unlimited-OCR,1521.3045615355204,1545.2060026385532,-23.901441103032766,23.901441103032766 +targeted_v2_warmup_5_explore_3_size_stop_3x,deepseek-ai/DeepSeek-OCR,1431.8866475038626,1451.9970235866688,-20.11037608280617,20.11037608280617 +targeted_v2_warmup_5_explore_3_size_stop_3x,deepseek-ai/DeepSeek-OCR-2,1396.756918375702,1380.2125479868414,16.54437038886067,16.54437038886067 +targeted_v2_warmup_5_explore_3_size_stop_3x,lightonai/LightOnOCR-2-1B,1687.7204464553765,1741.2525107847546,-53.532064329378045,53.532064329378045 +targeted_v2_warmup_5_explore_3_size_stop_3x,numind/NuExtract3,1636.636432568152,1689.6190357899554,-52.9826032218034,52.9826032218034 +targeted_v2_warmup_5_explore_3_size_stop_3x,rednote-hilab/dots.mocr,1802.3207792691564,1745.4252593866306,56.89551988252583,56.89551988252583 +targeted_v2_warmup_5_explore_3_size_stop_3x,rednote-hilab/dots.ocr,981.4488359073703,890.6755485577996,90.77328734957064,90.77328734957064 +targeted_v2_warmup_5_explore_3_size_stop_3x,tesseract-5,1181.4670720477113,1112.5865083646895,68.88056368302182,68.88056368302182 +targeted_v2_warmup_5_explore_3_size_stop_3x,zai-org/GLM-OCR,1729.0568328013956,1738.1794754621453,-9.122642660749761,9.122642660749761 diff --git a/experiments/adaptive-stopping/replay.py b/experiments/adaptive-stopping/replay.py index e0f9ae7..ec54a2b 100644 --- a/experiments/adaptive-stopping/replay.py +++ b/experiments/adaptive-stopping/replay.py @@ -50,6 +50,9 @@ class StrategyConfig: kind: StrategyKind size_tie_ratio: float | None = None size_tie_min_samples: int = 10 + warmup_min_per_pair: int | None = None + balanced_every_n_post_warmup_batches: int | None = None + size_rule_controls_sampling: bool = True @dataclass @@ -79,6 +82,43 @@ class ReplayResult: StrategyConfig("targeted_size_5_min_10", "targeted", 5.0, 10), ) +FOLLOWUP_CONFIGS = ( + StrategyConfig("targeted_v2_warmup_5", "targeted", warmup_min_per_pair=5), + StrategyConfig("targeted_v2_warmup_10", "targeted", warmup_min_per_pair=10), + StrategyConfig( + "targeted_v2_explore_every_3", + "targeted", + balanced_every_n_post_warmup_batches=3, + ), + StrategyConfig( + "targeted_v2_warmup_5_explore_3_annotate_3x", + "targeted", + size_tie_ratio=3.0, + size_tie_min_samples=10, + warmup_min_per_pair=5, + balanced_every_n_post_warmup_batches=3, + size_rule_controls_sampling=False, + ), + StrategyConfig( + "targeted_v2_warmup_10_explore_3_annotate_3x", + "targeted", + size_tie_ratio=3.0, + size_tie_min_samples=10, + warmup_min_per_pair=10, + balanced_every_n_post_warmup_batches=3, + size_rule_controls_sampling=False, + ), + StrategyConfig( + "targeted_v2_warmup_5_explore_3_size_stop_3x", + "targeted", + size_tie_ratio=3.0, + size_tie_min_samples=10, + warmup_min_per_pair=5, + balanced_every_n_post_warmup_batches=3, + size_rule_controls_sampling=True, + ), +) + def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) @@ -104,7 +144,12 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( "--skip-sensitivity", action="store_true", - help="Run only the four primary configurations.", + help="Skip the size ratio/min-evidence sensitivity configurations.", + ) + parser.add_argument( + "--skip-followup", + action="store_true", + help="Skip targeted-v2 warm-up and balanced-exploration configurations.", ) parser.add_argument( "--exclude-sentinel-comparisons", @@ -171,12 +216,17 @@ def _classify( board: Leaderboard, comparisons: Sequence[ComparisonResult], config: StrategyConfig, + *, + for_sampling: bool = False, ) -> list[AdjacentPairDecision]: """Call the production classifier with the production registry sizes.""" + size_tie_ratio = config.size_tie_ratio + if for_sampling and not config.size_rule_controls_sampling: + size_tie_ratio = None return classify_adjacent_pairs( board, comparison_pair_counts(comparisons), - size_tie_ratio=config.size_tie_ratio, + size_tie_ratio=size_tie_ratio, size_tie_min_samples=config.size_tie_min_samples, parameter_counts=model_parameter_counts(), ) @@ -191,6 +241,22 @@ def _status_counts(decisions: Sequence[AdjacentPairDecision]) -> dict[str, int]: } +def _warmup_ready( + pair_counts: Counter[tuple[str, str]], + *, + n_pairs: int, + total_comparisons: int, + min_before_check: int, + min_per_pair: int | None, +) -> bool: + """Whether aggregate and optional per-pair balanced warm-up gates are met.""" + if total_comparisons < min_before_check: + return False + if min_per_pair is None: + return True + return len(pair_counts) == n_pairs and min(pair_counts.values(), default=0) >= min_per_pair + + def replay_strategy( stored: Sequence[ComparisonResult], model_names: Sequence[str], @@ -199,11 +265,12 @@ def replay_strategy( n_bootstrap: int = 1000, batch_samples: int = BATCH_SAMPLES, ) -> ReplayResult: - """Replay one strategy with the production batching and decision rules. + """Replay one strategy with production decisions and configured allocation. - The first targeted allocation is balanced. As in ``cmd_judge``, decisions - are first checked after ``max(3 * n_pairs, 20)`` accumulated outcomes and - targeted allocation then follows the currently unresolved adjacent pairs. + The current strategy checks after ``max(3 * n_pairs, 20)`` outcomes. Follow-up + configurations may additionally require a per-pair warm-up floor, inject a + balanced batch after every N post-warm-up batches, or keep the size rule as an + annotation without allowing it to remove pairs from sampling. """ if not stored: raise ValueError("Cannot replay an empty comparison set") @@ -233,35 +300,60 @@ def replay_strategy( decisions: list[AdjacentPairDecision] = [] stopping_round = math.ceil(len(sample_order) / batch_samples) stopping_reason = "sample_batches_exhausted" + post_warmup_batches = 0 for round_number, start in enumerate(range(0, len(sample_order), batch_samples), start=1): batch_indices = sample_order[start : start + batch_samples] + allocation_pairs = active_pairs + if config.kind == "targeted" and active_pairs is None: + allocation_mode = "balanced-warmup" + elif config.kind == "targeted": + post_warmup_batches += 1 + exploration_every = config.balanced_every_n_post_warmup_batches + if exploration_every and post_warmup_batches % exploration_every == 0: + allocation_pairs = None + allocation_mode = "balanced-exploration" + else: + allocation_mode = "targeted" + else: + allocation_mode = "balanced" + batch = [ comparison for sample_idx in batch_indices for comparison in grouped.get(sample_idx, []) - if active_pairs is None - or normalize_model_pair(comparison.model_a, comparison.model_b) in active_pairs + if allocation_pairs is None + or normalize_model_pair(comparison.model_a, comparison.model_b) in allocation_pairs ] # Production skips the decision check when build_comparisons returns no rows. if not batch: continue accumulated.extend(batch) - if len(accumulated) < min_before_check: + pair_counts = comparison_pair_counts(accumulated) + if not _warmup_ready( + pair_counts, + n_pairs=n_pairs, + total_comparisons=len(accumulated), + min_before_check=min_before_check, + min_per_pair=config.warmup_min_per_pair, + ): round_history.append( { "round": round_number, "sample_indices": batch_indices, "batch_comparisons": len(batch), "cumulative_comparisons": len(accumulated), + "allocation_mode": allocation_mode, "active_pairs": None if active_pairs is None else len(active_pairs), + "observed_pairs": len(pair_counts), + "min_direct_comparisons": min(pair_counts.values(), default=0), "decision": "warm-up", } ) continue board = compute_elo(accumulated, list(model_names), n_bootstrap=n_bootstrap) - decisions = _classify(board, accumulated, config) + decisions = _classify(board, accumulated, config, for_sampling=True) next_pairs = unresolved_pairs(decisions) round_history.append( { @@ -269,6 +361,7 @@ def replay_strategy( "sample_indices": batch_indices, "batch_comparisons": len(batch), "cumulative_comparisons": len(accumulated), + "allocation_mode": allocation_mode, "active_pairs": None if active_pairs is None else len(active_pairs), "next_active_pairs": len(next_pairs), "decision": _status_counts(decisions), @@ -388,27 +481,48 @@ def strategy_metrics( } absolute_deltas = [abs(delta) for delta in elo_deltas.values()] statuses = _status_counts(result.decisions) + allocation_statuses = _status_counts( + _classify(result.board, result.comparisons, result.config, for_sampling=True) + ) if result.stopping_reason == "adaptive_criteria_met": resolution = ( - f"criteria met: {statuses['resolved']} statistically resolved, " - f"{statuses['prefer-smaller']} practical preferences" + f"allocation criteria met: {allocation_statuses['resolved']} statistically resolved, " + f"{allocation_statuses['prefer-smaller']} sampling preferences" ) elif result.config.kind == "full": resolution = "reference only; no adaptive stop" else: - resolution = f"sample batches exhausted: {statuses['unresolved']} adjacent pairs unresolved" + resolution = ( + "sample batches exhausted: " + f"{allocation_statuses['unresolved']} allocation pairs unresolved" + ) full_count = len(reference.comparisons) + percentage_saved = 100.0 * (full_count - len(result.comparisons)) / full_count top3_reference = reference_rank[:3] top3_strategy = strategy_rank[:3] + top3_order_matches = top3_strategy == top3_reference + max_absolute_delta = max(absolute_deltas) + followup_gate = { + "top3_order_exact": top3_order_matches, + "kendall_tau_at_least_0_95": float(kendall) >= 0.95, + "max_absolute_elo_delta_at_most_50": max_absolute_delta <= 50.0, + "percentage_saved_at_least_60": percentage_saved >= 60.0, + } + followup_gate["passed"] = all(followup_gate.values()) return { "name": result.config.name, "kind": result.config.kind, "size_tie_ratio": result.config.size_tie_ratio, "size_tie_min_samples": result.config.size_tie_min_samples, + "warmup_min_per_pair": result.config.warmup_min_per_pair, + "balanced_every_n_post_warmup_batches": ( + result.config.balanced_every_n_post_warmup_batches + ), + "size_rule_controls_sampling": result.config.size_rule_controls_sampling, "comparisons_consumed": len(result.comparisons), "comparisons_saved": full_count - len(result.comparisons), - "percentage_saved": 100.0 * (full_count - len(result.comparisons)) / full_count, + "percentage_saved": percentage_saved, "stopping_round": result.stopping_round, "stopping_reason": result.stopping_reason, "resolution_summary": resolution, @@ -418,15 +532,17 @@ def strategy_metrics( "top3": top3_strategy, "top3_membership_matches": set(top3_strategy) == set(top3_reference), "top3_membership_overlap": len(set(top3_strategy) & set(top3_reference)), - "top3_order_matches": top3_strategy == top3_reference, + "top3_order_matches": top3_order_matches, "elo": result.board.elo, "elo_ci": result.board.elo_ci, "elo_delta_from_full": elo_deltas, "median_absolute_elo_delta": statistics.median(absolute_deltas), - "max_absolute_elo_delta": max(absolute_deltas), + "max_absolute_elo_delta": max_absolute_delta, "max_absolute_elo_delta_model": max(elo_deltas, key=lambda model: abs(elo_deltas[model])), "decision_statuses": statuses, + "allocation_decision_statuses": allocation_statuses, "practical_preferences": _preference_rows(result.decisions), + "followup_acceptance_gate": followup_gate, "graph": graph_metrics(result.comparisons, model_names), "deterministic_across_repeats": deterministic, "round_history": result.round_history, @@ -447,13 +563,16 @@ def _write_csvs(output_dir: Path, metrics: Sequence[dict[str, Any]]) -> None: "top3_order_matches", "median_absolute_elo_delta", "max_absolute_elo_delta", + "followup_gate_passed", "deterministic_across_repeats", ] with (output_dir / "strategy-summary.csv").open("w", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=summary_fields, lineterminator="\n") writer.writeheader() for metric in metrics: - writer.writerow({field: metric[field] for field in summary_fields}) + row = {field: metric.get(field) for field in summary_fields} + row["followup_gate_passed"] = metric["followup_acceptance_gate"]["passed"] + writer.writerow(row) with (output_dir / "elo-deltas.csv").open("w", newline="") as handle: writer = csv.DictWriter( @@ -483,8 +602,10 @@ def _write_csvs(output_dir: Path, metrics: Sequence[dict[str, Any]]) -> None: "sample_indices", "batch_comparisons", "cumulative_comparisons", + "allocation_mode", "active_pairs", "next_active_pairs", + "min_direct_comparisons", "resolved", "prefer_smaller", "unresolved", @@ -503,8 +624,10 @@ def _write_csvs(output_dir: Path, metrics: Sequence[dict[str, Any]]) -> None: "sample_indices": json.dumps(row["sample_indices"]), "batch_comparisons": row["batch_comparisons"], "cumulative_comparisons": row["cumulative_comparisons"], + "allocation_mode": row.get("allocation_mode"), "active_pairs": row.get("active_pairs"), "next_active_pairs": row.get("next_active_pairs"), + "min_direct_comparisons": row.get("min_direct_comparisons"), "resolved": decision.get("resolved"), "prefer_smaller": decision.get("prefer-smaller"), "unresolved": decision.get("unresolved"), @@ -528,16 +651,20 @@ def main() -> None: configs = list(PRIMARY_CONFIGS) if not args.skip_sensitivity: configs.extend(SENSITIVITY_CONFIGS) + if not args.skip_followup: + configs.extend(FOLLOWUP_CONFIGS) results: dict[str, ReplayResult] = {} deterministic: dict[str, bool | None] = {} - primary_names = {config.name for config in PRIMARY_CONFIGS} + repeat_names = {config.name for config in PRIMARY_CONFIGS} | { + "targeted_v2_warmup_5_explore_3_annotate_3x" + } for config in configs: print(f"Replaying {config.name} ...", flush=True) first = replay_strategy(stored, model_names, config, n_bootstrap=args.bootstrap) results[config.name] = first signatures = [_determinism_signature(first)] - repeats = args.repeats if config.name in primary_names else 1 + repeats = args.repeats if config.name in repeat_names else 1 for _ in range(1, repeats): repeated = replay_strategy( stored, @@ -575,7 +702,14 @@ def main() -> None: "min_before_check": max(3 * (len(model_names) * (len(model_names) - 1) // 2), 20), "bootstrap_replicates": args.bootstrap, "bootstrap_seed": 42, - "primary_repeats": args.repeats, + "repeats": args.repeats, + "repeated_strategy_names": sorted(repeat_names & {config.name for config in configs}), + "followup_acceptance_gate": { + "top3_order_exact": True, + "kendall_tau_minimum": 0.95, + "max_absolute_elo_delta_maximum": 50.0, + "percentage_saved_minimum": 60.0, + }, "production_helpers": [ "compute_elo", "classify_adjacent_pairs", diff --git a/experiments/adaptive-stopping/results-sentinels-excluded.json b/experiments/adaptive-stopping/results-sentinels-excluded.json index 5897cb4..b8f33e9 100644 --- a/experiments/adaptive-stopping/results-sentinels-excluded.json +++ b/experiments/adaptive-stopping/results-sentinels-excluded.json @@ -4,6 +4,12 @@ "bootstrap_replicates": 1000, "bootstrap_seed": 42, "comparisons_replayed": 3665, + "followup_acceptance_gate": { + "kendall_tau_minimum": 0.95, + "max_absolute_elo_delta_maximum": 50.0, + "percentage_saved_minimum": 60.0, + "top3_order_exact": true + }, "historical_sentinel_comparisons": 628, "min_before_check": 234, "model_count": 13, @@ -22,7 +28,6 @@ "tesseract-5", "zai-org/GLM-OCR" ], - "primary_repeats": 1, "production_helpers": [ "compute_elo", "classify_adjacent_pairs", @@ -32,6 +37,14 @@ "practical_preferences", "unresolved_pairs" ], + "repeated_strategy_names": [ + "balanced_adaptive", + "full_stored_board", + "targeted", + "targeted_size_3_min_10", + "targeted_v2_warmup_5_explore_3_annotate_3x" + ], + "repeats": 1, "sample_count": 50, "sentinel_comparisons_excluded": true, "source_repo": "davanstrien/ocr-bench-britannica-results", @@ -40,6 +53,12 @@ }, "strategies": [ { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 8 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 3665, "comparisons_saved": 0, "decision_statuses": { @@ -147,6 +166,13 @@ "deepseek-ai/DeepSeek-OCR-2", "tesseract-5" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, "graph": { "connected": true, "connected_components": [ @@ -183,6 +209,7 @@ "practical_preferences": [], "resolution_summary": "reference only; no adaptive stop", "round_history": [], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 1.0, @@ -195,9 +222,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": true + "top3_order_matches": true, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 8 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 3665, "comparisons_saved": 0, "decision_statuses": { @@ -305,6 +339,13 @@ "deepseek-ai/DeepSeek-OCR-2", "tesseract-5" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, "graph": { "connected": true, "connected_components": [ @@ -339,10 +380,11 @@ "name": "balanced_adaptive", "percentage_saved": 0.0, "practical_preferences": [], - "resolution_summary": "sample batches exhausted: 8 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 8 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 267, "cumulative_comparisons": 267, "decision": { @@ -377,6 +419,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 315, "cumulative_comparisons": 582, "decision": { @@ -411,6 +454,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 388, "cumulative_comparisons": 970, "decision": { @@ -445,6 +489,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 390, "cumulative_comparisons": 1360, "decision": { @@ -479,6 +524,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 390, "cumulative_comparisons": 1750, "decision": { @@ -513,6 +559,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 389, "cumulative_comparisons": 2139, "decision": { @@ -547,6 +594,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 390, "cumulative_comparisons": 2529, "decision": { @@ -581,6 +629,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 390, "cumulative_comparisons": 2919, "decision": { @@ -615,6 +664,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 357, "cumulative_comparisons": 3276, "decision": { @@ -649,6 +699,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 389, "cumulative_comparisons": 3665, "decision": { @@ -682,6 +733,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 1.0, @@ -694,9 +746,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": true + "top3_order_matches": true, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 753, "comparisons_saved": 2912, "decision_statuses": { @@ -804,6 +863,13 @@ "PaddlePaddle/PP-OCRv6_medium", "tesseract-5" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, "graph": { "connected": true, "connected_components": [ @@ -838,10 +904,11 @@ "name": "targeted", "percentage_saved": 79.45429740791269, "practical_preferences": [], - "resolution_summary": "sample batches exhausted: 11 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced-warmup", "batch_comparisons": 267, "cumulative_comparisons": 267, "decision": { @@ -876,6 +943,7 @@ }, { "active_pairs": 12, + "allocation_mode": "targeted", "batch_comparisons": 52, "cumulative_comparisons": 319, "decision": { @@ -910,6 +978,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 54, "cumulative_comparisons": 373, "decision": { @@ -944,6 +1013,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 428, "decision": { @@ -978,6 +1048,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 483, "decision": { @@ -1012,6 +1083,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 538, "decision": { @@ -1046,6 +1118,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 593, "decision": { @@ -1080,6 +1153,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 648, "decision": { @@ -1114,6 +1188,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 50, "cumulative_comparisons": 698, "decision": { @@ -1148,6 +1223,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 753, "decision": { @@ -1181,6 +1257,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9835164835164836, @@ -1193,9 +1270,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": false + "top3_order_matches": false, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 5 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 624, "comparisons_saved": 3041, "decision_statuses": { @@ -1303,6 +1387,13 @@ "deepseek-ai/DeepSeek-OCR-2", "tesseract-5" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, "graph": { "connected": true, "connected_components": [ @@ -1374,10 +1465,11 @@ "smaller_model": "zai-org/GLM-OCR" } ], - "resolution_summary": "sample batches exhausted: 5 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 5 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced-warmup", "batch_comparisons": 267, "cumulative_comparisons": 267, "decision": { @@ -1412,6 +1504,7 @@ }, { "active_pairs": 12, + "allocation_mode": "targeted", "batch_comparisons": 52, "cumulative_comparisons": 319, "decision": { @@ -1446,6 +1539,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 54, "cumulative_comparisons": 373, "decision": { @@ -1480,6 +1574,7 @@ }, { "active_pairs": 9, + "allocation_mode": "targeted", "batch_comparisons": 45, "cumulative_comparisons": 418, "decision": { @@ -1514,6 +1609,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 40, "cumulative_comparisons": 458, "decision": { @@ -1548,6 +1644,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 40, "cumulative_comparisons": 498, "decision": { @@ -1582,6 +1679,7 @@ }, { "active_pairs": 9, + "allocation_mode": "targeted", "batch_comparisons": 45, "cumulative_comparisons": 543, "decision": { @@ -1616,6 +1714,7 @@ }, { "active_pairs": 6, + "allocation_mode": "targeted", "batch_comparisons": 30, "cumulative_comparisons": 573, "decision": { @@ -1650,6 +1749,7 @@ }, { "active_pairs": 6, + "allocation_mode": "targeted", "batch_comparisons": 26, "cumulative_comparisons": 599, "decision": { @@ -1684,6 +1784,7 @@ }, { "active_pairs": 5, + "allocation_mode": "targeted", "batch_comparisons": 25, "cumulative_comparisons": 624, "decision": { @@ -1717,6 +1818,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9725274725274725, @@ -1729,7 +1831,3125 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": false + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1016, + "comparisons_saved": 2649, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1594.1977167245018, + "PaddlePaddle/PP-OCRv6_medium": 1343.9363256863503, + "PaddlePaddle/PaddleOCR-VL-1.6": 1461.9855892384621, + "allenai/olmOCR-2-7B-1025-FP8": 1599.7840607374487, + "baidu/Qianfan-OCR": 1521.8442923817795, + "baidu/Unlimited-OCR": 1502.2638288058974, + "deepseek-ai/DeepSeek-OCR": 1421.495700939181, + "deepseek-ai/DeepSeek-OCR-2": 1375.5950554145513, + "lightonai/LightOnOCR-2-1B": 1660.0179980731882, + "numind/NuExtract3": 1573.741049016997, + "rednote-hilab/dots.mocr": 1771.482965377228, + "tesseract-5": 1014.0364064260923, + "zai-org/GLM-OCR": 1659.6190111783221 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1548.0122918324607, + 1657.8213976451166 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1272.534734219565, + 1422.2828517541896 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1408.6156899836799, + 1532.354585548627 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1548.954872012757, + 1669.2573603652158 + ], + "baidu/Qianfan-OCR": [ + 1472.1911367384866, + 1585.5581766648052 + ], + "baidu/Unlimited-OCR": [ + 1446.6147984291692, + 1568.387266502401 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1365.887231962342, + 1481.125756765516 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1308.6079900774423, + 1449.596877130258 + ], + "lightonai/LightOnOCR-2-1B": [ + 1601.0796917899452, + 1745.7944207667445 + ], + "numind/NuExtract3": [ + 1514.9415770736534, + 1644.9708996399656 + ], + "rednote-hilab/dots.mocr": [ + 1702.3977665602947, + 1878.194257286424 + ], + "tesseract-5": [ + 772.1083720578063, + 1138.0169569592026 + ], + "zai-org/GLM-OCR": [ + 1604.0197325174968, + 1733.054617156719 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 82.34296571280584, + "PaddlePaddle/PP-OCRv6_medium": -2.4210393629359714, + "PaddlePaddle/PaddleOCR-VL-1.6": 47.71000857756644, + "allenai/olmOCR-2-7B-1025-FP8": -73.96891853579382, + "baidu/Qianfan-OCR": -2.161756796373993, + "baidu/Unlimited-OCR": 4.220385314747318, + "deepseek-ai/DeepSeek-OCR": 12.66645583124182, + "deepseek-ai/DeepSeek-OCR-2": 38.72965873795442, + "lightonai/LightOnOCR-2-1B": -37.58525961033342, + "numind/NuExtract3": -76.01259241171533, + "rednote-hilab/dots.mocr": 71.39166507624122, + "tesseract-5": -28.596904699675406, + "zai-org/GLM-OCR": -36.31466783372889 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 49, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 5, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "targeted", + "max_absolute_elo_delta": 82.34296571280584, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 37.58525961033342, + "name": "targeted_v2_warmup_5", + "percentage_saved": 72.27830832196453, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 78, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 315, + "cumulative_comparisons": 582, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 636, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 691, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 746, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 801, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 856, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 911, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 50, + "cumulative_comparisons": 961, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1016, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9780219780219779, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1351, + "comparisons_saved": 2314, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1584.4540378390668, + "PaddlePaddle/PP-OCRv6_medium": 1370.087746133482, + "PaddlePaddle/PaddleOCR-VL-1.6": 1476.808075615833, + "allenai/olmOCR-2-7B-1025-FP8": 1618.3431895840686, + "baidu/Qianfan-OCR": 1535.452708568855, + "baidu/Unlimited-OCR": 1522.947359963196, + "deepseek-ai/DeepSeek-OCR": 1454.4407854931717, + "deepseek-ai/DeepSeek-OCR-2": 1426.8932568357582, + "lightonai/LightOnOCR-2-1B": 1661.907548415937, + "numind/NuExtract3": 1579.3964549847967, + "rednote-hilab/dots.mocr": 1665.3182348464015, + "tesseract-5": 960.6816984488148, + "zai-org/GLM-OCR": 1643.2689032706185 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1541.975049312204, + 1631.142775041068 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1314.0815158483088, + 1422.771239653849 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1433.3921057155708, + 1528.9764672617662 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1573.034774172818, + 1669.8448562058854 + ], + "baidu/Qianfan-OCR": [ + 1493.4932421468554, + 1586.3837736779778 + ], + "baidu/Unlimited-OCR": [ + 1476.18691598975, + 1570.159136712557 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1405.8032438666175, + 1504.806181540899 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1373.871017369812, + 1479.5813804195695 + ], + "lightonai/LightOnOCR-2-1B": [ + 1613.1387930424096, + 1722.0274752746095 + ], + "numind/NuExtract3": [ + 1533.0995289601428, + 1632.7697400846548 + ], + "rednote-hilab/dots.mocr": [ + 1614.2344301794028, + 1727.1556353651142 + ], + "tesseract-5": [ + 708.2146235618151, + 1068.6003544089108 + ], + "zai-org/GLM-OCR": [ + 1601.5690852762173, + 1700.1610627970576 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 72.59928682737086, + "PaddlePaddle/PP-OCRv6_medium": 23.730381084195642, + "PaddlePaddle/PaddleOCR-VL-1.6": 62.53249495493742, + "allenai/olmOCR-2-7B-1025-FP8": -55.40978968917398, + "baidu/Qianfan-OCR": 11.446659390701598, + "baidu/Unlimited-OCR": 24.90391647204592, + "deepseek-ai/DeepSeek-OCR": 45.6115403852325, + "deepseek-ai/DeepSeek-OCR-2": 90.02786015916126, + "lightonai/LightOnOCR-2-1B": -35.69570926758456, + "numind/NuExtract3": -70.35718644391568, + "rednote-hilab/dots.mocr": -34.77306545458532, + "tesseract-5": -81.95161267695289, + "zai-org/GLM-OCR": -52.66477574143255 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 49, + "median_direct_comparisons": 13.0, + "min_direct_comparisons": 10, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "targeted", + "max_absolute_elo_delta": 90.02786015916126, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 52.66477574143255, + "name": "targeted_v2_warmup_10", + "percentage_saved": 63.13778990450205, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 78, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 315, + "cumulative_comparisons": 582, + "decision": "warm-up", + "min_direct_comparisons": 5, + "observed_pairs": 78, + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 388, + "cumulative_comparisons": 970, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1025, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1080, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1135, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1190, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1245, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 51, + "cumulative_comparisons": 1296, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1351, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9780219780219779, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": 10 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 1756, + "comparisons_saved": 1909, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1578.7221953695243, + "PaddlePaddle/PP-OCRv6_medium": 1344.8714356962412, + "PaddlePaddle/PaddleOCR-VL-1.6": 1429.0904749243182, + "allenai/olmOCR-2-7B-1025-FP8": 1684.4595249185695, + "baidu/Qianfan-OCR": 1553.718918951271, + "baidu/Unlimited-OCR": 1492.4910380847436, + "deepseek-ai/DeepSeek-OCR": 1364.79279459873, + "deepseek-ai/DeepSeek-OCR-2": 1255.621293302505, + "lightonai/LightOnOCR-2-1B": 1725.5499489717706, + "numind/NuExtract3": 1662.1668878387213, + "rednote-hilab/dots.mocr": 1720.2453960707978, + "tesseract-5": 970.0674285255849, + "zai-org/GLM-OCR": 1718.2026627472223 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1533.94473347526, + 1624.0514932754495 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1294.39283603392, + 1394.8410542824838 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1386.879750150233, + 1473.3803090923095 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1639.8084920452216, + 1733.6149864248125 + ], + "baidu/Qianfan-OCR": [ + 1512.1751570247654, + 1597.5736276522393 + ], + "baidu/Unlimited-OCR": [ + 1449.3980558762755, + 1535.512243587908 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1320.6522881529013, + 1405.756173765873 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1190.5425432908667, + 1305.4338547938723 + ], + "lightonai/LightOnOCR-2-1B": [ + 1684.3156726411894, + 1779.375682668677 + ], + "numind/NuExtract3": [ + 1621.159322965108, + 1711.232791608624 + ], + "rednote-hilab/dots.mocr": [ + 1678.3331905855202, + 1775.2680663330157 + ], + "tesseract-5": [ + 842.6855465433207, + 1051.0483879190583 + ], + "zai-org/GLM-OCR": [ + 1674.1828879108189, + 1771.3680711869401 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 66.86744435782839, + "PaddlePaddle/PP-OCRv6_medium": -1.485929353045094, + "PaddlePaddle/PaddleOCR-VL-1.6": 14.814894263422502, + "allenai/olmOCR-2-7B-1025-FP8": 10.706545645326969, + "baidu/Qianfan-OCR": 29.712869773117518, + "baidu/Unlimited-OCR": -5.5524054064064785, + "deepseek-ai/DeepSeek-OCR": -44.03645050920909, + "deepseek-ai/DeepSeek-OCR-2": -81.24410337409199, + "lightonai/LightOnOCR-2-1B": 27.946691288248985, + "numind/NuExtract3": 12.413246410008924, + "rednote-hilab/dots.mocr": 20.154095769810965, + "tesseract-5": -72.56588260018282, + "zai-org/GLM-OCR": 22.268983735171332 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 49, + "median_direct_comparisons": 19.0, + "min_direct_comparisons": 17, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9487179487179485, + "kind": "targeted", + "max_absolute_elo_delta": 81.24410337409199, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 22.268983735171332, + "name": "targeted_v2_explore_every_3", + "percentage_saved": 52.08731241473397, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 52, + "cumulative_comparisons": 319, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 373, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 390, + "cumulative_comparisons": 763, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 818, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 872, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 390, + "cumulative_comparisons": 1262, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1317, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 50, + "cumulative_comparisons": 1367, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 389, + "cumulative_comparisons": 1756, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.989010989010989, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 1684, + "comparisons_saved": 1981, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 5 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1528.41957157949, + "PaddlePaddle/PP-OCRv6_medium": 1320.1231251382626, + "PaddlePaddle/PaddleOCR-VL-1.6": 1421.763606007078, + "allenai/olmOCR-2-7B-1025-FP8": 1632.9549673936333, + "baidu/Qianfan-OCR": 1523.643980465282, + "baidu/Unlimited-OCR": 1484.3914961825521, + "deepseek-ai/DeepSeek-OCR": 1399.7470360390994, + "deepseek-ai/DeepSeek-OCR-2": 1354.946370415816, + "lightonai/LightOnOCR-2-1B": 1661.992650991927, + "numind/NuExtract3": 1617.4093847305041, + "rednote-hilab/dots.mocr": 1757.1171941283797, + "tesseract-5": 1087.8128961744028, + "zai-org/GLM-OCR": 1709.677720753573 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1482.5652170872186, + 1571.9234237789242 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1267.7987192827172, + 1369.398152819473 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1377.7913909245426, + 1462.6550625412087 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1587.643712160444, + 1680.20393300897 + ], + "baidu/Qianfan-OCR": [ + 1482.4623193082723, + 1568.5055369026645 + ], + "baidu/Unlimited-OCR": [ + 1442.3753581734134, + 1526.6032308658766 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1357.4408833997363, + 1442.8625231635194 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1306.1912609145675, + 1400.8511830532318 + ], + "lightonai/LightOnOCR-2-1B": [ + 1614.7114310385173, + 1713.9165222974398 + ], + "numind/NuExtract3": [ + 1573.4454186810872, + 1665.1094207311894 + ], + "rednote-hilab/dots.mocr": [ + 1709.4691298320483, + 1814.8421704747725 + ], + "tesseract-5": [ + 983.8459938882305, + 1160.540437337229 + ], + "zai-org/GLM-OCR": [ + 1664.526158086622, + 1756.9781759997322 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 16.56482056779396, + "PaddlePaddle/PP-OCRv6_medium": -26.23423991102368, + "PaddlePaddle/PaddleOCR-VL-1.6": 7.488025346182212, + "allenai/olmOCR-2-7B-1025-FP8": -40.79801187960925, + "baidu/Qianfan-OCR": -0.3620687128714053, + "baidu/Unlimited-OCR": -13.651947308597983, + "deepseek-ai/DeepSeek-OCR": -9.082209068839802, + "deepseek-ai/DeepSeek-OCR-2": 18.08097373921919, + "lightonai/LightOnOCR-2-1B": -35.61060669159474, + "numind/NuExtract3": -32.344256698208255, + "rednote-hilab/dots.mocr": 57.02589382739279, + "tesseract-5": 45.17958504863509, + "zai-org/GLM-OCR": 13.744041741522096 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 49, + "median_direct_comparisons": 18.5, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "targeted", + "max_absolute_elo_delta": 57.02589382739279, + "max_absolute_elo_delta_model": "rednote-hilab/dots.mocr", + "median_absolute_elo_delta": 18.08097373921919, + "name": "targeted_v2_warmup_5_explore_3_annotate_3x", + "percentage_saved": 54.05184174624829, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR-2", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 8.3, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 10 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 78, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 315, + "cumulative_comparisons": 582, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 636, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 691, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 390, + "cumulative_comparisons": 1081, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 1135, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1190, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 390, + "cumulative_comparisons": 1580, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 50, + "cumulative_comparisons": 1630, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 1684, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": false, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9835164835164836, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 1990, + "comparisons_saved": 1675, + "decision_statuses": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 5 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1533.1159463461559, + "PaddlePaddle/PP-OCRv6_medium": 1356.4394547284237, + "PaddlePaddle/PaddleOCR-VL-1.6": 1456.6544017666968, + "allenai/olmOCR-2-7B-1025-FP8": 1625.4395483977885, + "baidu/Qianfan-OCR": 1530.3495913306213, + "baidu/Unlimited-OCR": 1513.3480345658143, + "deepseek-ai/DeepSeek-OCR": 1450.9921993413825, + "deepseek-ai/DeepSeek-OCR-2": 1402.5955569920488, + "lightonai/LightOnOCR-2-1B": 1675.317060191538, + "numind/NuExtract3": 1595.7458734953489, + "rednote-hilab/dots.mocr": 1668.452003795861, + "tesseract-5": 1046.225989143694, + "zai-org/GLM-OCR": 1645.3243399046264 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1494.8413048860393, + 1572.0690177651336 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1309.8835773489661, + 1400.316655752454 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1418.3529390528927, + 1496.8167539216047 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1588.0741786730778, + 1665.9166700878961 + ], + "baidu/Qianfan-OCR": [ + 1495.2274795617432, + 1571.7165158544522 + ], + "baidu/Unlimited-OCR": [ + 1477.7749893097453, + 1549.4542175566492 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1411.5939760602496, + 1487.1954934765208 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1362.6705142376954, + 1442.0109692493627 + ], + "lightonai/LightOnOCR-2-1B": [ + 1631.2202068103788, + 1720.933994561298 + ], + "numind/NuExtract3": [ + 1556.8076135971587, + 1637.3856912713704 + ], + "rednote-hilab/dots.mocr": [ + 1626.8756457830232, + 1713.8049446440773 + ], + "tesseract-5": [ + 942.7053230210765, + 1121.2198561394557 + ], + "zai-org/GLM-OCR": [ + 1609.1407552926914, + 1686.8362853113313 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 21.2611953344599, + "PaddlePaddle/PP-OCRv6_medium": 10.082089679137425, + "PaddlePaddle/PaddleOCR-VL-1.6": 42.37882110580108, + "allenai/olmOCR-2-7B-1025-FP8": -48.31343087545406, + "baidu/Qianfan-OCR": 6.343542152467762, + "baidu/Unlimited-OCR": 15.304591074664131, + "deepseek-ai/DeepSeek-OCR": 42.162954233443315, + "deepseek-ai/DeepSeek-OCR-2": 65.73016031545194, + "lightonai/LightOnOCR-2-1B": -22.286197491983557, + "numind/NuExtract3": -54.00776793336354, + "rednote-hilab/dots.mocr": -31.639296505125913, + "tesseract-5": 3.592678017926346, + "zai-org/GLM-OCR": -50.6093391074246 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 23.0, + "min_direct_comparisons": 18, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "targeted", + "max_absolute_elo_delta": 65.73016031545194, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 31.639296505125913, + "name": "targeted_v2_warmup_10_explore_3_annotate_3x", + "percentage_saved": 45.70259208731242, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR-2", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.0, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 78, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 315, + "cumulative_comparisons": 582, + "decision": "warm-up", + "min_direct_comparisons": 5, + "observed_pairs": 78, + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 388, + "cumulative_comparisons": 970, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1025, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1080, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 389, + "cumulative_comparisons": 1469, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1524, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1579, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 357, + "cumulative_comparisons": 1936, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 1990, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": false, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9835164835164836, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": 10 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 5 + }, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 1568, + "comparisons_saved": 2097, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 5 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1528.3672135547124, + "PaddlePaddle/PP-OCRv6_medium": 1318.8846421014914, + "PaddlePaddle/PaddleOCR-VL-1.6": 1425.111495714902, + "allenai/olmOCR-2-7B-1025-FP8": 1633.044808502267, + "baidu/Qianfan-OCR": 1525.067837174809, + "baidu/Unlimited-OCR": 1481.989400260938, + "deepseek-ai/DeepSeek-OCR": 1400.9864851554164, + "deepseek-ai/DeepSeek-OCR-2": 1359.45356432976, + "lightonai/LightOnOCR-2-1B": 1655.4354676840687, + "numind/NuExtract3": 1617.1646458973364, + "rednote-hilab/dots.mocr": 1766.0725472922893, + "tesseract-5": 1088.388606809264, + "zai-org/GLM-OCR": 1700.0332855227452 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1484.0454528025084, + 1573.1125988782476 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1261.2202552408899, + 1372.4989350481237 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1373.3318776687975, + 1473.75461406909 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1588.216489009013, + 1683.4046946126866 + ], + "baidu/Qianfan-OCR": [ + 1483.1317495050077, + 1566.4845816535224 + ], + "baidu/Unlimited-OCR": [ + 1439.3939330940739, + 1523.901475725061 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1354.5929361718495, + 1445.0884765324986 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1306.2506011106566, + 1407.5930432753114 + ], + "lightonai/LightOnOCR-2-1B": [ + 1610.00234940123, + 1709.543914205717 + ], + "numind/NuExtract3": [ + 1574.0732899103, + 1662.3109869025775 + ], + "rednote-hilab/dots.mocr": [ + 1715.8973109172186, + 1832.0174827266126 + ], + "tesseract-5": [ + 991.4306218314965, + 1155.459208818105 + ], + "zai-org/GLM-OCR": [ + 1658.1688789820614, + 1753.5207488708602 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 16.512462543016454, + "PaddlePaddle/PP-OCRv6_medium": -27.472722947794864, + "PaddlePaddle/PaddleOCR-VL-1.6": 10.835915054006364, + "allenai/olmOCR-2-7B-1025-FP8": -40.708170770975585, + "baidu/Qianfan-OCR": 1.0617879966555392, + "baidu/Unlimited-OCR": -16.054043230212073, + "deepseek-ai/DeepSeek-OCR": -7.842759952522783, + "deepseek-ai/DeepSeek-OCR-2": 22.588167653163055, + "lightonai/LightOnOCR-2-1B": -42.16778999945291, + "numind/NuExtract3": -32.588995531375986, + "rednote-hilab/dots.mocr": 65.98124699130244, + "tesseract-5": 45.75529568349634, + "zai-org/GLM-OCR": 4.09960651069423 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 49, + "median_direct_comparisons": 18.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "targeted", + "max_absolute_elo_delta": 65.98124699130244, + "max_absolute_elo_delta_model": "rednote-hilab/dots.mocr", + "median_absolute_elo_delta": 22.588167653163055, + "name": "targeted_v2_warmup_5_explore_3_size_stop_3x", + "percentage_saved": 57.21691678035471, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR-2", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 8.3, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 5 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 267, + "cumulative_comparisons": 267, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 78, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 315, + "cumulative_comparisons": 582, + "decision": { + "prefer-smaller": 1, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 10, + "allocation_mode": "targeted", + "batch_comparisons": 49, + "cumulative_comparisons": 631, + "decision": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 671, + "decision": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 390, + "cumulative_comparisons": 1061, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 30, + "cumulative_comparisons": 1091, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 30, + "cumulative_comparisons": 1121, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 390, + "cumulative_comparisons": 1511, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 28, + "cumulative_comparisons": 1539, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 29, + "cumulative_comparisons": 1568, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9835164835164836, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": 5 } ] } diff --git a/experiments/adaptive-stopping/results.json b/experiments/adaptive-stopping/results.json index 11680bc..9b0c514 100644 --- a/experiments/adaptive-stopping/results.json +++ b/experiments/adaptive-stopping/results.json @@ -4,6 +4,12 @@ "bootstrap_replicates": 1000, "bootstrap_seed": 42, "comparisons_replayed": 4293, + "followup_acceptance_gate": { + "kendall_tau_minimum": 0.95, + "max_absolute_elo_delta_maximum": 50.0, + "percentage_saved_minimum": 60.0, + "top3_order_exact": true + }, "historical_sentinel_comparisons": 628, "min_before_check": 273, "model_count": 14, @@ -23,7 +29,6 @@ "tesseract-5", "zai-org/GLM-OCR" ], - "primary_repeats": 2, "production_helpers": [ "compute_elo", "classify_adjacent_pairs", @@ -33,6 +38,14 @@ "practical_preferences", "unresolved_pairs" ], + "repeated_strategy_names": [ + "balanced_adaptive", + "full_stored_board", + "targeted", + "targeted_size_3_min_10", + "targeted_v2_warmup_5_explore_3_annotate_3x" + ], + "repeats": 2, "sample_count": 50, "sentinel_comparisons_excluded": false, "source_repo": "davanstrien/ocr-bench-britannica-results", @@ -41,6 +54,12 @@ }, "strategies": [ { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 4293, "comparisons_saved": 0, "decision_statuses": { @@ -155,6 +174,13 @@ "tesseract-5", "rednote-hilab/dots.ocr" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, "graph": { "connected": true, "connected_components": [ @@ -192,6 +218,7 @@ "practical_preferences": [], "resolution_summary": "reference only; no adaptive stop", "round_history": [], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 1.0, @@ -204,9 +231,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": true + "top3_order_matches": true, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 4293, "comparisons_saved": 0, "decision_statuses": { @@ -321,6 +355,13 @@ "tesseract-5", "rednote-hilab/dots.ocr" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, "graph": { "connected": true, "connected_components": [ @@ -356,10 +397,11 @@ "name": "balanced_adaptive", "percentage_saved": 0.0, "practical_preferences": [], - "resolution_summary": "sample batches exhausted: 9 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 9 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 320, "cumulative_comparisons": 320, "decision": { @@ -395,6 +437,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 373, "cumulative_comparisons": 693, "decision": { @@ -430,6 +473,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 453, "cumulative_comparisons": 1146, "decision": { @@ -465,6 +509,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 455, "cumulative_comparisons": 1601, "decision": { @@ -500,6 +545,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 455, "cumulative_comparisons": 2056, "decision": { @@ -535,6 +581,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 454, "cumulative_comparisons": 2510, "decision": { @@ -570,6 +617,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 455, "cumulative_comparisons": 2965, "decision": { @@ -605,6 +653,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 455, "cumulative_comparisons": 3420, "decision": { @@ -640,6 +689,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 419, "cumulative_comparisons": 3839, "decision": { @@ -675,6 +725,7 @@ }, { "active_pairs": null, + "allocation_mode": "balanced", "batch_comparisons": 454, "cumulative_comparisons": 4293, "decision": { @@ -709,6 +760,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 1.0, @@ -721,9 +773,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": true + "top3_order_matches": true, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 870, "comparisons_saved": 3423, "decision_statuses": { @@ -838,6 +897,13 @@ "tesseract-5", "rednote-hilab/dots.ocr" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, "graph": { "connected": true, "connected_components": [ @@ -873,10 +939,11 @@ "name": "targeted", "percentage_saved": 79.73445143256464, "practical_preferences": [], - "resolution_summary": "sample batches exhausted: 12 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 12 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced-warmup", "batch_comparisons": 320, "cumulative_comparisons": 320, "decision": { @@ -912,6 +979,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 56, "cumulative_comparisons": 376, "decision": { @@ -947,6 +1015,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 64, "cumulative_comparisons": 440, "decision": { @@ -982,6 +1051,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 65, "cumulative_comparisons": 505, "decision": { @@ -1017,6 +1087,7 @@ }, { "active_pairs": 12, + "allocation_mode": "targeted", "batch_comparisons": 60, "cumulative_comparisons": 565, "decision": { @@ -1052,6 +1123,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 65, "cumulative_comparisons": 630, "decision": { @@ -1087,6 +1159,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 65, "cumulative_comparisons": 695, "decision": { @@ -1122,6 +1195,7 @@ }, { "active_pairs": 12, + "allocation_mode": "targeted", "batch_comparisons": 60, "cumulative_comparisons": 755, "decision": { @@ -1157,6 +1231,7 @@ }, { "active_pairs": 12, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 810, "decision": { @@ -1192,6 +1267,7 @@ }, { "active_pairs": 12, + "allocation_mode": "targeted", "batch_comparisons": 60, "cumulative_comparisons": 870, "decision": { @@ -1226,6 +1302,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9824175824175824, @@ -1238,9 +1315,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": false + "top3_order_matches": false, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 6 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 736, "comparisons_saved": 3557, "decision_statuses": { @@ -1355,6 +1439,13 @@ "tesseract-5", "rednote-hilab/dots.ocr" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, "graph": { "connected": true, "connected_components": [ @@ -1427,10 +1518,11 @@ "smaller_model": "zai-org/GLM-OCR" } ], - "resolution_summary": "sample batches exhausted: 6 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 6 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced-warmup", "batch_comparisons": 320, "cumulative_comparisons": 320, "decision": { @@ -1466,6 +1558,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 56, "cumulative_comparisons": 376, "decision": { @@ -1501,6 +1594,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 64, "cumulative_comparisons": 440, "decision": { @@ -1536,6 +1630,7 @@ }, { "active_pairs": 10, + "allocation_mode": "targeted", "batch_comparisons": 50, "cumulative_comparisons": 490, "decision": { @@ -1571,6 +1666,7 @@ }, { "active_pairs": 10, + "allocation_mode": "targeted", "batch_comparisons": 50, "cumulative_comparisons": 540, "decision": { @@ -1606,6 +1702,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 595, "decision": { @@ -1641,6 +1738,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 40, "cumulative_comparisons": 635, "decision": { @@ -1676,6 +1774,7 @@ }, { "active_pairs": 7, + "allocation_mode": "targeted", "batch_comparisons": 35, "cumulative_comparisons": 670, "decision": { @@ -1711,6 +1810,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 36, "cumulative_comparisons": 706, "decision": { @@ -1746,6 +1846,7 @@ }, { "active_pairs": 6, + "allocation_mode": "targeted", "batch_comparisons": 30, "cumulative_comparisons": 736, "decision": { @@ -1780,6 +1881,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9780219780219781, @@ -1792,9 +1894,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": false + "top3_order_matches": false, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 9, + "resolved": 1, + "unresolved": 3 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 629, "comparisons_saved": 3664, "decision_statuses": { @@ -1909,6 +2018,13 @@ "tesseract-5", "rednote-hilab/dots.ocr" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, "graph": { "connected": true, "connected_components": [ @@ -1999,10 +2115,11 @@ "smaller_model": "zai-org/GLM-OCR" } ], - "resolution_summary": "sample batches exhausted: 3 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 3 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced-warmup", "batch_comparisons": 320, "cumulative_comparisons": 320, "decision": { @@ -2038,6 +2155,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 56, "cumulative_comparisons": 376, "decision": { @@ -2073,6 +2191,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 54, "cumulative_comparisons": 430, "decision": { @@ -2108,6 +2227,7 @@ }, { "active_pairs": 7, + "allocation_mode": "targeted", "batch_comparisons": 35, "cumulative_comparisons": 465, "decision": { @@ -2143,6 +2263,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 40, "cumulative_comparisons": 505, "decision": { @@ -2178,6 +2299,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 40, "cumulative_comparisons": 545, "decision": { @@ -2213,6 +2335,7 @@ }, { "active_pairs": 6, + "allocation_mode": "targeted", "batch_comparisons": 30, "cumulative_comparisons": 575, "decision": { @@ -2248,6 +2371,7 @@ }, { "active_pairs": 5, + "allocation_mode": "targeted", "batch_comparisons": 25, "cumulative_comparisons": 600, "decision": { @@ -2283,6 +2407,7 @@ }, { "active_pairs": 3, + "allocation_mode": "targeted", "batch_comparisons": 14, "cumulative_comparisons": 614, "decision": { @@ -2318,6 +2443,7 @@ }, { "active_pairs": 3, + "allocation_mode": "targeted", "batch_comparisons": 15, "cumulative_comparisons": 629, "decision": { @@ -2352,6 +2478,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": 2.0, "spearman_rho": 0.9516483516483516, @@ -2364,9 +2491,16 @@ ], "top3_membership_matches": false, "top3_membership_overlap": 2, - "top3_order_matches": false + "top3_order_matches": false, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 735, "comparisons_saved": 3558, "decision_statuses": { @@ -2481,6 +2615,13 @@ "tesseract-5", "rednote-hilab/dots.ocr" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, "graph": { "connected": true, "connected_components": [ @@ -2541,10 +2682,11 @@ "smaller_model": "zai-org/GLM-OCR" } ], - "resolution_summary": "sample batches exhausted: 8 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 8 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced-warmup", "batch_comparisons": 320, "cumulative_comparisons": 320, "decision": { @@ -2580,6 +2722,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 56, "cumulative_comparisons": 376, "decision": { @@ -2615,6 +2758,7 @@ }, { "active_pairs": 10, + "allocation_mode": "targeted", "batch_comparisons": 49, "cumulative_comparisons": 425, "decision": { @@ -2650,6 +2794,7 @@ }, { "active_pairs": 9, + "allocation_mode": "targeted", "batch_comparisons": 45, "cumulative_comparisons": 470, "decision": { @@ -2685,6 +2830,7 @@ }, { "active_pairs": 9, + "allocation_mode": "targeted", "batch_comparisons": 45, "cumulative_comparisons": 515, "decision": { @@ -2720,6 +2866,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 570, "decision": { @@ -2755,6 +2902,7 @@ }, { "active_pairs": 9, + "allocation_mode": "targeted", "batch_comparisons": 45, "cumulative_comparisons": 615, "decision": { @@ -2790,6 +2938,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 40, "cumulative_comparisons": 655, "decision": { @@ -2825,6 +2974,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 36, "cumulative_comparisons": 691, "decision": { @@ -2860,6 +3010,7 @@ }, { "active_pairs": 9, + "allocation_mode": "targeted", "batch_comparisons": 44, "cumulative_comparisons": 735, "decision": { @@ -2894,6 +3045,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 5, "size_tie_ratio": 3.0, "spearman_rho": 0.9120879120879121, @@ -2906,9 +3058,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": false + "top3_order_matches": false, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 786, "comparisons_saved": 3507, "decision_statuses": { @@ -3023,6 +3182,13 @@ "tesseract-5", "rednote-hilab/dots.ocr" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, "graph": { "connected": true, "connected_components": [ @@ -3089,10 +3255,11 @@ "smaller_model": "zai-org/GLM-OCR" } ], - "resolution_summary": "sample batches exhausted: 7 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 7 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced-warmup", "batch_comparisons": 320, "cumulative_comparisons": 320, "decision": { @@ -3128,6 +3295,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 56, "cumulative_comparisons": 376, "decision": { @@ -3163,6 +3331,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 64, "cumulative_comparisons": 440, "decision": { @@ -3198,6 +3367,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 65, "cumulative_comparisons": 505, "decision": { @@ -3233,6 +3403,7 @@ }, { "active_pairs": 10, + "allocation_mode": "targeted", "batch_comparisons": 50, "cumulative_comparisons": 555, "decision": { @@ -3268,6 +3439,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 610, "decision": { @@ -3303,6 +3475,7 @@ }, { "active_pairs": 12, + "allocation_mode": "targeted", "batch_comparisons": 60, "cumulative_comparisons": 670, "decision": { @@ -3338,6 +3511,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 40, "cumulative_comparisons": 710, "decision": { @@ -3373,6 +3547,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 36, "cumulative_comparisons": 746, "decision": { @@ -3408,6 +3583,7 @@ }, { "active_pairs": 8, + "allocation_mode": "targeted", "batch_comparisons": 40, "cumulative_comparisons": 786, "decision": { @@ -3442,6 +3618,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 15, "size_tie_ratio": 3.0, "spearman_rho": 0.9824175824175824, @@ -3454,9 +3631,16 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": false + "top3_order_matches": false, + "warmup_min_per_pair": null }, { + "allocation_decision_statuses": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 9 + }, + "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 810, "comparisons_saved": 3483, "decision_statuses": { @@ -3571,6 +3755,13 @@ "tesseract-5", "rednote-hilab/dots.ocr" ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, "graph": { "connected": true, "connected_components": [ @@ -3625,10 +3816,11 @@ "smaller_model": "lightonai/LightOnOCR-2-1B" } ], - "resolution_summary": "sample batches exhausted: 9 adjacent pairs unresolved", + "resolution_summary": "sample batches exhausted: 9 allocation pairs unresolved", "round_history": [ { "active_pairs": null, + "allocation_mode": "balanced-warmup", "batch_comparisons": 320, "cumulative_comparisons": 320, "decision": { @@ -3664,6 +3856,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 56, "cumulative_comparisons": 376, "decision": { @@ -3699,6 +3892,7 @@ }, { "active_pairs": 13, + "allocation_mode": "targeted", "batch_comparisons": 64, "cumulative_comparisons": 440, "decision": { @@ -3734,6 +3928,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 495, "decision": { @@ -3769,6 +3964,7 @@ }, { "active_pairs": 12, + "allocation_mode": "targeted", "batch_comparisons": 60, "cumulative_comparisons": 555, "decision": { @@ -3804,6 +4000,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 610, "decision": { @@ -3839,6 +4036,7 @@ }, { "active_pairs": 11, + "allocation_mode": "targeted", "batch_comparisons": 55, "cumulative_comparisons": 665, "decision": { @@ -3874,6 +4072,7 @@ }, { "active_pairs": 10, + "allocation_mode": "targeted", "batch_comparisons": 50, "cumulative_comparisons": 715, "decision": { @@ -3909,6 +4108,7 @@ }, { "active_pairs": 10, + "allocation_mode": "targeted", "batch_comparisons": 45, "cumulative_comparisons": 760, "decision": { @@ -3944,6 +4144,7 @@ }, { "active_pairs": 10, + "allocation_mode": "targeted", "batch_comparisons": 50, "cumulative_comparisons": 810, "decision": { @@ -3978,6 +4179,7 @@ ] } ], + "size_rule_controls_sampling": true, "size_tie_min_samples": 10, "size_tie_ratio": 5.0, "spearman_rho": 0.9824175824175824, @@ -3990,7 +4192,3220 @@ ], "top3_membership_matches": true, "top3_membership_overlap": 3, - "top3_order_matches": false + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1153, + "comparisons_saved": 3140, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1633.0661587781437, + "PaddlePaddle/PP-OCRv6_medium": 1378.5509421275347, + "PaddlePaddle/PaddleOCR-VL-1.6": 1489.5851365342287, + "allenai/olmOCR-2-7B-1025-FP8": 1617.7060474973778, + "baidu/Qianfan-OCR": 1539.3832484613636, + "baidu/Unlimited-OCR": 1520.2549659248068, + "deepseek-ai/DeepSeek-OCR": 1437.8082997396057, + "deepseek-ai/DeepSeek-OCR-2": 1393.7406984861343, + "lightonai/LightOnOCR-2-1B": 1663.2040351738883, + "numind/NuExtract3": 1583.5634855810165, + "rednote-hilab/dots.mocr": 1817.811063950996, + "rednote-hilab/dots.ocr": 1048.1461002900596, + "tesseract-5": 1219.1549769230849, + "zai-org/GLM-OCR": 1658.024840531759 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1578.4339017612608, + 1688.5272099787749 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1311.1430504759453, + 1436.904126244807 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1440.955284436211, + 1540.465589036037 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1570.177623914315, + 1676.4550711712125 + ], + "baidu/Qianfan-OCR": [ + 1489.4794850107228, + 1589.519674484426 + ], + "baidu/Unlimited-OCR": [ + 1473.5972395331396, + 1571.7045557156205 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1383.6560842389613, + 1488.3870717897364 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1331.6962765930052, + 1450.2273371936296 + ], + "lightonai/LightOnOCR-2-1B": [ + 1596.3000302026578, + 1732.9488683733 + ], + "numind/NuExtract3": [ + 1531.6102286866799, + 1640.8581894187055 + ], + "rednote-hilab/dots.mocr": [ + 1738.4685512482138, + 1925.8433613786888 + ], + "rednote-hilab/dots.ocr": [ + 902.7100761098048, + 1143.1185143857 + ], + "tesseract-5": [ + 1135.0290434143762, + 1285.6893855767862 + ], + "zai-org/GLM-OCR": [ + 1602.3605604199477, + 1723.4720524945826 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 75.61677209337472, + "PaddlePaddle/PP-OCRv6_medium": -18.54561780106701, + "PaddlePaddle/PaddleOCR-VL-1.6": 26.610584036839327, + "allenai/olmOCR-2-7B-1025-FP8": -101.52509053168933, + "baidu/Qianfan-OCR": -28.711201840770627, + "baidu/Unlimited-OCR": -24.951036713746362, + "deepseek-ai/DeepSeek-OCR": -14.188723847063102, + "deepseek-ai/DeepSeek-OCR-2": 13.528150499292906, + "lightonai/LightOnOCR-2-1B": -78.04847561086626, + "numind/NuExtract3": -106.05555020893894, + "rednote-hilab/dots.mocr": 72.38580456436534, + "rednote-hilab/dots.ocr": 157.47055173226, + "tesseract-5": 106.56846855839535, + "zai-org/GLM-OCR": -80.15463493038624 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 46, + "median_direct_comparisons": 9, + "min_direct_comparisons": 5, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9120879120879122, + "kind": "targeted", + "max_absolute_elo_delta": 157.47055173226, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 74.00128832887003, + "name": "targeted_v2_warmup_5", + "percentage_saved": 73.14232471465176, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 91, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 373, + "cumulative_comparisons": 693, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 753, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 813, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 873, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 928, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 983, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1043, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 56, + "cumulative_comparisons": 1099, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 1153, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9692307692307693, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1561, + "comparisons_saved": 2732, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1609.5530540751233, + "PaddlePaddle/PP-OCRv6_medium": 1405.5694788204771, + "PaddlePaddle/PaddleOCR-VL-1.6": 1504.841469006784, + "allenai/olmOCR-2-7B-1025-FP8": 1638.649188052229, + "baidu/Qianfan-OCR": 1561.8338533353342, + "baidu/Unlimited-OCR": 1550.500239335426, + "deepseek-ai/DeepSeek-OCR": 1473.6932550160866, + "deepseek-ai/DeepSeek-OCR-2": 1450.3162027749963, + "lightonai/LightOnOCR-2-1B": 1678.0896999616534, + "numind/NuExtract3": 1603.1064302580833, + "rednote-hilab/dots.mocr": 1703.374304097687, + "rednote-hilab/dots.ocr": 989.7414932324835, + "tesseract-5": 1160.5043899819825, + "zai-org/GLM-OCR": 1670.2269420516536 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1569.4568926210775, + 1654.0220124333316 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1351.4898012127262, + 1455.974487951902 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1460.3234134256506, + 1550.6702846635276 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1594.4757555301833, + 1684.7209663020526 + ], + "baidu/Qianfan-OCR": [ + 1513.3355905903593, + 1604.8756248890484 + ], + "baidu/Unlimited-OCR": [ + 1509.2767856769922, + 1596.6202054323169 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1425.291623399234, + 1519.5512247170175 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1398.0923855979674, + 1499.5799963122217 + ], + "lightonai/LightOnOCR-2-1B": [ + 1630.6862719440844, + 1738.4425162964008 + ], + "numind/NuExtract3": [ + 1559.811550520025, + 1657.429397964357 + ], + "rednote-hilab/dots.mocr": [ + 1651.066648983543, + 1764.45226389647 + ], + "rednote-hilab/dots.ocr": [ + 845.9671501136844, + 1077.1907401004844 + ], + "tesseract-5": [ + 1074.8393540090137, + 1220.9042250722184 + ], + "zai-org/GLM-OCR": [ + 1629.4378464309352, + 1722.9241263420615 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 52.1036673903543, + "PaddlePaddle/PP-OCRv6_medium": 8.472918891875452, + "PaddlePaddle/PaddleOCR-VL-1.6": 41.86691650939474, + "allenai/olmOCR-2-7B-1025-FP8": -80.58194997683813, + "baidu/Qianfan-OCR": -6.260596966799994, + "baidu/Unlimited-OCR": 5.294236696872758, + "deepseek-ai/DeepSeek-OCR": 21.69623142941782, + "deepseek-ai/DeepSeek-OCR-2": 70.10365478815493, + "lightonai/LightOnOCR-2-1B": -63.162810823101154, + "numind/NuExtract3": -86.51260553187217, + "rednote-hilab/dots.mocr": -42.05095528894367, + "rednote-hilab/dots.ocr": 99.06594467468392, + "tesseract-5": 47.917881617293006, + "zai-org/GLM-OCR": -67.9525334104917 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 49, + "median_direct_comparisons": 14, + "min_direct_comparisons": 10, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "targeted", + "max_absolute_elo_delta": 99.06594467468392, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 50.010774503823654, + "name": "targeted_v2_warmup_10", + "percentage_saved": 63.63848124854414, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 12 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 91, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 373, + "cumulative_comparisons": 693, + "decision": "warm-up", + "min_direct_comparisons": 5, + "observed_pairs": 91, + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 453, + "cumulative_comparisons": 1146, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1206, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1266, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1326, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1386, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1446, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1501, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1561, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9824175824175824, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": 10 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 2023, + "comparisons_saved": 2270, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1630.6920706317392, + "PaddlePaddle/PP-OCRv6_medium": 1399.2162696618263, + "PaddlePaddle/PaddleOCR-VL-1.6": 1482.4525331108623, + "allenai/olmOCR-2-7B-1025-FP8": 1734.9551475708581, + "baidu/Qianfan-OCR": 1599.5590661571557, + "baidu/Unlimited-OCR": 1545.0083820431003, + "deepseek-ai/DeepSeek-OCR": 1413.1052520605142, + "deepseek-ai/DeepSeek-OCR-2": 1305.1771911858696, + "lightonai/LightOnOCR-2-1B": 1777.5430173968446, + "numind/NuExtract3": 1717.252180625907, + "rednote-hilab/dots.mocr": 1772.3700975287675, + "rednote-hilab/dots.ocr": 792.1181443689077, + "tesseract-5": 1060.215191730965, + "zai-org/GLM-OCR": 1770.335455926683 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1588.9997798971945, + 1676.9697405492561 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1344.7414289484086, + 1447.629352807461 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1441.3567070096162, + 1524.7361098175095 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1690.970626840678, + 1790.2942459660972 + ], + "baidu/Qianfan-OCR": [ + 1557.0621814760348, + 1648.663148925247 + ], + "baidu/Unlimited-OCR": [ + 1503.8929787503748, + 1589.850554625217 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1369.5129617322136, + 1456.6889236057477 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1243.8666191686918, + 1360.027868384931 + ], + "lightonai/LightOnOCR-2-1B": [ + 1736.0695228990253, + 1831.1938905511581 + ], + "numind/NuExtract3": [ + 1673.9446469542804, + 1770.2148824806216 + ], + "rednote-hilab/dots.mocr": [ + 1729.2575564512088, + 1826.7440958982666 + ], + "rednote-hilab/dots.ocr": [ + 622.7785030108389, + 899.6919010022073 + ], + "tesseract-5": [ + 978.7566793453682, + 1130.1065492454106 + ], + "zai-org/GLM-OCR": [ + 1724.279709834768, + 1823.7445427931486 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 73.24268394697015, + "PaddlePaddle/PP-OCRv6_medium": 2.1197097332246813, + "PaddlePaddle/PaddleOCR-VL-1.6": 19.477980613472937, + "allenai/olmOCR-2-7B-1025-FP8": 15.72400954179102, + "baidu/Qianfan-OCR": 31.46461585502152, + "baidu/Unlimited-OCR": -0.19762059545291777, + "deepseek-ai/DeepSeek-OCR": -38.89177152615457, + "deepseek-ai/DeepSeek-OCR-2": -75.03535680097184, + "lightonai/LightOnOCR-2-1B": 36.290506612089985, + "numind/NuExtract3": 27.633144835951498, + "rednote-hilab/dots.mocr": 26.94483814213686, + "rednote-hilab/dots.ocr": -98.55740418889195, + "tesseract-5": -52.37131663372452, + "zai-org/GLM-OCR": 32.155980464537606 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 49, + "median_direct_comparisons": 20, + "min_direct_comparisons": 17, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "targeted", + "max_absolute_elo_delta": 98.55740418889195, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 31.810298159779563, + "name": "targeted_v2_explore_every_3", + "percentage_saved": 52.8767761472164, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 13, + "allocation_mode": "targeted", + "batch_comparisons": 56, + "cumulative_comparisons": 376, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 13, + "allocation_mode": "targeted", + "batch_comparisons": 64, + "cumulative_comparisons": 440, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 13, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 895, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 955, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 1009, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1464, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1519, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 50, + "cumulative_comparisons": 1569, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 454, + "cumulative_comparisons": 2023, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9912087912087912, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 1946, + "comparisons_saved": 2347, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1563.6216723985356, + "PaddlePaddle/PP-OCRv6_medium": 1359.1283790504326, + "PaddlePaddle/PaddleOCR-VL-1.6": 1462.1563493606204, + "allenai/olmOCR-2-7B-1025-FP8": 1678.5702750473943, + "baidu/Qianfan-OCR": 1563.059300514666, + "baidu/Unlimited-OCR": 1523.620796710941, + "deepseek-ai/DeepSeek-OCR": 1434.8844874590588, + "deepseek-ai/DeepSeek-OCR-2": 1389.3976050877043, + "lightonai/LightOnOCR-2-1B": 1695.4889972978024, + "numind/NuExtract3": 1636.9966241204509, + "rednote-hilab/dots.mocr": 1792.726987698778, + "rednote-hilab/dots.ocr": 978.8284730418867, + "tesseract-5": 1181.7368334994678, + "zai-org/GLM-OCR": 1739.7832187122617 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1519.9473533638115, + 1607.6592263489 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1308.041605401947, + 1403.7919760563188 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1417.961371491893, + 1505.6563418354976 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1635.3679608821587, + 1729.8416352877005 + ], + "baidu/Qianfan-OCR": [ + 1524.3615203735476, + 1606.1626953886607 + ], + "baidu/Unlimited-OCR": [ + 1482.0902841015352, + 1567.2319627850911 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1390.6626653170372, + 1476.3562124094487 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1343.1129706649006, + 1434.2723653249898 + ], + "lightonai/LightOnOCR-2-1B": [ + 1652.606975045897, + 1749.4705722837207 + ], + "numind/NuExtract3": [ + 1593.2294745610466, + 1685.1794422263827 + ], + "rednote-hilab/dots.mocr": [ + 1746.2603816213714, + 1850.0524231681413 + ], + "rednote-hilab/dots.ocr": [ + 861.283676771959, + 1061.1566885447094 + ], + "tesseract-5": [ + 1118.1941521191902, + 1235.629527794025 + ], + "zai-org/GLM-OCR": [ + 1696.2802031680687, + 1794.0320427339125 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 6.1722857137665414, + "PaddlePaddle/PP-OCRv6_medium": -37.96818087816905, + "PaddlePaddle/PaddleOCR-VL-1.6": -0.8182031367689433, + "allenai/olmOCR-2-7B-1025-FP8": -40.660862981672835, + "baidu/Qianfan-OCR": -5.035149787468299, + "baidu/Unlimited-OCR": -21.58520592761215, + "deepseek-ai/DeepSeek-OCR": -17.11253612760993, + "deepseek-ai/DeepSeek-OCR-2": 9.185057100862878, + "lightonai/LightOnOCR-2-1B": -45.763513486952206, + "numind/NuExtract3": -52.62241166950457, + "rednote-hilab/dots.mocr": 47.30172831214736, + "rednote-hilab/dots.ocr": 88.15292448408707, + "tesseract-5": 69.15032513477831, + "zai-org/GLM-OCR": 1.6037432501163948 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 45, + "median_direct_comparisons": 19, + "min_direct_comparisons": 15, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "targeted", + "max_absolute_elo_delta": 88.15292448408707, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 29.7766934028906, + "name": "targeted_v2_warmup_5_explore_3_annotate_3x", + "percentage_saved": 54.67039366410435, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR-2", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 8.3, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 91, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 373, + "cumulative_comparisons": 693, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 753, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 813, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1268, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 59, + "cumulative_comparisons": 1327, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1387, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1842, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 50, + "cumulative_comparisons": 1892, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 1946, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": false, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9868131868131869, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 2313, + "comparisons_saved": 1980, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1566.505914167425, + "PaddlePaddle/PP-OCRv6_medium": 1399.588726777184, + "PaddlePaddle/PaddleOCR-VL-1.6": 1493.208336041072, + "allenai/olmOCR-2-7B-1025-FP8": 1660.6962647727582, + "baidu/Qianfan-OCR": 1560.8388644628626, + "baidu/Unlimited-OCR": 1552.4905896931555, + "deepseek-ai/DeepSeek-OCR": 1481.1312209253567, + "deepseek-ai/DeepSeek-OCR-2": 1435.0430425115603, + "lightonai/LightOnOCR-2-1B": 1703.4194733050372, + "numind/NuExtract3": 1625.1807493413353, + "rednote-hilab/dots.mocr": 1704.1059067393662, + "rednote-hilab/dots.ocr": 990.3151628354453, + "tesseract-5": 1148.9315733513142, + "zai-org/GLM-OCR": 1678.5441750761272 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1529.210020715852, + 1604.6232939683875 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1359.1170764008261, + 1442.0550949118806 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1454.2334145543466, + 1529.4296773223068 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1622.5556150987331, + 1701.5357380031744 + ], + "baidu/Qianfan-OCR": [ + 1524.78291775698, + 1600.6015865160027 + ], + "baidu/Unlimited-OCR": [ + 1516.4199338971644, + 1590.0677258601654 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1444.2240885304577, + 1520.2071349526786 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1390.45289712047, + 1473.7994420844052 + ], + "lightonai/LightOnOCR-2-1B": [ + 1663.1764597426504, + 1749.0686418442176 + ], + "numind/NuExtract3": [ + 1583.8620393042518, + 1670.0089304531896 + ], + "rednote-hilab/dots.mocr": [ + 1664.2994280327837, + 1753.6785416420028 + ], + "rednote-hilab/dots.ocr": [ + 890.3089255199607, + 1066.1833591144534 + ], + "tesseract-5": [ + 1087.687818079124, + 1200.6161530222273 + ], + "zai-org/GLM-OCR": [ + 1640.712657699064, + 1723.182335119403 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 9.056527482656065, + "PaddlePaddle/PP-OCRv6_medium": 2.4921668485824284, + "PaddlePaddle/PaddleOCR-VL-1.6": 30.233783543682648, + "allenai/olmOCR-2-7B-1025-FP8": -58.534873256308856, + "baidu/Qianfan-OCR": -7.255585839271589, + "baidu/Unlimited-OCR": 7.284587054602298, + "deepseek-ai/DeepSeek-OCR": 29.134197338687954, + "deepseek-ai/DeepSeek-OCR-2": 54.830494524718915, + "lightonai/LightOnOCR-2-1B": -37.833037479717404, + "numind/NuExtract3": -64.43828644862015, + "rednote-hilab/dots.mocr": -41.319352647264395, + "rednote-hilab/dots.ocr": 99.6396142776457, + "tesseract-5": 36.34506498662472, + "zai-org/GLM-OCR": -59.635300386018116 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 24, + "min_direct_comparisons": 18, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "targeted", + "max_absolute_elo_delta": 99.6396142776457, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 37.08905123317106, + "name": "targeted_v2_warmup_10_explore_3_annotate_3x", + "percentage_saved": 46.121593291404615, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR-2", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.0, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 91, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 373, + "cumulative_comparisons": 693, + "decision": "warm-up", + "min_direct_comparisons": 5, + "observed_pairs": 91, + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 453, + "cumulative_comparisons": 1146, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1206, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1266, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 454, + "cumulative_comparisons": 1720, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1780, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1840, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 419, + "cumulative_comparisons": 2259, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 2313, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": false, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9912087912087912, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": 10 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 1820, + "comparisons_saved": 2473, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1568.1460930084588, + "PaddlePaddle/PP-OCRv6_medium": 1357.4682136865583, + "PaddlePaddle/PaddleOCR-VL-1.6": 1466.904200168557, + "allenai/olmOCR-2-7B-1025-FP8": 1674.0928720139548, + "baidu/Qianfan-OCR": 1564.7900946582236, + "baidu/Unlimited-OCR": 1521.3045615355204, + "deepseek-ai/DeepSeek-OCR": 1431.8866475038626, + "deepseek-ai/DeepSeek-OCR-2": 1396.756918375702, + "lightonai/LightOnOCR-2-1B": 1687.7204464553765, + "numind/NuExtract3": 1636.636432568152, + "rednote-hilab/dots.mocr": 1802.3207792691564, + "rednote-hilab/dots.ocr": 981.4488359073703, + "tesseract-5": 1181.4670720477113, + "zai-org/GLM-OCR": 1729.0568328013956 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1528.1807699596256, + 1613.9840684786827 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1307.9665070696833, + 1405.13968848872 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1423.3924065964677, + 1510.3515885363781 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1629.2901846365028, + 1722.9152587230114 + ], + "baidu/Qianfan-OCR": [ + 1520.9965109052466, + 1609.40571927851 + ], + "baidu/Unlimited-OCR": [ + 1479.4771418920911, + 1567.7091356270166 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1386.001658261216, + 1479.3209193647988 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1346.2760273639433, + 1445.417708324357 + ], + "lightonai/LightOnOCR-2-1B": [ + 1641.670695693238, + 1745.6891134896557 + ], + "numind/NuExtract3": [ + 1592.0106208552925, + 1686.7951923932571 + ], + "rednote-hilab/dots.mocr": [ + 1747.150046877756, + 1863.4124885725741 + ], + "rednote-hilab/dots.ocr": [ + 857.1382741052148, + 1063.7608456583407 + ], + "tesseract-5": [ + 1116.3430985430382, + 1233.8777855767057 + ], + "zai-org/GLM-OCR": [ + 1681.3668586217814, + 1786.011073303308 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 10.696706323689796, + "PaddlePaddle/PP-OCRv6_medium": -39.628346242043335, + "PaddlePaddle/PaddleOCR-VL-1.6": 3.929647671167686, + "allenai/olmOCR-2-7B-1025-FP8": -45.13826601511232, + "baidu/Qianfan-OCR": -3.3043556439106396, + "baidu/Unlimited-OCR": -23.901441103032766, + "deepseek-ai/DeepSeek-OCR": -20.11037608280617, + "deepseek-ai/DeepSeek-OCR-2": 16.54437038886067, + "lightonai/LightOnOCR-2-1B": -53.532064329378045, + "numind/NuExtract3": -52.9826032218034, + "rednote-hilab/dots.mocr": 56.89551988252583, + "rednote-hilab/dots.ocr": 90.77328734957064, + "tesseract-5": 68.88056368302182, + "zai-org/GLM-OCR": -9.122642660749761 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 45, + "median_direct_comparisons": 18, + "min_direct_comparisons": 15, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "targeted", + "max_absolute_elo_delta": 90.77328734957064, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 31.76489367253805, + "name": "targeted_v2_warmup_5_explore_3_size_stop_3x", + "percentage_saved": 57.60540414628465, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR-2", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 98.55072463768116, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 8.3, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 6 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 320, + "cumulative_comparisons": 320, + "decision": "warm-up", + "min_direct_comparisons": 2, + "observed_pairs": 91, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 373, + "cumulative_comparisons": 693, + "decision": { + "prefer-smaller": 1, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 748, + "decision": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 9, + "allocation_mode": "targeted", + "batch_comparisons": 45, + "cumulative_comparisons": 793, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1248, + "decision": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 30, + "cumulative_comparisons": 1278, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 30, + "cumulative_comparisons": 1308, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1763, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 28, + "cumulative_comparisons": 1791, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 29, + "cumulative_comparisons": 1820, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9868131868131869, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": 5 } ] } diff --git a/experiments/adaptive-stopping/round-history.csv b/experiments/adaptive-stopping/round-history.csv index e5df111..5cf5873 100644 --- a/experiments/adaptive-stopping/round-history.csv +++ b/experiments/adaptive-stopping/round-history.csv @@ -1,71 +1,131 @@ -strategy,round,sample_indices,batch_comparisons,cumulative_comparisons,active_pairs,next_active_pairs,resolved,prefer_smaller,unresolved -balanced_adaptive,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 -balanced_adaptive,2,"[5, 6, 7, 8, 9]",373,693,,12,1,0,12 -balanced_adaptive,3,"[10, 11, 12, 13, 14]",453,1146,,12,1,0,12 -balanced_adaptive,4,"[15, 16, 17, 18, 19]",455,1601,,12,1,0,12 -balanced_adaptive,5,"[20, 21, 22, 23, 24]",455,2056,,11,2,0,11 -balanced_adaptive,6,"[25, 26, 27, 28, 29]",454,2510,,10,3,0,10 -balanced_adaptive,7,"[30, 31, 32, 33, 34]",455,2965,,9,4,0,9 -balanced_adaptive,8,"[35, 36, 37, 38, 39]",455,3420,,9,4,0,9 -balanced_adaptive,9,"[40, 41, 42, 43, 44]",419,3839,,9,4,0,9 -balanced_adaptive,10,"[45, 46, 47, 48, 49]",454,4293,,9,4,0,9 -targeted,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 -targeted,2,"[5, 6, 7, 8, 9]",56,376,13,13,0,0,13 -targeted,3,"[10, 11, 12, 13, 14]",64,440,13,13,0,0,13 -targeted,4,"[15, 16, 17, 18, 19]",65,505,13,12,1,0,12 -targeted,5,"[20, 21, 22, 23, 24]",60,565,12,13,0,0,13 -targeted,6,"[25, 26, 27, 28, 29]",65,630,13,13,0,0,13 -targeted,7,"[30, 31, 32, 33, 34]",65,695,13,12,1,0,12 -targeted,8,"[35, 36, 37, 38, 39]",60,755,12,12,1,0,12 -targeted,9,"[40, 41, 42, 43, 44]",55,810,12,12,1,0,12 -targeted,10,"[45, 46, 47, 48, 49]",60,870,12,12,1,0,12 -targeted_size_3_min_10,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 -targeted_size_3_min_10,2,"[5, 6, 7, 8, 9]",56,376,13,13,0,0,13 -targeted_size_3_min_10,3,"[10, 11, 12, 13, 14]",64,440,13,10,0,3,10 -targeted_size_3_min_10,4,"[15, 16, 17, 18, 19]",50,490,10,10,1,2,10 -targeted_size_3_min_10,5,"[20, 21, 22, 23, 24]",50,540,10,11,0,2,11 -targeted_size_3_min_10,6,"[25, 26, 27, 28, 29]",55,595,11,8,0,5,8 -targeted_size_3_min_10,7,"[30, 31, 32, 33, 34]",40,635,8,7,1,5,7 -targeted_size_3_min_10,8,"[35, 36, 37, 38, 39]",35,670,7,8,0,5,8 -targeted_size_3_min_10,9,"[40, 41, 42, 43, 44]",36,706,8,6,1,6,6 -targeted_size_3_min_10,10,"[45, 46, 47, 48, 49]",30,736,6,6,1,6,6 -targeted_size_2_min_10,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 -targeted_size_2_min_10,2,"[5, 6, 7, 8, 9]",56,376,13,11,0,2,11 -targeted_size_2_min_10,3,"[10, 11, 12, 13, 14]",54,430,11,7,0,6,7 -targeted_size_2_min_10,4,"[15, 16, 17, 18, 19]",35,465,7,8,0,5,8 -targeted_size_2_min_10,5,"[20, 21, 22, 23, 24]",40,505,8,8,1,4,8 -targeted_size_2_min_10,6,"[25, 26, 27, 28, 29]",40,545,8,6,0,7,6 -targeted_size_2_min_10,7,"[30, 31, 32, 33, 34]",30,575,6,5,0,8,5 -targeted_size_2_min_10,8,"[35, 36, 37, 38, 39]",25,600,5,3,1,9,3 -targeted_size_2_min_10,9,"[40, 41, 42, 43, 44]",14,614,3,3,1,9,3 -targeted_size_2_min_10,10,"[45, 46, 47, 48, 49]",15,629,3,3,1,9,3 -targeted_size_3_min_5,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 -targeted_size_3_min_5,2,"[5, 6, 7, 8, 9]",56,376,13,10,0,3,10 -targeted_size_3_min_5,3,"[10, 11, 12, 13, 14]",49,425,10,9,0,4,9 -targeted_size_3_min_5,4,"[15, 16, 17, 18, 19]",45,470,9,9,1,3,9 -targeted_size_3_min_5,5,"[20, 21, 22, 23, 24]",45,515,9,11,0,2,11 -targeted_size_3_min_5,6,"[25, 26, 27, 28, 29]",55,570,11,9,0,4,9 -targeted_size_3_min_5,7,"[30, 31, 32, 33, 34]",45,615,9,8,1,4,8 -targeted_size_3_min_5,8,"[35, 36, 37, 38, 39]",40,655,8,8,1,4,8 -targeted_size_3_min_5,9,"[40, 41, 42, 43, 44]",36,691,8,9,0,4,9 -targeted_size_3_min_5,10,"[45, 46, 47, 48, 49]",44,735,9,8,1,4,8 -targeted_size_3_min_15,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 -targeted_size_3_min_15,2,"[5, 6, 7, 8, 9]",56,376,13,13,0,0,13 -targeted_size_3_min_15,3,"[10, 11, 12, 13, 14]",64,440,13,13,0,0,13 -targeted_size_3_min_15,4,"[15, 16, 17, 18, 19]",65,505,13,10,1,2,10 -targeted_size_3_min_15,5,"[20, 21, 22, 23, 24]",50,555,10,11,0,2,11 -targeted_size_3_min_15,6,"[25, 26, 27, 28, 29]",55,610,11,12,0,1,12 -targeted_size_3_min_15,7,"[30, 31, 32, 33, 34]",60,670,12,8,1,4,8 -targeted_size_3_min_15,8,"[35, 36, 37, 38, 39]",40,710,8,8,1,4,8 -targeted_size_3_min_15,9,"[40, 41, 42, 43, 44]",36,746,8,8,1,4,8 -targeted_size_3_min_15,10,"[45, 46, 47, 48, 49]",40,786,8,7,1,5,7 -targeted_size_5_min_10,1,"[0, 1, 2, 3, 4]",320,320,,13,0,0,13 -targeted_size_5_min_10,2,"[5, 6, 7, 8, 9]",56,376,13,13,0,0,13 -targeted_size_5_min_10,3,"[10, 11, 12, 13, 14]",64,440,13,11,0,2,11 -targeted_size_5_min_10,4,"[15, 16, 17, 18, 19]",55,495,11,12,0,1,12 -targeted_size_5_min_10,5,"[20, 21, 22, 23, 24]",60,555,12,11,1,1,11 -targeted_size_5_min_10,6,"[25, 26, 27, 28, 29]",55,610,11,11,0,2,11 -targeted_size_5_min_10,7,"[30, 31, 32, 33, 34]",55,665,11,10,1,2,10 -targeted_size_5_min_10,8,"[35, 36, 37, 38, 39]",50,715,10,10,1,2,10 -targeted_size_5_min_10,9,"[40, 41, 42, 43, 44]",45,760,10,10,1,2,10 -targeted_size_5_min_10,10,"[45, 46, 47, 48, 49]",50,810,10,9,1,3,9 +strategy,round,sample_indices,batch_comparisons,cumulative_comparisons,allocation_mode,active_pairs,next_active_pairs,min_direct_comparisons,resolved,prefer_smaller,unresolved +balanced_adaptive,1,"[0, 1, 2, 3, 4]",320,320,balanced,,13,,0,0,13 +balanced_adaptive,2,"[5, 6, 7, 8, 9]",373,693,balanced,,12,,1,0,12 +balanced_adaptive,3,"[10, 11, 12, 13, 14]",453,1146,balanced,,12,,1,0,12 +balanced_adaptive,4,"[15, 16, 17, 18, 19]",455,1601,balanced,,12,,1,0,12 +balanced_adaptive,5,"[20, 21, 22, 23, 24]",455,2056,balanced,,11,,2,0,11 +balanced_adaptive,6,"[25, 26, 27, 28, 29]",454,2510,balanced,,10,,3,0,10 +balanced_adaptive,7,"[30, 31, 32, 33, 34]",455,2965,balanced,,9,,4,0,9 +balanced_adaptive,8,"[35, 36, 37, 38, 39]",455,3420,balanced,,9,,4,0,9 +balanced_adaptive,9,"[40, 41, 42, 43, 44]",419,3839,balanced,,9,,4,0,9 +balanced_adaptive,10,"[45, 46, 47, 48, 49]",454,4293,balanced,,9,,4,0,9 +targeted,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,13,,0,0,13 +targeted,2,"[5, 6, 7, 8, 9]",56,376,targeted,13,13,,0,0,13 +targeted,3,"[10, 11, 12, 13, 14]",64,440,targeted,13,13,,0,0,13 +targeted,4,"[15, 16, 17, 18, 19]",65,505,targeted,13,12,,1,0,12 +targeted,5,"[20, 21, 22, 23, 24]",60,565,targeted,12,13,,0,0,13 +targeted,6,"[25, 26, 27, 28, 29]",65,630,targeted,13,13,,0,0,13 +targeted,7,"[30, 31, 32, 33, 34]",65,695,targeted,13,12,,1,0,12 +targeted,8,"[35, 36, 37, 38, 39]",60,755,targeted,12,12,,1,0,12 +targeted,9,"[40, 41, 42, 43, 44]",55,810,targeted,12,12,,1,0,12 +targeted,10,"[45, 46, 47, 48, 49]",60,870,targeted,12,12,,1,0,12 +targeted_size_3_min_10,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,13,,0,0,13 +targeted_size_3_min_10,2,"[5, 6, 7, 8, 9]",56,376,targeted,13,13,,0,0,13 +targeted_size_3_min_10,3,"[10, 11, 12, 13, 14]",64,440,targeted,13,10,,0,3,10 +targeted_size_3_min_10,4,"[15, 16, 17, 18, 19]",50,490,targeted,10,10,,1,2,10 +targeted_size_3_min_10,5,"[20, 21, 22, 23, 24]",50,540,targeted,10,11,,0,2,11 +targeted_size_3_min_10,6,"[25, 26, 27, 28, 29]",55,595,targeted,11,8,,0,5,8 +targeted_size_3_min_10,7,"[30, 31, 32, 33, 34]",40,635,targeted,8,7,,1,5,7 +targeted_size_3_min_10,8,"[35, 36, 37, 38, 39]",35,670,targeted,7,8,,0,5,8 +targeted_size_3_min_10,9,"[40, 41, 42, 43, 44]",36,706,targeted,8,6,,1,6,6 +targeted_size_3_min_10,10,"[45, 46, 47, 48, 49]",30,736,targeted,6,6,,1,6,6 +targeted_size_2_min_10,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,13,,0,0,13 +targeted_size_2_min_10,2,"[5, 6, 7, 8, 9]",56,376,targeted,13,11,,0,2,11 +targeted_size_2_min_10,3,"[10, 11, 12, 13, 14]",54,430,targeted,11,7,,0,6,7 +targeted_size_2_min_10,4,"[15, 16, 17, 18, 19]",35,465,targeted,7,8,,0,5,8 +targeted_size_2_min_10,5,"[20, 21, 22, 23, 24]",40,505,targeted,8,8,,1,4,8 +targeted_size_2_min_10,6,"[25, 26, 27, 28, 29]",40,545,targeted,8,6,,0,7,6 +targeted_size_2_min_10,7,"[30, 31, 32, 33, 34]",30,575,targeted,6,5,,0,8,5 +targeted_size_2_min_10,8,"[35, 36, 37, 38, 39]",25,600,targeted,5,3,,1,9,3 +targeted_size_2_min_10,9,"[40, 41, 42, 43, 44]",14,614,targeted,3,3,,1,9,3 +targeted_size_2_min_10,10,"[45, 46, 47, 48, 49]",15,629,targeted,3,3,,1,9,3 +targeted_size_3_min_5,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,13,,0,0,13 +targeted_size_3_min_5,2,"[5, 6, 7, 8, 9]",56,376,targeted,13,10,,0,3,10 +targeted_size_3_min_5,3,"[10, 11, 12, 13, 14]",49,425,targeted,10,9,,0,4,9 +targeted_size_3_min_5,4,"[15, 16, 17, 18, 19]",45,470,targeted,9,9,,1,3,9 +targeted_size_3_min_5,5,"[20, 21, 22, 23, 24]",45,515,targeted,9,11,,0,2,11 +targeted_size_3_min_5,6,"[25, 26, 27, 28, 29]",55,570,targeted,11,9,,0,4,9 +targeted_size_3_min_5,7,"[30, 31, 32, 33, 34]",45,615,targeted,9,8,,1,4,8 +targeted_size_3_min_5,8,"[35, 36, 37, 38, 39]",40,655,targeted,8,8,,1,4,8 +targeted_size_3_min_5,9,"[40, 41, 42, 43, 44]",36,691,targeted,8,9,,0,4,9 +targeted_size_3_min_5,10,"[45, 46, 47, 48, 49]",44,735,targeted,9,8,,1,4,8 +targeted_size_3_min_15,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,13,,0,0,13 +targeted_size_3_min_15,2,"[5, 6, 7, 8, 9]",56,376,targeted,13,13,,0,0,13 +targeted_size_3_min_15,3,"[10, 11, 12, 13, 14]",64,440,targeted,13,13,,0,0,13 +targeted_size_3_min_15,4,"[15, 16, 17, 18, 19]",65,505,targeted,13,10,,1,2,10 +targeted_size_3_min_15,5,"[20, 21, 22, 23, 24]",50,555,targeted,10,11,,0,2,11 +targeted_size_3_min_15,6,"[25, 26, 27, 28, 29]",55,610,targeted,11,12,,0,1,12 +targeted_size_3_min_15,7,"[30, 31, 32, 33, 34]",60,670,targeted,12,8,,1,4,8 +targeted_size_3_min_15,8,"[35, 36, 37, 38, 39]",40,710,targeted,8,8,,1,4,8 +targeted_size_3_min_15,9,"[40, 41, 42, 43, 44]",36,746,targeted,8,8,,1,4,8 +targeted_size_3_min_15,10,"[45, 46, 47, 48, 49]",40,786,targeted,8,7,,1,5,7 +targeted_size_5_min_10,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,13,,0,0,13 +targeted_size_5_min_10,2,"[5, 6, 7, 8, 9]",56,376,targeted,13,13,,0,0,13 +targeted_size_5_min_10,3,"[10, 11, 12, 13, 14]",64,440,targeted,13,11,,0,2,11 +targeted_size_5_min_10,4,"[15, 16, 17, 18, 19]",55,495,targeted,11,12,,0,1,12 +targeted_size_5_min_10,5,"[20, 21, 22, 23, 24]",60,555,targeted,12,11,,1,1,11 +targeted_size_5_min_10,6,"[25, 26, 27, 28, 29]",55,610,targeted,11,11,,0,2,11 +targeted_size_5_min_10,7,"[30, 31, 32, 33, 34]",55,665,targeted,11,10,,1,2,10 +targeted_size_5_min_10,8,"[35, 36, 37, 38, 39]",50,715,targeted,10,10,,1,2,10 +targeted_size_5_min_10,9,"[40, 41, 42, 43, 44]",45,760,targeted,10,10,,1,2,10 +targeted_size_5_min_10,10,"[45, 46, 47, 48, 49]",50,810,targeted,10,9,,1,3,9 +targeted_v2_warmup_5,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,,2,,, +targeted_v2_warmup_5,2,"[5, 6, 7, 8, 9]",373,693,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_5,3,"[10, 11, 12, 13, 14]",60,753,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,4,"[15, 16, 17, 18, 19]",60,813,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,5,"[20, 21, 22, 23, 24]",60,873,targeted,12,11,,2,0,11 +targeted_v2_warmup_5,6,"[25, 26, 27, 28, 29]",55,928,targeted,11,11,,2,0,11 +targeted_v2_warmup_5,7,"[30, 31, 32, 33, 34]",55,983,targeted,11,12,,1,0,12 +targeted_v2_warmup_5,8,"[35, 36, 37, 38, 39]",60,1043,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,9,"[40, 41, 42, 43, 44]",56,1099,targeted,12,11,,2,0,11 +targeted_v2_warmup_5,10,"[45, 46, 47, 48, 49]",54,1153,targeted,11,11,,2,0,11 +targeted_v2_warmup_10,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,,2,,, +targeted_v2_warmup_10,2,"[5, 6, 7, 8, 9]",373,693,balanced-warmup,,,5,,, +targeted_v2_warmup_10,3,"[10, 11, 12, 13, 14]",453,1146,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_10,4,"[15, 16, 17, 18, 19]",60,1206,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,5,"[20, 21, 22, 23, 24]",60,1266,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,6,"[25, 26, 27, 28, 29]",60,1326,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,7,"[30, 31, 32, 33, 34]",60,1386,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,8,"[35, 36, 37, 38, 39]",60,1446,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,9,"[40, 41, 42, 43, 44]",55,1501,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,10,"[45, 46, 47, 48, 49]",60,1561,targeted,12,12,,1,0,12 +targeted_v2_explore_every_3,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,13,,0,0,13 +targeted_v2_explore_every_3,2,"[5, 6, 7, 8, 9]",56,376,targeted,13,13,,0,0,13 +targeted_v2_explore_every_3,3,"[10, 11, 12, 13, 14]",64,440,targeted,13,13,,0,0,13 +targeted_v2_explore_every_3,4,"[15, 16, 17, 18, 19]",455,895,balanced-exploration,13,12,,1,0,12 +targeted_v2_explore_every_3,5,"[20, 21, 22, 23, 24]",60,955,targeted,12,11,,2,0,11 +targeted_v2_explore_every_3,6,"[25, 26, 27, 28, 29]",54,1009,targeted,11,11,,2,0,11 +targeted_v2_explore_every_3,7,"[30, 31, 32, 33, 34]",455,1464,balanced-exploration,11,11,,2,0,11 +targeted_v2_explore_every_3,8,"[35, 36, 37, 38, 39]",55,1519,targeted,11,11,,2,0,11 +targeted_v2_explore_every_3,9,"[40, 41, 42, 43, 44]",50,1569,targeted,11,11,,2,0,11 +targeted_v2_explore_every_3,10,"[45, 46, 47, 48, 49]",454,2023,balanced-exploration,11,11,,2,0,11 +targeted_v2_warmup_5_explore_3_annotate_3x,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,,2,,, +targeted_v2_warmup_5_explore_3_annotate_3x,2,"[5, 6, 7, 8, 9]",373,693,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,3,"[10, 11, 12, 13, 14]",60,753,targeted,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,4,"[15, 16, 17, 18, 19]",60,813,targeted,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,5,"[20, 21, 22, 23, 24]",455,1268,balanced-exploration,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,6,"[25, 26, 27, 28, 29]",59,1327,targeted,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,7,"[30, 31, 32, 33, 34]",60,1387,targeted,12,11,,2,0,11 +targeted_v2_warmup_5_explore_3_annotate_3x,8,"[35, 36, 37, 38, 39]",455,1842,balanced-exploration,11,11,,2,0,11 +targeted_v2_warmup_5_explore_3_annotate_3x,9,"[40, 41, 42, 43, 44]",50,1892,targeted,11,11,,2,0,11 +targeted_v2_warmup_5_explore_3_annotate_3x,10,"[45, 46, 47, 48, 49]",54,1946,targeted,11,11,,2,0,11 +targeted_v2_warmup_10_explore_3_annotate_3x,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,,2,,, +targeted_v2_warmup_10_explore_3_annotate_3x,2,"[5, 6, 7, 8, 9]",373,693,balanced-warmup,,,5,,, +targeted_v2_warmup_10_explore_3_annotate_3x,3,"[10, 11, 12, 13, 14]",453,1146,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_10_explore_3_annotate_3x,4,"[15, 16, 17, 18, 19]",60,1206,targeted,12,12,,1,0,12 +targeted_v2_warmup_10_explore_3_annotate_3x,5,"[20, 21, 22, 23, 24]",60,1266,targeted,12,12,,1,0,12 +targeted_v2_warmup_10_explore_3_annotate_3x,6,"[25, 26, 27, 28, 29]",454,1720,balanced-exploration,12,12,,1,0,12 +targeted_v2_warmup_10_explore_3_annotate_3x,7,"[30, 31, 32, 33, 34]",60,1780,targeted,12,12,,1,0,12 +targeted_v2_warmup_10_explore_3_annotate_3x,8,"[35, 36, 37, 38, 39]",60,1840,targeted,12,11,,2,0,11 +targeted_v2_warmup_10_explore_3_annotate_3x,9,"[40, 41, 42, 43, 44]",419,2259,balanced-exploration,11,11,,2,0,11 +targeted_v2_warmup_10_explore_3_annotate_3x,10,"[45, 46, 47, 48, 49]",54,2313,targeted,11,11,,2,0,11 +targeted_v2_warmup_5_explore_3_size_stop_3x,1,"[0, 1, 2, 3, 4]",320,320,balanced-warmup,,,2,,, +targeted_v2_warmup_5_explore_3_size_stop_3x,2,"[5, 6, 7, 8, 9]",373,693,balanced-warmup,,11,,1,1,11 +targeted_v2_warmup_5_explore_3_size_stop_3x,3,"[10, 11, 12, 13, 14]",55,748,targeted,11,9,,1,3,9 +targeted_v2_warmup_5_explore_3_size_stop_3x,4,"[15, 16, 17, 18, 19]",45,793,targeted,9,8,,1,4,8 +targeted_v2_warmup_5_explore_3_size_stop_3x,5,"[20, 21, 22, 23, 24]",455,1248,balanced-exploration,8,6,,1,6,6 +targeted_v2_warmup_5_explore_3_size_stop_3x,6,"[25, 26, 27, 28, 29]",30,1278,targeted,6,6,,2,5,6 +targeted_v2_warmup_5_explore_3_size_stop_3x,7,"[30, 31, 32, 33, 34]",30,1308,targeted,6,6,,2,5,6 +targeted_v2_warmup_5_explore_3_size_stop_3x,8,"[35, 36, 37, 38, 39]",455,1763,balanced-exploration,6,6,,2,5,6 +targeted_v2_warmup_5_explore_3_size_stop_3x,9,"[40, 41, 42, 43, 44]",28,1791,targeted,6,6,,2,5,6 +targeted_v2_warmup_5_explore_3_size_stop_3x,10,"[45, 46, 47, 48, 49]",29,1820,targeted,6,6,,2,5,6 diff --git a/experiments/adaptive-stopping/strategy-summary.csv b/experiments/adaptive-stopping/strategy-summary.csv index ed2c71b..5507d0c 100644 --- a/experiments/adaptive-stopping/strategy-summary.csv +++ b/experiments/adaptive-stopping/strategy-summary.csv @@ -1,9 +1,15 @@ -name,comparisons_consumed,comparisons_saved,percentage_saved,stopping_round,stopping_reason,kendall_tau,spearman_rho,top3_membership_matches,top3_order_matches,median_absolute_elo_delta,max_absolute_elo_delta,deterministic_across_repeats -full_stored_board,4293,0,0.0,10,reference_full_stored_board,1.0,1.0,True,True,0.0,0.0,True -balanced_adaptive,4293,0,0.0,10,sample_batches_exhausted,1.0,1.0,True,True,0.0,0.0,True -targeted,870,3423,79.73445143256464,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,36.324012616772166,85.48806834881134,True -targeted_size_3_min_10,736,3557,82.8558117866294,10,sample_batches_exhausted,0.9120879120879122,0.9780219780219781,True,False,58.990128796044814,105.94030058452245,True -targeted_size_2_min_10,629,3664,85.3482413230841,10,sample_batches_exhausted,0.8901098901098902,0.9516483516483516,False,False,79.17359272935346,167.3788996229182, -targeted_size_3_min_5,735,3558,82.87910552061496,10,sample_batches_exhausted,0.8241758241758242,0.9120879120879121,True,False,78.99465533836747,155.25138218007942, -targeted_size_3_min_15,786,3507,81.69112508735151,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,45.96421937350601,98.40518201923669, -targeted_size_5_min_10,810,3483,81.13207547169812,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,35.79769882564415,85.05249919270636, +name,comparisons_consumed,comparisons_saved,percentage_saved,stopping_round,stopping_reason,kendall_tau,spearman_rho,top3_membership_matches,top3_order_matches,median_absolute_elo_delta,max_absolute_elo_delta,followup_gate_passed,deterministic_across_repeats +full_stored_board,4293,0,0.0,10,reference_full_stored_board,1.0,1.0,True,True,0.0,0.0,False,True +balanced_adaptive,4293,0,0.0,10,sample_batches_exhausted,1.0,1.0,True,True,0.0,0.0,False,True +targeted,870,3423,79.73445143256464,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,36.324012616772166,85.48806834881134,False,True +targeted_size_3_min_10,736,3557,82.8558117866294,10,sample_batches_exhausted,0.9120879120879122,0.9780219780219781,True,False,58.990128796044814,105.94030058452245,False,True +targeted_size_2_min_10,629,3664,85.3482413230841,10,sample_batches_exhausted,0.8901098901098902,0.9516483516483516,False,False,79.17359272935346,167.3788996229182,False, +targeted_size_3_min_5,735,3558,82.87910552061496,10,sample_batches_exhausted,0.8241758241758242,0.9120879120879121,True,False,78.99465533836747,155.25138218007942,False, +targeted_size_3_min_15,786,3507,81.69112508735151,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,45.96421937350601,98.40518201923669,False, +targeted_size_5_min_10,810,3483,81.13207547169812,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,False,35.79769882564415,85.05249919270636,False, +targeted_v2_warmup_5,1153,3140,73.14232471465176,10,sample_batches_exhausted,0.9120879120879122,0.9692307692307693,True,True,74.00128832887003,157.47055173226,False, +targeted_v2_warmup_10,1561,2732,63.63848124854414,10,sample_batches_exhausted,0.9340659340659341,0.9824175824175824,True,True,50.010774503823654,99.06594467468392,False, +targeted_v2_explore_every_3,2023,2270,52.8767761472164,10,sample_batches_exhausted,0.9560439560439562,0.9912087912087912,True,False,31.810298159779563,98.55740418889195,False, +targeted_v2_warmup_5_explore_3_annotate_3x,1946,2347,54.67039366410435,10,sample_batches_exhausted,0.9340659340659341,0.9868131868131869,True,False,29.7766934028906,88.15292448408707,False,True +targeted_v2_warmup_10_explore_3_annotate_3x,2313,1980,46.121593291404615,10,sample_batches_exhausted,0.9560439560439562,0.9912087912087912,True,True,37.08905123317106,99.6396142776457,False, +targeted_v2_warmup_5_explore_3_size_stop_3x,1820,2473,57.60540414628465,10,sample_batches_exhausted,0.9340659340659341,0.9868131868131869,True,False,31.76489367253805,90.77328734957064,False, diff --git a/tests/test_adaptive_replay_experiment.py b/tests/test_adaptive_replay_experiment.py index 2bad9f0..67ad9a2 100644 --- a/tests/test_adaptive_replay_experiment.py +++ b/tests/test_adaptive_replay_experiment.py @@ -4,6 +4,7 @@ import importlib.util import sys +from collections import Counter from pathlib import Path from ocr_bench.elo import ComparisonResult, Leaderboard @@ -66,6 +67,73 @@ def test_targeted_replay_warms_up_balanced_then_filters_pairs(monkeypatch): assert targeted.stopping_reason == "sample_batches_exhausted" +def test_per_pair_warmup_requires_every_pair_to_reach_floor(): + counts = Counter({("a", "b"): 5, ("a", "c"): 5, ("b", "c"): 4}) + + assert replay._warmup_ready( + counts, + n_pairs=3, + total_comparisons=20, + min_before_check=20, + min_per_pair=None, + ) + assert not replay._warmup_ready( + counts, + n_pairs=3, + total_comparisons=20, + min_before_check=20, + min_per_pair=5, + ) + counts[("b", "c")] = 5 + assert replay._warmup_ready( + counts, + n_pairs=3, + total_comparisons=20, + min_before_check=20, + min_per_pair=5, + ) + + +def test_periodic_exploration_restores_a_balanced_batch(monkeypatch): + stored = _stored_grid(10) + monkeypatch.setattr(replay, "compute_elo", lambda *args, **kwargs: _fixed_board()) + + result = replay.replay_strategy( + stored, + ["a", "b", "c"], + replay.StrategyConfig( + "explore", + "targeted", + balanced_every_n_post_warmup_batches=3, + ), + n_bootstrap=1, + batch_samples=1, + ) + + assert len(result.comparisons) == 26 + assert result.round_history[-1]["allocation_mode"] == "balanced-exploration" + assert result.round_history[-1]["batch_comparisons"] == 3 + + +def test_size_rule_can_annotate_without_controlling_sampling(): + small = "zai-org/GLM-OCR" + large = "rednote-hilab/dots.mocr" + board = Leaderboard( + elo={large: 1510.0, small: 1500.0}, + elo_ci={large: (1450.0, 1570.0), small: (1440.0, 1560.0)}, + ) + comparisons = [ComparisonResult(i, large, small, "A") for i in range(10)] + config = replay.StrategyConfig( + "annotate-only", + "targeted", + size_tie_ratio=3.0, + size_rule_controls_sampling=False, + ) + + assert replay._classify(board, comparisons, config)[0].status == "prefer-smaller" + assert replay._classify(board, comparisons, config, for_sampling=True)[0].status == "unresolved" + + def test_graph_metrics_report_coverage_and_connectivity(): complete = replay.graph_metrics(_stored_grid(1), ["a", "b", "c"]) disconnected = replay.graph_metrics( From 46d4089c07b14a5ea0d3815bd970c6dcb11113a8 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Wed, 15 Jul 2026 22:52:30 +0100 Subject: [PATCH 3/5] Compare fixed outcome-independent allocations --- experiments/adaptive-stopping/README.md | 82 +- experiments/adaptive-stopping/elo-deltas.csv | 420 ++ experiments/adaptive-stopping/replay.py | 212 +- .../results-sentinels-excluded.json | 5444 ++++++++++++++++ experiments/adaptive-stopping/results.json | 5692 +++++++++++++++++ .../static-design-summary.csv | 7 + .../adaptive-stopping/strategy-summary.csv | 30 + tests/test_adaptive_replay_experiment.py | 36 + 8 files changed, 11901 insertions(+), 22 deletions(-) create mode 100644 experiments/adaptive-stopping/static-design-summary.csv diff --git a/experiments/adaptive-stopping/README.md b/experiments/adaptive-stopping/README.md index 0c60b09..006bbdc 100644 --- a/experiments/adaptive-stopping/README.md +++ b/experiments/adaptive-stopping/README.md @@ -1,8 +1,8 @@ # Adaptive stopping counterfactual replay -**Recommendation: keep targeted allocation experimental and do not run a live trial yet.** A -follow-up replay tested stronger per-pair warm-ups, periodic balanced exploration, and separating -size annotations from sampling. None passed the predeclared fidelity/savings gate. +**Recommendation: keep targeted allocation experimental and do not run a live trial.** Follow-up +replays tested stronger adaptive warm-ups/exploration and fixed outcome-independent designs over +five allocation seeds. No design passed the predeclared fidelity/savings gate robustly. The opt-in targeted allocator delivers the expected call reduction, but not by stopping early: it uses all 10 five-sample rounds while selecting fewer pairs per round. Against the full stored @@ -44,6 +44,8 @@ The replay: 5. Uses production `classify_adjacent_pairs`, `unresolved_pairs`, pair counting, model-size parsing, and practical-preference helpers. 6. Recomputes adjacency after every batch, just as the CLI does. +7. Separately evaluates fixed pair-balanced and mixed-random designs at budgets 700, 1,200, and + 2,000 over allocation seeds 42–46; these selectors never inspect winner values. The primary replay intentionally treats all 4,293 published rows as outcomes because that is the requested reference board and reproduces the historical leaderboard. The data includes 628 rows @@ -208,6 +210,38 @@ The exact order of the full board's top three is itself weakly identified—thei should not become a universal product criterion. It remains useful here as a deliberately strict counterfactual fidelity check, alongside full-order correlation and ELO drift. +## Fixed outcome-independent designs + +The next proposed direction was also replayed. Two fixed designs were evaluated at predeclared +budgets of 700, 1,200, and 2,000 outcomes over seeds 42–46: + +- **Pair-balanced:** shuffle outcomes within each model pair, then assign equal per-pair quotas. +- **Mixed-random:** retain the first five-page balanced warm-up, then fill the budget by seeded + pair-balanced exploration independent of interim winners. + +Both condition only on which valid stored rows exist; neither reads the winner when allocating. +The table reports medians and ranges across five seeds. Top-3 columns are counts out of five. + +| Design | Budget | Saved | Kendall τ median [range] | Top-3 set/order | Median abs(ΔELO), median | Max abs(ΔELO), median/worst | Min pair evidence | Gate passes | +|---|---:|---:|---|---:|---:|---:|---:|---:| +| Pair-balanced | 700 | 83.7% | 0.868 [0.758–0.978] | 1/1 | 34.9 | 75.2 / 116.4 | 7 | 0/5 | +| Pair-balanced | 1,200 | 72.0% | 0.890 [0.780–0.956] | 2/0 | 18.8 | 49.1 / 89.0 | 13 | 0/5 | +| Pair-balanced | 2,000 | 53.4% | 0.868 [0.846–0.956] | 1/1 | 17.6 | 37.7 / 61.2 | 21 | 0/5 | +| Mixed-random | 700 | 83.7% | 0.868 [0.868–0.890] | 5/1 | 48.2 | 146.7 / 191.3 | 7 | 0/5 | +| Mixed-random | 1,200 | 72.0% | 0.934 [0.934–0.956] | 5/1 | 28.7 | 79.8 / 122.8 | 13 | 0/5 | +| Mixed-random | 2,000 | 53.4% | 0.956 [0.934–0.978] | 5/2 | 13.7 | 46.3 / 61.4 | 21 | 0/5 | + +No fixed design passed the original gate. Mixed-random preserved top-3 **membership** in all 15 +runs and reached median Kendall τ 0.956 at budget 2,000, but that budget saved only 53.4%, exact +top-3 order held for 2/5 seeds, and worst-seed max ELO drift remained 61.4. At budget 1,200 it +saved 72.0%, but median max drift was 79.8. + +The seed ranges are as important as the medians: a single favorable seed would have overstated +fidelity. Outcome-independent allocation removes the targeted policy's optional-sampling feedback, +but does not make a small board equivalent to the full one. The fixed designs also show that +pair-count balance alone is insufficient; preserving a common balanced warm-up made top-3 +membership much more stable than independently sampling each pair. + ## Current sentinel-policy robustness check The primary result must use the requested 4,293-row stored board. However, current production @@ -225,7 +259,11 @@ All three adaptive runs still exhaust the sample batches. The targeted-v2 varian predeclared gate under current sentinel handling. The closest on ELO drift—warm-up 5 plus exploration 3, annotation-only—uses 1,684 outcomes (54.1% saved), has median/max absolute ELO deltas of 18.1/57.0, and still swaps the statistically unresolved models in ranks 2 and 3. -See [`results-sentinels-excluded.json`](results-sentinels-excluded.json) for full details. +For the fixed designs under current sentinel handling, mixed-random at budget 1,200 passed the +gate for 2/5 seeds but not robustly; no design/budget passed for every seed. Mixed-random at 2,000 +had median/worst max ELO drift of 22.3/32.3, but saved only 45.4% and reproduced exact top-3 order +in 0/5 seeds. See [`results-sentinels-excluded.json`](results-sentinels-excluded.json) for full +details. ## Determinism @@ -235,8 +273,11 @@ decisions, final annotations, stop round, and stop reason matched exactly. This deterministic execution in the tested environment, helped by the fixed bootstrap seed and deterministic equal-ELO tie-break. -It does not establish reproducibility across different SciPy/NumPy versions or statistical -validity under repeated data collection. +The fixed selectors are also deterministic for a fixed seed, with unit tests covering exact +budgets, pair balance, warm-up preservation, and repeat selection. Their five-seed spread measures +allocation sensitivity, not repeated-data statistical uncertainty. None of this establishes +reproducibility across different SciPy/NumPy versions or statistical validity under repeated data +collection. ## Limitations @@ -264,22 +305,20 @@ validity under repeated data collection. ## Recommendation and next experiment -**Keep `targeted` opt-in, retain `balanced` as the default, and do not implement the tested v2 -variants in production.** The follow-up falsified the simple “more warm-up plus periodic balanced -batches” revision under the predeclared gate. +**Keep `targeted` opt-in, retain `balanced` as the default, and do not implement the tested v2 or +fixed designs in production.** Neither threshold tuning nor the tested outcome-independent designs +passed the predeclared gate robustly. -The next useful direction is a different estimator/design rather than more threshold tuning: +Do not tune further on Britannica alone. The next useful work is methodological validation: -1. compare fixed balanced subsampling at matched budgets (for example 700, 1,200, and 2,000 - outcomes) against outcome-conditioned targeting; -2. test a predeclared mixed allocation with random exploration independent of interim winners; -3. evaluate top-k membership and pairwise decisions separately from exact ordering among - statistically unresolved models; and -4. use page-clustered, policy-aware uncertainty before considering a live adaptive trial. +1. replay the mixed-random design on several independent published boards; +2. predeclare a success criterion based on top-k membership and pairwise decisions, separately + from exact ordering among statistically unresolved models; +3. use page-clustered/design-aware uncertainty and report allocation-seed sensitivity; and +4. consider one live trial only if a design passes across collections and seeds. -Keep the 3× rule as a post-hoc deployment annotation only during that work. This board does not -support letting it influence sampling, making it a default, or treating it as statistical -resolution. +Keep the 3× rule as a post-hoc deployment annotation only. This board does not support letting it +influence sampling, making it a default, or treating it as statistical resolution. ## Reproduce @@ -290,8 +329,8 @@ uv sync --dev uv run python experiments/adaptive-stopping/replay.py ``` -The full primary, sensitivity, and targeted-v2 run takes about eleven minutes on the machine used -for this report. The pinned dataset +The full primary, sensitivity, targeted-v2, and fixed-design run takes about eleven minutes on the +machine used for this report. The pinned dataset revision is downloaded read-only. The script contains no judge backend construction and no Hub push/upload call. @@ -310,6 +349,7 @@ Generated artifacts: - [`strategy-summary.csv`](strategy-summary.csv): strategy-level metrics - [`elo-deltas.csv`](elo-deltas.csv): per-model ELO and deltas - [`round-history.csv`](round-history.csv): batch-by-batch allocation and decisions +- [`static-design-summary.csv`](static-design-summary.csv): five-seed fixed-design aggregates - [`results-sentinels-excluded.json`](results-sentinels-excluded.json): robustness replay Validation commands: diff --git a/experiments/adaptive-stopping/elo-deltas.csv b/experiments/adaptive-stopping/elo-deltas.csv index 85e0b39..4e93120 100644 --- a/experiments/adaptive-stopping/elo-deltas.csv +++ b/experiments/adaptive-stopping/elo-deltas.csv @@ -195,3 +195,423 @@ targeted_v2_warmup_5_explore_3_size_stop_3x,rednote-hilab/dots.mocr,1802.3207792 targeted_v2_warmup_5_explore_3_size_stop_3x,rednote-hilab/dots.ocr,981.4488359073703,890.6755485577996,90.77328734957064,90.77328734957064 targeted_v2_warmup_5_explore_3_size_stop_3x,tesseract-5,1181.4670720477113,1112.5865083646895,68.88056368302182,68.88056368302182 targeted_v2_warmup_5_explore_3_size_stop_3x,zai-org/GLM-OCR,1729.0568328013956,1738.1794754621453,-9.122642660749761,9.122642660749761 +pair-balanced_700_seed_42,FireRedTeam/FireRed-OCR,1608.3446805050585,1557.449386684769,50.89529382028945,50.89529382028945 +pair-balanced_700_seed_42,PaddlePaddle/PP-OCRv6_medium,1332.8846477743145,1397.0965599286017,-64.21191215428712,64.21191215428712 +pair-balanced_700_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1438.3575163571581,1462.9745524973894,-24.61703614023122,24.61703614023122 +pair-balanced_700_seed_42,allenai/olmOCR-2-7B-1025-FP8,1730.214953273804,1719.231138029067,10.983815244736888,10.983815244736888 +pair-balanced_700_seed_42,baidu/Qianfan-OCR,1581.4061124614927,1568.0944503021342,13.311662159358548,13.311662159358548 +pair-balanced_700_seed_42,baidu/Unlimited-OCR,1564.0356246826134,1545.2060026385532,18.829622044060216,18.829622044060216 +pair-balanced_700_seed_42,deepseek-ai/DeepSeek-OCR,1418.1917897223525,1451.9970235866688,-33.80523386431628,33.80523386431628 +pair-balanced_700_seed_42,deepseek-ai/DeepSeek-OCR-2,1385.1289084322368,1380.2125479868414,4.916360445395412,4.916360445395412 +pair-balanced_700_seed_42,lightonai/LightOnOCR-2-1B,1778.2195280154026,1741.2525107847546,36.967017230648025,36.967017230648025 +pair-balanced_700_seed_42,numind/NuExtract3,1737.608679123864,1689.6190357899554,47.98964333390859,47.98964333390859 +pair-balanced_700_seed_42,rednote-hilab/dots.mocr,1739.579581522016,1745.4252593866306,-5.845677864614572,5.845677864614572 +pair-balanced_700_seed_42,rednote-hilab/dots.ocr,845.042415658463,890.6755485577996,-45.6331328993366,45.6331328993366 +pair-balanced_700_seed_42,tesseract-5,1155.173699195922,1112.5865083646895,42.58719083123242,42.58719083123242 +pair-balanced_700_seed_42,zai-org/GLM-OCR,1685.8118632753017,1738.1794754621453,-52.367612186843644,52.367612186843644 +pair-balanced_700_seed_43,FireRedTeam/FireRed-OCR,1657.1329483356512,1557.449386684769,99.68356165088221,99.68356165088221 +pair-balanced_700_seed_43,PaddlePaddle/PP-OCRv6_medium,1378.9707197797002,1397.0965599286017,-18.12584014890149,18.12584014890149 +pair-balanced_700_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1346.5901715006312,1462.9745524973894,-116.38438099675818,116.38438099675818 +pair-balanced_700_seed_43,allenai/olmOCR-2-7B-1025-FP8,1727.613642061453,1719.231138029067,8.382504032385896,8.382504032385896 +pair-balanced_700_seed_43,baidu/Qianfan-OCR,1534.602018112062,1568.0944503021342,-33.49243219007212,33.49243219007212 +pair-balanced_700_seed_43,baidu/Unlimited-OCR,1585.4166207996852,1545.2060026385532,40.21061816113206,40.21061816113206 +pair-balanced_700_seed_43,deepseek-ai/DeepSeek-OCR,1380.324223342501,1451.9970235866688,-71.67280024416777,71.67280024416777 +pair-balanced_700_seed_43,deepseek-ai/DeepSeek-OCR-2,1388.941278698522,1380.2125479868414,8.728730711680555,8.728730711680555 +pair-balanced_700_seed_43,lightonai/LightOnOCR-2-1B,1783.4484061534754,1741.2525107847546,42.19589536872081,42.19589536872081 +pair-balanced_700_seed_43,numind/NuExtract3,1742.152908755929,1689.6190357899554,52.533872965973615,52.533872965973615 +pair-balanced_700_seed_43,rednote-hilab/dots.mocr,1762.0509140256804,1745.4252593866306,16.6256546390498,16.6256546390498 +pair-balanced_700_seed_43,rednote-hilab/dots.ocr,844.0078925036925,890.6755485577996,-46.66765605410717,46.66765605410717 +pair-balanced_700_seed_43,tesseract-5,1143.1918660877843,1112.5865083646895,30.605357723094812,30.605357723094812 +pair-balanced_700_seed_43,zai-org/GLM-OCR,1725.5563898432322,1738.1794754621453,-12.623085618913137,12.623085618913137 +pair-balanced_700_seed_44,FireRedTeam/FireRed-OCR,1549.1031763628914,1557.449386684769,-8.346210321877606,8.346210321877606 +pair-balanced_700_seed_44,PaddlePaddle/PP-OCRv6_medium,1387.1285844098475,1397.0965599286017,-9.967975518754201,9.967975518754201 +pair-balanced_700_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1466.2594932059958,1462.9745524973894,3.2849407086064275,3.2849407086064275 +pair-balanced_700_seed_44,allenai/olmOCR-2-7B-1025-FP8,1691.5775500285006,1719.231138029067,-27.653588000566515,27.653588000566515 +pair-balanced_700_seed_44,baidu/Qianfan-OCR,1563.2843283861919,1568.0944503021342,-4.810121915942318,4.810121915942318 +pair-balanced_700_seed_44,baidu/Unlimited-OCR,1557.0491256404791,1545.2060026385532,11.84312300192596,11.84312300192596 +pair-balanced_700_seed_44,deepseek-ai/DeepSeek-OCR,1406.808327693129,1451.9970235866688,-45.188695893539716,45.188695893539716 +pair-balanced_700_seed_44,deepseek-ai/DeepSeek-OCR-2,1364.972585224888,1380.2125479868414,-15.239962761953393,15.239962761953393 +pair-balanced_700_seed_44,lightonai/LightOnOCR-2-1B,1732.6001138796714,1741.2525107847546,-8.652396905083151,8.652396905083151 +pair-balanced_700_seed_44,numind/NuExtract3,1672.0071518119494,1689.6190357899554,-17.611883978006063,17.611883978006063 +pair-balanced_700_seed_44,rednote-hilab/dots.mocr,1787.4122936553579,1745.4252593866306,41.98703426872726,41.98703426872726 +pair-balanced_700_seed_44,rednote-hilab/dots.ocr,914.7807431316438,890.6755485577996,24.105194573844187,24.105194573844187 +pair-balanced_700_seed_44,tesseract-5,1185.162626437656,1112.5865083646895,72.57611807296644,72.57611807296644 +pair-balanced_700_seed_44,zai-org/GLM-OCR,1721.8539001317981,1738.1794754621453,-16.325575330347192,16.325575330347192 +pair-balanced_700_seed_45,FireRedTeam/FireRed-OCR,1494.4432451728896,1557.449386684769,-63.00614151187938,63.00614151187938 +pair-balanced_700_seed_45,PaddlePaddle/PP-OCRv6_medium,1400.5386642660735,1397.0965599286017,3.4421043374718465,3.4421043374718465 +pair-balanced_700_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1504.3086114691523,1462.9745524973894,41.33405897176294,41.33405897176294 +pair-balanced_700_seed_45,allenai/olmOCR-2-7B-1025-FP8,1832.0016417430204,1719.231138029067,112.7705037139533,112.7705037139533 +pair-balanced_700_seed_45,baidu/Qianfan-OCR,1587.7651831084647,1568.0944503021342,19.670732806330534,19.670732806330534 +pair-balanced_700_seed_45,baidu/Unlimited-OCR,1510.9300755021095,1545.2060026385532,-34.27592713644367,34.27592713644367 +pair-balanced_700_seed_45,deepseek-ai/DeepSeek-OCR,1440.0119728594623,1451.9970235866688,-11.985050727206499,11.985050727206499 +pair-balanced_700_seed_45,deepseek-ai/DeepSeek-OCR-2,1397.4088449651597,1380.2125479868414,17.196296978318287,17.196296978318287 +pair-balanced_700_seed_45,lightonai/LightOnOCR-2-1B,1772.1662715946009,1741.2525107847546,30.91376080984628,30.91376080984628 +pair-balanced_700_seed_45,numind/NuExtract3,1654.1202425151914,1689.6190357899554,-35.49879327476401,35.49879327476401 +pair-balanced_700_seed_45,rednote-hilab/dots.mocr,1758.8556429651162,1745.4252593866306,13.43038357848559,13.43038357848559 +pair-balanced_700_seed_45,rednote-hilab/dots.ocr,789.246045194604,890.6755485577996,-101.42950336319564,101.42950336319564 +pair-balanced_700_seed_45,tesseract-5,1168.1923471362472,1112.5865083646895,55.60583877155773,55.60583877155773 +pair-balanced_700_seed_45,zai-org/GLM-OCR,1690.011211507908,1738.1794754621453,-48.16826395423732,48.16826395423732 +pair-balanced_700_seed_46,FireRedTeam/FireRed-OCR,1532.9965763416174,1557.449386684769,-24.45281034315167,24.45281034315167 +pair-balanced_700_seed_46,PaddlePaddle/PP-OCRv6_medium,1350.6813473586062,1397.0965599286017,-46.41521256999545,46.41521256999545 +pair-balanced_700_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1483.4461686224597,1462.9745524973894,20.47161612507034,20.47161612507034 +pair-balanced_700_seed_46,allenai/olmOCR-2-7B-1025-FP8,1759.2535591141034,1719.231138029067,40.022421085036285,40.022421085036285 +pair-balanced_700_seed_46,baidu/Qianfan-OCR,1522.273532617472,1568.0944503021342,-45.82091768466216,45.82091768466216 +pair-balanced_700_seed_46,baidu/Unlimited-OCR,1570.4668927353853,1545.2060026385532,25.2608900968321,25.2608900968321 +pair-balanced_700_seed_46,deepseek-ai/DeepSeek-OCR,1447.614877087872,1451.9970235866688,-4.382146498796828,4.382146498796828 +pair-balanced_700_seed_46,deepseek-ai/DeepSeek-OCR-2,1427.3906577646069,1380.2125479868414,47.178109777765485,47.178109777765485 +pair-balanced_700_seed_46,lightonai/LightOnOCR-2-1B,1789.5080562081266,1741.2525107847546,48.25554542337204,48.25554542337204 +pair-balanced_700_seed_46,numind/NuExtract3,1732.0687549281085,1689.6190357899554,42.449719138153114,42.449719138153114 +pair-balanced_700_seed_46,rednote-hilab/dots.mocr,1752.0195012182132,1745.4252593866306,6.594241831582622,6.594241831582622 +pair-balanced_700_seed_46,rednote-hilab/dots.ocr,875.1068604120732,890.6755485577996,-15.568688145726469,15.568688145726469 +pair-balanced_700_seed_46,tesseract-5,1037.41488379773,1112.5865083646895,-75.17162456695951,75.17162456695951 +pair-balanced_700_seed_46,zai-org/GLM-OCR,1719.7583317936255,1738.1794754621453,-18.42114366851979,18.42114366851979 +pair-balanced_1200_seed_42,FireRedTeam/FireRed-OCR,1600.4726717794176,1557.449386684769,43.0232850946486,43.0232850946486 +pair-balanced_1200_seed_42,PaddlePaddle/PP-OCRv6_medium,1378.0985581446023,1397.0965599286017,-18.99800178399937,18.99800178399937 +pair-balanced_1200_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1437.665883952499,1462.9745524973894,-25.308668544890452,25.308668544890452 +pair-balanced_1200_seed_42,allenai/olmOCR-2-7B-1025-FP8,1711.533561624297,1719.231138029067,-7.697576404770189,7.697576404770189 +pair-balanced_1200_seed_42,baidu/Qianfan-OCR,1558.7249211818164,1568.0944503021342,-9.369529120317793,9.369529120317793 +pair-balanced_1200_seed_42,baidu/Unlimited-OCR,1554.7451470411702,1545.2060026385532,9.539144402617012,9.539144402617012 +pair-balanced_1200_seed_42,deepseek-ai/DeepSeek-OCR,1429.9817171297507,1451.9970235866688,-22.015306456918097,22.015306456918097 +pair-balanced_1200_seed_42,deepseek-ai/DeepSeek-OCR-2,1396.4126892988545,1380.2125479868414,16.200141312013102,16.200141312013102 +pair-balanced_1200_seed_42,lightonai/LightOnOCR-2-1B,1784.2069345273126,1741.2525107847546,42.95442374255799,42.95442374255799 +pair-balanced_1200_seed_42,numind/NuExtract3,1733.310681888583,1689.6190357899554,43.69164609862764,43.69164609862764 +pair-balanced_1200_seed_42,rednote-hilab/dots.mocr,1771.8173673153167,1745.4252593866306,26.392107928686073,26.392107928686073 +pair-balanced_1200_seed_42,rednote-hilab/dots.ocr,801.6805262528865,890.6755485577996,-88.99502230491316,88.99502230491316 +pair-balanced_1200_seed_42,tesseract-5,1131.1180934856711,1112.5865083646895,18.531585120981617,18.531585120981617 +pair-balanced_1200_seed_42,zai-org/GLM-OCR,1710.2312463778226,1738.1794754621453,-27.948229084322747,27.948229084322747 +pair-balanced_1200_seed_43,FireRedTeam/FireRed-OCR,1606.5699808139275,1557.449386684769,49.12059412915846,49.12059412915846 +pair-balanced_1200_seed_43,PaddlePaddle/PP-OCRv6_medium,1415.5873224553688,1397.0965599286017,18.490762526767185,18.490762526767185 +pair-balanced_1200_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1417.2122550103788,1462.9745524973894,-45.76229748701053,45.76229748701053 +pair-balanced_1200_seed_43,allenai/olmOCR-2-7B-1025-FP8,1700.6957270024277,1719.231138029067,-18.535411026639395,18.535411026639395 +pair-balanced_1200_seed_43,baidu/Qianfan-OCR,1551.367946962283,1568.0944503021342,-16.726503339851206,16.726503339851206 +pair-balanced_1200_seed_43,baidu/Unlimited-OCR,1558.7750426567673,1545.2060026385532,13.569040018214082,13.569040018214082 +pair-balanced_1200_seed_43,deepseek-ai/DeepSeek-OCR,1428.1867691315038,1451.9970235866688,-23.810254455165023,23.810254455165023 +pair-balanced_1200_seed_43,deepseek-ai/DeepSeek-OCR-2,1408.5674165777405,1380.2125479868414,28.35486859089906,28.35486859089906 +pair-balanced_1200_seed_43,lightonai/LightOnOCR-2-1B,1714.1415689383318,1741.2525107847546,-27.110941846422747,27.110941846422747 +pair-balanced_1200_seed_43,numind/NuExtract3,1677.1655861283741,1689.6190357899554,-12.453449661581317,12.453449661581317 +pair-balanced_1200_seed_43,rednote-hilab/dots.mocr,1743.8318867174214,1745.4252593866306,-1.5933726692092023,1.5933726692092023 +pair-balanced_1200_seed_43,rednote-hilab/dots.ocr,912.3362994163375,890.6755485577996,21.66075085853788,21.66075085853788 +pair-balanced_1200_seed_43,tesseract-5,1144.663740819601,1112.5865083646895,32.07723245491138,32.07723245491138 +pair-balanced_1200_seed_43,zai-org/GLM-OCR,1720.898457369537,1738.1794754621453,-17.281018092608292,17.281018092608292 +pair-balanced_1200_seed_44,FireRedTeam/FireRed-OCR,1573.5424064930826,1557.449386684769,16.093019808313556,16.093019808313556 +pair-balanced_1200_seed_44,PaddlePaddle/PP-OCRv6_medium,1395.8997498636347,1397.0965599286017,-1.1968100649669395,1.1968100649669395 +pair-balanced_1200_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1463.9790234713273,1462.9745524973894,1.0044709739379414,1.0044709739379414 +pair-balanced_1200_seed_44,allenai/olmOCR-2-7B-1025-FP8,1683.0223963923431,1719.231138029067,-36.20874163672397,36.20874163672397 +pair-balanced_1200_seed_44,baidu/Qianfan-OCR,1574.352219145655,1568.0944503021342,6.257768843520807,6.257768843520807 +pair-balanced_1200_seed_44,baidu/Unlimited-OCR,1529.156500455611,1545.2060026385532,-16.049502182942206,16.049502182942206 +pair-balanced_1200_seed_44,deepseek-ai/DeepSeek-OCR,1418.4389754414194,1451.9970235866688,-33.55804814524936,33.55804814524936 +pair-balanced_1200_seed_44,deepseek-ai/DeepSeek-OCR-2,1336.4144529728567,1380.2125479868414,-43.79809501398472,43.79809501398472 +pair-balanced_1200_seed_44,lightonai/LightOnOCR-2-1B,1732.9551869653317,1741.2525107847546,-8.2973238194229,8.2973238194229 +pair-balanced_1200_seed_44,numind/NuExtract3,1710.3223213798774,1689.6190357899554,20.70328558992196,20.70328558992196 +pair-balanced_1200_seed_44,rednote-hilab/dots.mocr,1788.4854627634636,1745.4252593866306,43.06020337683299,43.06020337683299 +pair-balanced_1200_seed_44,rednote-hilab/dots.ocr,903.9097693046194,890.6755485577996,13.234220746819801,13.234220746819801 +pair-balanced_1200_seed_44,tesseract-5,1145.3288761166018,1112.5865083646895,32.7423677519123,32.7423677519123 +pair-balanced_1200_seed_44,zai-org/GLM-OCR,1744.1926592341763,1738.1794754621453,6.01318377203097,6.01318377203097 +pair-balanced_1200_seed_45,FireRedTeam/FireRed-OCR,1545.724521818796,1557.449386684769,-11.724864865973132,11.724864865973132 +pair-balanced_1200_seed_45,PaddlePaddle/PP-OCRv6_medium,1383.900840635146,1397.0965599286017,-13.195719293455568,13.195719293455568 +pair-balanced_1200_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1464.4178359389718,1462.9745524973894,1.44328344158248,1.44328344158248 +pair-balanced_1200_seed_45,allenai/olmOCR-2-7B-1025-FP8,1807.1318714731017,1719.231138029067,87.90073344403459,87.90073344403459 +pair-balanced_1200_seed_45,baidu/Qianfan-OCR,1578.2968420561524,1568.0944503021342,10.202391754018208,10.202391754018208 +pair-balanced_1200_seed_45,baidu/Unlimited-OCR,1513.311255633662,1545.2060026385532,-31.894747004891087,31.894747004891087 +pair-balanced_1200_seed_45,deepseek-ai/DeepSeek-OCR,1431.653750732666,1451.9970235866688,-20.343272854002862,20.343272854002862 +pair-balanced_1200_seed_45,deepseek-ai/DeepSeek-OCR-2,1410.6557947711547,1380.2125479868414,30.44324678431326,30.44324678431326 +pair-balanced_1200_seed_45,lightonai/LightOnOCR-2-1B,1727.1786591904113,1741.2525107847546,-14.073851594343296,14.073851594343296 +pair-balanced_1200_seed_45,numind/NuExtract3,1694.3839038775739,1689.6190357899554,4.764868087618424,4.764868087618424 +pair-balanced_1200_seed_45,rednote-hilab/dots.mocr,1732.8897793121141,1745.4252593866306,-12.5354800745165,12.5354800745165 +pair-balanced_1200_seed_45,rednote-hilab/dots.ocr,809.7273002286885,890.6755485577996,-80.94824832911115,80.94824832911115 +pair-balanced_1200_seed_45,tesseract-5,1169.762673008484,1112.5865083646895,57.176164643794436,57.176164643794436 +pair-balanced_1200_seed_45,zai-org/GLM-OCR,1730.9649713230774,1738.1794754621453,-7.2145041390679125,7.2145041390679125 +pair-balanced_1200_seed_46,FireRedTeam/FireRed-OCR,1526.5101874998254,1557.449386684769,-30.939199184943618,30.939199184943618 +pair-balanced_1200_seed_46,PaddlePaddle/PP-OCRv6_medium,1374.302046872808,1397.0965599286017,-22.794513055793686,22.794513055793686 +pair-balanced_1200_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1479.9544584515177,1462.9745524973894,16.9799059541283,16.9799059541283 +pair-balanced_1200_seed_46,allenai/olmOCR-2-7B-1025-FP8,1724.771396416001,1719.231138029067,5.540258386933829,5.540258386933829 +pair-balanced_1200_seed_46,baidu/Qianfan-OCR,1547.3750745751424,1568.0944503021342,-20.719375726991757,20.719375726991757 +pair-balanced_1200_seed_46,baidu/Unlimited-OCR,1551.1095763124817,1545.2060026385532,5.903573673928577,5.903573673928577 +pair-balanced_1200_seed_46,deepseek-ai/DeepSeek-OCR,1465.5134908836683,1451.9970235866688,13.516467296999508,13.516467296999508 +pair-balanced_1200_seed_46,deepseek-ai/DeepSeek-OCR-2,1403.4914746396144,1380.2125479868414,23.278926652772952,23.278926652772952 +pair-balanced_1200_seed_46,lightonai/LightOnOCR-2-1B,1728.362116552952,1741.2525107847546,-12.890394231802475,12.890394231802475 +pair-balanced_1200_seed_46,numind/NuExtract3,1736.236829719455,1689.6190357899554,46.61779392949961,46.61779392949961 +pair-balanced_1200_seed_46,rednote-hilab/dots.mocr,1719.1750827872102,1745.4252593866306,-26.250176599420456,26.250176599420456 +pair-balanced_1200_seed_46,rednote-hilab/dots.ocr,905.1330347400462,890.6755485577996,14.457486182246612,14.457486182246612 +pair-balanced_1200_seed_46,tesseract-5,1121.3977460985457,1112.5865083646895,8.81123773385616,8.81123773385616 +pair-balanced_1200_seed_46,zai-org/GLM-OCR,1716.6674844507322,1738.1794754621453,-21.5119910114131,21.5119910114131 +pair-balanced_2000_seed_42,FireRedTeam/FireRed-OCR,1596.1874707393995,1557.449386684769,38.73808405463046,38.73808405463046 +pair-balanced_2000_seed_42,PaddlePaddle/PP-OCRv6_medium,1394.3910182438551,1397.0965599286017,-2.7055416847465494,2.7055416847465494 +pair-balanced_2000_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1452.1281104188165,1462.9745524973894,-10.846442078572863,10.846442078572863 +pair-balanced_2000_seed_42,allenai/olmOCR-2-7B-1025-FP8,1701.8443541143245,1719.231138029067,-17.386783914742637,17.386783914742637 +pair-balanced_2000_seed_42,baidu/Qianfan-OCR,1547.5002410008585,1568.0944503021342,-20.59420930127567,20.59420930127567 +pair-balanced_2000_seed_42,baidu/Unlimited-OCR,1563.9847142297087,1545.2060026385532,18.77871159115557,18.77871159115557 +pair-balanced_2000_seed_42,deepseek-ai/DeepSeek-OCR,1414.660622351955,1451.9970235866688,-37.336401234713776,37.336401234713776 +pair-balanced_2000_seed_42,deepseek-ai/DeepSeek-OCR-2,1371.7335689048398,1380.2125479868414,-8.478979082001615,8.478979082001615 +pair-balanced_2000_seed_42,lightonai/LightOnOCR-2-1B,1759.0179249803687,1741.2525107847546,17.765414195614085,17.765414195614085 +pair-balanced_2000_seed_42,numind/NuExtract3,1706.7482173550318,1689.6190357899554,17.12918156507635,17.12918156507635 +pair-balanced_2000_seed_42,rednote-hilab/dots.mocr,1787.052865705835,1745.4252593866306,41.62760631920446,41.62760631920446 +pair-balanced_2000_seed_42,rednote-hilab/dots.ocr,867.795044383654,890.6755485577996,-22.880504174145585,22.880504174145585 +pair-balanced_2000_seed_42,tesseract-5,1113.81008423597,1112.5865083646895,1.2235758712804454,1.2235758712804454 +pair-balanced_2000_seed_42,zai-org/GLM-OCR,1723.1457633353828,1738.1794754621453,-15.033712126762566,15.033712126762566 +pair-balanced_2000_seed_43,FireRedTeam/FireRed-OCR,1566.6268490438601,1557.449386684769,9.177462359091123,9.177462359091123 +pair-balanced_2000_seed_43,PaddlePaddle/PP-OCRv6_medium,1405.067456468896,1397.0965599286017,7.970896540294234,7.970896540294234 +pair-balanced_2000_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1435.3969521958059,1462.9745524973894,-27.577600301583516,27.577600301583516 +pair-balanced_2000_seed_43,allenai/olmOCR-2-7B-1025-FP8,1739.3632219889735,1719.231138029067,20.132083959906367,20.132083959906367 +pair-balanced_2000_seed_43,baidu/Qianfan-OCR,1541.852759968683,1568.0944503021342,-26.241690333451288,26.241690333451288 +pair-balanced_2000_seed_43,baidu/Unlimited-OCR,1544.7706986544954,1545.2060026385532,-0.43530398405778215,0.43530398405778215 +pair-balanced_2000_seed_43,deepseek-ai/DeepSeek-OCR,1424.8021970795628,1451.9970235866688,-27.194826507105972,27.194826507105972 +pair-balanced_2000_seed_43,deepseek-ai/DeepSeek-OCR-2,1413.163607752931,1380.2125479868414,32.95105976608966,32.95105976608966 +pair-balanced_2000_seed_43,lightonai/LightOnOCR-2-1B,1729.7323362983238,1741.2525107847546,-11.520174486430733,11.520174486430733 +pair-balanced_2000_seed_43,numind/NuExtract3,1705.055919186711,1689.6190357899554,15.436883396755547,15.436883396755547 +pair-balanced_2000_seed_43,rednote-hilab/dots.mocr,1728.175047511091,1745.4252593866306,-17.250211875539662,17.250211875539662 +pair-balanced_2000_seed_43,rednote-hilab/dots.ocr,901.4422346135815,890.6755485577996,10.76668605578186,10.76668605578186 +pair-balanced_2000_seed_43,tesseract-5,1149.6709102633217,1112.5865083646895,37.08440189863222,37.08440189863222 +pair-balanced_2000_seed_43,zai-org/GLM-OCR,1714.8798089737636,1738.1794754621453,-23.299666488381717,23.299666488381717 +pair-balanced_2000_seed_44,FireRedTeam/FireRed-OCR,1567.8680530085896,1557.449386684769,10.418666323820617,10.418666323820617 +pair-balanced_2000_seed_44,PaddlePaddle/PP-OCRv6_medium,1399.2210253836206,1397.0965599286017,2.1244654550189352,2.1244654550189352 +pair-balanced_2000_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1465.6951681945661,1462.9745524973894,2.720615697176754,2.720615697176754 +pair-balanced_2000_seed_44,allenai/olmOCR-2-7B-1025-FP8,1731.3880025296849,1719.231138029067,12.156864500617758,12.156864500617758 +pair-balanced_2000_seed_44,baidu/Qianfan-OCR,1581.2568165397106,1568.0944503021342,13.162366237576407,13.162366237576407 +pair-balanced_2000_seed_44,baidu/Unlimited-OCR,1530.5499399383973,1545.2060026385532,-14.656062700155871,14.656062700155871 +pair-balanced_2000_seed_44,deepseek-ai/DeepSeek-OCR,1424.6298671830664,1451.9970235866688,-27.367156403602394,27.367156403602394 +pair-balanced_2000_seed_44,deepseek-ai/DeepSeek-OCR-2,1397.8377687978734,1380.2125479868414,17.625220811032023,17.625220811032023 +pair-balanced_2000_seed_44,lightonai/LightOnOCR-2-1B,1718.141749111582,1741.2525107847546,-23.110761673172647,23.110761673172647 +pair-balanced_2000_seed_44,numind/NuExtract3,1699.6515194109957,1689.6190357899554,10.032483621040228,10.032483621040228 +pair-balanced_2000_seed_44,rednote-hilab/dots.mocr,1771.1307005874357,1745.4252593866306,25.705441200805126,25.705441200805126 +pair-balanced_2000_seed_44,rednote-hilab/dots.ocr,829.455793281588,890.6755485577996,-61.219755276211686,61.219755276211686 +pair-balanced_2000_seed_44,tesseract-5,1113.8310195500526,1112.5865083646895,1.2445111853630806,1.2445111853630806 +pair-balanced_2000_seed_44,zai-org/GLM-OCR,1769.3425764828376,1738.1794754621453,31.16310102069224,31.16310102069224 +pair-balanced_2000_seed_45,FireRedTeam/FireRed-OCR,1523.076413895152,1557.449386684769,-34.37297278961705,34.37297278961705 +pair-balanced_2000_seed_45,PaddlePaddle/PP-OCRv6_medium,1382.7661165365973,1397.0965599286017,-14.330443392004327,14.330443392004327 +pair-balanced_2000_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1467.3477795142187,1462.9745524973894,4.373227016829333,4.373227016829333 +pair-balanced_2000_seed_45,allenai/olmOCR-2-7B-1025-FP8,1756.9393632316828,1719.231138029067,37.70822520261572,37.70822520261572 +pair-balanced_2000_seed_45,baidu/Qianfan-OCR,1570.5522982800758,1568.0944503021342,2.457847977941583,2.457847977941583 +pair-balanced_2000_seed_45,baidu/Unlimited-OCR,1521.005251333356,1545.2060026385532,-24.200751305197173,24.200751305197173 +pair-balanced_2000_seed_45,deepseek-ai/DeepSeek-OCR,1469.3437300248895,1451.9970235866688,17.346706438220735,17.346706438220735 +pair-balanced_2000_seed_45,deepseek-ai/DeepSeek-OCR-2,1379.9810876191825,1380.2125479868414,-0.23146036765888311,0.23146036765888311 +pair-balanced_2000_seed_45,lightonai/LightOnOCR-2-1B,1771.2060108113592,1741.2525107847546,29.953500026604615,29.953500026604615 +pair-balanced_2000_seed_45,numind/NuExtract3,1719.444754476605,1689.6190357899554,29.82571868664968,29.82571868664968 +pair-balanced_2000_seed_45,rednote-hilab/dots.mocr,1717.7916860569073,1745.4252593866306,-27.633573329723276,27.633573329723276 +pair-balanced_2000_seed_45,rednote-hilab/dots.ocr,857.7745469808318,890.6755485577996,-32.901001576967815,32.901001576967815 +pair-balanced_2000_seed_45,tesseract-5,1136.673621269437,1112.5865083646895,24.087112904747528,24.087112904747528 +pair-balanced_2000_seed_45,zai-org/GLM-OCR,1726.0973399697052,1738.1794754621453,-12.082135492440102,12.082135492440102 +pair-balanced_2000_seed_46,FireRedTeam/FireRed-OCR,1545.7340873487474,1557.449386684769,-11.715299336021644,11.715299336021644 +pair-balanced_2000_seed_46,PaddlePaddle/PP-OCRv6_medium,1380.7241712145735,1397.0965599286017,-16.372388714028148,16.372388714028148 +pair-balanced_2000_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1462.4311167863614,1462.9745524973894,-0.5434357110279961,0.5434357110279961 +pair-balanced_2000_seed_46,allenai/olmOCR-2-7B-1025-FP8,1744.625375941635,1719.231138029067,25.39423791256786,25.39423791256786 +pair-balanced_2000_seed_46,baidu/Qianfan-OCR,1547.1158782626403,1568.0944503021342,-20.978572039493883,20.978572039493883 +pair-balanced_2000_seed_46,baidu/Unlimited-OCR,1558.090204217733,1545.2060026385532,12.884201579179717,12.884201579179717 +pair-balanced_2000_seed_46,deepseek-ai/DeepSeek-OCR,1451.666963770977,1451.9970235866688,-0.3300598156918113,0.3300598156918113 +pair-balanced_2000_seed_46,deepseek-ai/DeepSeek-OCR-2,1385.0112013089922,1380.2125479868414,4.7986533221508125,4.7986533221508125 +pair-balanced_2000_seed_46,lightonai/LightOnOCR-2-1B,1734.4262547193853,1741.2525107847546,-6.826256065369307,6.826256065369307 +pair-balanced_2000_seed_46,numind/NuExtract3,1693.6490592512116,1689.6190357899554,4.030023461256178,4.030023461256178 +pair-balanced_2000_seed_46,rednote-hilab/dots.mocr,1726.0623646860977,1745.4252593866306,-19.362894700532934,19.362894700532934 +pair-balanced_2000_seed_46,rednote-hilab/dots.ocr,926.5510180906398,890.6755485577996,35.87546953284016,35.87546953284016 +pair-balanced_2000_seed_46,tesseract-5,1131.0013126596991,1112.5865083646895,18.414804295009617,18.414804295009617 +pair-balanced_2000_seed_46,zai-org/GLM-OCR,1712.910991741307,1738.1794754621453,-25.268483720838276,25.268483720838276 +mixed-random_700_seed_42,FireRedTeam/FireRed-OCR,1620.3032487466087,1557.449386684769,62.85386206183966,62.85386206183966 +mixed-random_700_seed_42,PaddlePaddle/PP-OCRv6_medium,1448.0801157254782,1397.0965599286017,50.983555796876544,50.983555796876544 +mixed-random_700_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1531.8207137997033,1462.9745524973894,68.84616130231393,68.84616130231393 +mixed-random_700_seed_42,allenai/olmOCR-2-7B-1025-FP8,1634.0400417494823,1719.231138029067,-85.19109627958483,85.19109627958483 +mixed-random_700_seed_42,baidu/Qianfan-OCR,1538.2836316036683,1568.0944503021342,-29.810818698465937,29.810818698465937 +mixed-random_700_seed_42,baidu/Unlimited-OCR,1517.702660840959,1545.2060026385532,-27.503341797594203,27.503341797594203 +mixed-random_700_seed_42,deepseek-ai/DeepSeek-OCR,1392.137677876126,1451.9970235866688,-59.85934571054281,59.85934571054281 +mixed-random_700_seed_42,deepseek-ai/DeepSeek-OCR-2,1349.4205303204378,1380.2125479868414,-30.79201766640358,30.79201766640358 +mixed-random_700_seed_42,lightonai/LightOnOCR-2-1B,1775.7835608664172,1741.2525107847546,34.53105008166267,34.53105008166267 +mixed-random_700_seed_42,numind/NuExtract3,1561.6520916534273,1689.6190357899554,-127.96694413652813,127.96694413652813 +mixed-random_700_seed_42,rednote-hilab/dots.mocr,1754.0975135727124,1745.4252593866306,8.672254186081773,8.672254186081773 +mixed-random_700_seed_42,rednote-hilab/dots.ocr,1082.0146077190846,890.6755485577996,191.33905916128492,191.33905916128492 +mixed-random_700_seed_42,tesseract-5,1101.9883922886106,1112.5865083646895,-10.598116076078895,10.598116076078895 +mixed-random_700_seed_42,zai-org/GLM-OCR,1692.6752132372842,1738.1794754621453,-45.50426222486112,45.50426222486112 +mixed-random_700_seed_43,FireRedTeam/FireRed-OCR,1658.1300409873988,1557.449386684769,100.68065430262982,100.68065430262982 +mixed-random_700_seed_43,PaddlePaddle/PP-OCRv6_medium,1483.074712843407,1397.0965599286017,85.97815291480538,85.97815291480538 +mixed-random_700_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1504.3991933461061,1462.9745524973894,41.42464084871676,41.42464084871676 +mixed-random_700_seed_43,allenai/olmOCR-2-7B-1025-FP8,1603.1041639443386,1719.231138029067,-116.12697408472854,116.12697408472854 +mixed-random_700_seed_43,baidu/Qianfan-OCR,1547.2499525646024,1568.0944503021342,-20.84449773753181,20.84449773753181 +mixed-random_700_seed_43,baidu/Unlimited-OCR,1516.6814051771173,1545.2060026385532,-28.52459746143586,28.52459746143586 +mixed-random_700_seed_43,deepseek-ai/DeepSeek-OCR,1445.4342070163211,1451.9970235866688,-6.56281657034765,6.56281657034765 +mixed-random_700_seed_43,deepseek-ai/DeepSeek-OCR-2,1374.882382805725,1380.2125479868414,-5.330165181116399,5.330165181116399 +mixed-random_700_seed_43,lightonai/LightOnOCR-2-1B,1675.399591311468,1741.2525107847546,-65.85291947328665,65.85291947328665 +mixed-random_700_seed_43,numind/NuExtract3,1542.9342406766941,1689.6190357899554,-146.68479511326132,146.68479511326132 +mixed-random_700_seed_43,rednote-hilab/dots.mocr,1784.7941812902018,1745.4252593866306,39.36892190357116,39.36892190357116 +mixed-random_700_seed_43,rednote-hilab/dots.ocr,1025.3764009255913,890.6755485577996,134.70085236779164,134.70085236779164 +mixed-random_700_seed_43,tesseract-5,1146.4334230110414,1112.5865083646895,33.846914646351934,33.846914646351934 +mixed-random_700_seed_43,zai-org/GLM-OCR,1692.1061040999869,1738.1794754621453,-46.07337136215847,46.07337136215847 +mixed-random_700_seed_44,FireRedTeam/FireRed-OCR,1624.4696530798947,1557.449386684769,67.02026639512565,67.02026639512565 +mixed-random_700_seed_44,PaddlePaddle/PP-OCRv6_medium,1405.3038896649575,1397.0965599286017,8.20732973635586,8.20732973635586 +mixed-random_700_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1515.3186853885575,1462.9745524973894,52.3441328911681,52.3441328911681 +mixed-random_700_seed_44,allenai/olmOCR-2-7B-1025-FP8,1605.9509594267513,1719.231138029067,-113.2801786023158,113.2801786023158 +mixed-random_700_seed_44,baidu/Qianfan-OCR,1565.7232148546693,1568.0944503021342,-2.371235447464869,2.371235447464869 +mixed-random_700_seed_44,baidu/Unlimited-OCR,1549.8136788437002,1545.2060026385532,4.607676205147072,4.607676205147072 +mixed-random_700_seed_44,deepseek-ai/DeepSeek-OCR,1390.7373139823922,1451.9970235866688,-61.25970960427662,61.25970960427662 +mixed-random_700_seed_44,deepseek-ai/DeepSeek-OCR-2,1338.3686805976035,1380.2125479868414,-41.84386738923786,41.84386738923786 +mixed-random_700_seed_44,lightonai/LightOnOCR-2-1B,1760.9006761240214,1741.2525107847546,19.648165339266825,19.648165339266825 +mixed-random_700_seed_44,numind/NuExtract3,1547.9771212128735,1689.6190357899554,-141.64191457708193,141.64191457708193 +mixed-random_700_seed_44,rednote-hilab/dots.mocr,1800.2793063127092,1745.4252593866306,54.85404692607858,54.85404692607858 +mixed-random_700_seed_44,rednote-hilab/dots.ocr,1013.415930167874,890.6755485577996,122.74038161007434,122.74038161007434 +mixed-random_700_seed_44,tesseract-5,1126.3948639615478,1112.5865083646895,13.808355596858291,13.808355596858291 +mixed-random_700_seed_44,zai-org/GLM-OCR,1755.3460263824475,1738.1794754621453,17.166550920302143,17.166550920302143 +mixed-random_700_seed_45,FireRedTeam/FireRed-OCR,1616.6764539348237,1557.449386684769,59.22706725005469,59.22706725005469 +mixed-random_700_seed_45,PaddlePaddle/PP-OCRv6_medium,1432.3243092383013,1397.0965599286017,35.227749309699675,35.227749309699675 +mixed-random_700_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1556.5163594615397,1462.9745524973894,93.54180696415028,93.54180696415028 +mixed-random_700_seed_45,allenai/olmOCR-2-7B-1025-FP8,1632.8025445032843,1719.231138029067,-86.42859352578284,86.42859352578284 +mixed-random_700_seed_45,baidu/Qianfan-OCR,1502.7868187381414,1568.0944503021342,-65.3076315639928,65.3076315639928 +mixed-random_700_seed_45,baidu/Unlimited-OCR,1464.6781744964755,1545.2060026385532,-80.52782814207762,80.52782814207762 +mixed-random_700_seed_45,deepseek-ai/DeepSeek-OCR,1385.0531735380455,1451.9970235866688,-66.94385004862329,66.94385004862329 +mixed-random_700_seed_45,deepseek-ai/DeepSeek-OCR-2,1356.308505076121,1380.2125479868414,-23.904042910720364,23.904042910720364 +mixed-random_700_seed_45,lightonai/LightOnOCR-2-1B,1760.0696261769076,1741.2525107847546,18.817115392153028,18.817115392153028 +mixed-random_700_seed_45,numind/NuExtract3,1563.7280181280119,1689.6190357899554,-125.89101766194358,125.89101766194358 +mixed-random_700_seed_45,rednote-hilab/dots.mocr,1786.5992043467038,1745.4252593866306,41.173944960073186,41.173944960073186 +mixed-random_700_seed_45,rednote-hilab/dots.ocr,1048.8151416940684,890.6755485577996,158.1395931362688,158.1395931362688 +mixed-random_700_seed_45,tesseract-5,1117.5498702412406,1112.5865083646895,4.963361876551062,4.963361876551062 +mixed-random_700_seed_45,zai-org/GLM-OCR,1776.0918004263349,1738.1794754621453,37.912324964189565,37.912324964189565 +mixed-random_700_seed_46,FireRedTeam/FireRed-OCR,1642.8296635263614,1557.449386684769,85.38027684159238,85.38027684159238 +mixed-random_700_seed_46,PaddlePaddle/PP-OCRv6_medium,1461.4593175069006,1397.0965599286017,64.3627575782989,64.3627575782989 +mixed-random_700_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1532.4211100911155,1462.9745524973894,69.44655759372608,69.44655759372608 +mixed-random_700_seed_46,allenai/olmOCR-2-7B-1025-FP8,1587.6087802667914,1719.231138029067,-131.62235776227567,131.62235776227567 +mixed-random_700_seed_46,baidu/Qianfan-OCR,1574.9096795204657,1568.0944503021342,6.815229218331524,6.815229218331524 +mixed-random_700_seed_46,baidu/Unlimited-OCR,1492.1369624804809,1545.2060026385532,-53.06904015807231,53.06904015807231 +mixed-random_700_seed_46,deepseek-ai/DeepSeek-OCR,1410.315359324659,1451.9970235866688,-41.68166426200969,41.68166426200969 +mixed-random_700_seed_46,deepseek-ai/DeepSeek-OCR-2,1305.8875377897534,1380.2125479868414,-74.32501019708798,74.32501019708798 +mixed-random_700_seed_46,lightonai/LightOnOCR-2-1B,1708.4227527517337,1741.2525107847546,-32.82975803302088,32.82975803302088 +mixed-random_700_seed_46,numind/NuExtract3,1581.9261120589224,1689.6190357899554,-107.69292373103303,107.69292373103303 +mixed-random_700_seed_46,rednote-hilab/dots.mocr,1821.1999097944224,1745.4252593866306,75.7746504077918,75.7746504077918 +mixed-random_700_seed_46,rednote-hilab/dots.ocr,1028.2352296473787,890.6755485577996,137.55968108957904,137.55968108957904 +mixed-random_700_seed_46,tesseract-5,1133.394792135793,1112.5865083646895,20.80828377110356,20.80828377110356 +mixed-random_700_seed_46,zai-org/GLM-OCR,1719.252793105222,1738.1794754621453,-18.926682356923266,18.926682356923266 +mixed-random_1200_seed_42,FireRedTeam/FireRed-OCR,1602.6332985272588,1557.449386684769,45.183911842489806,45.183911842489806 +mixed-random_1200_seed_42,PaddlePaddle/PP-OCRv6_medium,1424.7257969345728,1397.0965599286017,27.629237005971163,27.629237005971163 +mixed-random_1200_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1516.124996622898,1462.9745524973894,53.15044412550856,53.15044412550856 +mixed-random_1200_seed_42,allenai/olmOCR-2-7B-1025-FP8,1648.3748813019154,1719.231138029067,-70.85625672715173,70.85625672715173 +mixed-random_1200_seed_42,baidu/Qianfan-OCR,1552.6509244692784,1568.0944503021342,-15.443525832855812,15.443525832855812 +mixed-random_1200_seed_42,baidu/Unlimited-OCR,1522.4957724935928,1545.2060026385532,-22.710230144960406,22.710230144960406 +mixed-random_1200_seed_42,deepseek-ai/DeepSeek-OCR,1394.151233443908,1451.9970235866688,-57.845790142760734,57.845790142760734 +mixed-random_1200_seed_42,deepseek-ai/DeepSeek-OCR-2,1360.345624298629,1380.2125479868414,-19.86692368821241,19.86692368821241 +mixed-random_1200_seed_42,lightonai/LightOnOCR-2-1B,1750.5142877977216,1741.2525107847546,9.261777012967059,9.261777012967059 +mixed-random_1200_seed_42,numind/NuExtract3,1603.8898649357998,1689.6190357899554,-85.72917085415565,85.72917085415565 +mixed-random_1200_seed_42,rednote-hilab/dots.mocr,1743.8541972952437,1745.4252593866306,-1.5710620913869207,1.5710620913869207 +mixed-random_1200_seed_42,rednote-hilab/dots.ocr,1013.5235693711902,890.6755485577996,122.84802081339058,122.84802081339058 +mixed-random_1200_seed_42,tesseract-5,1142.3217385902733,1112.5865083646895,29.73523022558379,29.73523022558379 +mixed-random_1200_seed_42,zai-org/GLM-OCR,1724.393813917718,1738.1794754621453,-13.785661544427285,13.785661544427285 +mixed-random_1200_seed_43,FireRedTeam/FireRed-OCR,1631.5684374061764,1557.449386684769,74.11905072140735,74.11905072140735 +mixed-random_1200_seed_43,PaddlePaddle/PP-OCRv6_medium,1457.9282436766819,1397.0965599286017,60.831683748080195,60.831683748080195 +mixed-random_1200_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1459.9798813197353,1462.9745524973894,-2.994671177654027,2.994671177654027 +mixed-random_1200_seed_43,allenai/olmOCR-2-7B-1025-FP8,1659.9896745564783,1719.231138029067,-59.24146347258875,59.24146347258875 +mixed-random_1200_seed_43,baidu/Qianfan-OCR,1548.3166728591757,1568.0944503021342,-19.777777442958495,19.777777442958495 +mixed-random_1200_seed_43,baidu/Unlimited-OCR,1530.4885461241554,1545.2060026385532,-14.717456514397782,14.717456514397782 +mixed-random_1200_seed_43,deepseek-ai/DeepSeek-OCR,1437.7503045020208,1451.9970235866688,-14.24671908464802,14.24671908464802 +mixed-random_1200_seed_43,deepseek-ai/DeepSeek-OCR-2,1403.7121240107635,1380.2125479868414,23.499576023922145,23.499576023922145 +mixed-random_1200_seed_43,lightonai/LightOnOCR-2-1B,1713.0175030039131,1741.2525107847546,-28.235007780841443,28.235007780841443 +mixed-random_1200_seed_43,numind/NuExtract3,1624.1387113427645,1689.6190357899554,-65.48032444719092,65.48032444719092 +mixed-random_1200_seed_43,rednote-hilab/dots.mocr,1737.8189614779017,1745.4252593866306,-7.606297908728948,7.606297908728948 +mixed-random_1200_seed_43,rednote-hilab/dots.ocr,964.5251465139563,890.6755485577996,73.84959795615669,73.84959795615669 +mixed-random_1200_seed_43,tesseract-5,1132.5432951930675,1112.5865083646895,19.95678682837797,19.95678682837797 +mixed-random_1200_seed_43,zai-org/GLM-OCR,1698.2224980132096,1738.1794754621453,-39.95697744893573,39.95697744893573 +mixed-random_1200_seed_44,FireRedTeam/FireRed-OCR,1607.7861805557245,1557.449386684769,50.33679387095549,50.33679387095549 +mixed-random_1200_seed_44,PaddlePaddle/PP-OCRv6_medium,1396.3957751748949,1397.0965599286017,-0.7007847537067846,0.7007847537067846 +mixed-random_1200_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1474.8905419565415,1462.9745524973894,11.915989459152115,11.915989459152115 +mixed-random_1200_seed_44,allenai/olmOCR-2-7B-1025-FP8,1692.3902735308052,1719.231138029067,-26.840864498261908,26.840864498261908 +mixed-random_1200_seed_44,baidu/Qianfan-OCR,1581.621324913027,1568.0944503021342,13.526874610892719,13.526874610892719 +mixed-random_1200_seed_44,baidu/Unlimited-OCR,1542.7564460217004,1545.2060026385532,-2.4495566168527603,2.4495566168527603 +mixed-random_1200_seed_44,deepseek-ai/DeepSeek-OCR,1424.7527198097753,1451.9970235866688,-27.244303776893503,27.244303776893503 +mixed-random_1200_seed_44,deepseek-ai/DeepSeek-OCR-2,1371.9917610209986,1380.2125479868414,-8.220786965842763,8.220786965842763 +mixed-random_1200_seed_44,lightonai/LightOnOCR-2-1B,1781.3505143431964,1741.2525107847546,40.09800355844186,40.09800355844186 +mixed-random_1200_seed_44,numind/NuExtract3,1609.7988324831279,1689.6190357899554,-79.82020330682758,79.82020330682758 +mixed-random_1200_seed_44,rednote-hilab/dots.mocr,1780.0746048393153,1745.4252593866306,34.649345452684656,34.649345452684656 +mixed-random_1200_seed_44,rednote-hilab/dots.ocr,893.0590984780348,890.6755485577996,2.3835499202351684,2.3835499202351684 +mixed-random_1200_seed_44,tesseract-5,1091.0106174371163,1112.5865083646895,-21.575890927573255,21.575890927573255 +mixed-random_1200_seed_44,zai-org/GLM-OCR,1752.1213094357418,1738.1794754621453,13.941833973596431,13.941833973596431 +mixed-random_1200_seed_45,FireRedTeam/FireRed-OCR,1597.3609110094637,1557.449386684769,39.91152432469471,39.91152432469471 +mixed-random_1200_seed_45,PaddlePaddle/PP-OCRv6_medium,1405.0333502757255,1397.0965599286017,7.936790347123861,7.936790347123861 +mixed-random_1200_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1495.0041767638872,1462.9745524973894,32.029624266497876,32.029624266497876 +mixed-random_1200_seed_45,allenai/olmOCR-2-7B-1025-FP8,1656.2434826753345,1719.231138029067,-62.987655353732634,62.987655353732634 +mixed-random_1200_seed_45,baidu/Qianfan-OCR,1526.4743702957785,1568.0944503021342,-41.62008000635569,41.62008000635569 +mixed-random_1200_seed_45,baidu/Unlimited-OCR,1510.2801888072604,1545.2060026385532,-34.92581383129277,34.92581383129277 +mixed-random_1200_seed_45,deepseek-ai/DeepSeek-OCR,1405.3825103322545,1451.9970235866688,-46.614513254414305,46.614513254414305 +mixed-random_1200_seed_45,deepseek-ai/DeepSeek-OCR-2,1377.0004345945476,1380.2125479868414,-3.212113392293759,3.212113392293759 +mixed-random_1200_seed_45,lightonai/LightOnOCR-2-1B,1724.0564789789507,1741.2525107847546,-17.19603180580384,17.19603180580384 +mixed-random_1200_seed_45,numind/NuExtract3,1607.894369759667,1689.6190357899554,-81.72466603028852,81.72466603028852 +mixed-random_1200_seed_45,rednote-hilab/dots.mocr,1775.0819393897812,1745.4252593866306,29.65668000315054,29.65668000315054 +mixed-random_1200_seed_45,rednote-hilab/dots.ocr,996.843465884866,890.6755485577996,106.16791732706633,106.16791732706633 +mixed-random_1200_seed_45,tesseract-5,1140.1993705525779,1112.5865083646895,27.612862187888368,27.612862187888368 +mixed-random_1200_seed_45,zai-org/GLM-OCR,1783.1449506799054,1738.1794754621453,44.96547521776006,44.96547521776006 +mixed-random_1200_seed_46,FireRedTeam/FireRed-OCR,1610.6885206648683,1557.449386684769,53.23913398009927,53.23913398009927 +mixed-random_1200_seed_46,PaddlePaddle/PP-OCRv6_medium,1422.268006598586,1397.0965599286017,25.171446669984334,25.171446669984334 +mixed-random_1200_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1513.5332381287021,1462.9745524973894,50.558685631312756,50.558685631312756 +mixed-random_1200_seed_46,allenai/olmOCR-2-7B-1025-FP8,1649.7151580440377,1719.231138029067,-69.51597998502939,69.51597998502939 +mixed-random_1200_seed_46,baidu/Qianfan-OCR,1547.7945560822852,1568.0944503021342,-20.299894219848966,20.299894219848966 +mixed-random_1200_seed_46,baidu/Unlimited-OCR,1519.2845133601986,1545.2060026385532,-25.921489278354557,25.921489278354557 +mixed-random_1200_seed_46,deepseek-ai/DeepSeek-OCR,1425.793536080444,1451.9970235866688,-26.203487506224747,26.203487506224747 +mixed-random_1200_seed_46,deepseek-ai/DeepSeek-OCR-2,1349.9327173323106,1380.2125479868414,-30.27983065453077,30.27983065453077 +mixed-random_1200_seed_46,lightonai/LightOnOCR-2-1B,1698.5968052265146,1741.2525107847546,-42.655705558240015,42.655705558240015 +mixed-random_1200_seed_46,numind/NuExtract3,1637.8008835293765,1689.6190357899554,-51.81815226057893,51.81815226057893 +mixed-random_1200_seed_46,rednote-hilab/dots.mocr,1799.9482948879072,1745.4252593866306,54.52303550127658,54.52303550127658 +mixed-random_1200_seed_46,rednote-hilab/dots.ocr,953.249266842184,890.6755485577996,62.57371828438443,62.57371828438443 +mixed-random_1200_seed_46,tesseract-5,1124.9178798966157,1112.5865083646895,12.331371531926152,12.331371531926152 +mixed-random_1200_seed_46,zai-org/GLM-OCR,1746.4766233259697,1738.1794754621453,8.29714786382442,8.29714786382442 +mixed-random_2000_seed_42,FireRedTeam/FireRed-OCR,1573.663528450672,1557.449386684769,16.214141765902923,16.214141765902923 +mixed-random_2000_seed_42,PaddlePaddle/PP-OCRv6_medium,1411.9252749410973,1397.0965599286017,14.82871501249565,14.82871501249565 +mixed-random_2000_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1499.7002230467394,1462.9745524973894,36.72567054935007,36.72567054935007 +mixed-random_2000_seed_42,allenai/olmOCR-2-7B-1025-FP8,1687.7519284077068,1719.231138029067,-31.47920962136027,31.47920962136027 +mixed-random_2000_seed_42,baidu/Qianfan-OCR,1552.0426056461395,1568.0944503021342,-16.051844655994728,16.051844655994728 +mixed-random_2000_seed_42,baidu/Unlimited-OCR,1528.1241985753716,1545.2060026385532,-17.081804063181607,17.081804063181607 +mixed-random_2000_seed_42,deepseek-ai/DeepSeek-OCR,1445.5209314599153,1451.9970235866688,-6.476092126753429,6.476092126753429 +mixed-random_2000_seed_42,deepseek-ai/DeepSeek-OCR-2,1381.0494265611999,1380.2125479868414,0.8368785743584795,0.8368785743584795 +mixed-random_2000_seed_42,lightonai/LightOnOCR-2-1B,1731.1734656434533,1741.2525107847546,-10.079045141301322,10.079045141301322 +mixed-random_2000_seed_42,numind/NuExtract3,1649.639397023911,1689.6190357899554,-39.979638766044445,39.979638766044445 +mixed-random_2000_seed_42,rednote-hilab/dots.mocr,1755.0098525318556,1745.4252593866306,9.584593145224972,9.584593145224972 +mixed-random_2000_seed_42,rednote-hilab/dots.ocr,936.9457261023799,890.6755485577996,46.27017754458029,46.27017754458029 +mixed-random_2000_seed_42,tesseract-5,1108.6631036916806,1112.5865083646895,-3.9234046730089176,3.9234046730089176 +mixed-random_2000_seed_42,zai-org/GLM-OCR,1738.7903379178777,1738.1794754621453,0.610862455732331,0.610862455732331 +mixed-random_2000_seed_43,FireRedTeam/FireRed-OCR,1590.9924199371674,1557.449386684769,33.543033252398345,33.543033252398345 +mixed-random_2000_seed_43,PaddlePaddle/PP-OCRv6_medium,1450.1605591024454,1397.0965599286017,53.06399917384374,53.06399917384374 +mixed-random_2000_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1467.060276748374,1462.9745524973894,4.08572425098464,4.08572425098464 +mixed-random_2000_seed_43,allenai/olmOCR-2-7B-1025-FP8,1698.817107093526,1719.231138029067,-20.414030935541177,20.414030935541177 +mixed-random_2000_seed_43,baidu/Qianfan-OCR,1578.0969388006488,1568.0944503021342,10.002488498514595,10.002488498514595 +mixed-random_2000_seed_43,baidu/Unlimited-OCR,1547.9903036612516,1545.2060026385532,2.7843010226984006,2.7843010226984006 +mixed-random_2000_seed_43,deepseek-ai/DeepSeek-OCR,1440.7954324639893,1451.9970235866688,-11.201591122679474,11.201591122679474 +mixed-random_2000_seed_43,deepseek-ai/DeepSeek-OCR-2,1393.6801466160546,1380.2125479868414,13.467598629213171,13.467598629213171 +mixed-random_2000_seed_43,lightonai/LightOnOCR-2-1B,1720.9364431326946,1741.2525107847546,-20.316067652059928,20.316067652059928 +mixed-random_2000_seed_43,numind/NuExtract3,1675.7194457018418,1689.6190357899554,-13.899590088113655,13.899590088113655 +mixed-random_2000_seed_43,rednote-hilab/dots.mocr,1738.7947057543856,1745.4252593866306,-6.6305536322449825,6.6305536322449825 +mixed-random_2000_seed_43,rednote-hilab/dots.ocr,858.6865807119874,890.6755485577996,-31.988967845812226,31.988967845812226 +mixed-random_2000_seed_43,tesseract-5,1117.3331971429388,1112.5865083646895,4.746688778249336,4.746688778249336 +mixed-random_2000_seed_43,zai-org/GLM-OCR,1720.9364431326946,1738.1794754621453,-17.24303232945067,17.24303232945067 +mixed-random_2000_seed_44,FireRedTeam/FireRed-OCR,1582.9877386813391,1557.449386684769,25.5383519965701,25.5383519965701 +mixed-random_2000_seed_44,PaddlePaddle/PP-OCRv6_medium,1407.578088781668,1397.0965599286017,10.481528853066266,10.481528853066266 +mixed-random_2000_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1458.3012179453053,1462.9745524973894,-4.673334552084043,4.673334552084043 +mixed-random_2000_seed_44,allenai/olmOCR-2-7B-1025-FP8,1719.28716691742,1719.231138029067,0.05602888835301201,0.05602888835301201 +mixed-random_2000_seed_44,baidu/Qianfan-OCR,1572.6889266294816,1568.0944503021342,4.594476327347365,4.594476327347365 +mixed-random_2000_seed_44,baidu/Unlimited-OCR,1547.5331435011065,1545.2060026385532,2.3271408625532786,2.3271408625532786 +mixed-random_2000_seed_44,deepseek-ai/DeepSeek-OCR,1453.1308594475618,1451.9970235866688,1.1338358608929866,1.1338358608929866 +mixed-random_2000_seed_44,deepseek-ai/DeepSeek-OCR-2,1360.1062743256673,1380.2125479868414,-20.1062736611741,20.1062736611741 +mixed-random_2000_seed_44,lightonai/LightOnOCR-2-1B,1758.0960844763797,1741.2525107847546,16.843573691625124,16.843573691625124 +mixed-random_2000_seed_44,numind/NuExtract3,1676.3284471989311,1689.6190357899554,-13.290588591024289,13.290588591024289 +mixed-random_2000_seed_44,rednote-hilab/dots.mocr,1775.8457889268475,1745.4252593866306,30.420529540216876,30.420529540216876 +mixed-random_2000_seed_44,rednote-hilab/dots.ocr,851.6248064535293,890.6755485577996,-39.050742104270284,39.050742104270284 +mixed-random_2000_seed_44,tesseract-5,1115.566066733883,1112.5865083646895,2.9795583691934553,2.9795583691934553 +mixed-random_2000_seed_44,zai-org/GLM-OCR,1720.9253899808805,1738.1794754621453,-17.254085481264838,17.254085481264838 +mixed-random_2000_seed_45,FireRedTeam/FireRed-OCR,1569.896491674002,1557.449386684769,12.447104989232912,12.447104989232912 +mixed-random_2000_seed_45,PaddlePaddle/PP-OCRv6_medium,1378.565197068115,1397.0965599286017,-18.53136286048675,18.53136286048675 +mixed-random_2000_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1484.3891354817986,1462.9745524973894,21.414582984409208,21.414582984409208 +mixed-random_2000_seed_45,allenai/olmOCR-2-7B-1025-FP8,1686.3534853065385,1719.231138029067,-32.87765272252864,32.87765272252864 +mixed-random_2000_seed_45,baidu/Qianfan-OCR,1547.945597283799,1568.0944503021342,-20.148853018335103,20.148853018335103 +mixed-random_2000_seed_45,baidu/Unlimited-OCR,1528.2567330710913,1545.2060026385532,-16.949269567461897,16.949269567461897 +mixed-random_2000_seed_45,deepseek-ai/DeepSeek-OCR,1434.2127739677405,1451.9970235866688,-17.784249618928243,17.784249618928243 +mixed-random_2000_seed_45,deepseek-ai/DeepSeek-OCR-2,1370.0687765517825,1380.2125479868414,-10.143771435058852,10.143771435058852 +mixed-random_2000_seed_45,lightonai/LightOnOCR-2-1B,1717.9643101264915,1741.2525107847546,-23.288200658263122,23.288200658263122 +mixed-random_2000_seed_45,numind/NuExtract3,1670.028774534308,1689.6190357899554,-19.59026125564742,19.59026125564742 +mixed-random_2000_seed_45,rednote-hilab/dots.mocr,1760.819926775015,1745.4252593866306,15.394667388384278,15.394667388384278 +mixed-random_2000_seed_45,rednote-hilab/dots.ocr,952.050229833096,890.6755485577996,61.374681275296325,61.374681275296325 +mixed-random_2000_seed_45,tesseract-5,1131.9011179110926,1112.5865083646895,19.314609546403062,19.314609546403062 +mixed-random_2000_seed_45,zai-org/GLM-OCR,1767.5474504151298,1738.1794754621453,29.36797495298447,29.36797495298447 +mixed-random_2000_seed_46,FireRedTeam/FireRed-OCR,1592.7633358261285,1557.449386684769,35.313949141359444,35.313949141359444 +mixed-random_2000_seed_46,PaddlePaddle/PP-OCRv6_medium,1391.1568376958744,1397.0965599286017,-5.939722232727263,5.939722232727263 +mixed-random_2000_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1476.7399278109592,1462.9745524973894,13.765375313569848,13.765375313569848 +mixed-random_2000_seed_46,allenai/olmOCR-2-7B-1025-FP8,1695.2128061736842,1719.231138029067,-24.01833185538294,24.01833185538294 +mixed-random_2000_seed_46,baidu/Qianfan-OCR,1567.7775392389688,1568.0944503021342,-0.3169110631654348,0.3169110631654348 +mixed-random_2000_seed_46,baidu/Unlimited-OCR,1539.8106470676487,1545.2060026385532,-5.395355570904485,5.395355570904485 +mixed-random_2000_seed_46,deepseek-ai/DeepSeek-OCR,1444.2131492114418,1451.9970235866688,-7.783874375227015,7.783874375227015 +mixed-random_2000_seed_46,deepseek-ai/DeepSeek-OCR-2,1375.8677515569764,1380.2125479868414,-4.344796429865028,4.344796429865028 +mixed-random_2000_seed_46,lightonai/LightOnOCR-2-1B,1711.6085948995449,1741.2525107847546,-29.643915885209708,29.643915885209708 +mixed-random_2000_seed_46,numind/NuExtract3,1686.1783751467542,1689.6190357899554,-3.4406606432012268,3.4406606432012268 +mixed-random_2000_seed_46,rednote-hilab/dots.mocr,1786.0139731582658,1745.4252593866306,40.588713771635184,40.588713771635184 +mixed-random_2000_seed_46,rednote-hilab/dots.ocr,886.2632525158735,890.6755485577996,-4.4122960419260835,4.4122960419260835 +mixed-random_2000_seed_46,tesseract-5,1105.0174638175795,1112.5865083646895,-7.569044547110025,7.569044547110025 +mixed-random_2000_seed_46,zai-org/GLM-OCR,1741.3763458803003,1738.1794754621453,3.1968704181549583,3.1968704181549583 diff --git a/experiments/adaptive-stopping/replay.py b/experiments/adaptive-stopping/replay.py index ec54a2b..2bc81f9 100644 --- a/experiments/adaptive-stopping/replay.py +++ b/experiments/adaptive-stopping/replay.py @@ -12,6 +12,7 @@ import csv import json import math +import random import statistics from collections import Counter, defaultdict from collections.abc import Iterable, Sequence @@ -39,7 +40,7 @@ DEFAULT_REVISION = "48a0f42de26009892d2784a3a97d6d61525f4040" BATCH_SAMPLES = 5 -StrategyKind = Literal["full", "balanced", "targeted"] +StrategyKind = Literal["full", "balanced", "targeted", "pair-balanced", "mixed-random"] @dataclass(frozen=True) @@ -53,6 +54,8 @@ class StrategyConfig: warmup_min_per_pair: int | None = None balanced_every_n_post_warmup_batches: int | None = None size_rule_controls_sampling: bool = True + static_budget: int | None = None + allocation_seed: int = 42 @dataclass @@ -82,6 +85,21 @@ class ReplayResult: StrategyConfig("targeted_size_5_min_10", "targeted", 5.0, 10), ) +STATIC_BUDGETS = (700, 1200, 2000) +STATIC_SEEDS = (42, 43, 44, 45, 46) + +STATIC_CONFIGS = tuple( + StrategyConfig( + f"{kind}_{budget}_seed_{seed}", + kind, + static_budget=budget, + allocation_seed=seed, + ) + for kind in ("pair-balanced", "mixed-random") + for budget in STATIC_BUDGETS + for seed in STATIC_SEEDS +) + FOLLOWUP_CONFIGS = ( StrategyConfig("targeted_v2_warmup_5", "targeted", warmup_min_per_pair=5), StrategyConfig("targeted_v2_warmup_10", "targeted", warmup_min_per_pair=10), @@ -151,6 +169,11 @@ def _parse_args() -> argparse.Namespace: action="store_true", help="Skip targeted-v2 warm-up and balanced-exploration configurations.", ) + parser.add_argument( + "--skip-static", + action="store_true", + help="Skip fixed pair-balanced and outcome-independent mixed designs.", + ) parser.add_argument( "--exclude-sentinel-comparisons", action="store_true", @@ -257,6 +280,82 @@ def _warmup_ready( return len(pair_counts) == n_pairs and min(pair_counts.values(), default=0) >= min_per_pair +def _fixed_pair_balanced_sample( + stored: Sequence[ComparisonResult], + *, + budget: int, + seed: int, + required: Sequence[ComparisonResult] = (), +) -> list[ComparisonResult]: + """Select a fixed-budget, outcome-independent sample balanced across pairs. + + ``required`` supports the mixed design's first balanced page batch. Remaining + rows are shuffled within each pair using the fixed allocation seed, then added + to the currently least-sampled pairs until the exact budget is reached. Winner + values never participate in allocation. + """ + if budget < len(required): + raise ValueError(f"Budget {budget} is smaller than {len(required)} required rows") + if budget > len(stored): + raise ValueError(f"Budget {budget} exceeds {len(stored)} stored rows") + + grouped = _group_by_pair(stored) + rng = random.Random(seed) + required_keys = {_comparison_key(comparison) for comparison in required} + if len(required_keys) != len(required): + raise ValueError("Required comparisons contain duplicate pair/sample keys") + + queues: dict[tuple[str, str], list[ComparisonResult]] = {} + counts: Counter[tuple[str, str]] = Counter() + for comparison in required: + counts[normalize_model_pair(comparison.model_a, comparison.model_b)] += 1 + for pair, comparisons in sorted(grouped.items()): + remaining = [ + comparison + for comparison in comparisons + if _comparison_key(comparison) not in required_keys + ] + rng.shuffle(remaining) + queues[pair] = remaining + counts.setdefault(pair, 0) + + selected = list(required) + while len(selected) < budget: + eligible = [pair for pair, queue in queues.items() if queue] + if not eligible: + raise ValueError(f"Only {len(selected)} unique rows available for budget {budget}") + minimum = min(counts[pair] for pair in eligible) + candidates = sorted(pair for pair in eligible if counts[pair] == minimum) + rng.shuffle(candidates) + for pair in candidates: + if len(selected) >= budget: + break + selected.append(queues[pair].pop()) + counts[pair] += 1 + return selected + + +def _comparison_key(comparison: ComparisonResult) -> tuple[int, tuple[str, str]]: + return ( + comparison.sample_idx, + normalize_model_pair(comparison.model_a, comparison.model_b), + ) + + +def _group_by_pair( + comparisons: Sequence[ComparisonResult], +) -> dict[tuple[str, str], list[ComparisonResult]]: + grouped: dict[tuple[str, str], list[ComparisonResult]] = defaultdict(list) + seen: set[tuple[int, tuple[str, str]]] = set() + for comparison in comparisons: + key = _comparison_key(comparison) + if key in seen: + raise ValueError(f"Duplicate pair/sample comparison: {key}") + seen.add(key) + grouped[key[1]].append(comparison) + return dict(grouped) + + def replay_strategy( stored: Sequence[ComparisonResult], model_names: Sequence[str], @@ -279,6 +378,33 @@ def replay_strategy( n_pairs = len(model_names) * (len(model_names) - 1) // 2 min_before_check = max(3 * n_pairs, 20) + if config.kind in {"pair-balanced", "mixed-random"}: + if config.static_budget is None: + raise ValueError(f"{config.kind} requires a static budget") + required: list[ComparisonResult] = [] + if config.kind == "mixed-random": + warmup_indices = set(sample_order[:batch_samples]) + required = [ + comparison for comparison in stored if comparison.sample_idx in warmup_indices + ] + comparisons = _fixed_pair_balanced_sample( + stored, + budget=config.static_budget, + seed=config.allocation_seed, + required=required, + ) + board = compute_elo(comparisons, list(model_names), n_bootstrap=n_bootstrap) + decisions = _classify(board, comparisons, config) + return ReplayResult( + config=config, + comparisons=comparisons, + board=board, + decisions=decisions, + stopping_round=0, + stopping_reason="predeclared_static_budget", + round_history=[], + ) + if config.kind == "full": comparisons = list(stored) board = compute_elo(comparisons, list(model_names), n_bootstrap=n_bootstrap) @@ -491,6 +617,8 @@ def strategy_metrics( ) elif result.config.kind == "full": resolution = "reference only; no adaptive stop" + elif result.stopping_reason == "predeclared_static_budget": + resolution = "predeclared outcome-independent budget; no adaptive stop" else: resolution = ( "sample batches exhausted: " @@ -520,6 +648,8 @@ def strategy_metrics( result.config.balanced_every_n_post_warmup_batches ), "size_rule_controls_sampling": result.config.size_rule_controls_sampling, + "static_budget": result.config.static_budget, + "allocation_seed": result.config.allocation_seed, "comparisons_consumed": len(result.comparisons), "comparisons_saved": full_count - len(result.comparisons), "percentage_saved": percentage_saved, @@ -549,6 +679,72 @@ def strategy_metrics( } +def _static_design_aggregates(metrics: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + """Aggregate fixed-design results across predeclared allocation seeds.""" + grouped: dict[tuple[str, int], list[dict[str, Any]]] = defaultdict(list) + for metric in metrics: + budget = metric.get("static_budget") + if metric["kind"] in {"pair-balanced", "mixed-random"} and isinstance(budget, int): + grouped[(metric["kind"], budget)].append(metric) + + aggregates: list[dict[str, Any]] = [] + for (kind, budget), rows in sorted(grouped.items()): + taus = [row["kendall_tau"] for row in rows] + rhos = [row["spearman_rho"] for row in rows] + med_deltas = [row["median_absolute_elo_delta"] for row in rows] + max_deltas = [row["max_absolute_elo_delta"] for row in rows] + aggregates.append( + { + "kind": kind, + "budget": budget, + "seeds": [row["allocation_seed"] for row in rows], + "percentage_saved": rows[0]["percentage_saved"], + "kendall_tau_median": statistics.median(taus), + "kendall_tau_min": min(taus), + "kendall_tau_max": max(taus), + "spearman_rho_median": statistics.median(rhos), + "spearman_rho_min": min(rhos), + "spearman_rho_max": max(rhos), + "top3_membership_matches": sum(row["top3_membership_matches"] for row in rows), + "top3_order_matches": sum(row["top3_order_matches"] for row in rows), + "median_absolute_elo_delta_median": statistics.median(med_deltas), + "max_absolute_elo_delta_median": statistics.median(max_deltas), + "max_absolute_elo_delta_worst": max(max_deltas), + "minimum_pair_evidence": min( + row["graph"]["min_direct_comparisons"] for row in rows + ), + "gate_passes": sum(row["followup_acceptance_gate"]["passed"] for row in rows), + "runs": len(rows), + } + ) + return aggregates + + +def _write_static_csv(output_dir: Path, aggregates: Sequence[dict[str, Any]]) -> None: + fields = [ + "kind", + "budget", + "percentage_saved", + "kendall_tau_median", + "kendall_tau_min", + "kendall_tau_max", + "spearman_rho_median", + "top3_membership_matches", + "top3_order_matches", + "median_absolute_elo_delta_median", + "max_absolute_elo_delta_median", + "max_absolute_elo_delta_worst", + "minimum_pair_evidence", + "gate_passes", + "runs", + ] + with (output_dir / "static-design-summary.csv").open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") + writer.writeheader() + for aggregate in aggregates: + writer.writerow({field: aggregate[field] for field in fields}) + + def _write_csvs(output_dir: Path, metrics: Sequence[dict[str, Any]]) -> None: summary_fields = [ "name", @@ -653,6 +849,8 @@ def main() -> None: configs.extend(SENSITIVITY_CONFIGS) if not args.skip_followup: configs.extend(FOLLOWUP_CONFIGS) + if not args.skip_static: + configs.extend(STATIC_CONFIGS) results: dict[str, ReplayResult] = {} deterministic: dict[str, bool | None] = {} @@ -687,6 +885,7 @@ def main() -> None: ) for config in configs ] + static_aggregates = _static_design_aggregates(metrics) payload = { "experiment": { "source_repo": args.repo, @@ -704,6 +903,15 @@ def main() -> None: "bootstrap_seed": 42, "repeats": args.repeats, "repeated_strategy_names": sorted(repeat_names & {config.name for config in configs}), + "static_design": { + "budgets": list(STATIC_BUDGETS), + "allocation_seeds": list(STATIC_SEEDS), + "pair_balanced": "fixed equal per-pair quotas across all stored samples", + "mixed_random": ( + "first five-sample balanced batch, then seeded outcome-independent " + "pair-balanced exploration" + ), + }, "followup_acceptance_gate": { "top3_order_exact": True, "kendall_tau_minimum": 0.95, @@ -721,6 +929,7 @@ def main() -> None: ], }, "strategies": metrics, + "static_design_aggregates": static_aggregates, } args.output_dir.mkdir(parents=True, exist_ok=True) @@ -729,6 +938,7 @@ def main() -> None: json_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") if not args.exclude_sentinel_comparisons: _write_csvs(args.output_dir, metrics) + _write_static_csv(args.output_dir, static_aggregates) print(f"Wrote {json_path}") diff --git a/experiments/adaptive-stopping/results-sentinels-excluded.json b/experiments/adaptive-stopping/results-sentinels-excluded.json index b8f33e9..1f91daf 100644 --- a/experiments/adaptive-stopping/results-sentinels-excluded.json +++ b/experiments/adaptive-stopping/results-sentinels-excluded.json @@ -49,8 +49,182 @@ "sentinel_comparisons_excluded": true, "source_repo": "davanstrien/ocr-bench-britannica-results", "source_revision": "48a0f42de26009892d2784a3a97d6d61525f4040", + "static_design": { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "budgets": [ + 700, + 1200, + 2000 + ], + "mixed_random": "first five-sample balanced batch, then seeded outcome-independent pair-balanced exploration", + "pair_balanced": "fixed equal per-pair quotas across all stored samples" + }, "stored_comparisons_at_revision": 4293 }, + "static_design_aggregates": [ + { + "budget": 700, + "gate_passes": 0, + "kendall_tau_max": 0.9487179487179485, + "kendall_tau_median": 0.8974358974358972, + "kendall_tau_min": 0.846153846153846, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 90.57457123464246, + "max_absolute_elo_delta_worst": 110.50920721914326, + "median_absolute_elo_delta_median": 43.21362959287171, + "minimum_pair_evidence": 8, + "percentage_saved": 80.90040927694406, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.989010989010989, + "spearman_rho_median": 0.9725274725274725, + "spearman_rho_min": 0.945054945054945, + "top3_membership_matches": 5, + "top3_order_matches": 5 + }, + { + "budget": 1200, + "gate_passes": 2, + "kendall_tau_max": 0.9743589743589742, + "kendall_tau_median": 0.9230769230769229, + "kendall_tau_min": 0.8974358974358972, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 42.428365126298104, + "max_absolute_elo_delta_worst": 57.316679880515494, + "median_absolute_elo_delta_median": 18.909684828118543, + "minimum_pair_evidence": 15, + "percentage_saved": 67.25784447476126, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9945054945054945, + "spearman_rho_median": 0.9835164835164836, + "spearman_rho_min": 0.967032967032967, + "top3_membership_matches": 5, + "top3_order_matches": 2 + }, + { + "budget": 2000, + "gate_passes": 0, + "kendall_tau_max": 0.9743589743589742, + "kendall_tau_median": 0.9487179487179485, + "kendall_tau_min": 0.8717948717948716, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 22.317428443707513, + "max_absolute_elo_delta_worst": 32.32713115053002, + "median_absolute_elo_delta_median": 9.930725048769546, + "minimum_pair_evidence": 25, + "percentage_saved": 45.429740791268756, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9945054945054945, + "spearman_rho_median": 0.989010989010989, + "spearman_rho_min": 0.967032967032967, + "top3_membership_matches": 5, + "top3_order_matches": 0 + }, + { + "budget": 700, + "gate_passes": 0, + "kendall_tau_max": 0.9230769230769229, + "kendall_tau_median": 0.846153846153846, + "kendall_tau_min": 0.7948717948717947, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 72.42371763917004, + "max_absolute_elo_delta_worst": 102.34154125719442, + "median_absolute_elo_delta_median": 22.75104611031111, + "minimum_pair_evidence": 8, + "percentage_saved": 80.90040927694406, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9835164835164836, + "spearman_rho_median": 0.9505494505494506, + "spearman_rho_min": 0.9340659340659341, + "top3_membership_matches": 3, + "top3_order_matches": 0 + }, + { + "budget": 1200, + "gate_passes": 1, + "kendall_tau_max": 0.9743589743589742, + "kendall_tau_median": 0.8974358974358972, + "kendall_tau_min": 0.8717948717948716, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 52.410771855799794, + "max_absolute_elo_delta_worst": 74.96168169932707, + "median_absolute_elo_delta_median": 16.47916806097669, + "minimum_pair_evidence": 15, + "percentage_saved": 67.25784447476126, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9945054945054945, + "spearman_rho_median": 0.9725274725274725, + "spearman_rho_min": 0.9560439560439561, + "top3_membership_matches": 4, + "top3_order_matches": 1 + }, + { + "budget": 2000, + "gate_passes": 0, + "kendall_tau_max": 0.9487179487179485, + "kendall_tau_median": 0.8974358974358972, + "kendall_tau_min": 0.846153846153846, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 29.35659645917508, + "max_absolute_elo_delta_worst": 37.693908638986386, + "median_absolute_elo_delta_median": 9.460307540245594, + "minimum_pair_evidence": 25, + "percentage_saved": 45.429740791268756, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.989010989010989, + "spearman_rho_median": 0.9725274725274725, + "spearman_rho_min": 0.945054945054945, + "top3_membership_matches": 4, + "top3_order_matches": 0 + } + ], "strategies": [ { "allocation_decision_statuses": { @@ -58,6 +232,7 @@ "resolved": 4, "unresolved": 8 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 3665, "comparisons_saved": 0, @@ -213,6 +388,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 1.0, + "static_budget": null, "stopping_reason": "reference_full_stored_board", "stopping_round": 10, "top3": [ @@ -231,6 +407,7 @@ "resolved": 4, "unresolved": 8 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 3665, "comparisons_saved": 0, @@ -737,6 +914,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 1.0, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -755,6 +933,7 @@ "resolved": 1, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 753, "comparisons_saved": 2912, @@ -1261,6 +1440,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9835164835164836, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -1279,6 +1459,7 @@ "resolved": 1, "unresolved": 5 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 624, "comparisons_saved": 3041, @@ -1822,6 +2003,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9725274725274725, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -1840,6 +2022,7 @@ "resolved": 1, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 1016, "comparisons_saved": 2649, @@ -2328,6 +2511,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9780219780219779, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -2346,6 +2530,7 @@ "resolved": 1, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 1351, "comparisons_saved": 2314, @@ -2816,6 +3001,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9780219780219779, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -2834,6 +3020,7 @@ "resolved": 1, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": 3, "comparisons_consumed": 1756, "comparisons_saved": 1909, @@ -3340,6 +3527,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.989010989010989, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -3358,6 +3546,7 @@ "resolved": 2, "unresolved": 10 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": 3, "comparisons_consumed": 1684, "comparisons_saved": 1981, @@ -3877,6 +4066,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9835164835164836, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -3895,6 +4085,7 @@ "resolved": 1, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": 3, "comparisons_consumed": 1990, "comparisons_saved": 1675, @@ -4402,6 +4593,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9835164835164836, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -4420,6 +4612,7 @@ "resolved": 2, "unresolved": 5 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": 3, "comparisons_consumed": 1568, "comparisons_saved": 2097, @@ -4939,6 +5132,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9835164835164836, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -4950,6 +5144,5256 @@ "top3_membership_overlap": 3, "top3_order_matches": false, "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1543.0853037454192, + "PaddlePaddle/PP-OCRv6_medium": 1383.2205896308137, + "PaddlePaddle/PaddleOCR-VL-1.6": 1405.8076830862049, + "allenai/olmOCR-2-7B-1025-FP8": 1664.4750629895334, + "baidu/Qianfan-OCR": 1480.800164146396, + "baidu/Unlimited-OCR": 1506.106657322795, + "deepseek-ai/DeepSeek-OCR": 1410.6564592176926, + "deepseek-ai/DeepSeek-OCR-2": 1321.3236440930846, + "lightonai/LightOnOCR-2-1B": 1722.7967070663603, + "numind/NuExtract3": 1672.5046875390235, + "rednote-hilab/dots.mocr": 1688.832657435086, + "tesseract-5": 969.8331381647815, + "zai-org/GLM-OCR": 1730.5572455628094 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1466.045249315543, + 1622.3820935521 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1309.6839983179932, + 1456.6324026903649 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1334.2895444966712, + 1477.8798544064111 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1596.5129217575181, + 1751.0388422613619 + ], + "baidu/Qianfan-OCR": [ + 1415.7797846004482, + 1549.2534573873018 + ], + "baidu/Unlimited-OCR": [ + 1440.9752250686329, + 1578.7883452115877 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1341.0551412900286, + 1477.2528386932254 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1231.2143665664712, + 1399.0989776604451 + ], + "lightonai/LightOnOCR-2-1B": [ + 1647.590191523521, + 1814.949400771358 + ], + "numind/NuExtract3": [ + 1597.3858323092925, + 1762.135365605007 + ], + "rednote-hilab/dots.mocr": [ + 1628.0704606859856, + 1771.2599796342322 + ], + "tesseract-5": [ + 706.0699894010556, + 1085.190521422589 + ], + "zai-org/GLM-OCR": [ + 1666.6576671962607, + 1817.1791634900815 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 31.230552733723243, + "PaddlePaddle/PP-OCRv6_medium": 36.86322458152745, + "PaddlePaddle/PaddleOCR-VL-1.6": -8.467897574690824, + "allenai/olmOCR-2-7B-1025-FP8": -9.277916283709146, + "baidu/Qianfan-OCR": -43.20588503175759, + "baidu/Unlimited-OCR": 8.063213831644816, + "deepseek-ai/DeepSeek-OCR": 1.8272141097534131, + "deepseek-ai/DeepSeek-OCR-2": -15.541752583512334, + "lightonai/LightOnOCR-2-1B": 25.193449382838708, + "numind/NuExtract3": 22.75104611031111, + "rednote-hilab/dots.mocr": -11.25864286590081, + "tesseract-5": -72.80017296098617, + "zai-org/GLM-OCR": 34.62356655075837 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8205128205128204, + "kind": "pair-balanced", + "max_absolute_elo_delta": 72.80017296098617, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 22.75104611031111, + "name": "pair-balanced_700_seed_42", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9505494505494506, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1574.7921366784847, + "PaddlePaddle/PP-OCRv6_medium": 1340.4724390656875, + "PaddlePaddle/PaddleOCR-VL-1.6": 1398.6657139152867, + "allenai/olmOCR-2-7B-1025-FP8": 1669.6224468701632, + "baidu/Qianfan-OCR": 1430.2964101747075, + "baidu/Unlimited-OCR": 1499.1555485858391, + "deepseek-ai/DeepSeek-OCR": 1340.4724390656875, + "deepseek-ai/DeepSeek-OCR-2": 1367.8803083191708, + "lightonai/LightOnOCR-2-1B": 1664.2173264404153, + "numind/NuExtract3": 1645.1015194404895, + "rednote-hilab/dots.mocr": 1725.8410441553094, + "tesseract-5": 1045.2074470195134, + "zai-org/GLM-OCR": 1798.2752202692454 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1494.544911079249, + 1655.008076502896 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1261.3042221000546, + 1405.3726997594952 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1322.3968020932748, + 1470.1368657080388 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1594.312975262157, + 1753.5396582980752 + ], + "baidu/Qianfan-OCR": [ + 1353.5773229347055, + 1503.2414058536729 + ], + "baidu/Unlimited-OCR": [ + 1426.5544578710858, + 1571.5189072701098 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1269.4874097627999, + 1399.1977359119624 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1281.1795186508998, + 1439.881870247457 + ], + "lightonai/LightOnOCR-2-1B": [ + 1595.7875542373508, + 1742.412652586529 + ], + "numind/NuExtract3": [ + 1563.9509717704464, + 1735.212793116971 + ], + "rednote-hilab/dots.mocr": [ + 1658.307221264548, + 1818.2074791808568 + ], + "tesseract-5": [ + 872.2726715171863, + 1145.9269178980076 + ], + "zai-org/GLM-OCR": [ + 1726.82952875956, + 1901.6131963003188 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 62.937385666788714, + "PaddlePaddle/PP-OCRv6_medium": -5.884925983598805, + "PaddlePaddle/PaddleOCR-VL-1.6": -15.609866745608997, + "allenai/olmOCR-2-7B-1025-FP8": -4.1305324030793145, + "baidu/Qianfan-OCR": -93.709639003446, + "baidu/Unlimited-OCR": 1.1121050946890136, + "deepseek-ai/DeepSeek-OCR": -68.35680604225172, + "deepseek-ai/DeepSeek-OCR-2": 31.014911642573907, + "lightonai/LightOnOCR-2-1B": -33.38593124310637, + "numind/NuExtract3": -4.652121988222916, + "rednote-hilab/dots.mocr": 25.74974385432256, + "tesseract-5": 2.5741358937457335, + "zai-org/GLM-OCR": 102.34154125719442 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.7948717948717947, + "kind": "pair-balanced", + "max_absolute_elo_delta": 102.34154125719442, + "max_absolute_elo_delta_model": "zai-org/GLM-OCR", + "median_absolute_elo_delta": 25.74974385432256, + "name": "pair-balanced_700_seed_43", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9340659340659341, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1512.8863843583702, + "PaddlePaddle/PP-OCRv6_medium": 1359.3840135696803, + "PaddlePaddle/PaddleOCR-VL-1.6": 1479.017396280434, + "allenai/olmOCR-2-7B-1025-FP8": 1639.0527964984053, + "baidu/Qianfan-OCR": 1511.4006741156516, + "baidu/Unlimited-OCR": 1480.6048144058043, + "deepseek-ai/DeepSeek-OCR": 1389.1375735715262, + "deepseek-ai/DeepSeek-OCR-2": 1323.0492471239172, + "lightonai/LightOnOCR-2-1B": 1715.837557249851, + "numind/NuExtract3": 1631.4417427030041, + "rednote-hilab/dots.mocr": 1678.286516697602, + "tesseract-5": 1078.7160531944837, + "zai-org/GLM-OCR": 1701.18523023127 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1440.8223270518254, + 1588.6483537369256 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1279.0518574032308, + 1429.0258836970718 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1415.1775624514637, + 1543.4600697147237 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1569.3290409333927, + 1726.8948913752497 + ], + "baidu/Qianfan-OCR": [ + 1441.3244273434702, + 1585.183393878468 + ], + "baidu/Unlimited-OCR": [ + 1411.1360512190627, + 1545.6997208292007 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1311.7641517877544, + 1453.4899452903237 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1234.7409388892474, + 1396.9001864596535 + ], + "lightonai/LightOnOCR-2-1B": [ + 1647.3952468244509, + 1802.6605364824763 + ], + "numind/NuExtract3": [ + 1559.0478550879304, + 1714.0606920846744 + ], + "rednote-hilab/dots.mocr": [ + 1620.4494514036846, + 1752.270701939007 + ], + "tesseract-5": [ + 932.6965649090969, + 1166.84360762251 + ], + "zai-org/GLM-OCR": [ + 1633.079470625357, + 1787.8866464630755 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 1.031633346674198, + "PaddlePaddle/PP-OCRv6_medium": 13.02664852039402, + "PaddlePaddle/PaddleOCR-VL-1.6": 64.74181561953833, + "allenai/olmOCR-2-7B-1025-FP8": -34.70018277483723, + "baidu/Qianfan-OCR": -12.605375062501935, + "baidu/Unlimited-OCR": -17.438629085345838, + "deepseek-ai/DeepSeek-OCR": -19.69167153641297, + "deepseek-ai/DeepSeek-OCR-2": -13.81614955267969, + "lightonai/LightOnOCR-2-1B": 18.234299566329355, + "numind/NuExtract3": -18.31189872570826, + "rednote-hilab/dots.mocr": -21.80478360338475, + "tesseract-5": 36.082742068716016, + "zai-org/GLM-OCR": 5.251551219218982 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "pair-balanced", + "max_absolute_elo_delta": 64.74181561953833, + "max_absolute_elo_delta_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "median_absolute_elo_delta": 18.234299566329355, + "name": "pair-balanced_700_seed_44", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9780219780219779, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1464.16895734602, + "PaddlePaddle/PP-OCRv6_medium": 1337.328001168623, + "PaddlePaddle/PaddleOCR-VL-1.6": 1456.7808660106632, + "allenai/olmOCR-2-7B-1025-FP8": 1648.5956919064101, + "baidu/Qianfan-OCR": 1515.2190723660028, + "baidu/Unlimited-OCR": 1500.7135625399076, + "deepseek-ai/DeepSeek-OCR": 1415.3770515845151, + "deepseek-ai/DeepSeek-OCR-2": 1338.7670919288284, + "lightonai/LightOnOCR-2-1B": 1722.9541336433388, + "numind/NuExtract3": 1628.5686934952653, + "rednote-hilab/dots.mocr": 1718.56084159372, + "tesseract-5": 1059.2507318356475, + "zai-org/GLM-OCR": 1693.715304581058 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1377.4038536749513, + 1535.4068064042242 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1256.047714755538, + 1406.4141851294328 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1389.9114159560297, + 1523.4545979363024 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1581.2015323811443, + 1736.0509245789842 + ], + "baidu/Qianfan-OCR": [ + 1454.1752561463052, + 1583.6496670182676 + ], + "baidu/Unlimited-OCR": [ + 1439.0909483328019, + 1563.7248866311295 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1345.7852516672972, + 1477.576251819647 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1244.1726844414409, + 1409.8982946176322 + ], + "lightonai/LightOnOCR-2-1B": [ + 1654.0940052359365, + 1809.013911350066 + ], + "numind/NuExtract3": [ + 1559.788774463211, + 1710.5013592023004 + ], + "rednote-hilab/dots.mocr": [ + 1655.5928859093326, + 1801.4765677819962 + ], + "tesseract-5": [ + 860.3163307607956, + 1155.0126441161904 + ], + "zai-org/GLM-OCR": [ + 1621.5360265984705, + 1772.4547149298999 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -47.685793665675874, + "PaddlePaddle/PP-OCRv6_medium": -9.029363880663368, + "PaddlePaddle/PaddleOCR-VL-1.6": 42.50528534976752, + "allenai/olmOCR-2-7B-1025-FP8": -25.157287366832406, + "baidu/Qianfan-OCR": -8.78697681215067, + "baidu/Unlimited-OCR": 2.670119048757442, + "deepseek-ai/DeepSeek-OCR": 6.547806476575943, + "deepseek-ai/DeepSeek-OCR-2": 1.9016952522315478, + "lightonai/LightOnOCR-2-1B": 25.35087595981713, + "numind/NuExtract3": -21.18494793344712, + "rednote-hilab/dots.mocr": 18.469541292733084, + "tesseract-5": 16.617420709879752, + "zai-org/GLM-OCR": -2.2183744309929807 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "pair-balanced", + "max_absolute_elo_delta": 47.685793665675874, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 16.617420709879752, + "name": "pair-balanced_700_seed_45", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9835164835164836, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1448.9443247812612, + "PaddlePaddle/PP-OCRv6_medium": 1340.0792124885024, + "PaddlePaddle/PaddleOCR-VL-1.6": 1413.5984847161192, + "allenai/olmOCR-2-7B-1025-FP8": 1746.1766969124126, + "baidu/Qianfan-OCR": 1552.296404700822, + "baidu/Unlimited-OCR": 1437.2572110941214, + "deepseek-ai/DeepSeek-OCR": 1376.0664061117, + "deepseek-ai/DeepSeek-OCR-2": 1325.8981760418128, + "lightonai/LightOnOCR-2-1B": 1696.6063227541244, + "numind/NuExtract3": 1679.4964304623486, + "rednote-hilab/dots.mocr": 1691.2848643406123, + "tesseract-5": 1046.1187686837504, + "zai-org/GLM-OCR": 1746.1766969124126 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1368.6765419961835, + 1524.4039398624093 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1248.2778249749335, + 1414.3599327104557 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1338.6343967799205, + 1486.058993188094 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1672.3956866439821, + 1851.4431166178958 + ], + "baidu/Qianfan-OCR": [ + 1484.7265386817367, + 1617.9877061285774 + ], + "baidu/Unlimited-OCR": [ + 1369.0518234214767, + 1502.3171510449208 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1296.8844782456201, + 1442.3235625751624 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1233.8398109886257, + 1404.408385077436 + ], + "lightonai/LightOnOCR-2-1B": [ + 1631.2314308345467, + 1784.622190982807 + ], + "numind/NuExtract3": [ + 1600.8820055477165, + 1769.6098646328123 + ], + "rednote-hilab/dots.mocr": [ + 1622.3816843967938, + 1772.317500987505 + ], + "tesseract-5": [ + 880.1074907624222, + 1148.3780857067377 + ], + "zai-org/GLM-OCR": [ + 1673.9621222535159, + 1833.5349645593951 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -62.91042623043472, + "PaddlePaddle/PP-OCRv6_medium": -6.2781525607838375, + "PaddlePaddle/PaddleOCR-VL-1.6": -0.6770959447765108, + "allenai/olmOCR-2-7B-1025-FP8": 72.42371763917004, + "baidu/Qianfan-OCR": 28.290355522668506, + "baidu/Unlimited-OCR": -60.7862323970287, + "deepseek-ai/DeepSeek-OCR": -32.762838996239225, + "deepseek-ai/DeepSeek-OCR-2": -10.967220634784098, + "lightonai/LightOnOCR-2-1B": -0.9969349293971845, + "numind/NuExtract3": 29.7427890336362, + "rednote-hilab/dots.mocr": -8.806435960374529, + "tesseract-5": 3.485457557982727, + "zai-org/GLM-OCR": 50.243017900361565 + }, + "final_rank_order": [ + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.846153846153846, + "kind": "pair-balanced", + "max_absolute_elo_delta": 72.42371763917004, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 28.290355522668506, + "name": "pair-balanced_700_seed_46", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.945054945054945, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1547.3228097547958, + "PaddlePaddle/PP-OCRv6_medium": 1367.614025640857, + "PaddlePaddle/PaddleOCR-VL-1.6": 1399.473651845994, + "allenai/olmOCR-2-7B-1025-FP8": 1662.4845389730026, + "baidu/Qianfan-OCR": 1522.1895051795816, + "baidu/Unlimited-OCR": 1483.4461099341195, + "deepseek-ai/DeepSeek-OCR": 1373.0737034403448, + "deepseek-ai/DeepSeek-OCR-2": 1312.4397570411727, + "lightonai/LightOnOCR-2-1B": 1741.63011614823, + "numind/NuExtract3": 1663.6672299682727, + "rednote-hilab/dots.mocr": 1713.7010315528473, + "tesseract-5": 993.6650874385243, + "zai-org/GLM-OCR": 1719.292433082258 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1489.0890536988481, + 1601.7546487385234 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1299.2871685714892, + 1425.3115639702698 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1340.8980672996072, + 1454.8969231659053 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1612.9858584985582, + 1719.3818791458064 + ], + "baidu/Qianfan-OCR": [ + 1475.6859765051402, + 1570.005098511732 + ], + "baidu/Unlimited-OCR": [ + 1428.9520326188335, + 1534.3809474722475 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1313.0743576665368, + 1429.308629519766 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1248.5348082030284, + 1370.2436024936842 + ], + "lightonai/LightOnOCR-2-1B": [ + 1684.1291985190585, + 1803.5016518745329 + ], + "numind/NuExtract3": [ + 1609.3065535349651, + 1729.1641823155987 + ], + "rednote-hilab/dots.mocr": [ + 1664.280790014529, + 1769.7536852679652 + ], + "tesseract-5": [ + 855.3476737109352, + 1085.9462518426121 + ], + "zai-org/GLM-OCR": [ + 1664.5086859824953, + 1787.6494532885354 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 35.468058743099846, + "PaddlePaddle/PP-OCRv6_medium": 21.256660591570835, + "PaddlePaddle/PaddleOCR-VL-1.6": -14.801928814901657, + "allenai/olmOCR-2-7B-1025-FP8": -11.268440300239945, + "baidu/Qianfan-OCR": -1.8165439985718876, + "baidu/Unlimited-OCR": -14.597333557030652, + "deepseek-ai/DeepSeek-OCR": -35.75554166759434, + "deepseek-ai/DeepSeek-OCR-2": -24.425639635424204, + "lightonai/LightOnOCR-2-1B": 44.02685846470831, + "numind/NuExtract3": 13.913588539560351, + "rednote-hilab/dots.mocr": 13.609731251860467, + "tesseract-5": -48.96822368724338, + "zai-org/GLM-OCR": 23.358754070206942 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "pair-balanced", + "max_absolute_elo_delta": 48.96822368724338, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 21.256660591570835, + "name": "pair-balanced_1200_seed_42", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9725274725274725, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1551.7349136238274, + "PaddlePaddle/PP-OCRv6_medium": 1339.0153766639796, + "PaddlePaddle/PaddleOCR-VL-1.6": 1399.1357609573547, + "allenai/olmOCR-2-7B-1025-FP8": 1692.0462580233188, + "baidu/Qianfan-OCR": 1471.5952773223537, + "baidu/Unlimited-OCR": 1486.4535648600647, + "deepseek-ai/DeepSeek-OCR": 1390.6583979431405, + "deepseek-ai/DeepSeek-OCR-2": 1325.3796256029702, + "lightonai/LightOnOCR-2-1B": 1682.0025336876813, + "numind/NuExtract3": 1651.6801391725207, + "rednote-hilab/dots.mocr": 1720.4714958091326, + "tesseract-5": 1052.8107786651858, + "zai-org/GLM-OCR": 1737.01587766847 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1497.1005016764639, + 1612.5125797204664 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1279.2319835319195, + 1396.6316589301775 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1343.923605406217, + 1450.1854194128719 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1639.1698021602708, + 1755.679983190647 + ], + "baidu/Qianfan-OCR": [ + 1420.3619583617697, + 1524.9171322982404 + ], + "baidu/Unlimited-OCR": [ + 1434.6409185566858, + 1539.130680079567 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1337.618076104956, + 1440.4919694685407 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1257.7399524184427, + 1384.843617809355 + ], + "lightonai/LightOnOCR-2-1B": [ + 1629.7405793001967, + 1742.524500814699 + ], + "numind/NuExtract3": [ + 1595.7163062038476, + 1720.3218785829445 + ], + "rednote-hilab/dots.mocr": [ + 1665.0771367783823, + 1785.267289341399 + ], + "tesseract-5": [ + 935.6234734821086, + 1134.3936426564674 + ], + "zai-org/GLM-OCR": [ + 1685.062517761773, + 1800.624770374371 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 39.88016261213147, + "PaddlePaddle/PP-OCRv6_medium": -7.341988385306649, + "PaddlePaddle/PaddleOCR-VL-1.6": -15.13981970354098, + "allenai/olmOCR-2-7B-1025-FP8": 18.293278750076297, + "baidu/Qianfan-OCR": -52.410771855799794, + "baidu/Unlimited-OCR": -11.589878631085412, + "deepseek-ai/DeepSeek-OCR": -18.170847164798715, + "deepseek-ai/DeepSeek-OCR-2": -11.485771073626665, + "lightonai/LightOnOCR-2-1B": -15.600723995840326, + "numind/NuExtract3": 1.9264977438083406, + "rednote-hilab/dots.mocr": 20.38019550814579, + "tesseract-5": 10.177467539418103, + "zai-org/GLM-OCR": 41.082198656418996 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8717948717948716, + "kind": "pair-balanced", + "max_absolute_elo_delta": 52.410771855799794, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 15.600723995840326, + "name": "pair-balanced_1200_seed_43", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9560439560439561, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1527.7167110632138, + "PaddlePaddle/PP-OCRv6_medium": 1337.1781342276731, + "PaddlePaddle/PaddleOCR-VL-1.6": 1453.4783830936715, + "allenai/olmOCR-2-7B-1025-FP8": 1641.6110671619392, + "baidu/Qianfan-OCR": 1508.424857301885, + "baidu/Unlimited-OCR": 1496.0549358764658, + "deepseek-ai/DeepSeek-OCR": 1379.14059902558, + "deepseek-ai/DeepSeek-OCR-2": 1318.3661810110073, + "lightonai/LightOnOCR-2-1B": 1709.5907081811063, + "numind/NuExtract3": 1649.4373293405176, + "rednote-hilab/dots.mocr": 1687.7108101624913, + "tesseract-5": 1100.7348520040146, + "zai-org/GLM-OCR": 1690.5554315504344 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1472.1512315733326, + 1583.3507273768362 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1274.7273836144732, + 1388.1753438657247 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1399.2842969190287, + 1507.7959019201237 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1587.973049479066, + 1702.7638325835626 + ], + "baidu/Qianfan-OCR": [ + 1456.420346162897, + 1561.5128184598805 + ], + "baidu/Unlimited-OCR": [ + 1440.0829543918019, + 1548.827840491773 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1325.6492124159627, + 1426.7510102013691 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1253.8526197658773, + 1377.5543568506664 + ], + "lightonai/LightOnOCR-2-1B": [ + 1656.4579383807568, + 1776.9567300692459 + ], + "numind/NuExtract3": [ + 1594.5421584035598, + 1714.7558674664144 + ], + "rednote-hilab/dots.mocr": [ + 1637.9815371183342, + 1741.0134174540603 + ], + "tesseract-5": [ + 996.3041024231352, + 1170.699141004421 + ], + "zai-org/GLM-OCR": [ + 1638.0680451740366, + 1752.080307125966 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 15.861960051517826, + "PaddlePaddle/PP-OCRv6_medium": -9.179230821613146, + "PaddlePaddle/PaddleOCR-VL-1.6": 39.2028024327758, + "allenai/olmOCR-2-7B-1025-FP8": -32.14191211130333, + "baidu/Qianfan-OCR": -15.581191876268576, + "baidu/Unlimited-OCR": -1.9885076146842948, + "deepseek-ai/DeepSeek-OCR": -29.688646082359128, + "deepseek-ai/DeepSeek-OCR-2": -18.499215665589645, + "lightonai/LightOnOCR-2-1B": 11.987450497584632, + "numind/NuExtract3": -0.3163120881947634, + "rednote-hilab/dots.mocr": -12.380490138495588, + "tesseract-5": 58.10154087824685, + "zai-org/GLM-OCR": -5.378247461616638 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "pair-balanced", + "max_absolute_elo_delta": 58.10154087824685, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 15.581191876268576, + "name": "pair-balanced_1200_seed_44", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9725274725274725, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1507.0882022800595, + "PaddlePaddle/PP-OCRv6_medium": 1307.4299549870389, + "PaddlePaddle/PaddleOCR-VL-1.6": 1451.2613235479266, + "allenai/olmOCR-2-7B-1025-FP8": 1685.3874723343267, + "baidu/Qianfan-OCR": 1525.656715249995, + "baidu/Unlimited-OCR": 1474.8591947916398, + "deepseek-ai/DeepSeek-OCR": 1392.3500770469625, + "deepseek-ai/DeepSeek-OCR-2": 1359.9235404178542, + "lightonai/LightOnOCR-2-1B": 1704.6925445570932, + "numind/NuExtract3": 1659.174710936634, + "rednote-hilab/dots.mocr": 1720.121826265433, + "tesseract-5": 1010.3644839503829, + "zai-org/GLM-OCR": 1701.6899536346539 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1450.5488522646758, + 1564.5215828455684 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1244.2761291331637, + 1359.6865799227073 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1396.3207880458588, + 1504.0320926978943 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1632.82254173296, + 1746.6560100273991 + ], + "baidu/Qianfan-OCR": [ + 1475.5147973746277, + 1582.035200435875 + ], + "baidu/Unlimited-OCR": [ + 1426.9288805623446, + 1523.9541535388898 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1335.0102156259356, + 1445.663334472061 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1299.2758265730465, + 1417.2257932098732 + ], + "lightonai/LightOnOCR-2-1B": [ + 1648.1983753968666, + 1775.7609608705627 + ], + "numind/NuExtract3": [ + 1608.233445224852, + 1722.1646657017905 + ], + "rednote-hilab/dots.mocr": [ + 1669.0685650761827, + 1786.6624210531377 + ], + "tesseract-5": [ + 874.8443084918018, + 1095.6765831944635 + ], + "zai-org/GLM-OCR": [ + 1654.198116776979, + 1763.4195060266547 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -4.766548731636476, + "PaddlePaddle/PP-OCRv6_medium": -38.92741006224742, + "PaddlePaddle/PaddleOCR-VL-1.6": 36.98574288703094, + "allenai/olmOCR-2-7B-1025-FP8": 11.634493061084186, + "baidu/Qianfan-OCR": 1.6506660718414423, + "baidu/Unlimited-OCR": -23.184248699510363, + "deepseek-ai/DeepSeek-OCR": -16.47916806097669, + "deepseek-ai/DeepSeek-OCR-2": 23.058143741257254, + "lightonai/LightOnOCR-2-1B": 7.089286873571609, + "numind/NuExtract3": 9.42106950792163, + "rednote-hilab/dots.mocr": 20.030525964446042, + "tesseract-5": -32.268827175384786, + "zai-org/GLM-OCR": 5.756274622602859 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": true, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9743589743589742, + "kind": "pair-balanced", + "max_absolute_elo_delta": 38.92741006224742, + "max_absolute_elo_delta_model": "PaddlePaddle/PP-OCRv6_medium", + "median_absolute_elo_delta": 16.47916806097669, + "name": "pair-balanced_1200_seed_45", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9945054945054945, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1473.9975452834951, + "PaddlePaddle/PP-OCRv6_medium": 1349.1478341210855, + "PaddlePaddle/PaddleOCR-VL-1.6": 1404.9969901112029, + "allenai/olmOCR-2-7B-1025-FP8": 1688.3086825002492, + "baidu/Qianfan-OCR": 1522.708140669344, + "baidu/Unlimited-OCR": 1475.8005702021514, + "deepseek-ai/DeepSeek-OCR": 1347.8108880183515, + "deepseek-ai/DeepSeek-OCR-2": 1304.6092072192018, + "lightonai/LightOnOCR-2-1B": 1689.4922760955706, + "numind/NuExtract3": 1685.8775892812982, + "rednote-hilab/dots.mocr": 1711.5585295555043, + "tesseract-5": 1074.7963862311672, + "zai-org/GLM-OCR": 1770.895360711378 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1415.2364291739018, + 1531.187683394298 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1290.75351074685, + 1404.1376214836387 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1347.9654963094656, + 1459.6765903489345 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1636.5150172825583, + 1749.5506251030138 + ], + "baidu/Qianfan-OCR": [ + 1470.906754489091, + 1575.0617775633414 + ], + "baidu/Unlimited-OCR": [ + 1427.6232617631929, + 1526.9448537816827 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1290.5818512265516, + 1398.6836911655355 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1236.9960731438723, + 1359.896618964595 + ], + "lightonai/LightOnOCR-2-1B": [ + 1634.5585552465113, + 1752.3715467809893 + ], + "numind/NuExtract3": [ + 1629.3000514758191, + 1751.7825227349028 + ], + "rednote-hilab/dots.mocr": [ + 1667.3089520826893, + 1769.7785250046854 + ], + "tesseract-5": [ + 974.6551553662375, + 1149.8932838512928 + ], + "zai-org/GLM-OCR": [ + 1716.640298376799, + 1837.4414612207738 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -37.857205728200825, + "PaddlePaddle/PP-OCRv6_medium": 2.79046907179918, + "PaddlePaddle/PaddleOCR-VL-1.6": -9.278590549692808, + "allenai/olmOCR-2-7B-1025-FP8": 14.555703227006688, + "baidu/Qianfan-OCR": -1.2979085088095417, + "baidu/Unlimited-OCR": -22.24287328899868, + "deepseek-ai/DeepSeek-OCR": -61.01835708958765, + "deepseek-ai/DeepSeek-OCR-2": -32.25618945739507, + "lightonai/LightOnOCR-2-1B": -8.110981587951073, + "numind/NuExtract3": 36.12394785258584, + "rednote-hilab/dots.mocr": 11.467229254517406, + "tesseract-5": 32.16307510539946, + "zai-org/GLM-OCR": 74.96168169932707 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "pair-balanced", + "max_absolute_elo_delta": 74.96168169932707, + "max_absolute_elo_delta_model": "zai-org/GLM-OCR", + "median_absolute_elo_delta": 22.24287328899868, + "name": "pair-balanced_1200_seed_46", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9725274725274725, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1538.857583317062, + "PaddlePaddle/PP-OCRv6_medium": 1353.4379218563915, + "PaddlePaddle/PaddleOCR-VL-1.6": 1392.623887199742, + "allenai/olmOCR-2-7B-1025-FP8": 1682.72120876899, + "baidu/Qianfan-OCR": 1529.3704679298387, + "baidu/Unlimited-OCR": 1491.5465685365257, + "deepseek-ai/DeepSeek-OCR": 1408.4806373466429, + "deepseek-ai/DeepSeek-OCR-2": 1326.473045104206, + "lightonai/LightOnOCR-2-1B": 1697.9898265432876, + "numind/NuExtract3": 1634.967697705266, + "rednote-hilab/dots.mocr": 1693.8404494575589, + "tesseract-5": 1031.3931966696923, + "zai-org/GLM-OCR": 1718.2975095647969 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1494.7505478839082, + 1580.2323111328824 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1307.698772420618, + 1396.5846887670746 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1350.957576893178, + 1433.3788298069123 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1645.3677666094577, + 1727.6091411285893 + ], + "baidu/Qianfan-OCR": [ + 1492.3392738210323, + 1566.9154810072905 + ], + "baidu/Unlimited-OCR": [ + 1451.01039505498, + 1528.298711480445 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1366.283363046864, + 1449.139316391026 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1278.8383041230527, + 1374.8428696028807 + ], + "lightonai/LightOnOCR-2-1B": [ + 1659.9653054567696, + 1747.982911236205 + ], + "numind/NuExtract3": [ + 1593.8905512418435, + 1679.6400905461055 + ], + "rednote-hilab/dots.mocr": [ + 1652.215744260047, + 1738.1381950215948 + ], + "tesseract-5": [ + 935.1959267150929, + 1094.9443950951277 + ], + "zai-org/GLM-OCR": [ + 1677.0667595607442, + 1766.356899189886 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 27.002832305366155, + "PaddlePaddle/PP-OCRv6_medium": 7.080556807105268, + "PaddlePaddle/PaddleOCR-VL-1.6": -21.651693461153627, + "allenai/olmOCR-2-7B-1025-FP8": 8.968229495747437, + "baidu/Qianfan-OCR": 5.364418751685207, + "baidu/Unlimited-OCR": -6.496874954624445, + "deepseek-ai/DeepSeek-OCR": -0.3486077612963072, + "deepseek-ai/DeepSeek-OCR-2": -10.392351572390908, + "lightonai/LightOnOCR-2-1B": 0.38656885976593003, + "numind/NuExtract3": -14.785943723446508, + "rednote-hilab/dots.mocr": -6.2508508434279975, + "tesseract-5": -11.24011445607539, + "zai-org/GLM-OCR": 22.363830552745867 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8717948717948716, + "kind": "pair-balanced", + "max_absolute_elo_delta": 27.002832305366155, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 8.968229495747437, + "name": "pair-balanced_2000_seed_42", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.967032967032967, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1521.7873516451434, + "PaddlePaddle/PP-OCRv6_medium": 1350.8776242485956, + "PaddlePaddle/PaddleOCR-VL-1.6": 1398.77607829941, + "allenai/olmOCR-2-7B-1025-FP8": 1703.1095757324176, + "baidu/Qianfan-OCR": 1500.1895890750861, + "baidu/Unlimited-OCR": 1490.7558125754242, + "deepseek-ai/DeepSeek-OCR": 1403.7816912782237, + "deepseek-ai/DeepSeek-OCR-2": 1325.5231978332424, + "lightonai/LightOnOCR-2-1B": 1679.4657777930283, + "numind/NuExtract3": 1664.1522085782112, + "rednote-hilab/dots.mocr": 1697.3664968587675, + "tesseract-5": 1038.9617923935948, + "zai-org/GLM-OCR": 1725.2528036888555 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1477.9220882255418, + 1560.9077964158332 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1305.9225874128538, + 1395.0462423031468 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1357.8919707213138, + 1439.2924601151954 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1660.7810254033582, + 1750.9697397243494 + ], + "baidu/Qianfan-OCR": [ + 1459.7885648263148, + 1543.4408567150228 + ], + "baidu/Unlimited-OCR": [ + 1455.8384453020121, + 1529.2022222704777 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1362.3432654814562, + 1439.8420144555942 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1279.4394297225429, + 1372.9301209814896 + ], + "lightonai/LightOnOCR-2-1B": [ + 1638.2311729366154, + 1724.1073848833275 + ], + "numind/NuExtract3": [ + 1622.3309192623628, + 1712.3684802355542 + ], + "rednote-hilab/dots.mocr": [ + 1658.590883139834, + 1744.4416236104396 + ], + "tesseract-5": [ + 950.6969989591065, + 1100.545596366913 + ], + "zai-org/GLM-OCR": [ + 1685.242574653562, + 1771.0715595807144 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 9.932600633447464, + "PaddlePaddle/PP-OCRv6_medium": 4.520259199309294, + "PaddlePaddle/PaddleOCR-VL-1.6": -15.499502361485611, + "allenai/olmOCR-2-7B-1025-FP8": 29.35659645917508, + "baidu/Qianfan-OCR": -23.816460103067357, + "baidu/Unlimited-OCR": -7.287630915725913, + "deepseek-ai/DeepSeek-OCR": -5.047553829715525, + "deepseek-ai/DeepSeek-OCR-2": -11.342198843354481, + "lightonai/LightOnOCR-2-1B": -18.137479890493296, + "numind/NuExtract3": 14.398567149498831, + "rednote-hilab/dots.mocr": -2.724803442219354, + "tesseract-5": -3.671518732172899, + "zai-org/GLM-OCR": 29.31912467680445 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.846153846153846, + "kind": "pair-balanced", + "max_absolute_elo_delta": 29.35659645917508, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 11.342198843354481, + "name": "pair-balanced_2000_seed_43", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.945054945054945, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1524.165845793165, + "PaddlePaddle/PP-OCRv6_medium": 1345.6834162636842, + "PaddlePaddle/PaddleOCR-VL-1.6": 1442.2993313650516, + "allenai/olmOCR-2-7B-1025-FP8": 1675.8186523501097, + "baidu/Qianfan-OCR": 1507.35207576401, + "baidu/Unlimited-OCR": 1486.9103797506498, + "deepseek-ai/DeepSeek-OCR": 1393.2232487410406, + "deepseek-ai/DeepSeek-OCR-2": 1326.8153235658697, + "lightonai/LightOnOCR-2-1B": 1701.7710686061757, + "numind/NuExtract3": 1670.254563988339, + "rednote-hilab/dots.mocr": 1693.8104922469818, + "tesseract-5": 1047.612459834112, + "zai-org/GLM-OCR": 1684.2831417308107 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1476.4702880353739, + 1567.476873266488 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1300.9579615128414, + 1388.0583319235175 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1399.9123229370134, + 1481.1017778598155 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1629.2104178096715, + 1726.1061424756986 + ], + "baidu/Qianfan-OCR": [ + 1470.308005312484, + 1549.2741895164052 + ], + "baidu/Unlimited-OCR": [ + 1449.2449384997462, + 1527.8067161728113 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1352.0160861710015, + 1436.1618946539413 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1277.0644543867377, + 1371.4502999507124 + ], + "lightonai/LightOnOCR-2-1B": [ + 1659.9331165791177, + 1751.1231369297077 + ], + "numind/NuExtract3": [ + 1628.1925437725486, + 1714.320949784218 + ], + "rednote-hilab/dots.mocr": [ + 1656.4547513411744, + 1741.808965554938 + ], + "tesseract-5": [ + 959.8792854436724, + 1111.30325902957 + ], + "zai-org/GLM-OCR": [ + 1642.1379043457646, + 1727.126780488196 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 12.311094781469137, + "PaddlePaddle/PP-OCRv6_medium": -0.6739487856020787, + "PaddlePaddle/PaddleOCR-VL-1.6": 28.023750704155873, + "allenai/olmOCR-2-7B-1025-FP8": 2.0656730768671423, + "baidu/Qianfan-OCR": -16.653973414143593, + "baidu/Unlimited-OCR": -11.133063740500347, + "deepseek-ai/DeepSeek-OCR": -15.60599636689858, + "deepseek-ai/DeepSeek-OCR-2": -10.050073110727226, + "lightonai/LightOnOCR-2-1B": 4.167810922654098, + "numind/NuExtract3": 20.500922559626588, + "rednote-hilab/dots.mocr": -6.280808054005092, + "tesseract-5": 4.979148708344383, + "zai-org/GLM-OCR": -11.650537281240304 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9487179487179485, + "kind": "pair-balanced", + "max_absolute_elo_delta": 28.023750704155873, + "max_absolute_elo_delta_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "median_absolute_elo_delta": 11.133063740500347, + "name": "pair-balanced_2000_seed_44", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.989010989010989, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1491.1997693554488, + "PaddlePaddle/PP-OCRv6_medium": 1348.769130022881, + "PaddlePaddle/PaddleOCR-VL-1.6": 1440.3745190642467, + "allenai/olmOCR-2-7B-1025-FP8": 1682.7527475192317, + "baidu/Qianfan-OCR": 1524.534186598011, + "baidu/Unlimited-OCR": 1498.2724714345277, + "deepseek-ai/DeepSeek-OCR": 1417.0689124545254, + "deepseek-ai/DeepSeek-OCR-2": 1352.424540828758, + "lightonai/LightOnOCR-2-1B": 1699.0811991844396, + "numind/NuExtract3": 1660.2056326921897, + "rednote-hilab/dots.mocr": 1686.7713698803502, + "tesseract-5": 1004.9394024867813, + "zai-org/GLM-OCR": 1693.6061184786092 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1443.346903290222, + 1536.090790626268 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1300.1114295713753, + 1391.3365295387339 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1399.8915293336129, + 1482.3570260433362 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1643.6559749763096, + 1730.7832919011566 + ], + "baidu/Qianfan-OCR": [ + 1487.8025028995573, + 1563.842539541392 + ], + "baidu/Unlimited-OCR": [ + 1460.3537665395197, + 1536.492111152623 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1375.785778578611, + 1456.6425077749514 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1306.5248088169192, + 1399.4023296829137 + ], + "lightonai/LightOnOCR-2-1B": [ + 1659.1023809375245, + 1744.642808355164 + ], + "numind/NuExtract3": [ + 1617.5624319845904, + 1704.8442509695028 + ], + "rednote-hilab/dots.mocr": [ + 1646.3208175119648, + 1733.6457170317678 + ], + "tesseract-5": [ + 906.3657464615923, + 1075.0239955546404 + ], + "zai-org/GLM-OCR": [ + 1654.7466074699862, + 1737.5545056226724 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -20.654981656247173, + "PaddlePaddle/PP-OCRv6_medium": 2.4117649735946998, + "PaddlePaddle/PaddleOCR-VL-1.6": 26.098938403350985, + "allenai/olmOCR-2-7B-1025-FP8": 8.999768245989117, + "baidu/Qianfan-OCR": 0.5281374198575577, + "baidu/Unlimited-OCR": 0.2290279433775595, + "deepseek-ai/DeepSeek-OCR": 8.239667346586202, + "deepseek-ai/DeepSeek-OCR-2": 15.559144152161025, + "lightonai/LightOnOCR-2-1B": 1.4779415009179502, + "numind/NuExtract3": 10.451991263477339, + "rednote-hilab/dots.mocr": -13.319930420636638, + "tesseract-5": -37.693908638986386, + "zai-org/GLM-OCR": -2.3275605334417833 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "pair-balanced", + "max_absolute_elo_delta": 37.693908638986386, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 8.999768245989117, + "name": "pair-balanced_2000_seed_45", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9725274725274725, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1493.668874417381, + "PaddlePaddle/PP-OCRv6_medium": 1368.1079283592187, + "PaddlePaddle/PaddleOCR-VL-1.6": 1407.5377552284708, + "allenai/olmOCR-2-7B-1025-FP8": 1684.761324012015, + "baidu/Qianfan-OCR": 1530.9660740724503, + "baidu/Unlimited-OCR": 1494.063714500483, + "deepseek-ai/DeepSeek-OCR": 1373.9442532554858, + "deepseek-ai/DeepSeek-OCR-2": 1315.3094674528193, + "lightonai/LightOnOCR-2-1B": 1692.2421007900132, + "numind/NuExtract3": 1652.1510807850323, + "rednote-hilab/dots.mocr": 1709.5516078412325, + "tesseract-5": 1046.0163995282362, + "zai-org/GLM-OCR": 1731.679419757162 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1450.095961038773, + 1538.017840426799 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1321.4293648214912, + 1409.244570108095 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1366.9658134273302, + 1449.9893067725357 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1644.5060493856463, + 1730.458251379344 + ], + "baidu/Qianfan-OCR": [ + 1494.1141577328522, + 1568.4325567872977 + ], + "baidu/Unlimited-OCR": [ + 1452.0196886713504, + 1531.857353234493 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1331.2558932693016, + 1413.995622068187 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1266.8957274190134, + 1364.6116497774183 + ], + "lightonai/LightOnOCR-2-1B": [ + 1651.799663309388, + 1739.3541811968144 + ], + "numind/NuExtract3": [ + 1608.2351652789444, + 1700.5774477203788 + ], + "rednote-hilab/dots.mocr": [ + 1671.5900196312284, + 1755.6988874825554 + ], + "tesseract-5": [ + 962.8870515783669, + 1106.2276734802024 + ], + "zai-org/GLM-OCR": [ + 1688.0122651847669, + 1784.2963390107006 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -18.185876594315005, + "PaddlePaddle/PP-OCRv6_medium": 21.75056330993243, + "PaddlePaddle/PaddleOCR-VL-1.6": -6.73782543242487, + "allenai/olmOCR-2-7B-1025-FP8": 11.008344738772394, + "baidu/Qianfan-OCR": 6.960024894296794, + "baidu/Unlimited-OCR": -3.9797289906671267, + "deepseek-ai/DeepSeek-OCR": -34.88499185245337, + "deepseek-ai/DeepSeek-OCR-2": -21.55592922377764, + "lightonai/LightOnOCR-2-1B": -5.361156893508451, + "numind/NuExtract3": 2.3974393563198646, + "rednote-hilab/dots.mocr": 9.460307540245594, + "tesseract-5": 3.383088402468502, + "zai-org/GLM-OCR": 35.74574074511088 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "pair-balanced", + "max_absolute_elo_delta": 35.74574074511088, + "max_absolute_elo_delta_model": "zai-org/GLM-OCR", + "median_absolute_elo_delta": 9.460307540245594, + "name": "pair-balanced_2000_seed_46", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9780219780219779, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1568.3757724381894, + "PaddlePaddle/PP-OCRv6_medium": 1372.6037660424795, + "PaddlePaddle/PaddleOCR-VL-1.6": 1457.4892102537674, + "allenai/olmOCR-2-7B-1025-FP8": 1614.3477910012941, + "baidu/Qianfan-OCR": 1541.0898563482947, + "baidu/Unlimited-OCR": 1515.822715600171, + "deepseek-ai/DeepSeek-OCR": 1362.2153634275935, + "deepseek-ai/DeepSeek-OCR-2": 1312.7402175488392, + "lightonai/LightOnOCR-2-1B": 1731.8007034875268, + "numind/NuExtract3": 1559.17907019407, + "rednote-hilab/dots.mocr": 1764.4251393752568, + "tesseract-5": 1055.33672269336, + "zai-org/GLM-OCR": 1644.5736715891576 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1496.3547402393422, + 1648.8433022876313 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1290.7691539706468, + 1447.7349290855161 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1392.1410811116982, + 1521.088426838953 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1541.5939270938645, + 1704.3921322017554 + ], + "baidu/Qianfan-OCR": [ + 1478.661833340298, + 1605.7169773607923 + ], + "baidu/Unlimited-OCR": [ + 1449.9471703220042, + 1586.3009912501102 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1287.5264137939591, + 1423.899044589451 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1222.9357344317048, + 1386.988613524616 + ], + "lightonai/LightOnOCR-2-1B": [ + 1664.1863142765455, + 1819.7426928985083 + ], + "numind/NuExtract3": [ + 1493.4043754771592, + 1631.403840399588 + ], + "rednote-hilab/dots.mocr": [ + 1699.212516081395, + 1859.192926870087 + ], + "tesseract-5": [ + 890.9562318279883, + 1141.6790342969243 + ], + "zai-org/GLM-OCR": [ + 1581.817951475659, + 1725.9754546610106 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 56.52102142649346, + "PaddlePaddle/PP-OCRv6_medium": 26.24640099319322, + "PaddlePaddle/PaddleOCR-VL-1.6": 43.21362959287171, + "allenai/olmOCR-2-7B-1025-FP8": -59.40518827194842, + "baidu/Qianfan-OCR": 17.0838071701412, + "baidu/Unlimited-OCR": 17.779272109020894, + "deepseek-ai/DeepSeek-OCR": -46.613881680345685, + "deepseek-ai/DeepSeek-OCR-2": -24.125179127757747, + "lightonai/LightOnOCR-2-1B": 34.197445804005156, + "numind/NuExtract3": -90.57457123464246, + "rednote-hilab/dots.mocr": 64.33383907426992, + "tesseract-5": 12.703411567592184, + "zai-org/GLM-OCR": -51.36000742289343 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "mixed-random", + "max_absolute_elo_delta": 90.57457123464246, + "max_absolute_elo_delta_model": "numind/NuExtract3", + "median_absolute_elo_delta": 43.21362959287171, + "name": "mixed-random_700_seed_42", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9780219780219779, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1622.3639582308392, + "PaddlePaddle/PP-OCRv6_medium": 1363.2004436785749, + "PaddlePaddle/PaddleOCR-VL-1.6": 1482.9399814877208, + "allenai/olmOCR-2-7B-1025-FP8": 1591.7638685849531, + "baidu/Qianfan-OCR": 1521.3465930257587, + "baidu/Unlimited-OCR": 1450.5643292612099, + "deepseek-ai/DeepSeek-OCR": 1338.8554476431773, + "deepseek-ai/DeepSeek-OCR-2": 1358.9796608417946, + "lightonai/LightOnOCR-2-1B": 1715.8441711987007, + "numind/NuExtract3": 1554.536382443841, + "rednote-hilab/dots.mocr": 1791.688717805248, + "tesseract-5": 1000.7676763323984, + "zai-org/GLM-OCR": 1707.1487694657835 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1554.7083559567911, + 1703.2632627067576 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1282.5604799914254, + 1434.6533122401245 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1412.8485727053098, + 1549.8415170872393 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1520.6335928841825, + 1673.292178649122 + ], + "baidu/Qianfan-OCR": [ + 1453.8121283475193, + 1590.681990026472 + ], + "baidu/Unlimited-OCR": [ + 1382.291385526643, + 1519.4771589362122 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1249.3922243826955, + 1411.0767386536086 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1267.531994853869, + 1443.7263508759709 + ], + "lightonai/LightOnOCR-2-1B": [ + 1650.343256648463, + 1801.1043049885423 + ], + "numind/NuExtract3": [ + 1487.040500977108, + 1637.3314655027746 + ], + "rednote-hilab/dots.mocr": [ + 1715.8501937005003, + 1904.290313752627 + ], + "tesseract-5": [ + 810.6160457503508, + 1101.6364742807891 + ], + "zai-org/GLM-OCR": [ + 1643.1404499762884, + 1794.973657541364 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 110.50920721914326, + "PaddlePaddle/PP-OCRv6_medium": 16.843078629288584, + "PaddlePaddle/PaddleOCR-VL-1.6": 68.66440082682516, + "allenai/olmOCR-2-7B-1025-FP8": -81.98911068828943, + "baidu/Qianfan-OCR": -2.659456152394796, + "baidu/Unlimited-OCR": -47.47911422994025, + "deepseek-ai/DeepSeek-OCR": -69.9737974647619, + "deepseek-ai/DeepSeek-OCR-2": 22.11426416519771, + "lightonai/LightOnOCR-2-1B": 18.240913515179045, + "numind/NuExtract3": -95.21725898487148, + "rednote-hilab/dots.mocr": 91.5974175042611, + "tesseract-5": -41.86563479336928, + "zai-org/GLM-OCR": 11.2150904537325 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.846153846153846, + "kind": "mixed-random", + "max_absolute_elo_delta": 110.50920721914326, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 47.47911422994025, + "name": "mixed-random_700_seed_43", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.945054945054945, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1552.3361144375635, + "PaddlePaddle/PP-OCRv6_medium": 1380.2230399407642, + "PaddlePaddle/PaddleOCR-VL-1.6": 1462.7814852704466, + "allenai/olmOCR-2-7B-1025-FP8": 1584.907048969294, + "baidu/Qianfan-OCR": 1523.718989319802, + "baidu/Unlimited-OCR": 1455.5179210293159, + "deepseek-ai/DeepSeek-OCR": 1347.9936569341887, + "deepseek-ai/DeepSeek-OCR-2": 1377.3423583284175, + "lightonai/LightOnOCR-2-1B": 1700.7532963850583, + "numind/NuExtract3": 1570.3597410322045, + "rednote-hilab/dots.mocr": 1770.5600655637809, + "tesseract-5": 1080.2462803212627, + "zai-org/GLM-OCR": 1693.260002467901 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1483.588142810978, + 1628.5705711364437 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1302.7633478116786, + 1444.1531551495086 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1391.2948902277958, + 1535.84929115151 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1517.8955227007111, + 1663.8159006800574 + ], + "baidu/Qianfan-OCR": [ + 1456.8459416146193, + 1588.7477701372572 + ], + "baidu/Unlimited-OCR": [ + 1397.6989591220072, + 1525.8979424580248 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1266.712431435876, + 1417.6698925876744 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1291.8094864965956, + 1455.3981883431748 + ], + "lightonai/LightOnOCR-2-1B": [ + 1629.659895499414, + 1792.682426618349 + ], + "numind/NuExtract3": [ + 1504.4589733738942, + 1650.969867809129 + ], + "rednote-hilab/dots.mocr": [ + 1701.6443649403611, + 1864.8212967833938 + ], + "tesseract-5": [ + 929.9924444271518, + 1173.315710826951 + ], + "zai-org/GLM-OCR": [ + 1626.6888432782316, + 1775.1024622052391 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 40.481363425867585, + "PaddlePaddle/PP-OCRv6_medium": 33.865674891477966, + "PaddlePaddle/PaddleOCR-VL-1.6": 48.50590460955095, + "allenai/olmOCR-2-7B-1025-FP8": -88.84593030394853, + "baidu/Qianfan-OCR": -0.28705985835154024, + "baidu/Unlimited-OCR": -42.52552246183427, + "deepseek-ai/DeepSeek-OCR": -60.835588173750466, + "deepseek-ai/DeepSeek-OCR-2": 40.4769616518206, + "lightonai/LightOnOCR-2-1B": 3.150038701536687, + "numind/NuExtract3": -79.39390039650789, + "rednote-hilab/dots.mocr": 70.46876526279402, + "tesseract-5": 37.612969195495, + "zai-org/GLM-OCR": -2.673676544150112 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "mixed-random", + "max_absolute_elo_delta": 88.84593030394853, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 40.481363425867585, + "name": "mixed-random_700_seed_44", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9725274725274725, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1574.1980891844792, + "PaddlePaddle/PP-OCRv6_medium": 1428.5449755534469, + "PaddlePaddle/PaddleOCR-VL-1.6": 1476.803446811303, + "allenai/olmOCR-2-7B-1025-FP8": 1622.052253698868, + "baidu/Qianfan-OCR": 1556.130413953367, + "baidu/Unlimited-OCR": 1473.151527471005, + "deepseek-ai/DeepSeek-OCR": 1357.189073847404, + "deepseek-ai/DeepSeek-OCR-2": 1283.755241089148, + "lightonai/LightOnOCR-2-1B": 1690.1423556478014, + "numind/NuExtract3": 1540.2789890621548, + "rednote-hilab/dots.mocr": 1754.013304240449, + "tesseract-5": 1059.356874479118, + "zai-org/GLM-OCR": 1684.3834549614555 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1505.1686136521687, + 1650.907826550792 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1357.9027353482095, + 1494.044443203509 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1409.165574412068, + 1546.6476567043285 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1549.3350001507965, + 1707.1113334445222 + ], + "baidu/Qianfan-OCR": [ + 1491.145698781296, + 1625.499055122809 + ], + "baidu/Unlimited-OCR": [ + 1401.6094005954362, + 1540.8218023676934 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1278.6855592036768, + 1427.707792872336 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1188.0816228220724, + 1361.5512135537163 + ], + "lightonai/LightOnOCR-2-1B": [ + 1625.0891046943523, + 1771.8953768983483 + ], + "numind/NuExtract3": [ + 1465.8094639878063, + 1614.8377510328721 + ], + "rednote-hilab/dots.mocr": [ + 1688.983705247469, + 1843.7598706405752 + ], + "tesseract-5": [ + 910.3883946026476, + 1145.19588667258 + ], + "zai-org/GLM-OCR": [ + 1616.7957600226764, + 1766.1502938665328 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 62.343338172783206, + "PaddlePaddle/PP-OCRv6_medium": 82.1876105041606, + "PaddlePaddle/PaddleOCR-VL-1.6": 62.52786615040736, + "allenai/olmOCR-2-7B-1025-FP8": -51.700725574374474, + "baidu/Qianfan-OCR": 32.12436477521351, + "baidu/Unlimited-OCR": -24.891916020145118, + "deepseek-ai/DeepSeek-OCR": -51.640171260535226, + "deepseek-ai/DeepSeek-OCR-2": -53.110155587448844, + "lightonai/LightOnOCR-2-1B": -7.46090203572021, + "numind/NuExtract3": -109.47465236655762, + "rednote-hilab/dots.mocr": 53.92200393946223, + "tesseract-5": 16.7235633533503, + "zai-org/GLM-OCR": -11.550224050595489 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8717948717948716, + "kind": "mixed-random", + "max_absolute_elo_delta": 109.47465236655762, + "max_absolute_elo_delta_model": "numind/NuExtract3", + "median_absolute_elo_delta": 51.700725574374474, + "name": "mixed-random_700_seed_45", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.967032967032967, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 2965, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1581.724770859182, + "PaddlePaddle/PP-OCRv6_medium": 1373.4795593190702, + "PaddlePaddle/PaddleOCR-VL-1.6": 1473.319483064947, + "allenai/olmOCR-2-7B-1025-FP8": 1618.9957597180755, + "baidu/Qianfan-OCR": 1520.5568869193578, + "baidu/Unlimited-OCR": 1465.9593975757891, + "deepseek-ai/DeepSeek-OCR": 1381.5926058078226, + "deepseek-ai/DeepSeek-OCR-2": 1299.6610887445422, + "lightonai/LightOnOCR-2-1B": 1692.9120868932694, + "numind/NuExtract3": 1618.9957597180755, + "rednote-hilab/dots.mocr": 1727.8533746725998, + "tesseract-5": 1055.7385954711401, + "zai-org/GLM-OCR": 1689.2106312361288 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1513.1526855645286, + 1657.3646772635414 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1295.7747121581199, + 1439.572619229316 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1400.55264919696, + 1538.5312859626893 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1550.9871995041055, + 1695.5259097587375 + ], + "baidu/Qianfan-OCR": [ + 1456.9042394633, + 1583.0780917370782 + ], + "baidu/Unlimited-OCR": [ + 1400.6532565993557, + 1529.397325986483 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1304.4793889966727, + 1452.8497533707289 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1205.4723377632806, + 1379.4863410629396 + ], + "lightonai/LightOnOCR-2-1B": [ + 1627.0023098645079, + 1771.588503606129 + ], + "numind/NuExtract3": [ + 1548.8604650994473, + 1691.8070879611935 + ], + "rednote-hilab/dots.mocr": [ + 1654.206002939988, + 1822.2015254954906 + ], + "tesseract-5": [ + 883.4305881582985, + 1149.032935787121 + ], + "zai-org/GLM-OCR": [ + 1615.6196052872967, + 1776.425865923446 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 69.87001984748599, + "PaddlePaddle/PP-OCRv6_medium": 27.122194269783904, + "PaddlePaddle/PaddleOCR-VL-1.6": 59.04390240405132, + "allenai/olmOCR-2-7B-1025-FP8": -54.75721955516701, + "baidu/Qianfan-OCR": -3.4491622587956954, + "baidu/Unlimited-OCR": -32.084045915361, + "deepseek-ai/DeepSeek-OCR": -27.23663930011662, + "deepseek-ai/DeepSeek-OCR-2": -37.20430793205469, + "lightonai/LightOnOCR-2-1B": -4.691170790252272, + "numind/NuExtract3": -30.757881710636866, + "rednote-hilab/dots.mocr": 27.762074371612925, + "tesseract-5": 13.10528434537241, + "zai-org/GLM-OCR": -6.7230477759221685 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 9, + "median_direct_comparisons": 9.0, + "min_direct_comparisons": 8, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9487179487179485, + "kind": "mixed-random", + "max_absolute_elo_delta": 69.87001984748599, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 27.762074371612925, + "name": "mixed-random_700_seed_46", + "percentage_saved": 80.90040927694406, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.989010989010989, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1542.2731534480074, + "PaddlePaddle/PP-OCRv6_medium": 1382.291356420484, + "PaddlePaddle/PaddleOCR-VL-1.6": 1426.0889761592027, + "allenai/olmOCR-2-7B-1025-FP8": 1642.042263243064, + "baidu/Qianfan-OCR": 1518.4803305893438, + "baidu/Unlimited-OCR": 1516.195289490195, + "deepseek-ai/DeepSeek-OCR": 1389.9195602798206, + "deepseek-ai/DeepSeek-OCR-2": 1322.2778553166315, + "lightonai/LightOnOCR-2-1B": 1711.4527261826724, + "numind/NuExtract3": 1620.6754071255164, + "rednote-hilab/dots.mocr": 1740.5536595747546, + "tesseract-5": 1000.2049459994696, + "zai-org/GLM-OCR": 1687.544476170838 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1485.8996057490763, + 1599.3520794028423 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1321.2852105548461, + 1433.734069471369 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1371.8201333005281, + 1478.3677820068328 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1581.3976592864024, + 1707.9396991315373 + ], + "baidu/Qianfan-OCR": [ + 1466.8963319920836, + 1573.5692170904779 + ], + "baidu/Unlimited-OCR": [ + 1463.4085733065629, + 1568.5396889669098 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1338.9070126068314, + 1439.00419631303 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1256.8421348767001, + 1386.7006805197536 + ], + "lightonai/LightOnOCR-2-1B": [ + 1661.2181970395723, + 1772.970724632413 + ], + "numind/NuExtract3": [ + 1565.7145302939534, + 1683.7900394381052 + ], + "rednote-hilab/dots.mocr": [ + 1691.3440161647995, + 1799.6982576875512 + ], + "tesseract-5": [ + 867.8041501788628, + 1095.6333089284033 + ], + "zai-org/GLM-OCR": [ + 1635.68868674211, + 1750.4648126716709 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 30.418402436311453, + "PaddlePaddle/PP-OCRv6_medium": 35.933991371197635, + "PaddlePaddle/PaddleOCR-VL-1.6": 11.813395498307045, + "allenai/olmOCR-2-7B-1025-FP8": -31.710716030178446, + "baidu/Qianfan-OCR": -5.525718588809696, + "baidu/Unlimited-OCR": 18.15184599904478, + "deepseek-ai/DeepSeek-OCR": -18.909684828118543, + "deepseek-ai/DeepSeek-OCR-2": -14.587541359965371, + "lightonai/LightOnOCR-2-1B": 13.849468499150817, + "numind/NuExtract3": -29.07823430319604, + "rednote-hilab/dots.mocr": 40.46235927376779, + "tesseract-5": -42.428365126298104, + "zai-org/GLM-OCR": -8.389202841213091 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": true, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9743589743589742, + "kind": "mixed-random", + "max_absolute_elo_delta": 42.428365126298104, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 18.909684828118543, + "name": "mixed-random_1200_seed_42", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9945054945054945, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1569.1714308922114, + "PaddlePaddle/PP-OCRv6_medium": 1359.3915639665847, + "PaddlePaddle/PaddleOCR-VL-1.6": 1419.2478138109495, + "allenai/olmOCR-2-7B-1025-FP8": 1623.96015573092, + "baidu/Qianfan-OCR": 1519.1174171344292, + "baidu/Unlimited-OCR": 1492.2312304002107, + "deepseek-ai/DeepSeek-OCR": 1353.6007170597934, + "deepseek-ai/DeepSeek-OCR-2": 1337.1508314261428, + "lightonai/LightOnOCR-2-1B": 1696.0407600997837, + "numind/NuExtract3": 1625.6646806220829, + "rednote-hilab/dots.mocr": 1744.473083052518, + "tesseract-5": 1057.3664809912548, + "zai-org/GLM-OCR": 1702.583834813119 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1513.6465118390133, + 1624.8411394984337 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1295.045911927137, + 1414.3894949861112 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1364.4773899632696, + 1467.97269280944 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1570.2706195953403, + 1684.4318233376619 + ], + "baidu/Qianfan-OCR": [ + 1468.6519977366952, + 1570.293446819824 + ], + "baidu/Unlimited-OCR": [ + 1443.5345332356258, + 1542.968989464483 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1290.0553139824422, + 1406.4265737728708 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1269.506660935207, + 1395.2976962955306 + ], + "lightonai/LightOnOCR-2-1B": [ + 1646.2946232512365, + 1757.9851595660728 + ], + "numind/NuExtract3": [ + 1567.8999289967057, + 1684.8864648271763 + ], + "rednote-hilab/dots.mocr": [ + 1688.545286060514, + 1810.7767215024596 + ], + "tesseract-5": [ + 948.3393427962795, + 1131.2003703829998 + ], + "zai-org/GLM-OCR": [ + 1652.5115729291774, + 1766.5741345000856 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 57.316679880515494, + "PaddlePaddle/PP-OCRv6_medium": 13.034198917298454, + "PaddlePaddle/PaddleOCR-VL-1.6": 4.972233150053853, + "allenai/olmOCR-2-7B-1025-FP8": -49.792823542322594, + "baidu/Qianfan-OCR": -4.8886320437243285, + "baidu/Unlimited-OCR": -5.8122130909393945, + "deepseek-ai/DeepSeek-OCR": -55.22852804814579, + "deepseek-ai/DeepSeek-OCR-2": 0.2854347495458569, + "lightonai/LightOnOCR-2-1B": -1.5624975837379225, + "numind/NuExtract3": -24.08896080662953, + "rednote-hilab/dots.mocr": 44.38178275153109, + "tesseract-5": 14.73316986548707, + "zai-org/GLM-OCR": 6.650155801067967 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "mixed-random", + "max_absolute_elo_delta": 57.316679880515494, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 13.034198917298454, + "name": "mixed-random_1200_seed_43", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9780219780219779, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1540.0770371859935, + "PaddlePaddle/PP-OCRv6_medium": 1364.3902334965335, + "PaddlePaddle/PaddleOCR-VL-1.6": 1419.0553216231738, + "allenai/olmOCR-2-7B-1025-FP8": 1635.9228720998954, + "baidu/Qianfan-OCR": 1517.7393003953875, + "baidu/Unlimited-OCR": 1461.007776842755, + "deepseek-ai/DeepSeek-OCR": 1388.7676529111636, + "deepseek-ai/DeepSeek-OCR-2": 1363.5974039981409, + "lightonai/LightOnOCR-2-1B": 1705.135646851379, + "numind/NuExtract3": 1621.2681692059696, + "rednote-hilab/dots.mocr": 1731.689038778754, + "tesseract-5": 1049.1914123784838, + "zai-org/GLM-OCR": 1702.1581342323707 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1488.4114761427913, + 1590.7653301092737 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1303.3106287042747, + 1423.6805971874105 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1364.912071722028, + 1473.4370582981423 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1578.8812733975985, + 1695.6837036073482 + ], + "baidu/Qianfan-OCR": [ + 1467.0213936428381, + 1568.0200335478496 + ], + "baidu/Unlimited-OCR": [ + 1410.068535180301, + 1513.5552827786407 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1331.674668862896, + 1438.1041655634394 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1303.7413604777335, + 1421.7076829987209 + ], + "lightonai/LightOnOCR-2-1B": [ + 1654.1367053511024, + 1764.6209629699024 + ], + "numind/NuExtract3": [ + 1569.4477160994538, + 1677.4546771857981 + ], + "rednote-hilab/dots.mocr": [ + 1677.5328622738377, + 1796.231750446716 + ], + "tesseract-5": [ + 934.8434079470228, + 1129.963008382917 + ], + "zai-org/GLM-OCR": [ + 1652.6982107604317, + 1767.1625793090604 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 28.22228617429755, + "PaddlePaddle/PP-OCRv6_medium": 18.03286844724721, + "PaddlePaddle/PaddleOCR-VL-1.6": 4.779740962278083, + "allenai/olmOCR-2-7B-1025-FP8": -37.83010717334719, + "baidu/Qianfan-OCR": -6.266748782765944, + "baidu/Unlimited-OCR": -37.035666648395136, + "deepseek-ai/DeepSeek-OCR": -20.061592196775564, + "deepseek-ai/DeepSeek-OCR-2": 26.73200732154396, + "lightonai/LightOnOCR-2-1B": 7.5323891678574455, + "numind/NuExtract3": -28.48547222274283, + "rednote-hilab/dots.mocr": 31.597738477767052, + "tesseract-5": 6.558101252716142, + "zai-org/GLM-OCR": 6.224455220319669 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": true, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9743589743589742, + "kind": "mixed-random", + "max_absolute_elo_delta": 37.83010717334719, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 20.061592196775564, + "name": "mixed-random_1200_seed_44", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9945054945054945, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1548.3979093981743, + "PaddlePaddle/PP-OCRv6_medium": 1381.9625337837633, + "PaddlePaddle/PaddleOCR-VL-1.6": 1427.7316232773803, + "allenai/olmOCR-2-7B-1025-FP8": 1690.0293852405443, + "baidu/Qianfan-OCR": 1537.4354559583417, + "baidu/Unlimited-OCR": 1457.8129162428863, + "deepseek-ai/DeepSeek-OCR": 1371.8367744758652, + "deepseek-ai/DeepSeek-OCR-2": 1283.5321362017203, + "lightonai/LightOnOCR-2-1B": 1700.4411366191428, + "numind/NuExtract3": 1593.1453568796262, + "rednote-hilab/dots.mocr": 1749.721196183658, + "tesseract-5": 1051.6106431247508, + "zai-org/GLM-OCR": 1706.342932614146 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1489.4865879567567, + 1601.2344158729964 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1321.3189838346245, + 1430.9452825240653 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1374.2315914995168, + 1484.0732515150687 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1638.1864269235318, + 1760.5378420766503 + ], + "baidu/Qianfan-OCR": [ + 1482.2598001330014, + 1590.80176697829 + ], + "baidu/Unlimited-OCR": [ + 1406.4922405858558, + 1511.4709346654959 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1315.8659481833197, + 1422.8671792649602 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1210.254823910572, + 1343.2631290155257 + ], + "lightonai/LightOnOCR-2-1B": [ + 1650.4276400857102, + 1762.2977252441447 + ], + "numind/NuExtract3": [ + 1538.4476528042085, + 1654.9364307698668 + ], + "rednote-hilab/dots.mocr": [ + 1699.1634417846153, + 1814.7852640293 + ], + "tesseract-5": [ + 950.8618261930715, + 1126.3453231965984 + ], + "zai-org/GLM-OCR": [ + 1653.0772851725924, + 1769.4966726297996 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 36.54315838647835, + "PaddlePaddle/PP-OCRv6_medium": 35.60516873447705, + "PaddlePaddle/PaddleOCR-VL-1.6": 13.456042616484638, + "allenai/olmOCR-2-7B-1025-FP8": 16.2764059673018, + "baidu/Qianfan-OCR": 13.429406780188174, + "baidu/Unlimited-OCR": -40.2305272482638, + "deepseek-ai/DeepSeek-OCR": -36.992470632073946, + "deepseek-ai/DeepSeek-OCR-2": -53.333260474876624, + "lightonai/LightOnOCR-2-1B": 2.83787893562112, + "numind/NuExtract3": -56.60828454908619, + "rednote-hilab/dots.mocr": 49.629895882671235, + "tesseract-5": 8.977331998983118, + "zai-org/GLM-OCR": 10.409253602095077 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9230769230769229, + "kind": "mixed-random", + "max_absolute_elo_delta": 56.60828454908619, + "max_absolute_elo_delta_model": "numind/NuExtract3", + "median_absolute_elo_delta": 35.60516873447705, + "name": "mixed-random_1200_seed_45", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9835164835164836, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 2465, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1545.758935357603, + "PaddlePaddle/PP-OCRv6_medium": 1360.8703221953983, + "PaddlePaddle/PaddleOCR-VL-1.6": 1417.5380247578462, + "allenai/olmOCR-2-7B-1025-FP8": 1647.8708595287917, + "baidu/Qianfan-OCR": 1512.1829443852555, + "baidu/Unlimited-OCR": 1513.4081948336227, + "deepseek-ai/DeepSeek-OCR": 1389.2344717440421, + "deepseek-ai/DeepSeek-OCR-2": 1345.2940881565203, + "lightonai/LightOnOCR-2-1B": 1662.2584323422798, + "numind/NuExtract3": 1637.5223337086848, + "rednote-hilab/dots.mocr": 1686.9492603822323, + "tesseract-5": 1075.231849288151, + "zai-org/GLM-OCR": 1705.8802833195728 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1489.1163378574377, + 1601.1962498420417 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1301.0244063635464, + 1410.5807510065104 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1362.697022835376, + 1468.4809528198257 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1595.2052892497752, + 1707.6476603695799 + ], + "baidu/Qianfan-OCR": [ + 1462.0899415922613, + 1566.9226775945958 + ], + "baidu/Unlimited-OCR": [ + 1464.788501399085, + 1565.091284798886 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1334.7858387323529, + 1442.7427103210557 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1278.5340603811019, + 1401.0533429225782 + ], + "lightonai/LightOnOCR-2-1B": [ + 1614.4776821782257, + 1722.0558464549222 + ], + "numind/NuExtract3": [ + 1582.737047005323, + 1698.1494892586138 + ], + "rednote-hilab/dots.mocr": [ + 1637.4495129659942, + 1746.8082524984302 + ], + "tesseract-5": [ + 971.6806939370014, + 1150.9493885266747 + ], + "zai-org/GLM-OCR": [ + 1651.8896526712972, + 1765.9491401603334 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 33.90418434590697, + "PaddlePaddle/PP-OCRv6_medium": 14.512957146112058, + "PaddlePaddle/PaddleOCR-VL-1.6": 3.262444096950503, + "allenai/olmOCR-2-7B-1025-FP8": -25.882119744450847, + "baidu/Qianfan-OCR": -11.823104792897993, + "baidu/Unlimited-OCR": 15.364751342472573, + "deepseek-ai/DeepSeek-OCR": -19.594773363897048, + "deepseek-ai/DeepSeek-OCR-2": 8.428691479923373, + "lightonai/LightOnOCR-2-1B": -35.34482534124186, + "numind/NuExtract3": -12.231307720027644, + "rednote-hilab/dots.mocr": -13.142039918754563, + "tesseract-5": 32.59853816238319, + "zai-org/GLM-OCR": 9.946604307521739 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 16, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 15, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "mixed-random", + "max_absolute_elo_delta": 35.34482534124186, + "max_absolute_elo_delta_model": "lightonai/LightOnOCR-2-1B", + "median_absolute_elo_delta": 14.512957146112058, + "name": "mixed-random_1200_seed_46", + "percentage_saved": 67.25784447476126, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.967032967032967, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1524.4731094141437, + "PaddlePaddle/PP-OCRv6_medium": 1353.4103087098065, + "PaddlePaddle/PaddleOCR-VL-1.6": 1409.3660358316768, + "allenai/olmOCR-2-7B-1025-FP8": 1660.5236011229763, + "baidu/Qianfan-OCR": 1521.629958929006, + "baidu/Unlimited-OCR": 1505.3930928257362, + "deepseek-ai/DeepSeek-OCR": 1397.1412651577061, + "deepseek-ai/DeepSeek-OCR-2": 1338.6489534186155, + "lightonai/LightOnOCR-2-1B": 1691.5456052457262, + "numind/NuExtract3": 1644.0893439803604, + "rednote-hilab/dots.mocr": 1708.9163260389782, + "tesseract-5": 1041.6784062956938, + "zai-org/GLM-OCR": 1703.1839930295744 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1484.6431803967298, + 1566.05053161491 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1305.2714498590876, + 1397.4896859437192 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1369.5180977374869, + 1445.5974360312252 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1617.0339979434013, + 1707.5491049812822 + ], + "baidu/Qianfan-OCR": [ + 1481.8480815582093, + 1561.1289731181053 + ], + "baidu/Unlimited-OCR": [ + 1465.1360770501678, + 1549.5479594107903 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1356.048697348841, + 1437.714996891418 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1290.4970271272873, + 1382.329184324439 + ], + "lightonai/LightOnOCR-2-1B": [ + 1652.7807957863388, + 1736.7759283810406 + ], + "numind/NuExtract3": [ + 1599.1847982011748, + 1691.8448945663529 + ], + "rednote-hilab/dots.mocr": [ + 1669.055943876028, + 1752.3012113252753 + ], + "tesseract-5": [ + 952.8531822385969, + 1105.4219792013923 + ], + "zai-org/GLM-OCR": [ + 1661.4601460744732, + 1748.9324377705852 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 12.618358402447711, + "PaddlePaddle/PP-OCRv6_medium": 7.052943660520214, + "PaddlePaddle/PaddleOCR-VL-1.6": -4.90954482921893, + "allenai/olmOCR-2-7B-1025-FP8": -13.229378150266257, + "baidu/Qianfan-OCR": -2.37609024914741, + "baidu/Unlimited-OCR": 7.349649334586047, + "deepseek-ai/DeepSeek-OCR": -11.687979950233057, + "deepseek-ai/DeepSeek-OCR-2": 1.783556742018618, + "lightonai/LightOnOCR-2-1B": -6.057652437795468, + "numind/NuExtract3": -5.664297448352045, + "rednote-hilab/dots.mocr": 8.825025737991382, + "tesseract-5": -0.9549048300739287, + "zai-org/GLM-OCR": 7.250314017523351 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9487179487179485, + "kind": "mixed-random", + "max_absolute_elo_delta": 13.229378150266257, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 7.052943660520214, + "name": "mixed-random_2000_seed_42", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.989010989010989, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1524.9667899861865, + "PaddlePaddle/PP-OCRv6_medium": 1356.7313764522748, + "PaddlePaddle/PaddleOCR-VL-1.6": 1410.8610201716253, + "allenai/olmOCR-2-7B-1025-FP8": 1651.435550829535, + "baidu/Qianfan-OCR": 1505.847117583739, + "baidu/Unlimited-OCR": 1503.9440897789925, + "deepseek-ai/DeepSeek-OCR": 1394.9493673281327, + "deepseek-ai/DeepSeek-OCR-2": 1337.6340915497055, + "lightonai/LightOnOCR-2-1B": 1699.1831145055246, + "numind/NuExtract3": 1657.6540603406665, + "rednote-hilab/dots.mocr": 1697.725218402506, + "tesseract-5": 1048.2455348571812, + "zai-org/GLM-OCR": 1710.8226682139307 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1484.58101686882, + 1568.236251653002 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1312.852809810413, + 1396.623165112017 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1369.4581422051726, + 1451.1192412706891 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1609.2111670644863, + 1693.3406074761172 + ], + "baidu/Qianfan-OCR": [ + 1466.7233762947005, + 1548.3655087059144 + ], + "baidu/Unlimited-OCR": [ + 1463.9545321410023, + 1541.7179873756372 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1351.6660846122993, + 1437.6950839393319 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1291.3080029918697, + 1386.431868491243 + ], + "lightonai/LightOnOCR-2-1B": [ + 1659.464101375074, + 1742.5949902534016 + ], + "numind/NuExtract3": [ + 1615.5659444444086, + 1702.3240419609579 + ], + "rednote-hilab/dots.mocr": [ + 1654.0679922793377, + 1743.9714924637265 + ], + "tesseract-5": [ + 968.7186318018415, + 1105.5573283397157 + ], + "zai-org/GLM-OCR": [ + 1668.320053478764, + 1757.0266893247438 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 13.112038974490588, + "PaddlePaddle/PP-OCRv6_medium": 10.374011402988572, + "PaddlePaddle/PaddleOCR-VL-1.6": -3.414560489270343, + "allenai/olmOCR-2-7B-1025-FP8": -22.317428443707513, + "baidu/Qianfan-OCR": -18.158931594414526, + "baidu/Unlimited-OCR": 5.900646287842392, + "deepseek-ai/DeepSeek-OCR": -13.87987777980652, + "deepseek-ai/DeepSeek-OCR-2": 0.7686948731086432, + "lightonai/LightOnOCR-2-1B": 1.5798568220029665, + "numind/NuExtract3": 7.900418911954148, + "rednote-hilab/dots.mocr": -2.366081898480843, + "tesseract-5": 5.612223731413451, + "zai-org/GLM-OCR": 14.888989201879667 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8717948717948716, + "kind": "mixed-random", + "max_absolute_elo_delta": 22.317428443707513, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 7.900418911954148, + "name": "mixed-random_2000_seed_43", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.967032967032967, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1520.8638550728301, + "PaddlePaddle/PP-OCRv6_medium": 1343.2962443607219, + "PaddlePaddle/PaddleOCR-VL-1.6": 1414.367979019664, + "allenai/olmOCR-2-7B-1025-FP8": 1661.1707832520488, + "baidu/Qianfan-OCR": 1538.8514797580187, + "baidu/Unlimited-OCR": 1489.0996074523973, + "deepseek-ai/DeepSeek-OCR": 1376.5021139574092, + "deepseek-ai/DeepSeek-OCR-2": 1351.4810722098198, + "lightonai/LightOnOCR-2-1B": 1696.8848585455028, + "numind/NuExtract3": 1630.5684174846483, + "rednote-hilab/dots.mocr": 1715.0711387453173, + "tesseract-5": 1064.9139021340566, + "zai-org/GLM-OCR": 1696.9285480075655 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1475.5856503346379, + 1565.3056364853994 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1295.3241038822075, + 1384.8050389649948 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1372.6036617366249, + 1453.6446103366673 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1615.6524125856915, + 1707.1328941224044 + ], + "baidu/Qianfan-OCR": [ + 1500.0408051498903, + 1580.0646434908265 + ], + "baidu/Unlimited-OCR": [ + 1448.8323185665822, + 1526.5774010480927 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1331.7701498944716, + 1418.2583660486944 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1304.2176337010092, + 1396.8578657554622 + ], + "lightonai/LightOnOCR-2-1B": [ + 1656.6464086871865, + 1741.260631943609 + ], + "numind/NuExtract3": [ + 1589.425416843075, + 1678.1564867094228 + ], + "rednote-hilab/dots.mocr": [ + 1674.569978268076, + 1758.4478404657457 + ], + "tesseract-5": [ + 980.5698429017463, + 1123.3178935676267 + ], + "zai-org/GLM-OCR": [ + 1656.7876426117132, + 1745.7735529779718 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 9.009104061134167, + "PaddlePaddle/PP-OCRv6_medium": -3.061120688564415, + "PaddlePaddle/PaddleOCR-VL-1.6": 0.09239835876837788, + "allenai/olmOCR-2-7B-1025-FP8": -12.582196021193795, + "baidu/Qianfan-OCR": 14.845430579865251, + "baidu/Unlimited-OCR": -8.943836038752806, + "deepseek-ai/DeepSeek-OCR": -32.32713115053002, + "deepseek-ai/DeepSeek-OCR-2": 14.615675533222884, + "lightonai/LightOnOCR-2-1B": -0.7183991380188672, + "numind/NuExtract3": -19.18522394406409, + "rednote-hilab/dots.mocr": 14.979838444330426, + "tesseract-5": 22.280591008288866, + "zai-org/GLM-OCR": 0.99486899551448 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9487179487179485, + "kind": "mixed-random", + "max_absolute_elo_delta": 32.32713115053002, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR", + "median_absolute_elo_delta": 12.582196021193795, + "name": "mixed-random_2000_seed_44", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.989010989010989, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1512.1977173593375, + "PaddlePaddle/PP-OCRv6_medium": 1353.0733820781556, + "PaddlePaddle/PaddleOCR-VL-1.6": 1424.2063057096652, + "allenai/olmOCR-2-7B-1025-FP8": 1677.4324114490018, + "baidu/Qianfan-OCR": 1536.9673544915072, + "baidu/Unlimited-OCR": 1495.5756835649372, + "deepseek-ai/DeepSeek-OCR": 1390.678698123616, + "deepseek-ai/DeepSeek-OCR-2": 1320.2351432098117, + "lightonai/LightOnOCR-2-1B": 1680.234374186804, + "numind/NuExtract3": 1638.962260570605, + "rednote-hilab/dots.mocr": 1705.4371282992859, + "tesseract-5": 1061.8852527674437, + "zai-org/GLM-OCR": 1703.1142881898288 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1469.9085193476883, + 1554.038025855893 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1308.936053724141, + 1394.799586840276 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1382.4878790848027, + 1462.582864513801 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1633.846528469744, + 1721.8422717792382 + ], + "baidu/Qianfan-OCR": [ + 1497.8701502609244, + 1578.6012994450916 + ], + "baidu/Unlimited-OCR": [ + 1455.0579527510604, + 1534.8018274178976 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1345.594886431943, + 1428.805233695043 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1268.0026305082524, + 1365.8947082543182 + ], + "lightonai/LightOnOCR-2-1B": [ + 1642.061796984405, + 1726.0407199121069 + ], + "numind/NuExtract3": [ + 1597.7171919427267, + 1683.2997693228428 + ], + "rednote-hilab/dots.mocr": [ + 1662.825228661662, + 1750.272535647071 + ], + "tesseract-5": [ + 981.7823626848402, + 1122.3934827105309 + ], + "zai-org/GLM-OCR": [ + 1664.5582731619652, + 1751.7506175433668 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 0.3429663476415499, + "PaddlePaddle/PP-OCRv6_medium": 6.7160170288693735, + "PaddlePaddle/PaddleOCR-VL-1.6": 9.930725048769546, + "allenai/olmOCR-2-7B-1025-FP8": 3.6794321757593025, + "baidu/Qianfan-OCR": 12.961305313353705, + "baidu/Unlimited-OCR": -2.4677599262129206, + "deepseek-ai/DeepSeek-OCR": -18.150546984323228, + "deepseek-ai/DeepSeek-OCR-2": -16.630253466785234, + "lightonai/LightOnOCR-2-1B": -17.36888349671767, + "numind/NuExtract3": -10.791380858107459, + "rednote-hilab/dots.mocr": 5.345827998299001, + "tesseract-5": 19.251941641676012, + "zai-org/GLM-OCR": 7.1806091777777965 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.9743589743589742, + "kind": "mixed-random", + "max_absolute_elo_delta": 19.251941641676012, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 9.930725048769546, + "name": "mixed-random_2000_seed_45", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9945054945054945, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 1665, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1533.8119622807478, + "PaddlePaddle/PP-OCRv6_medium": 1345.7031598372882, + "PaddlePaddle/PaddleOCR-VL-1.6": 1409.4419781885404, + "allenai/olmOCR-2-7B-1025-FP8": 1650.760752569424, + "baidu/Qianfan-OCR": 1514.6923036476044, + "baidu/Unlimited-OCR": 1519.4640866019377, + "deepseek-ai/DeepSeek-OCR": 1394.5645842349468, + "deepseek-ai/DeepSeek-OCR-2": 1354.3306676775535, + "lightonai/LightOnOCR-2-1B": 1680.2666749146024, + "numind/NuExtract3": 1640.1872347568808, + "rednote-hilab/dots.mocr": 1694.7231872871903, + "tesseract-5": 1067.7968394963352, + "zai-org/GLM-OCR": 1694.256568506948 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1491.9687979530636, + 1575.5016140373652 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1297.7068794502168, + 1389.8858322647707 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1370.1402218192918, + 1446.9416954646042 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1606.6382973762575, + 1695.078052810251 + ], + "baidu/Qianfan-OCR": [ + 1474.281920126229, + 1552.7179756261191 + ], + "baidu/Unlimited-OCR": [ + 1484.493481831039, + 1559.7636218262035 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1349.7613298590338, + 1436.3483288125153 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1308.4971509808863, + 1399.9337514013191 + ], + "lightonai/LightOnOCR-2-1B": [ + 1640.1519037051426, + 1727.2833418638625 + ], + "numind/NuExtract3": [ + 1596.8696122089577, + 1685.364577128291 + ], + "rednote-hilab/dots.mocr": [ + 1656.1415438309468, + 1741.5898336516393 + ], + "tesseract-5": [ + 988.9832247563841, + 1127.0661717058106 + ], + "zai-org/GLM-OCR": [ + 1653.4441466244011, + 1739.5497166206278 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 21.957211269051868, + "PaddlePaddle/PP-OCRv6_medium": -0.6542052119980326, + "PaddlePaddle/PaddleOCR-VL-1.6": -4.833602472355324, + "allenai/olmOCR-2-7B-1025-FP8": -22.99222670381846, + "baidu/Qianfan-OCR": -9.313745530549113, + "baidu/Unlimited-OCR": 21.420643110787523, + "deepseek-ai/DeepSeek-OCR": -14.264660872992408, + "deepseek-ai/DeepSeek-OCR-2": 17.46527100095659, + "lightonai/LightOnOCR-2-1B": -17.336582768919243, + "numind/NuExtract3": -9.5664066718316, + "rednote-hilab/dots.mocr": -5.368113013796574, + "tesseract-5": 25.16352837056752, + "zai-org/GLM-OCR": -1.6771105051029735 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 26, + "median_direct_comparisons": 26.0, + "min_direct_comparisons": 25, + "observed_pairs": 78, + "pair_coverage_pct": 100.0, + "possible_pairs": 78 + }, + "kendall_tau": 0.8974358974358972, + "kind": "mixed-random", + "max_absolute_elo_delta": 25.16352837056752, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 14.264660872992408, + "name": "mixed-random_2000_seed_46", + "percentage_saved": 45.429740791268756, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9725274725274725, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null } ] } diff --git a/experiments/adaptive-stopping/results.json b/experiments/adaptive-stopping/results.json index 9b0c514..5a00bcb 100644 --- a/experiments/adaptive-stopping/results.json +++ b/experiments/adaptive-stopping/results.json @@ -50,8 +50,182 @@ "sentinel_comparisons_excluded": false, "source_repo": "davanstrien/ocr-bench-britannica-results", "source_revision": "48a0f42de26009892d2784a3a97d6d61525f4040", + "static_design": { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "budgets": [ + 700, + 1200, + 2000 + ], + "mixed_random": "first five-sample balanced batch, then seeded outcome-independent pair-balanced exploration", + "pair_balanced": "fixed equal per-pair quotas across all stored samples" + }, "stored_comparisons_at_revision": 4293 }, + "static_design_aggregates": [ + { + "budget": 700, + "gate_passes": 0, + "kendall_tau_max": 0.8901098901098902, + "kendall_tau_median": 0.8681318681318682, + "kendall_tau_min": 0.8681318681318682, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 146.68479511326132, + "max_absolute_elo_delta_worst": 191.33905916128492, + "median_absolute_elo_delta_median": 48.24390901086883, + "minimum_pair_evidence": 7, + "percentage_saved": 83.69438621010949, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9736263736263736, + "spearman_rho_median": 0.9604395604395607, + "spearman_rho_min": 0.9516483516483516, + "top3_membership_matches": 5, + "top3_order_matches": 1 + }, + { + "budget": 1200, + "gate_passes": 0, + "kendall_tau_max": 0.9560439560439562, + "kendall_tau_median": 0.9340659340659341, + "kendall_tau_min": 0.9340659340659341, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 79.82020330682758, + "max_absolute_elo_delta_worst": 122.84802081339058, + "median_absolute_elo_delta_median": 28.682233615777477, + "minimum_pair_evidence": 13, + "percentage_saved": 72.04751921733053, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9912087912087912, + "spearman_rho_median": 0.9868131868131869, + "spearman_rho_min": 0.9824175824175824, + "top3_membership_matches": 5, + "top3_order_matches": 1 + }, + { + "budget": 2000, + "gate_passes": 0, + "kendall_tau_max": 0.9780219780219781, + "kendall_tau_median": 0.9560439560439562, + "kendall_tau_min": 0.9340659340659341, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 46.27017754458029, + "max_absolute_elo_delta_worst": 61.374681275296325, + "median_absolute_elo_delta_median": 13.683594358663413, + "minimum_pair_evidence": 21, + "percentage_saved": 53.41253202888423, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9956043956043955, + "spearman_rho_median": 0.9912087912087912, + "spearman_rho_min": 0.9824175824175824, + "top3_membership_matches": 5, + "top3_order_matches": 2 + }, + { + "budget": 700, + "gate_passes": 0, + "kendall_tau_max": 0.9780219780219781, + "kendall_tau_median": 0.8681318681318682, + "kendall_tau_min": 0.7582417582417583, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 75.17162456695951, + "max_absolute_elo_delta_worst": 116.38438099675818, + "median_absolute_elo_delta_median": 34.88736020560384, + "minimum_pair_evidence": 7, + "percentage_saved": 83.69438621010949, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9956043956043955, + "spearman_rho_median": 0.956043956043956, + "spearman_rho_min": 0.9252747252747253, + "top3_membership_matches": 1, + "top3_order_matches": 1 + }, + { + "budget": 1200, + "gate_passes": 0, + "kendall_tau_max": 0.9560439560439562, + "kendall_tau_median": 0.8901098901098902, + "kendall_tau_min": 0.7802197802197802, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 49.12059412915846, + "max_absolute_elo_delta_worst": 88.99502230491316, + "median_absolute_elo_delta_median": 18.849640840560028, + "minimum_pair_evidence": 13, + "percentage_saved": 72.04751921733053, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9912087912087912, + "spearman_rho_median": 0.9692307692307693, + "spearman_rho_min": 0.9164835164835166, + "top3_membership_matches": 2, + "top3_order_matches": 0 + }, + { + "budget": 2000, + "gate_passes": 0, + "kendall_tau_max": 0.9560439560439562, + "kendall_tau_median": 0.8681318681318682, + "kendall_tau_min": 0.8461538461538461, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 37.70822520261572, + "max_absolute_elo_delta_worst": 61.219755276211686, + "median_absolute_elo_delta_median": 17.57609905517836, + "minimum_pair_evidence": 21, + "percentage_saved": 53.41253202888423, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9868131868131869, + "spearman_rho_median": 0.9516483516483516, + "spearman_rho_min": 0.9472527472527472, + "top3_membership_matches": 1, + "top3_order_matches": 1 + } + ], "strategies": [ { "allocation_decision_statuses": { @@ -59,6 +233,7 @@ "resolved": 4, "unresolved": 9 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 4293, "comparisons_saved": 0, @@ -222,6 +397,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 1.0, + "static_budget": null, "stopping_reason": "reference_full_stored_board", "stopping_round": 10, "top3": [ @@ -240,6 +416,7 @@ "resolved": 4, "unresolved": 9 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 4293, "comparisons_saved": 0, @@ -764,6 +941,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 1.0, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -782,6 +960,7 @@ "resolved": 1, "unresolved": 12 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 870, "comparisons_saved": 3423, @@ -1306,6 +1485,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9824175824175824, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -1324,6 +1504,7 @@ "resolved": 1, "unresolved": 6 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 736, "comparisons_saved": 3557, @@ -1885,6 +2066,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9780219780219781, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -1903,6 +2085,7 @@ "resolved": 1, "unresolved": 3 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 629, "comparisons_saved": 3664, @@ -2482,6 +2665,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 2.0, "spearman_rho": 0.9516483516483516, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -2500,6 +2684,7 @@ "resolved": 1, "unresolved": 8 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 735, "comparisons_saved": 3558, @@ -3049,6 +3234,7 @@ "size_tie_min_samples": 5, "size_tie_ratio": 3.0, "spearman_rho": 0.9120879120879121, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -3067,6 +3253,7 @@ "resolved": 1, "unresolved": 7 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 786, "comparisons_saved": 3507, @@ -3622,6 +3809,7 @@ "size_tie_min_samples": 15, "size_tie_ratio": 3.0, "spearman_rho": 0.9824175824175824, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -3640,6 +3828,7 @@ "resolved": 1, "unresolved": 9 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 810, "comparisons_saved": 3483, @@ -4183,6 +4372,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 5.0, "spearman_rho": 0.9824175824175824, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -4201,6 +4391,7 @@ "resolved": 2, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 1153, "comparisons_saved": 3140, @@ -4706,6 +4897,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9692307692307693, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -4724,6 +4916,7 @@ "resolved": 1, "unresolved": 12 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": null, "comparisons_consumed": 1561, "comparisons_saved": 2732, @@ -5210,6 +5403,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9824175824175824, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -5228,6 +5422,7 @@ "resolved": 2, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": 3, "comparisons_consumed": 2023, "comparisons_saved": 2270, @@ -5752,6 +5947,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": null, "spearman_rho": 0.9912087912087912, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -5770,6 +5966,7 @@ "resolved": 2, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": 3, "comparisons_consumed": 1946, "comparisons_saved": 2347, @@ -6306,6 +6503,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9868131868131869, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -6324,6 +6522,7 @@ "resolved": 2, "unresolved": 11 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": 3, "comparisons_consumed": 2313, "comparisons_saved": 1980, @@ -6841,6 +7040,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9912087912087912, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -6859,6 +7059,7 @@ "resolved": 2, "unresolved": 6 }, + "allocation_seed": 42, "balanced_every_n_post_warmup_batches": 3, "comparisons_consumed": 1820, "comparisons_saved": 2473, @@ -7395,6 +7596,7 @@ "size_tie_min_samples": 10, "size_tie_ratio": 3.0, "spearman_rho": 0.9868131868131869, + "static_budget": null, "stopping_reason": "sample_batches_exhausted", "stopping_round": 10, "top3": [ @@ -7406,6 +7608,5496 @@ "top3_membership_overlap": 3, "top3_order_matches": false, "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1608.3446805050585, + "PaddlePaddle/PP-OCRv6_medium": 1332.8846477743145, + "PaddlePaddle/PaddleOCR-VL-1.6": 1438.3575163571581, + "allenai/olmOCR-2-7B-1025-FP8": 1730.214953273804, + "baidu/Qianfan-OCR": 1581.4061124614927, + "baidu/Unlimited-OCR": 1564.0356246826134, + "deepseek-ai/DeepSeek-OCR": 1418.1917897223525, + "deepseek-ai/DeepSeek-OCR-2": 1385.1289084322368, + "lightonai/LightOnOCR-2-1B": 1778.2195280154026, + "numind/NuExtract3": 1737.608679123864, + "rednote-hilab/dots.mocr": 1739.579581522016, + "rednote-hilab/dots.ocr": 845.042415658463, + "tesseract-5": 1155.173699195922, + "zai-org/GLM-OCR": 1685.8118632753017 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1534.9150590489728, + 1807.4608279544132 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1241.523927862069, + 1514.2109426637105 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1355.6552540805403, + 1606.107738135891 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1660.0122772237298, + 1935.393010564465 + ], + "baidu/Qianfan-OCR": [ + 1508.1833545914426, + 1788.0163580347207 + ], + "baidu/Unlimited-OCR": [ + 1496.295663931427, + 1751.617832417686 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1328.7785050030668, + 1594.6804146772197 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1298.1061235564296, + 1578.5003012086534 + ], + "lightonai/LightOnOCR-2-1B": [ + 1700.6202380060438, + 1976.8982943133926 + ], + "numind/NuExtract3": [ + 1666.4688869072734, + 1937.2442018262807 + ], + "rednote-hilab/dots.mocr": [ + 1670.433432107287, + 1946.4184365287997 + ], + "rednote-hilab/dots.ocr": [ + -1636.4043842554938, + 991.6025480941214 + ], + "tesseract-5": [ + 1014.7259000654316, + 1343.9878886013662 + ], + "zai-org/GLM-OCR": [ + 1616.2856183746385, + 1895.3771247016134 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 50.89529382028945, + "PaddlePaddle/PP-OCRv6_medium": -64.21191215428712, + "PaddlePaddle/PaddleOCR-VL-1.6": -24.61703614023122, + "allenai/olmOCR-2-7B-1025-FP8": 10.983815244736888, + "baidu/Qianfan-OCR": 13.311662159358548, + "baidu/Unlimited-OCR": 18.829622044060216, + "deepseek-ai/DeepSeek-OCR": -33.80523386431628, + "deepseek-ai/DeepSeek-OCR-2": 4.916360445395412, + "lightonai/LightOnOCR-2-1B": 36.967017230648025, + "numind/NuExtract3": 47.98964333390859, + "rednote-hilab/dots.mocr": -5.845677864614572, + "rednote-hilab/dots.ocr": -45.6331328993366, + "tesseract-5": 42.58719083123242, + "zai-org/GLM-OCR": -52.367612186843644 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "pair-balanced", + "max_absolute_elo_delta": 64.21191215428712, + "max_absolute_elo_delta_model": "PaddlePaddle/PP-OCRv6_medium", + "median_absolute_elo_delta": 35.38612554748215, + "name": "pair-balanced_700_seed_42", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9692307692307693, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1657.1329483356512, + "PaddlePaddle/PP-OCRv6_medium": 1378.9707197797002, + "PaddlePaddle/PaddleOCR-VL-1.6": 1346.5901715006312, + "allenai/olmOCR-2-7B-1025-FP8": 1727.613642061453, + "baidu/Qianfan-OCR": 1534.602018112062, + "baidu/Unlimited-OCR": 1585.4166207996852, + "deepseek-ai/DeepSeek-OCR": 1380.324223342501, + "deepseek-ai/DeepSeek-OCR-2": 1388.941278698522, + "lightonai/LightOnOCR-2-1B": 1783.4484061534754, + "numind/NuExtract3": 1742.152908755929, + "rednote-hilab/dots.mocr": 1762.0509140256804, + "rednote-hilab/dots.ocr": 844.0078925036925, + "tesseract-5": 1143.1918660877843, + "zai-org/GLM-OCR": 1725.5563898432322 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1576.578705042873, + 1869.8414374506115 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1297.837087734854, + 1575.631757851565 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1268.7611841047797, + 1533.0100514293295 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1652.813809714065, + 1930.6562361769597 + ], + "baidu/Qianfan-OCR": [ + 1456.4027480274842, + 1752.8653997741242 + ], + "baidu/Unlimited-OCR": [ + 1509.704477504483, + 1791.6809907283532 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1288.9748518021624, + 1589.072652521325 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1287.6211235228243, + 1595.8363433858137 + ], + "lightonai/LightOnOCR-2-1B": [ + 1706.7524155114177, + 1990.403715590343 + ], + "numind/NuExtract3": [ + 1667.3867014730786, + 1948.815013801288 + ], + "rednote-hilab/dots.mocr": [ + 1696.3251776779098, + 1986.9696860441895 + ], + "rednote-hilab/dots.ocr": [ + -1690.9527120693788, + 995.216649668997 + ], + "tesseract-5": [ + 1002.7155896651695, + 1308.9778963944125 + ], + "zai-org/GLM-OCR": [ + 1649.557837230462, + 1935.05230350082 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 99.68356165088221, + "PaddlePaddle/PP-OCRv6_medium": -18.12584014890149, + "PaddlePaddle/PaddleOCR-VL-1.6": -116.38438099675818, + "allenai/olmOCR-2-7B-1025-FP8": 8.382504032385896, + "baidu/Qianfan-OCR": -33.49243219007212, + "baidu/Unlimited-OCR": 40.21061816113206, + "deepseek-ai/DeepSeek-OCR": -71.67280024416777, + "deepseek-ai/DeepSeek-OCR-2": 8.728730711680555, + "lightonai/LightOnOCR-2-1B": 42.19589536872081, + "numind/NuExtract3": 52.533872965973615, + "rednote-hilab/dots.mocr": 16.6256546390498, + "rednote-hilab/dots.ocr": -46.66765605410717, + "tesseract-5": 30.605357723094812, + "zai-org/GLM-OCR": -12.623085618913137 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7582417582417583, + "kind": "pair-balanced", + "max_absolute_elo_delta": 116.38438099675818, + "max_absolute_elo_delta_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "median_absolute_elo_delta": 36.85152517560209, + "name": "pair-balanced_700_seed_43", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9252747252747253, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1549.1031763628914, + "PaddlePaddle/PP-OCRv6_medium": 1387.1285844098475, + "PaddlePaddle/PaddleOCR-VL-1.6": 1466.2594932059958, + "allenai/olmOCR-2-7B-1025-FP8": 1691.5775500285006, + "baidu/Qianfan-OCR": 1563.2843283861919, + "baidu/Unlimited-OCR": 1557.0491256404791, + "deepseek-ai/DeepSeek-OCR": 1406.808327693129, + "deepseek-ai/DeepSeek-OCR-2": 1364.972585224888, + "lightonai/LightOnOCR-2-1B": 1732.6001138796714, + "numind/NuExtract3": 1672.0071518119494, + "rednote-hilab/dots.mocr": 1787.4122936553579, + "rednote-hilab/dots.ocr": 914.7807431316438, + "tesseract-5": 1185.162626437656, + "zai-org/GLM-OCR": 1721.8539001317981 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1463.5726643396492, + 1648.0210138782195 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1306.303765490901, + 1467.7125530647845 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1396.0614835214349, + 1552.0484006506624 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1610.1678800628251, + 1805.5812885181517 + ], + "baidu/Qianfan-OCR": [ + 1491.5547562867255, + 1648.0042460066693 + ], + "baidu/Unlimited-OCR": [ + 1486.99748083875, + 1645.14323761808 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1324.2136492448424, + 1502.4069770650856 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1272.6773510553207, + 1463.5232242701188 + ], + "lightonai/LightOnOCR-2-1B": [ + 1656.1746274244183, + 1840.4726863716453 + ], + "numind/NuExtract3": [ + 1598.6685081643914, + 1778.5507074037962 + ], + "rednote-hilab/dots.mocr": [ + 1720.4087066625605, + 1894.00649225669 + ], + "rednote-hilab/dots.ocr": [ + 638.590170667781, + 1054.7027117595617 + ], + "tesseract-5": [ + 1076.2674147073892, + 1279.3651442188905 + ], + "zai-org/GLM-OCR": [ + 1647.8135130132978, + 1826.1412315083212 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -8.346210321877606, + "PaddlePaddle/PP-OCRv6_medium": -9.967975518754201, + "PaddlePaddle/PaddleOCR-VL-1.6": 3.2849407086064275, + "allenai/olmOCR-2-7B-1025-FP8": -27.653588000566515, + "baidu/Qianfan-OCR": -4.810121915942318, + "baidu/Unlimited-OCR": 11.84312300192596, + "deepseek-ai/DeepSeek-OCR": -45.188695893539716, + "deepseek-ai/DeepSeek-OCR-2": -15.239962761953393, + "lightonai/LightOnOCR-2-1B": -8.652396905083151, + "numind/NuExtract3": -17.611883978006063, + "rednote-hilab/dots.mocr": 41.98703426872726, + "rednote-hilab/dots.ocr": 24.105194573844187, + "tesseract-5": 72.57611807296644, + "zai-org/GLM-OCR": -16.325575330347192 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9780219780219781, + "kind": "pair-balanced", + "max_absolute_elo_delta": 72.57611807296644, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 15.782769046150293, + "name": "pair-balanced_700_seed_44", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9956043956043955, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1494.4432451728896, + "PaddlePaddle/PP-OCRv6_medium": 1400.5386642660735, + "PaddlePaddle/PaddleOCR-VL-1.6": 1504.3086114691523, + "allenai/olmOCR-2-7B-1025-FP8": 1832.0016417430204, + "baidu/Qianfan-OCR": 1587.7651831084647, + "baidu/Unlimited-OCR": 1510.9300755021095, + "deepseek-ai/DeepSeek-OCR": 1440.0119728594623, + "deepseek-ai/DeepSeek-OCR-2": 1397.4088449651597, + "lightonai/LightOnOCR-2-1B": 1772.1662715946009, + "numind/NuExtract3": 1654.1202425151914, + "rednote-hilab/dots.mocr": 1758.8556429651162, + "rednote-hilab/dots.ocr": 789.246045194604, + "tesseract-5": 1168.1923471362472, + "zai-org/GLM-OCR": 1690.011211507908 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1410.9487984360248, + 1709.6212652013835 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1312.2326177255811, + 1619.9571284171848 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1428.6799596541516, + 1726.1066136058066 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1749.2304750311073, + 2064.264552380834 + ], + "baidu/Qianfan-OCR": [ + 1520.4949747071532, + 1810.4290842139603 + ], + "baidu/Unlimited-OCR": [ + 1444.5429500880327, + 1730.1235372876367 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1356.7513578905055, + 1652.4850493469253 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1317.5618861878152, + 1613.2636887728338 + ], + "lightonai/LightOnOCR-2-1B": [ + 1697.7130417736557, + 1985.490662678676 + ], + "numind/NuExtract3": [ + 1575.9034214504081, + 1878.0570616573425 + ], + "rednote-hilab/dots.mocr": [ + 1691.7260458133323, + 1977.9992769357461 + ], + "rednote-hilab/dots.ocr": [ + -1704.4523328257937, + 965.0845019419426 + ], + "tesseract-5": [ + 1055.6054338315946, + 1389.0494580368768 + ], + "zai-org/GLM-OCR": [ + 1611.1492951019125, + 1944.2828027603352 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -63.00614151187938, + "PaddlePaddle/PP-OCRv6_medium": 3.4421043374718465, + "PaddlePaddle/PaddleOCR-VL-1.6": 41.33405897176294, + "allenai/olmOCR-2-7B-1025-FP8": 112.7705037139533, + "baidu/Qianfan-OCR": 19.670732806330534, + "baidu/Unlimited-OCR": -34.27592713644367, + "deepseek-ai/DeepSeek-OCR": -11.985050727206499, + "deepseek-ai/DeepSeek-OCR-2": 17.196296978318287, + "lightonai/LightOnOCR-2-1B": 30.91376080984628, + "numind/NuExtract3": -35.49879327476401, + "rednote-hilab/dots.mocr": 13.43038357848559, + "rednote-hilab/dots.ocr": -101.42950336319564, + "tesseract-5": 55.60583877155773, + "zai-org/GLM-OCR": -48.16826395423732 + }, + "final_rank_order": [ + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "pair-balanced", + "max_absolute_elo_delta": 112.7705037139533, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 34.88736020560384, + "name": "pair-balanced_700_seed_45", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.956043956043956, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1532.9965763416174, + "PaddlePaddle/PP-OCRv6_medium": 1350.6813473586062, + "PaddlePaddle/PaddleOCR-VL-1.6": 1483.4461686224597, + "allenai/olmOCR-2-7B-1025-FP8": 1759.2535591141034, + "baidu/Qianfan-OCR": 1522.273532617472, + "baidu/Unlimited-OCR": 1570.4668927353853, + "deepseek-ai/DeepSeek-OCR": 1447.614877087872, + "deepseek-ai/DeepSeek-OCR-2": 1427.3906577646069, + "lightonai/LightOnOCR-2-1B": 1789.5080562081266, + "numind/NuExtract3": 1732.0687549281085, + "rednote-hilab/dots.mocr": 1752.0195012182132, + "rednote-hilab/dots.ocr": 875.1068604120732, + "tesseract-5": 1037.41488379773, + "zai-org/GLM-OCR": 1719.7583317936255 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1456.7765644732815, + 1644.4678879413632 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1266.6657706830665, + 1446.3443843791322 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1408.9312722960826, + 1581.6089196604564 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1682.7709022091701, + 1881.4519232057862 + ], + "baidu/Qianfan-OCR": [ + 1440.2445051048235, + 1635.167902089953 + ], + "baidu/Unlimited-OCR": [ + 1499.8107903765117, + 1667.2455484417912 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1365.8325001281632, + 1549.4354397662742 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1338.226909198498, + 1548.5461440007223 + ], + "lightonai/LightOnOCR-2-1B": [ + 1711.2186238775178, + 1916.9815816263458 + ], + "numind/NuExtract3": [ + 1650.8560944492217, + 1867.506870753834 + ], + "rednote-hilab/dots.mocr": [ + 1685.3943296493055, + 1859.4257515723746 + ], + "rednote-hilab/dots.ocr": [ + 536.543274893219, + 1036.549623843551 + ], + "tesseract-5": [ + 856.1009759202418, + 1153.667110681044 + ], + "zai-org/GLM-OCR": [ + 1647.6032674317312, + 1849.5597524165867 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -24.45281034315167, + "PaddlePaddle/PP-OCRv6_medium": -46.41521256999545, + "PaddlePaddle/PaddleOCR-VL-1.6": 20.47161612507034, + "allenai/olmOCR-2-7B-1025-FP8": 40.022421085036285, + "baidu/Qianfan-OCR": -45.82091768466216, + "baidu/Unlimited-OCR": 25.2608900968321, + "deepseek-ai/DeepSeek-OCR": -4.382146498796828, + "deepseek-ai/DeepSeek-OCR-2": 47.178109777765485, + "lightonai/LightOnOCR-2-1B": 48.25554542337204, + "numind/NuExtract3": 42.449719138153114, + "rednote-hilab/dots.mocr": 6.594241831582622, + "rednote-hilab/dots.ocr": -15.568688145726469, + "tesseract-5": -75.17162456695951, + "zai-org/GLM-OCR": -18.42114366851979 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "zai-org/GLM-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "pair-balanced", + "max_absolute_elo_delta": 75.17162456695951, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 32.64165559093419, + "name": "pair-balanced_700_seed_46", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9472527472527472, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1600.4726717794176, + "PaddlePaddle/PP-OCRv6_medium": 1378.0985581446023, + "PaddlePaddle/PaddleOCR-VL-1.6": 1437.665883952499, + "allenai/olmOCR-2-7B-1025-FP8": 1711.533561624297, + "baidu/Qianfan-OCR": 1558.7249211818164, + "baidu/Unlimited-OCR": 1554.7451470411702, + "deepseek-ai/DeepSeek-OCR": 1429.9817171297507, + "deepseek-ai/DeepSeek-OCR-2": 1396.4126892988545, + "lightonai/LightOnOCR-2-1B": 1784.2069345273126, + "numind/NuExtract3": 1733.310681888583, + "rednote-hilab/dots.mocr": 1771.8173673153167, + "rednote-hilab/dots.ocr": 801.6805262528865, + "tesseract-5": 1131.1180934856711, + "zai-org/GLM-OCR": 1710.2312463778226 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1542.2203668030318, + 1699.7297144292208 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1313.927064354395, + 1469.107606292131 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1378.8661276362416, + 1536.582297913028 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1650.8009109727052, + 1823.7730029467127 + ], + "baidu/Qianfan-OCR": [ + 1505.7869556394312, + 1665.1304349079726 + ], + "baidu/Unlimited-OCR": [ + 1503.1566696175214, + 1645.9352172261697 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1366.252618093495, + 1533.7960802852178 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1329.8638755860975, + 1516.7959508499675 + ], + "lightonai/LightOnOCR-2-1B": [ + 1724.8133798546692, + 1912.5152809783895 + ], + "numind/NuExtract3": [ + 1674.103202426341, + 1841.4629933469619 + ], + "rednote-hilab/dots.mocr": [ + 1716.9910442116152, + 1880.6804555627198 + ], + "rednote-hilab/dots.ocr": [ + 480.1214246160183, + 927.612804625696 + ], + "tesseract-5": [ + 1026.8154391971943, + 1223.066616801513 + ], + "zai-org/GLM-OCR": [ + 1654.4894345029243, + 1831.4441241775794 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 43.0232850946486, + "PaddlePaddle/PP-OCRv6_medium": -18.99800178399937, + "PaddlePaddle/PaddleOCR-VL-1.6": -25.308668544890452, + "allenai/olmOCR-2-7B-1025-FP8": -7.697576404770189, + "baidu/Qianfan-OCR": -9.369529120317793, + "baidu/Unlimited-OCR": 9.539144402617012, + "deepseek-ai/DeepSeek-OCR": -22.015306456918097, + "deepseek-ai/DeepSeek-OCR-2": 16.200141312013102, + "lightonai/LightOnOCR-2-1B": 42.95442374255799, + "numind/NuExtract3": 43.69164609862764, + "rednote-hilab/dots.mocr": 26.392107928686073, + "rednote-hilab/dots.ocr": -88.99502230491316, + "tesseract-5": 18.531585120981617, + "zai-org/GLM-OCR": -27.948229084322747 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "pair-balanced", + "max_absolute_elo_delta": 88.99502230491316, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 23.661987500904274, + "name": "pair-balanced_1200_seed_42", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9692307692307693, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "numind/NuExtract3" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1606.5699808139275, + "PaddlePaddle/PP-OCRv6_medium": 1415.5873224553688, + "PaddlePaddle/PaddleOCR-VL-1.6": 1417.2122550103788, + "allenai/olmOCR-2-7B-1025-FP8": 1700.6957270024277, + "baidu/Qianfan-OCR": 1551.367946962283, + "baidu/Unlimited-OCR": 1558.7750426567673, + "deepseek-ai/DeepSeek-OCR": 1428.1867691315038, + "deepseek-ai/DeepSeek-OCR-2": 1408.5674165777405, + "lightonai/LightOnOCR-2-1B": 1714.1415689383318, + "numind/NuExtract3": 1677.1655861283741, + "rednote-hilab/dots.mocr": 1743.8318867174214, + "rednote-hilab/dots.ocr": 912.3362994163375, + "tesseract-5": 1144.663740819601, + "zai-org/GLM-OCR": 1720.898457369537 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1547.3315343460652, + 1668.3339427527296 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1350.6450576745751, + 1477.7100789466006 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1362.3370912347966, + 1471.5578715112592 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1642.000823281285, + 1768.2612277806284 + ], + "baidu/Qianfan-OCR": [ + 1497.0813921262065, + 1610.7595520148395 + ], + "baidu/Unlimited-OCR": [ + 1504.6429953865397, + 1616.5974910397965 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1368.1612493712614, + 1484.8655008255566 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1339.8756226829996, + 1474.778685266347 + ], + "lightonai/LightOnOCR-2-1B": [ + 1661.2000789705005, + 1781.5822901536844 + ], + "numind/NuExtract3": [ + 1622.1747558827528, + 1743.980652012283 + ], + "rednote-hilab/dots.mocr": [ + 1691.4793480224962, + 1812.7856832979235 + ], + "rednote-hilab/dots.ocr": [ + 709.9292909791816, + 1017.8429111219252 + ], + "tesseract-5": [ + 1047.1883574019791, + 1208.0580020400705 + ], + "zai-org/GLM-OCR": [ + 1664.8548729747213, + 1793.7931444392466 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 49.12059412915846, + "PaddlePaddle/PP-OCRv6_medium": 18.490762526767185, + "PaddlePaddle/PaddleOCR-VL-1.6": -45.76229748701053, + "allenai/olmOCR-2-7B-1025-FP8": -18.535411026639395, + "baidu/Qianfan-OCR": -16.726503339851206, + "baidu/Unlimited-OCR": 13.569040018214082, + "deepseek-ai/DeepSeek-OCR": -23.810254455165023, + "deepseek-ai/DeepSeek-OCR-2": 28.35486859089906, + "lightonai/LightOnOCR-2-1B": -27.110941846422747, + "numind/NuExtract3": -12.453449661581317, + "rednote-hilab/dots.mocr": -1.5933726692092023, + "rednote-hilab/dots.ocr": 21.66075085853788, + "tesseract-5": 32.07723245491138, + "zai-org/GLM-OCR": -17.281018092608292 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9120879120879122, + "kind": "pair-balanced", + "max_absolute_elo_delta": 49.12059412915846, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 20.098080942588638, + "name": "pair-balanced_1200_seed_43", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9780219780219781, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1573.5424064930826, + "PaddlePaddle/PP-OCRv6_medium": 1395.8997498636347, + "PaddlePaddle/PaddleOCR-VL-1.6": 1463.9790234713273, + "allenai/olmOCR-2-7B-1025-FP8": 1683.0223963923431, + "baidu/Qianfan-OCR": 1574.352219145655, + "baidu/Unlimited-OCR": 1529.156500455611, + "deepseek-ai/DeepSeek-OCR": 1418.4389754414194, + "deepseek-ai/DeepSeek-OCR-2": 1336.4144529728567, + "lightonai/LightOnOCR-2-1B": 1732.9551869653317, + "numind/NuExtract3": 1710.3223213798774, + "rednote-hilab/dots.mocr": 1788.4854627634636, + "rednote-hilab/dots.ocr": 903.9097693046194, + "tesseract-5": 1145.3288761166018, + "zai-org/GLM-OCR": 1744.1926592341763 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1513.992687523148, + 1635.2688432048178 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1333.9015487173867, + 1457.7467663797509 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1406.6405605442965, + 1521.0415202212932 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1625.0960274759325, + 1751.0942738145636 + ], + "baidu/Qianfan-OCR": [ + 1521.7004561396157, + 1638.2967710792816 + ], + "baidu/Unlimited-OCR": [ + 1473.268628512455, + 1588.6647669807858 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1348.6977520956593, + 1482.4752394709008 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1266.6188759272409, + 1403.8049032327538 + ], + "lightonai/LightOnOCR-2-1B": [ + 1665.4038944151555, + 1809.8237851204085 + ], + "numind/NuExtract3": [ + 1646.8892670164073, + 1784.026337611934 + ], + "rednote-hilab/dots.mocr": [ + 1732.1590779520593, + 1861.2939064539232 + ], + "rednote-hilab/dots.ocr": [ + 686.4775028747939, + 1017.2376855446747 + ], + "tesseract-5": [ + 1053.5644836676151, + 1210.8033174809675 + ], + "zai-org/GLM-OCR": [ + 1691.7664822406882, + 1818.321959329725 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 16.093019808313556, + "PaddlePaddle/PP-OCRv6_medium": -1.1968100649669395, + "PaddlePaddle/PaddleOCR-VL-1.6": 1.0044709739379414, + "allenai/olmOCR-2-7B-1025-FP8": -36.20874163672397, + "baidu/Qianfan-OCR": 6.257768843520807, + "baidu/Unlimited-OCR": -16.049502182942206, + "deepseek-ai/DeepSeek-OCR": -33.55804814524936, + "deepseek-ai/DeepSeek-OCR-2": -43.79809501398472, + "lightonai/LightOnOCR-2-1B": -8.2973238194229, + "numind/NuExtract3": 20.70328558992196, + "rednote-hilab/dots.mocr": 43.06020337683299, + "rednote-hilab/dots.ocr": 13.234220746819801, + "tesseract-5": 32.7423677519123, + "zai-org/GLM-OCR": 6.01318377203097 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "pair-balanced", + "max_absolute_elo_delta": 43.79809501398472, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 16.07126099562788, + "name": "pair-balanced_1200_seed_44", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9912087912087912, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1545.724521818796, + "PaddlePaddle/PP-OCRv6_medium": 1383.900840635146, + "PaddlePaddle/PaddleOCR-VL-1.6": 1464.4178359389718, + "allenai/olmOCR-2-7B-1025-FP8": 1807.1318714731017, + "baidu/Qianfan-OCR": 1578.2968420561524, + "baidu/Unlimited-OCR": 1513.311255633662, + "deepseek-ai/DeepSeek-OCR": 1431.653750732666, + "deepseek-ai/DeepSeek-OCR-2": 1410.6557947711547, + "lightonai/LightOnOCR-2-1B": 1727.1786591904113, + "numind/NuExtract3": 1694.3839038775739, + "rednote-hilab/dots.mocr": 1732.8897793121141, + "rednote-hilab/dots.ocr": 809.7273002286885, + "tesseract-5": 1169.762673008484, + "zai-org/GLM-OCR": 1730.9649713230774 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1486.179737437875, + 1642.1178392585762 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1325.1787883015725, + 1471.707723988396 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1403.8691771818199, + 1544.7034425120678 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1743.442169850833, + 1921.7567254303779 + ], + "baidu/Qianfan-OCR": [ + 1523.413504483978, + 1671.9464740614117 + ], + "baidu/Unlimited-OCR": [ + 1460.2935403620575, + 1590.655092363093 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1368.0432266893674, + 1514.8554106413408 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1342.0254021520573, + 1499.7492441173451 + ], + "lightonai/LightOnOCR-2-1B": [ + 1668.4120992043383, + 1824.709009952776 + ], + "numind/NuExtract3": [ + 1630.074270776628, + 1802.1258752369258 + ], + "rednote-hilab/dots.mocr": [ + 1686.256732880936, + 1827.3424547194443 + ], + "rednote-hilab/dots.ocr": [ + 530.3129013220425, + 948.4338832610994 + ], + "tesseract-5": [ + 1076.270420755164, + 1260.9733296229451 + ], + "zai-org/GLM-OCR": [ + 1671.3011302424322, + 1822.7381900469732 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -11.724864865973132, + "PaddlePaddle/PP-OCRv6_medium": -13.195719293455568, + "PaddlePaddle/PaddleOCR-VL-1.6": 1.44328344158248, + "allenai/olmOCR-2-7B-1025-FP8": 87.90073344403459, + "baidu/Qianfan-OCR": 10.202391754018208, + "baidu/Unlimited-OCR": -31.894747004891087, + "deepseek-ai/DeepSeek-OCR": -20.343272854002862, + "deepseek-ai/DeepSeek-OCR-2": 30.44324678431326, + "lightonai/LightOnOCR-2-1B": -14.073851594343296, + "numind/NuExtract3": 4.764868087618424, + "rednote-hilab/dots.mocr": -12.5354800745165, + "rednote-hilab/dots.ocr": -80.94824832911115, + "tesseract-5": 57.176164643794436, + "zai-org/GLM-OCR": -7.2145041390679125 + }, + "final_rank_order": [ + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8901098901098902, + "kind": "pair-balanced", + "max_absolute_elo_delta": 87.90073344403459, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 13.634785443899432, + "name": "pair-balanced_1200_seed_45", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9648351648351648, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1526.5101874998254, + "PaddlePaddle/PP-OCRv6_medium": 1374.302046872808, + "PaddlePaddle/PaddleOCR-VL-1.6": 1479.9544584515177, + "allenai/olmOCR-2-7B-1025-FP8": 1724.771396416001, + "baidu/Qianfan-OCR": 1547.3750745751424, + "baidu/Unlimited-OCR": 1551.1095763124817, + "deepseek-ai/DeepSeek-OCR": 1465.5134908836683, + "deepseek-ai/DeepSeek-OCR-2": 1403.4914746396144, + "lightonai/LightOnOCR-2-1B": 1728.362116552952, + "numind/NuExtract3": 1736.236829719455, + "rednote-hilab/dots.mocr": 1719.1750827872102, + "rednote-hilab/dots.ocr": 905.1330347400462, + "tesseract-5": 1121.3977460985457, + "zai-org/GLM-OCR": 1716.6674844507322 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1465.3033899283962, + 1596.9408040729209 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1314.837474351731, + 1426.9619405307228 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1418.1491601773241, + 1539.5986717037422 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1669.4494124835296, + 1791.6166142752927 + ], + "baidu/Qianfan-OCR": [ + 1490.4619499424787, + 1603.5148205183084 + ], + "baidu/Unlimited-OCR": [ + 1495.9658329186116, + 1611.182475631006 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1411.6964287560277, + 1518.5336395535019 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1337.2199241896249, + 1463.487834543042 + ], + "lightonai/LightOnOCR-2-1B": [ + 1673.8933353628906, + 1799.0954700035586 + ], + "numind/NuExtract3": [ + 1668.8056190475966, + 1817.1915110321904 + ], + "rednote-hilab/dots.mocr": [ + 1664.3937499261958, + 1782.0089337911795 + ], + "rednote-hilab/dots.ocr": [ + 692.2952275294781, + 1023.0182269157588 + ], + "tesseract-5": [ + 1019.6243191802291, + 1195.3600751123415 + ], + "zai-org/GLM-OCR": [ + 1660.0263474838935, + 1784.1235319710063 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -30.939199184943618, + "PaddlePaddle/PP-OCRv6_medium": -22.794513055793686, + "PaddlePaddle/PaddleOCR-VL-1.6": 16.9799059541283, + "allenai/olmOCR-2-7B-1025-FP8": 5.540258386933829, + "baidu/Qianfan-OCR": -20.719375726991757, + "baidu/Unlimited-OCR": 5.903573673928577, + "deepseek-ai/DeepSeek-OCR": 13.516467296999508, + "deepseek-ai/DeepSeek-OCR-2": 23.278926652772952, + "lightonai/LightOnOCR-2-1B": -12.890394231802475, + "numind/NuExtract3": 46.61779392949961, + "rednote-hilab/dots.mocr": -26.250176599420456, + "rednote-hilab/dots.ocr": 14.457486182246612, + "tesseract-5": 8.81123773385616, + "zai-org/GLM-OCR": -21.5119910114131 + }, + "final_rank_order": [ + "numind/NuExtract3", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "pair-balanced", + "max_absolute_elo_delta": 46.61779392949961, + "max_absolute_elo_delta_model": "numind/NuExtract3", + "median_absolute_elo_delta": 18.849640840560028, + "name": "pair-balanced_1200_seed_46", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9164835164835166, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "numind/NuExtract3", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 1, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1596.1874707393995, + "PaddlePaddle/PP-OCRv6_medium": 1394.3910182438551, + "PaddlePaddle/PaddleOCR-VL-1.6": 1452.1281104188165, + "allenai/olmOCR-2-7B-1025-FP8": 1701.8443541143245, + "baidu/Qianfan-OCR": 1547.5002410008585, + "baidu/Unlimited-OCR": 1563.9847142297087, + "deepseek-ai/DeepSeek-OCR": 1414.660622351955, + "deepseek-ai/DeepSeek-OCR-2": 1371.7335689048398, + "lightonai/LightOnOCR-2-1B": 1759.0179249803687, + "numind/NuExtract3": 1706.7482173550318, + "rednote-hilab/dots.mocr": 1787.052865705835, + "rednote-hilab/dots.ocr": 867.795044383654, + "tesseract-5": 1113.81008423597, + "zai-org/GLM-OCR": 1723.1457633353828 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1550.1848725497307, + 1644.7047386074573 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1343.6449455108968, + 1444.1390353845557 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1407.7185384477452, + 1494.2256203648874 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1652.1273206917253, + 1754.2548742227284 + ], + "baidu/Qianfan-OCR": [ + 1506.122761788161, + 1592.842946263691 + ], + "baidu/Unlimited-OCR": [ + 1521.8368459618378, + 1609.331514098942 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1368.1957360848935, + 1460.6614511643832 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1316.9125846421266, + 1423.7457157828928 + ], + "lightonai/LightOnOCR-2-1B": [ + 1710.3001292926276, + 1813.5988822683355 + ], + "numind/NuExtract3": [ + 1664.3332107702836, + 1758.7573874650739 + ], + "rednote-hilab/dots.mocr": [ + 1745.2668968880394, + 1841.5091995912505 + ], + "rednote-hilab/dots.ocr": [ + 712.0519362352653, + 963.1865579125184 + ], + "tesseract-5": [ + 1036.6209299489942, + 1174.857542296048 + ], + "zai-org/GLM-OCR": [ + 1677.3847635011607, + 1774.3644695345495 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 38.73808405463046, + "PaddlePaddle/PP-OCRv6_medium": -2.7055416847465494, + "PaddlePaddle/PaddleOCR-VL-1.6": -10.846442078572863, + "allenai/olmOCR-2-7B-1025-FP8": -17.386783914742637, + "baidu/Qianfan-OCR": -20.59420930127567, + "baidu/Unlimited-OCR": 18.77871159115557, + "deepseek-ai/DeepSeek-OCR": -37.336401234713776, + "deepseek-ai/DeepSeek-OCR-2": -8.478979082001615, + "lightonai/LightOnOCR-2-1B": 17.765414195614085, + "numind/NuExtract3": 17.12918156507635, + "rednote-hilab/dots.mocr": 41.62760631920446, + "rednote-hilab/dots.ocr": -22.880504174145585, + "tesseract-5": 1.2235758712804454, + "zai-org/GLM-OCR": -15.033712126762566 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "pair-balanced", + "max_absolute_elo_delta": 41.62760631920446, + "max_absolute_elo_delta_model": "rednote-hilab/dots.mocr", + "median_absolute_elo_delta": 17.57609905517836, + "name": "pair-balanced_2000_seed_42", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9824175824175824, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 4, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1566.6268490438601, + "PaddlePaddle/PP-OCRv6_medium": 1405.067456468896, + "PaddlePaddle/PaddleOCR-VL-1.6": 1435.3969521958059, + "allenai/olmOCR-2-7B-1025-FP8": 1739.3632219889735, + "baidu/Qianfan-OCR": 1541.852759968683, + "baidu/Unlimited-OCR": 1544.7706986544954, + "deepseek-ai/DeepSeek-OCR": 1424.8021970795628, + "deepseek-ai/DeepSeek-OCR-2": 1413.163607752931, + "lightonai/LightOnOCR-2-1B": 1729.7323362983238, + "numind/NuExtract3": 1705.055919186711, + "rednote-hilab/dots.mocr": 1728.175047511091, + "rednote-hilab/dots.ocr": 901.4422346135815, + "tesseract-5": 1149.6709102633217, + "zai-org/GLM-OCR": 1714.8798089737636 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1523.9267308783524, + 1617.6028568689826 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1355.870971176614, + 1448.030867022724 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1392.7352421745497, + 1478.4304118500042 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1694.9467058314217, + 1790.6646570935773 + ], + "baidu/Qianfan-OCR": [ + 1496.3352179972117, + 1588.545416933346 + ], + "baidu/Unlimited-OCR": [ + 1503.961810458935, + 1587.5768264146675 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1379.4969798323837, + 1469.3831233949606 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1362.193311625837, + 1464.608599769165 + ], + "lightonai/LightOnOCR-2-1B": [ + 1682.7188866194724, + 1784.4645427130013 + ], + "numind/NuExtract3": [ + 1657.1986594774917, + 1763.586023374784 + ], + "rednote-hilab/dots.mocr": [ + 1687.722654024963, + 1775.8809980447068 + ], + "rednote-hilab/dots.ocr": [ + 764.1967152892386, + 994.1659942417493 + ], + "tesseract-5": [ + 1077.4545824580014, + 1200.4449938246626 + ], + "zai-org/GLM-OCR": [ + 1668.5312185755263, + 1764.348020267041 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 9.177462359091123, + "PaddlePaddle/PP-OCRv6_medium": 7.970896540294234, + "PaddlePaddle/PaddleOCR-VL-1.6": -27.577600301583516, + "allenai/olmOCR-2-7B-1025-FP8": 20.132083959906367, + "baidu/Qianfan-OCR": -26.241690333451288, + "baidu/Unlimited-OCR": -0.43530398405778215, + "deepseek-ai/DeepSeek-OCR": -27.194826507105972, + "deepseek-ai/DeepSeek-OCR-2": 32.95105976608966, + "lightonai/LightOnOCR-2-1B": -11.520174486430733, + "numind/NuExtract3": 15.436883396755547, + "rednote-hilab/dots.mocr": -17.250211875539662, + "rednote-hilab/dots.ocr": 10.76668605578186, + "tesseract-5": 37.08440189863222, + "zai-org/GLM-OCR": -23.299666488381717 + }, + "final_rank_order": [ + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "pair-balanced", + "max_absolute_elo_delta": 37.08440189863222, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 18.691147917723015, + "name": "pair-balanced_2000_seed_43", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9516483516483516, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1567.8680530085896, + "PaddlePaddle/PP-OCRv6_medium": 1399.2210253836206, + "PaddlePaddle/PaddleOCR-VL-1.6": 1465.6951681945661, + "allenai/olmOCR-2-7B-1025-FP8": 1731.3880025296849, + "baidu/Qianfan-OCR": 1581.2568165397106, + "baidu/Unlimited-OCR": 1530.5499399383973, + "deepseek-ai/DeepSeek-OCR": 1424.6298671830664, + "deepseek-ai/DeepSeek-OCR-2": 1397.8377687978734, + "lightonai/LightOnOCR-2-1B": 1718.141749111582, + "numind/NuExtract3": 1699.6515194109957, + "rednote-hilab/dots.mocr": 1771.1307005874357, + "rednote-hilab/dots.ocr": 829.455793281588, + "tesseract-5": 1113.8310195500526, + "zai-org/GLM-OCR": 1769.3425764828376 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1520.657527361017, + 1619.3413924927586 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1352.9077918422113, + 1447.7769272030866 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1419.9501240773193, + 1513.2858170466293 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1686.966981803942, + 1786.680829646697 + ], + "baidu/Qianfan-OCR": [ + 1539.7892878606906, + 1628.6905218579166 + ], + "baidu/Unlimited-OCR": [ + 1489.6442729790342, + 1574.6280252662496 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1372.4738512733284, + 1473.8475966519813 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1352.1499099603955, + 1449.9768499286213 + ], + "lightonai/LightOnOCR-2-1B": [ + 1672.3598242760531, + 1769.7378596256099 + ], + "numind/NuExtract3": [ + 1653.2162273738456, + 1754.316404415539 + ], + "rednote-hilab/dots.mocr": [ + 1727.7693366315737, + 1822.7508599919115 + ], + "rednote-hilab/dots.ocr": [ + 659.6063678780697, + 930.7774114205268 + ], + "tesseract-5": [ + 1029.2611371792957, + 1169.8070716304564 + ], + "zai-org/GLM-OCR": [ + 1723.9131501437266, + 1825.281900949477 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 10.418666323820617, + "PaddlePaddle/PP-OCRv6_medium": 2.1244654550189352, + "PaddlePaddle/PaddleOCR-VL-1.6": 2.720615697176754, + "allenai/olmOCR-2-7B-1025-FP8": 12.156864500617758, + "baidu/Qianfan-OCR": 13.162366237576407, + "baidu/Unlimited-OCR": -14.656062700155871, + "deepseek-ai/DeepSeek-OCR": -27.367156403602394, + "deepseek-ai/DeepSeek-OCR-2": 17.625220811032023, + "lightonai/LightOnOCR-2-1B": -23.110761673172647, + "numind/NuExtract3": 10.032483621040228, + "rednote-hilab/dots.mocr": 25.705441200805126, + "rednote-hilab/dots.ocr": -61.219755276211686, + "tesseract-5": 1.2445111853630806, + "zai-org/GLM-OCR": 31.16310102069224 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "pair-balanced", + "max_absolute_elo_delta": 61.219755276211686, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 13.90921446886614, + "name": "pair-balanced_2000_seed_44", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9868131868131869, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1523.076413895152, + "PaddlePaddle/PP-OCRv6_medium": 1382.7661165365973, + "PaddlePaddle/PaddleOCR-VL-1.6": 1467.3477795142187, + "allenai/olmOCR-2-7B-1025-FP8": 1756.9393632316828, + "baidu/Qianfan-OCR": 1570.5522982800758, + "baidu/Unlimited-OCR": 1521.005251333356, + "deepseek-ai/DeepSeek-OCR": 1469.3437300248895, + "deepseek-ai/DeepSeek-OCR-2": 1379.9810876191825, + "lightonai/LightOnOCR-2-1B": 1771.2060108113592, + "numind/NuExtract3": 1719.444754476605, + "rednote-hilab/dots.mocr": 1717.7916860569073, + "rednote-hilab/dots.ocr": 857.7745469808318, + "tesseract-5": 1136.673621269437, + "zai-org/GLM-OCR": 1726.0973399697052 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1478.7062590551777, + 1571.3507983274476 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1332.6775600028748, + 1431.6858257965223 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1425.7199990188437, + 1512.7170758443895 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1708.1801127808246, + 1813.4511209702887 + ], + "baidu/Qianfan-OCR": [ + 1523.0837095791262, + 1619.4699053089532 + ], + "baidu/Unlimited-OCR": [ + 1477.1955346584805, + 1565.147750037454 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1425.6134331938076, + 1516.5275471864381 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1329.0420936937933, + 1433.276836777751 + ], + "lightonai/LightOnOCR-2-1B": [ + 1725.9268410344416, + 1825.1451704593562 + ], + "numind/NuExtract3": [ + 1668.520954085699, + 1778.7712443706819 + ], + "rednote-hilab/dots.mocr": [ + 1678.3159446706857, + 1768.5359891567168 + ], + "rednote-hilab/dots.ocr": [ + 679.8556180175032, + 956.0004653943233 + ], + "tesseract-5": [ + 1068.0880508296236, + 1189.703263446844 + ], + "zai-org/GLM-OCR": [ + 1684.7289793067914, + 1777.6673090939944 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -34.37297278961705, + "PaddlePaddle/PP-OCRv6_medium": -14.330443392004327, + "PaddlePaddle/PaddleOCR-VL-1.6": 4.373227016829333, + "allenai/olmOCR-2-7B-1025-FP8": 37.70822520261572, + "baidu/Qianfan-OCR": 2.457847977941583, + "baidu/Unlimited-OCR": -24.200751305197173, + "deepseek-ai/DeepSeek-OCR": 17.346706438220735, + "deepseek-ai/DeepSeek-OCR-2": -0.23146036765888311, + "lightonai/LightOnOCR-2-1B": 29.953500026604615, + "numind/NuExtract3": 29.82571868664968, + "rednote-hilab/dots.mocr": -27.633573329723276, + "rednote-hilab/dots.ocr": -32.901001576967815, + "tesseract-5": 24.087112904747528, + "zai-org/GLM-OCR": -12.082135492440102 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "pair-balanced", + "max_absolute_elo_delta": 37.70822520261572, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 24.14393210497235, + "name": "pair-balanced_2000_seed_45", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9472527472527472, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1545.7340873487474, + "PaddlePaddle/PP-OCRv6_medium": 1380.7241712145735, + "PaddlePaddle/PaddleOCR-VL-1.6": 1462.4311167863614, + "allenai/olmOCR-2-7B-1025-FP8": 1744.625375941635, + "baidu/Qianfan-OCR": 1547.1158782626403, + "baidu/Unlimited-OCR": 1558.090204217733, + "deepseek-ai/DeepSeek-OCR": 1451.666963770977, + "deepseek-ai/DeepSeek-OCR-2": 1385.0112013089922, + "lightonai/LightOnOCR-2-1B": 1734.4262547193853, + "numind/NuExtract3": 1693.6490592512116, + "rednote-hilab/dots.mocr": 1726.0623646860977, + "rednote-hilab/dots.ocr": 926.5510180906398, + "tesseract-5": 1131.0013126596991, + "zai-org/GLM-OCR": 1712.910991741307 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1494.1026455196736, + 1592.8376802793996 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1335.1703950263973, + 1426.6288311310682 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1415.8806727821893, + 1502.3334115691105 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1699.0657710002363, + 1799.0943959490937 + ], + "baidu/Qianfan-OCR": [ + 1504.068201713156, + 1586.9929562507712 + ], + "baidu/Unlimited-OCR": [ + 1514.7067202314317, + 1602.5745555747283 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1406.5228485318305, + 1496.3314813478094 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1331.9516664369353, + 1433.8368257977502 + ], + "lightonai/LightOnOCR-2-1B": [ + 1688.9208282570062, + 1784.4957388879316 + ], + "numind/NuExtract3": [ + 1651.8430223292187, + 1747.7353401377318 + ], + "rednote-hilab/dots.mocr": [ + 1684.410448902808, + 1774.5086017948818 + ], + "rednote-hilab/dots.ocr": [ + 798.2941062345981, + 1013.020604521334 + ], + "tesseract-5": [ + 1066.6018323134228, + 1185.9892968933138 + ], + "zai-org/GLM-OCR": [ + 1672.4954589097922, + 1764.7491776989702 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -11.715299336021644, + "PaddlePaddle/PP-OCRv6_medium": -16.372388714028148, + "PaddlePaddle/PaddleOCR-VL-1.6": -0.5434357110279961, + "allenai/olmOCR-2-7B-1025-FP8": 25.39423791256786, + "baidu/Qianfan-OCR": -20.978572039493883, + "baidu/Unlimited-OCR": 12.884201579179717, + "deepseek-ai/DeepSeek-OCR": -0.3300598156918113, + "deepseek-ai/DeepSeek-OCR-2": 4.7986533221508125, + "lightonai/LightOnOCR-2-1B": -6.826256065369307, + "numind/NuExtract3": 4.030023461256178, + "rednote-hilab/dots.mocr": -19.362894700532934, + "rednote-hilab/dots.ocr": 35.87546953284016, + "tesseract-5": 18.414804295009617, + "zai-org/GLM-OCR": -25.268483720838276 + }, + "final_rank_order": [ + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "pair-balanced", + "max_absolute_elo_delta": 35.87546953284016, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 14.628295146603932, + "name": "pair-balanced_2000_seed_46", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9516483516483516, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "allenai/olmOCR-2-7B-1025-FP8", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1620.3032487466087, + "PaddlePaddle/PP-OCRv6_medium": 1448.0801157254782, + "PaddlePaddle/PaddleOCR-VL-1.6": 1531.8207137997033, + "allenai/olmOCR-2-7B-1025-FP8": 1634.0400417494823, + "baidu/Qianfan-OCR": 1538.2836316036683, + "baidu/Unlimited-OCR": 1517.702660840959, + "deepseek-ai/DeepSeek-OCR": 1392.137677876126, + "deepseek-ai/DeepSeek-OCR-2": 1349.4205303204378, + "lightonai/LightOnOCR-2-1B": 1775.7835608664172, + "numind/NuExtract3": 1561.6520916534273, + "rednote-hilab/dots.mocr": 1754.0975135727124, + "rednote-hilab/dots.ocr": 1082.0146077190846, + "tesseract-5": 1101.9883922886106, + "zai-org/GLM-OCR": 1692.6752132372842 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1547.3125639904288, + 1698.2167398287695 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1362.159073849008, + 1522.0770050535214 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1455.7252268848404, + 1611.8718494678294 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1551.8026190902992, + 1719.4244047609363 + ], + "baidu/Qianfan-OCR": [ + 1462.2724019929278, + 1613.8568419271776 + ], + "baidu/Unlimited-OCR": [ + 1454.670099773985, + 1585.731484054117 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1305.825726209014, + 1472.270248271712 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1246.9787568427257, + 1429.018059484472 + ], + "lightonai/LightOnOCR-2-1B": [ + 1697.7034067580173, + 1881.900869538341 + ], + "numind/NuExtract3": [ + 1482.9698212073456, + 1649.6531805867266 + ], + "rednote-hilab/dots.mocr": [ + 1678.9596772856496, + 1847.7140425631173 + ], + "rednote-hilab/dots.ocr": [ + 930.9917784139569, + 1178.9500302644296 + ], + "tesseract-5": [ + 969.651113892322, + 1185.7606528243928 + ], + "zai-org/GLM-OCR": [ + 1624.418212245822, + 1777.9945911570733 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 62.85386206183966, + "PaddlePaddle/PP-OCRv6_medium": 50.983555796876544, + "PaddlePaddle/PaddleOCR-VL-1.6": 68.84616130231393, + "allenai/olmOCR-2-7B-1025-FP8": -85.19109627958483, + "baidu/Qianfan-OCR": -29.810818698465937, + "baidu/Unlimited-OCR": -27.503341797594203, + "deepseek-ai/DeepSeek-OCR": -59.85934571054281, + "deepseek-ai/DeepSeek-OCR-2": -30.79201766640358, + "lightonai/LightOnOCR-2-1B": 34.53105008166267, + "numind/NuExtract3": -127.96694413652813, + "rednote-hilab/dots.mocr": 8.672254186081773, + "rednote-hilab/dots.ocr": 191.33905916128492, + "tesseract-5": -10.598116076078895, + "zai-org/GLM-OCR": -45.50426222486112 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8901098901098902, + "kind": "mixed-random", + "max_absolute_elo_delta": 191.33905916128492, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 48.24390901086883, + "name": "mixed-random_700_seed_42", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9736263736263736, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1658.1300409873988, + "PaddlePaddle/PP-OCRv6_medium": 1483.074712843407, + "PaddlePaddle/PaddleOCR-VL-1.6": 1504.3991933461061, + "allenai/olmOCR-2-7B-1025-FP8": 1603.1041639443386, + "baidu/Qianfan-OCR": 1547.2499525646024, + "baidu/Unlimited-OCR": 1516.6814051771173, + "deepseek-ai/DeepSeek-OCR": 1445.4342070163211, + "deepseek-ai/DeepSeek-OCR-2": 1374.882382805725, + "lightonai/LightOnOCR-2-1B": 1675.399591311468, + "numind/NuExtract3": 1542.9342406766941, + "rednote-hilab/dots.mocr": 1784.7941812902018, + "rednote-hilab/dots.ocr": 1025.3764009255913, + "tesseract-5": 1146.4334230110414, + "zai-org/GLM-OCR": 1692.1061040999869 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1584.8501642704066, + 1740.0589094116433 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1408.1740219837297, + 1554.1472578837452 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1436.9135374472771, + 1572.2449292036165 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1532.1173774701028, + 1683.0766494866873 + ], + "baidu/Qianfan-OCR": [ + 1474.401264972803, + 1622.233411152382 + ], + "baidu/Unlimited-OCR": [ + 1451.577807248523, + 1585.297119300334 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1368.8240050231493, + 1520.4879140374192 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1280.2881680407195, + 1461.3899296474635 + ], + "lightonai/LightOnOCR-2-1B": [ + 1605.6574629298982, + 1762.210240192047 + ], + "numind/NuExtract3": [ + 1469.987442017479, + 1615.2986462715292 + ], + "rednote-hilab/dots.mocr": [ + 1708.6858479930859, + 1895.9777357504443 + ], + "rednote-hilab/dots.ocr": [ + 804.5278961224793, + 1133.8808973060059 + ], + "tesseract-5": [ + 1039.2428139882581, + 1220.4789440898053 + ], + "zai-org/GLM-OCR": [ + 1620.1967246861323, + 1776.0570647799593 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 100.68065430262982, + "PaddlePaddle/PP-OCRv6_medium": 85.97815291480538, + "PaddlePaddle/PaddleOCR-VL-1.6": 41.42464084871676, + "allenai/olmOCR-2-7B-1025-FP8": -116.12697408472854, + "baidu/Qianfan-OCR": -20.84449773753181, + "baidu/Unlimited-OCR": -28.52459746143586, + "deepseek-ai/DeepSeek-OCR": -6.56281657034765, + "deepseek-ai/DeepSeek-OCR-2": -5.330165181116399, + "lightonai/LightOnOCR-2-1B": -65.85291947328665, + "numind/NuExtract3": -146.68479511326132, + "rednote-hilab/dots.mocr": 39.36892190357116, + "rednote-hilab/dots.ocr": 134.70085236779164, + "tesseract-5": 33.846914646351934, + "zai-org/GLM-OCR": -46.07337136215847 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "mixed-random", + "max_absolute_elo_delta": 146.68479511326132, + "max_absolute_elo_delta_model": "numind/NuExtract3", + "median_absolute_elo_delta": 43.749006105437616, + "name": "mixed-random_700_seed_43", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9604395604395607, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1624.4696530798947, + "PaddlePaddle/PP-OCRv6_medium": 1405.3038896649575, + "PaddlePaddle/PaddleOCR-VL-1.6": 1515.3186853885575, + "allenai/olmOCR-2-7B-1025-FP8": 1605.9509594267513, + "baidu/Qianfan-OCR": 1565.7232148546693, + "baidu/Unlimited-OCR": 1549.8136788437002, + "deepseek-ai/DeepSeek-OCR": 1390.7373139823922, + "deepseek-ai/DeepSeek-OCR-2": 1338.3686805976035, + "lightonai/LightOnOCR-2-1B": 1760.9006761240214, + "numind/NuExtract3": 1547.9771212128735, + "rednote-hilab/dots.mocr": 1800.2793063127092, + "rednote-hilab/dots.ocr": 1013.415930167874, + "tesseract-5": 1126.3948639615478, + "zai-org/GLM-OCR": 1755.3460263824475 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1552.3120132811791, + 1698.1568108792103 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1323.2176514274104, + 1478.6771271764271 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1436.4406141876552, + 1593.3053029966013 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1535.6953576960793, + 1689.2284311201536 + ], + "baidu/Qianfan-OCR": [ + 1495.818996541234, + 1640.4485489732194 + ], + "baidu/Unlimited-OCR": [ + 1482.4940897037382, + 1628.1588019088106 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1301.8829140321386, + 1474.2296731030156 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1222.5402534232119, + 1431.0076018494185 + ], + "lightonai/LightOnOCR-2-1B": [ + 1692.4541371584407, + 1848.5641512040409 + ], + "numind/NuExtract3": [ + 1462.5371165958463, + 1641.2074117302784 + ], + "rednote-hilab/dots.mocr": [ + 1722.755085335763, + 1905.778641104726 + ], + "rednote-hilab/dots.ocr": [ + 799.6969369620936, + 1129.028715343885 + ], + "tesseract-5": [ + 1016.6422915171202, + 1200.39318989499 + ], + "zai-org/GLM-OCR": [ + 1671.4554810953903, + 1871.0297097451457 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 67.02026639512565, + "PaddlePaddle/PP-OCRv6_medium": 8.20732973635586, + "PaddlePaddle/PaddleOCR-VL-1.6": 52.3441328911681, + "allenai/olmOCR-2-7B-1025-FP8": -113.2801786023158, + "baidu/Qianfan-OCR": -2.371235447464869, + "baidu/Unlimited-OCR": 4.607676205147072, + "deepseek-ai/DeepSeek-OCR": -61.25970960427662, + "deepseek-ai/DeepSeek-OCR-2": -41.84386738923786, + "lightonai/LightOnOCR-2-1B": 19.648165339266825, + "numind/NuExtract3": -141.64191457708193, + "rednote-hilab/dots.mocr": 54.85404692607858, + "rednote-hilab/dots.ocr": 122.74038161007434, + "tesseract-5": 13.808355596858291, + "zai-org/GLM-OCR": 17.166550920302143 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "mixed-random", + "max_absolute_elo_delta": 141.64191457708193, + "max_absolute_elo_delta_model": "numind/NuExtract3", + "median_absolute_elo_delta": 47.09400014020298, + "name": "mixed-random_700_seed_44", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9516483516483516, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1616.6764539348237, + "PaddlePaddle/PP-OCRv6_medium": 1432.3243092383013, + "PaddlePaddle/PaddleOCR-VL-1.6": 1556.5163594615397, + "allenai/olmOCR-2-7B-1025-FP8": 1632.8025445032843, + "baidu/Qianfan-OCR": 1502.7868187381414, + "baidu/Unlimited-OCR": 1464.6781744964755, + "deepseek-ai/DeepSeek-OCR": 1385.0531735380455, + "deepseek-ai/DeepSeek-OCR-2": 1356.308505076121, + "lightonai/LightOnOCR-2-1B": 1760.0696261769076, + "numind/NuExtract3": 1563.7280181280119, + "rednote-hilab/dots.mocr": 1786.5992043467038, + "rednote-hilab/dots.ocr": 1048.8151416940684, + "tesseract-5": 1117.5498702412406, + "zai-org/GLM-OCR": 1776.0918004263349 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1536.5595729206152, + 1702.2314570793415 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1355.279829501517, + 1506.5690546928693 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1487.143506374889, + 1639.268176103961 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1541.7678111747384, + 1728.753346014942 + ], + "baidu/Qianfan-OCR": [ + 1423.1345037769797, + 1578.1134022874298 + ], + "baidu/Unlimited-OCR": [ + 1384.2190139539337, + 1535.3553340528613 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1299.594154010431, + 1464.1114247277858 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1265.79426552804, + 1433.326208368318 + ], + "lightonai/LightOnOCR-2-1B": [ + 1692.4215491345037, + 1853.8275794261658 + ], + "numind/NuExtract3": [ + 1482.8143628294292, + 1647.8941518538327 + ], + "rednote-hilab/dots.mocr": [ + 1715.771687541492, + 1889.1399882493522 + ], + "rednote-hilab/dots.ocr": [ + 873.3390900480958, + 1144.0314758689615 + ], + "tesseract-5": [ + 1001.0754800959435, + 1198.3621093721736 + ], + "zai-org/GLM-OCR": [ + 1694.356227339741, + 1879.7853302922977 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 59.22706725005469, + "PaddlePaddle/PP-OCRv6_medium": 35.227749309699675, + "PaddlePaddle/PaddleOCR-VL-1.6": 93.54180696415028, + "allenai/olmOCR-2-7B-1025-FP8": -86.42859352578284, + "baidu/Qianfan-OCR": -65.3076315639928, + "baidu/Unlimited-OCR": -80.52782814207762, + "deepseek-ai/DeepSeek-OCR": -66.94385004862329, + "deepseek-ai/DeepSeek-OCR-2": -23.904042910720364, + "lightonai/LightOnOCR-2-1B": 18.817115392153028, + "numind/NuExtract3": -125.89101766194358, + "rednote-hilab/dots.mocr": 41.173944960073186, + "rednote-hilab/dots.ocr": 158.1395931362688, + "tesseract-5": 4.963361876551062, + "zai-org/GLM-OCR": 37.912324964189565 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "mixed-random", + "max_absolute_elo_delta": 158.1395931362688, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 62.267349407023744, + "name": "mixed-random_700_seed_45", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9604395604395607, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3593, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1642.8296635263614, + "PaddlePaddle/PP-OCRv6_medium": 1461.4593175069006, + "PaddlePaddle/PaddleOCR-VL-1.6": 1532.4211100911155, + "allenai/olmOCR-2-7B-1025-FP8": 1587.6087802667914, + "baidu/Qianfan-OCR": 1574.9096795204657, + "baidu/Unlimited-OCR": 1492.1369624804809, + "deepseek-ai/DeepSeek-OCR": 1410.315359324659, + "deepseek-ai/DeepSeek-OCR-2": 1305.8875377897534, + "lightonai/LightOnOCR-2-1B": 1708.4227527517337, + "numind/NuExtract3": 1581.9261120589224, + "rednote-hilab/dots.mocr": 1821.1999097944224, + "rednote-hilab/dots.ocr": 1028.2352296473787, + "tesseract-5": 1133.394792135793, + "zai-org/GLM-OCR": 1719.252793105222 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1569.0638960651027, + 1726.4870430751896 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1377.2726572935971, + 1538.8563836301685 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1456.3496157736467, + 1613.2004563900825 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1513.3255687283624, + 1667.111590241618 + ], + "baidu/Qianfan-OCR": [ + 1512.7461055931542, + 1650.204474847995 + ], + "baidu/Unlimited-OCR": [ + 1420.5399928490497, + 1565.9616084908462 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1325.0917821456926, + 1481.046501997631 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1192.8327537968362, + 1392.260184781307 + ], + "lightonai/LightOnOCR-2-1B": [ + 1638.3730724389352, + 1797.6971539809301 + ], + "numind/NuExtract3": [ + 1508.5488249375042, + 1663.36410674218 + ], + "rednote-hilab/dots.mocr": [ + 1742.1511751008313, + 1935.551745487085 + ], + "rednote-hilab/dots.ocr": [ + 841.5514275006605, + 1128.4446579218663 + ], + "tesseract-5": [ + 1029.0284648452664, + 1202.800626738709 + ], + "zai-org/GLM-OCR": [ + 1648.244017172841, + 1811.471493732675 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 85.38027684159238, + "PaddlePaddle/PP-OCRv6_medium": 64.3627575782989, + "PaddlePaddle/PaddleOCR-VL-1.6": 69.44655759372608, + "allenai/olmOCR-2-7B-1025-FP8": -131.62235776227567, + "baidu/Qianfan-OCR": 6.815229218331524, + "baidu/Unlimited-OCR": -53.06904015807231, + "deepseek-ai/DeepSeek-OCR": -41.68166426200969, + "deepseek-ai/DeepSeek-OCR-2": -74.32501019708798, + "lightonai/LightOnOCR-2-1B": -32.82975803302088, + "numind/NuExtract3": -107.69292373103303, + "rednote-hilab/dots.mocr": 75.7746504077918, + "rednote-hilab/dots.ocr": 137.55968108957904, + "tesseract-5": 20.80828377110356, + "zai-org/GLM-OCR": -18.926682356923266 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "mixed-random", + "max_absolute_elo_delta": 137.55968108957904, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 66.90465758601249, + "name": "mixed-random_700_seed_46", + "percentage_saved": 83.69438621010949, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9604395604395607, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1602.6332985272588, + "PaddlePaddle/PP-OCRv6_medium": 1424.7257969345728, + "PaddlePaddle/PaddleOCR-VL-1.6": 1516.124996622898, + "allenai/olmOCR-2-7B-1025-FP8": 1648.3748813019154, + "baidu/Qianfan-OCR": 1552.6509244692784, + "baidu/Unlimited-OCR": 1522.4957724935928, + "deepseek-ai/DeepSeek-OCR": 1394.151233443908, + "deepseek-ai/DeepSeek-OCR-2": 1360.345624298629, + "lightonai/LightOnOCR-2-1B": 1750.5142877977216, + "numind/NuExtract3": 1603.8898649357998, + "rednote-hilab/dots.mocr": 1743.8541972952437, + "rednote-hilab/dots.ocr": 1013.5235693711902, + "tesseract-5": 1142.3217385902733, + "zai-org/GLM-OCR": 1724.393813917718 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1547.1868421581753, + 1658.083684859092 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1364.8755326429919, + 1482.0311921917908 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1459.0049188517078, + 1573.0640506545435 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1593.9378444126637, + 1715.4911792232097 + ], + "baidu/Qianfan-OCR": [ + 1491.9987022563357, + 1612.1056241225956 + ], + "baidu/Unlimited-OCR": [ + 1472.3960456548314, + 1572.8733316556052 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1332.0391065665722, + 1448.1957525457278 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1288.0123902996113, + 1425.9672705578694 + ], + "lightonai/LightOnOCR-2-1B": [ + 1698.2917890650292, + 1818.3240585054948 + ], + "numind/NuExtract3": [ + 1543.3511365453817, + 1668.549370236066 + ], + "rednote-hilab/dots.mocr": [ + 1688.306217201919, + 1812.9883802102222 + ], + "rednote-hilab/dots.ocr": [ + 879.0945126684292, + 1105.1034161227387 + ], + "tesseract-5": [ + 1049.0919052373642, + 1210.115641513951 + ], + "zai-org/GLM-OCR": [ + 1665.78633067882, + 1789.0330868360963 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 45.183911842489806, + "PaddlePaddle/PP-OCRv6_medium": 27.629237005971163, + "PaddlePaddle/PaddleOCR-VL-1.6": 53.15044412550856, + "allenai/olmOCR-2-7B-1025-FP8": -70.85625672715173, + "baidu/Qianfan-OCR": -15.443525832855812, + "baidu/Unlimited-OCR": -22.710230144960406, + "deepseek-ai/DeepSeek-OCR": -57.845790142760734, + "deepseek-ai/DeepSeek-OCR-2": -19.86692368821241, + "lightonai/LightOnOCR-2-1B": 9.261777012967059, + "numind/NuExtract3": -85.72917085415565, + "rednote-hilab/dots.mocr": -1.5710620913869207, + "rednote-hilab/dots.ocr": 122.84802081339058, + "tesseract-5": 29.73523022558379, + "zai-org/GLM-OCR": -13.785661544427285 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "mixed-random", + "max_absolute_elo_delta": 122.84802081339058, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 28.682233615777477, + "name": "mixed-random_1200_seed_42", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9868131868131869, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1631.5684374061764, + "PaddlePaddle/PP-OCRv6_medium": 1457.9282436766819, + "PaddlePaddle/PaddleOCR-VL-1.6": 1459.9798813197353, + "allenai/olmOCR-2-7B-1025-FP8": 1659.9896745564783, + "baidu/Qianfan-OCR": 1548.3166728591757, + "baidu/Unlimited-OCR": 1530.4885461241554, + "deepseek-ai/DeepSeek-OCR": 1437.7503045020208, + "deepseek-ai/DeepSeek-OCR-2": 1403.7121240107635, + "lightonai/LightOnOCR-2-1B": 1713.0175030039131, + "numind/NuExtract3": 1624.1387113427645, + "rednote-hilab/dots.mocr": 1737.8189614779017, + "rednote-hilab/dots.ocr": 964.5251465139563, + "tesseract-5": 1132.5432951930675, + "zai-org/GLM-OCR": 1698.2224980132096 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1573.4353067582008, + 1690.7073468521546 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1404.103520281381, + 1515.2199534701542 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1405.8547731583046, + 1514.7086463107485 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1602.785848528315, + 1725.1725511766185 + ], + "baidu/Qianfan-OCR": [ + 1494.8274308766408, + 1606.8949559277414 + ], + "baidu/Unlimited-OCR": [ + 1478.955263437513, + 1584.840678963274 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1375.002452454808, + 1502.1414655213384 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1335.54177301643, + 1463.4511437939614 + ], + "lightonai/LightOnOCR-2-1B": [ + 1657.1322316665905, + 1779.6841692967892 + ], + "numind/NuExtract3": [ + 1571.6851119861262, + 1694.3888693877263 + ], + "rednote-hilab/dots.mocr": [ + 1685.7921149120843, + 1799.7619258972545 + ], + "rednote-hilab/dots.ocr": [ + 805.3995661310506, + 1069.5053290316982 + ], + "tesseract-5": [ + 1035.3898677019586, + 1197.2371283444177 + ], + "zai-org/GLM-OCR": [ + 1647.4029167991205, + 1752.9753694529895 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 74.11905072140735, + "PaddlePaddle/PP-OCRv6_medium": 60.831683748080195, + "PaddlePaddle/PaddleOCR-VL-1.6": -2.994671177654027, + "allenai/olmOCR-2-7B-1025-FP8": -59.24146347258875, + "baidu/Qianfan-OCR": -19.777777442958495, + "baidu/Unlimited-OCR": -14.717456514397782, + "deepseek-ai/DeepSeek-OCR": -14.24671908464802, + "deepseek-ai/DeepSeek-OCR-2": 23.499576023922145, + "lightonai/LightOnOCR-2-1B": -28.235007780841443, + "numind/NuExtract3": -65.48032444719092, + "rednote-hilab/dots.mocr": -7.606297908728948, + "rednote-hilab/dots.ocr": 73.84959795615669, + "tesseract-5": 19.95678682837797, + "zai-org/GLM-OCR": -39.95697744893573 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "mixed-random", + "max_absolute_elo_delta": 74.11905072140735, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 25.867291902381794, + "name": "mixed-random_1200_seed_43", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9824175824175824, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1607.7861805557245, + "PaddlePaddle/PP-OCRv6_medium": 1396.3957751748949, + "PaddlePaddle/PaddleOCR-VL-1.6": 1474.8905419565415, + "allenai/olmOCR-2-7B-1025-FP8": 1692.3902735308052, + "baidu/Qianfan-OCR": 1581.621324913027, + "baidu/Unlimited-OCR": 1542.7564460217004, + "deepseek-ai/DeepSeek-OCR": 1424.7527198097753, + "deepseek-ai/DeepSeek-OCR-2": 1371.9917610209986, + "lightonai/LightOnOCR-2-1B": 1781.3505143431964, + "numind/NuExtract3": 1609.7988324831279, + "rednote-hilab/dots.mocr": 1780.0746048393153, + "rednote-hilab/dots.ocr": 893.0590984780348, + "tesseract-5": 1091.0106174371163, + "zai-org/GLM-OCR": 1752.1213094357418 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1558.46013442249, + 1669.3464490328768 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1329.3803431319416, + 1457.1324927673236 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1421.2509377046067, + 1539.9443677354561 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1632.2628718399326, + 1763.967464852086 + ], + "baidu/Qianfan-OCR": [ + 1525.4648063850277, + 1648.9342342663074 + ], + "baidu/Unlimited-OCR": [ + 1487.5086530527135, + 1602.8289726173984 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1358.863683642829, + 1485.880007206843 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1293.6615890066257, + 1441.409011676811 + ], + "lightonai/LightOnOCR-2-1B": [ + 1727.1469161110683, + 1855.1810579650878 + ], + "numind/NuExtract3": [ + 1550.5944143719125, + 1679.5409250820162 + ], + "rednote-hilab/dots.mocr": [ + 1723.3151530180435, + 1859.6687857236882 + ], + "rednote-hilab/dots.ocr": [ + 665.6572889436363, + 1015.6959233977989 + ], + "tesseract-5": [ + 991.3580606694954, + 1152.9741283947646 + ], + "zai-org/GLM-OCR": [ + 1692.120411465703, + 1830.0116536677306 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 50.33679387095549, + "PaddlePaddle/PP-OCRv6_medium": -0.7007847537067846, + "PaddlePaddle/PaddleOCR-VL-1.6": 11.915989459152115, + "allenai/olmOCR-2-7B-1025-FP8": -26.840864498261908, + "baidu/Qianfan-OCR": 13.526874610892719, + "baidu/Unlimited-OCR": -2.4495566168527603, + "deepseek-ai/DeepSeek-OCR": -27.244303776893503, + "deepseek-ai/DeepSeek-OCR-2": -8.220786965842763, + "lightonai/LightOnOCR-2-1B": 40.09800355844186, + "numind/NuExtract3": -79.82020330682758, + "rednote-hilab/dots.mocr": 34.649345452684656, + "rednote-hilab/dots.ocr": 2.3835499202351684, + "tesseract-5": -21.575890927573255, + "zai-org/GLM-OCR": 13.941833973596431 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "mixed-random", + "max_absolute_elo_delta": 79.82020330682758, + "max_absolute_elo_delta_model": "numind/NuExtract3", + "median_absolute_elo_delta": 17.758862450584843, + "name": "mixed-random_1200_seed_44", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9912087912087912, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1597.3609110094637, + "PaddlePaddle/PP-OCRv6_medium": 1405.0333502757255, + "PaddlePaddle/PaddleOCR-VL-1.6": 1495.0041767638872, + "allenai/olmOCR-2-7B-1025-FP8": 1656.2434826753345, + "baidu/Qianfan-OCR": 1526.4743702957785, + "baidu/Unlimited-OCR": 1510.2801888072604, + "deepseek-ai/DeepSeek-OCR": 1405.3825103322545, + "deepseek-ai/DeepSeek-OCR-2": 1377.0004345945476, + "lightonai/LightOnOCR-2-1B": 1724.0564789789507, + "numind/NuExtract3": 1607.894369759667, + "rednote-hilab/dots.mocr": 1775.0819393897812, + "rednote-hilab/dots.ocr": 996.843465884866, + "tesseract-5": 1140.1993705525779, + "zai-org/GLM-OCR": 1783.1449506799054 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1543.4344716028954, + 1658.6901176953113 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1344.4970590877915, + 1457.032844444523 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1436.7720823449881, + 1554.0052404786823 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1598.690388800887, + 1714.1771619976594 + ], + "baidu/Qianfan-OCR": [ + 1475.7888512777006, + 1581.9126461727046 + ], + "baidu/Unlimited-OCR": [ + 1458.4205612088213, + 1568.3854802179876 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1339.9443596036665, + 1461.0487297910138 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1311.4549201320817, + 1438.2648946756124 + ], + "lightonai/LightOnOCR-2-1B": [ + 1669.539722194586, + 1790.9345913701393 + ], + "numind/NuExtract3": [ + 1549.793072563139, + 1672.1643246727403 + ], + "rednote-hilab/dots.mocr": [ + 1721.3058609005786, + 1847.3352516445987 + ], + "rednote-hilab/dots.ocr": [ + 854.6455112109763, + 1095.2329212539912 + ], + "tesseract-5": [ + 1048.1346520330735, + 1201.8575971509667 + ], + "zai-org/GLM-OCR": [ + 1724.6934697271952, + 1860.0187416424014 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 39.91152432469471, + "PaddlePaddle/PP-OCRv6_medium": 7.936790347123861, + "PaddlePaddle/PaddleOCR-VL-1.6": 32.029624266497876, + "allenai/olmOCR-2-7B-1025-FP8": -62.987655353732634, + "baidu/Qianfan-OCR": -41.62008000635569, + "baidu/Unlimited-OCR": -34.92581383129277, + "deepseek-ai/DeepSeek-OCR": -46.614513254414305, + "deepseek-ai/DeepSeek-OCR-2": -3.212113392293759, + "lightonai/LightOnOCR-2-1B": -17.19603180580384, + "numind/NuExtract3": -81.72466603028852, + "rednote-hilab/dots.mocr": 29.65668000315054, + "rednote-hilab/dots.ocr": 106.16791732706633, + "tesseract-5": 27.612862187888368, + "zai-org/GLM-OCR": 44.96547521776006 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "mixed-random", + "max_absolute_elo_delta": 106.16791732706633, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 37.41866907799374, + "name": "mixed-random_1200_seed_45", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9824175824175824, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3093, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1610.6885206648683, + "PaddlePaddle/PP-OCRv6_medium": 1422.268006598586, + "PaddlePaddle/PaddleOCR-VL-1.6": 1513.5332381287021, + "allenai/olmOCR-2-7B-1025-FP8": 1649.7151580440377, + "baidu/Qianfan-OCR": 1547.7945560822852, + "baidu/Unlimited-OCR": 1519.2845133601986, + "deepseek-ai/DeepSeek-OCR": 1425.793536080444, + "deepseek-ai/DeepSeek-OCR-2": 1349.9327173323106, + "lightonai/LightOnOCR-2-1B": 1698.5968052265146, + "numind/NuExtract3": 1637.8008835293765, + "rednote-hilab/dots.mocr": 1799.9482948879072, + "rednote-hilab/dots.ocr": 953.249266842184, + "tesseract-5": 1124.9178798966157, + "zai-org/GLM-OCR": 1746.4766233259697 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1551.56627635339, + 1675.740997513204 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1360.8931846483756, + 1479.8812991826553 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1460.6488856314668, + 1574.2564717417229 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1592.6652407342287, + 1712.7898425363935 + ], + "baidu/Qianfan-OCR": [ + 1495.4468816183933, + 1606.5046826112628 + ], + "baidu/Unlimited-OCR": [ + 1465.1251310687212, + 1574.7837372332679 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1367.180323954122, + 1481.9754183860057 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1263.2323212912538, + 1424.6097523668398 + ], + "lightonai/LightOnOCR-2-1B": [ + 1643.1190601153985, + 1767.1443360261978 + ], + "numind/NuExtract3": [ + 1581.1201150887593, + 1701.5740417540248 + ], + "rednote-hilab/dots.mocr": [ + 1734.4361220158705, + 1885.0527589162898 + ], + "rednote-hilab/dots.ocr": [ + 779.1402550144655, + 1058.888953564272 + ], + "tesseract-5": [ + 1042.2455307359592, + 1187.1329147955396 + ], + "zai-org/GLM-OCR": [ + 1692.4171745866988, + 1813.4967163523647 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 53.23913398009927, + "PaddlePaddle/PP-OCRv6_medium": 25.171446669984334, + "PaddlePaddle/PaddleOCR-VL-1.6": 50.558685631312756, + "allenai/olmOCR-2-7B-1025-FP8": -69.51597998502939, + "baidu/Qianfan-OCR": -20.299894219848966, + "baidu/Unlimited-OCR": -25.921489278354557, + "deepseek-ai/DeepSeek-OCR": -26.203487506224747, + "deepseek-ai/DeepSeek-OCR-2": -30.27983065453077, + "lightonai/LightOnOCR-2-1B": -42.655705558240015, + "numind/NuExtract3": -51.81815226057893, + "rednote-hilab/dots.mocr": 54.52303550127658, + "rednote-hilab/dots.ocr": 62.57371828438443, + "tesseract-5": 12.331371531926152, + "zai-org/GLM-OCR": 8.29714786382442 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "mixed-random", + "max_absolute_elo_delta": 69.51597998502939, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 36.46776810638539, + "name": "mixed-random_1200_seed_46", + "percentage_saved": 72.04751921733053, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9912087912087912, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1573.663528450672, + "PaddlePaddle/PP-OCRv6_medium": 1411.9252749410973, + "PaddlePaddle/PaddleOCR-VL-1.6": 1499.7002230467394, + "allenai/olmOCR-2-7B-1025-FP8": 1687.7519284077068, + "baidu/Qianfan-OCR": 1552.0426056461395, + "baidu/Unlimited-OCR": 1528.1241985753716, + "deepseek-ai/DeepSeek-OCR": 1445.5209314599153, + "deepseek-ai/DeepSeek-OCR-2": 1381.0494265611999, + "lightonai/LightOnOCR-2-1B": 1731.1734656434533, + "numind/NuExtract3": 1649.639397023911, + "rednote-hilab/dots.mocr": 1755.0098525318556, + "rednote-hilab/dots.ocr": 936.9457261023799, + "tesseract-5": 1108.6631036916806, + "zai-org/GLM-OCR": 1738.7903379178777 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1529.569397149427, + 1621.2536407943467 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1366.0835945265592, + 1458.587798521135 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1459.5671265807384, + 1548.3356206747562 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1643.1929587757354, + 1741.6087224855562 + ], + "baidu/Qianfan-OCR": [ + 1507.8275763951635, + 1597.0213044199663 + ], + "baidu/Unlimited-OCR": [ + 1486.3755956225157, + 1571.4954784926401 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1399.8181040919412, + 1489.7348864878172 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1330.6953197795008, + 1430.6398518589624 + ], + "lightonai/LightOnOCR-2-1B": [ + 1689.3254004365142, + 1786.5756726091809 + ], + "numind/NuExtract3": [ + 1601.6536973146628, + 1698.4564275648684 + ], + "rednote-hilab/dots.mocr": [ + 1711.8502551037045, + 1804.23210891879 + ], + "rednote-hilab/dots.ocr": [ + 814.4337651048638, + 1020.7291924988103 + ], + "tesseract-5": [ + 1036.4727690965276, + 1165.7291087283056 + ], + "zai-org/GLM-OCR": [ + 1698.0024882318548, + 1789.4757130506891 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 16.214141765902923, + "PaddlePaddle/PP-OCRv6_medium": 14.82871501249565, + "PaddlePaddle/PaddleOCR-VL-1.6": 36.72567054935007, + "allenai/olmOCR-2-7B-1025-FP8": -31.47920962136027, + "baidu/Qianfan-OCR": -16.051844655994728, + "baidu/Unlimited-OCR": -17.081804063181607, + "deepseek-ai/DeepSeek-OCR": -6.476092126753429, + "deepseek-ai/DeepSeek-OCR-2": 0.8368785743584795, + "lightonai/LightOnOCR-2-1B": -10.079045141301322, + "numind/NuExtract3": -39.979638766044445, + "rednote-hilab/dots.mocr": 9.584593145224972, + "rednote-hilab/dots.ocr": 46.27017754458029, + "tesseract-5": -3.9234046730089176, + "zai-org/GLM-OCR": 0.610862455732331 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "mixed-random", + "max_absolute_elo_delta": 46.27017754458029, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 15.440279834245189, + "name": "mixed-random_2000_seed_42", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9912087912087912, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1590.9924199371674, + "PaddlePaddle/PP-OCRv6_medium": 1450.1605591024454, + "PaddlePaddle/PaddleOCR-VL-1.6": 1467.060276748374, + "allenai/olmOCR-2-7B-1025-FP8": 1698.817107093526, + "baidu/Qianfan-OCR": 1578.0969388006488, + "baidu/Unlimited-OCR": 1547.9903036612516, + "deepseek-ai/DeepSeek-OCR": 1440.7954324639893, + "deepseek-ai/DeepSeek-OCR-2": 1393.6801466160546, + "lightonai/LightOnOCR-2-1B": 1720.9364431326946, + "numind/NuExtract3": 1675.7194457018418, + "rednote-hilab/dots.mocr": 1738.7947057543856, + "rednote-hilab/dots.ocr": 858.6865807119874, + "tesseract-5": 1117.3331971429388, + "zai-org/GLM-OCR": 1720.9364431326946 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1545.4348547617815, + 1640.6544909907113 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1408.0437785093047, + 1494.5087089736428 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1423.7103662534214, + 1509.7755253410878 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1655.9243010348455, + 1747.7430506114354 + ], + "baidu/Qianfan-OCR": [ + 1533.0735742788609, + 1622.746410906176 + ], + "baidu/Unlimited-OCR": [ + 1506.626550882564, + 1592.1316509072283 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1395.3998457750868, + 1489.611594236466 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1343.3716319222112, + 1440.5655614327559 + ], + "lightonai/LightOnOCR-2-1B": [ + 1679.1746665525934, + 1770.8938146958985 + ], + "numind/NuExtract3": [ + 1630.1992089505409, + 1724.6427283477008 + ], + "rednote-hilab/dots.mocr": [ + 1693.54653199798, + 1793.6098811572786 + ], + "rednote-hilab/dots.ocr": [ + 697.0937014379675, + 961.6314354893002 + ], + "tesseract-5": [ + 1044.3535773275744, + 1173.8312771675635 + ], + "zai-org/GLM-OCR": [ + 1679.4655632457857, + 1770.7206703331613 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 33.543033252398345, + "PaddlePaddle/PP-OCRv6_medium": 53.06399917384374, + "PaddlePaddle/PaddleOCR-VL-1.6": 4.08572425098464, + "allenai/olmOCR-2-7B-1025-FP8": -20.414030935541177, + "baidu/Qianfan-OCR": 10.002488498514595, + "baidu/Unlimited-OCR": 2.7843010226984006, + "deepseek-ai/DeepSeek-OCR": -11.201591122679474, + "deepseek-ai/DeepSeek-OCR-2": 13.467598629213171, + "lightonai/LightOnOCR-2-1B": -20.316067652059928, + "numind/NuExtract3": -13.899590088113655, + "rednote-hilab/dots.mocr": -6.6305536322449825, + "rednote-hilab/dots.ocr": -31.988967845812226, + "tesseract-5": 4.746688778249336, + "zai-org/GLM-OCR": -17.24303232945067 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "mixed-random", + "max_absolute_elo_delta": 53.06399917384374, + "max_absolute_elo_delta_model": "PaddlePaddle/PP-OCRv6_medium", + "median_absolute_elo_delta": 13.683594358663413, + "name": "mixed-random_2000_seed_43", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9912087912087912, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1582.9877386813391, + "PaddlePaddle/PP-OCRv6_medium": 1407.578088781668, + "PaddlePaddle/PaddleOCR-VL-1.6": 1458.3012179453053, + "allenai/olmOCR-2-7B-1025-FP8": 1719.28716691742, + "baidu/Qianfan-OCR": 1572.6889266294816, + "baidu/Unlimited-OCR": 1547.5331435011065, + "deepseek-ai/DeepSeek-OCR": 1453.1308594475618, + "deepseek-ai/DeepSeek-OCR-2": 1360.1062743256673, + "lightonai/LightOnOCR-2-1B": 1758.0960844763797, + "numind/NuExtract3": 1676.3284471989311, + "rednote-hilab/dots.mocr": 1775.8457889268475, + "rednote-hilab/dots.ocr": 851.6248064535293, + "tesseract-5": 1115.566066733883, + "zai-org/GLM-OCR": 1720.9253899808805 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1537.9726139519505, + 1630.5862192922746 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1358.2333365411469, + 1456.7056768971108 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1415.3312211217692, + 1503.9139662483947 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1670.4523037850436, + 1776.4321265618973 + ], + "baidu/Qianfan-OCR": [ + 1525.8039135509252, + 1620.2211749857693 + ], + "baidu/Unlimited-OCR": [ + 1509.1739889790754, + 1592.1573922926189 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1406.284630574299, + 1502.1575027626545 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1303.8917026809522, + 1411.939957105943 + ], + "lightonai/LightOnOCR-2-1B": [ + 1712.1894461505112, + 1809.7498352795283 + ], + "numind/NuExtract3": [ + 1628.1400724672535, + 1730.0986797174546 + ], + "rednote-hilab/dots.mocr": [ + 1731.2197509085818, + 1827.8558145364534 + ], + "rednote-hilab/dots.ocr": [ + 686.0956020820535, + 964.3992747061824 + ], + "tesseract-5": [ + 1042.672174159757, + 1171.5929337203586 + ], + "zai-org/GLM-OCR": [ + 1676.626258689866, + 1771.1750930859403 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 25.5383519965701, + "PaddlePaddle/PP-OCRv6_medium": 10.481528853066266, + "PaddlePaddle/PaddleOCR-VL-1.6": -4.673334552084043, + "allenai/olmOCR-2-7B-1025-FP8": 0.05602888835301201, + "baidu/Qianfan-OCR": 4.594476327347365, + "baidu/Unlimited-OCR": 2.3271408625532786, + "deepseek-ai/DeepSeek-OCR": 1.1338358608929866, + "deepseek-ai/DeepSeek-OCR-2": -20.1062736611741, + "lightonai/LightOnOCR-2-1B": 16.843573691625124, + "numind/NuExtract3": -13.290588591024289, + "rednote-hilab/dots.mocr": 30.420529540216876, + "rednote-hilab/dots.ocr": -39.050742104270284, + "tesseract-5": 2.9795583691934553, + "zai-org/GLM-OCR": -17.254085481264838 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9780219780219781, + "kind": "mixed-random", + "max_absolute_elo_delta": 39.050742104270284, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 11.886058722045277, + "name": "mixed-random_2000_seed_44", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9956043956043955, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 3, + "unresolved": 10 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1569.896491674002, + "PaddlePaddle/PP-OCRv6_medium": 1378.565197068115, + "PaddlePaddle/PaddleOCR-VL-1.6": 1484.3891354817986, + "allenai/olmOCR-2-7B-1025-FP8": 1686.3534853065385, + "baidu/Qianfan-OCR": 1547.945597283799, + "baidu/Unlimited-OCR": 1528.2567330710913, + "deepseek-ai/DeepSeek-OCR": 1434.2127739677405, + "deepseek-ai/DeepSeek-OCR-2": 1370.0687765517825, + "lightonai/LightOnOCR-2-1B": 1717.9643101264915, + "numind/NuExtract3": 1670.028774534308, + "rednote-hilab/dots.mocr": 1760.819926775015, + "rednote-hilab/dots.ocr": 952.050229833096, + "tesseract-5": 1131.9011179110926, + "zai-org/GLM-OCR": 1767.5474504151298 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1523.2704436388044, + 1617.6028988217959 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1327.108509393474, + 1424.3074697146894 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1439.158804858982, + 1528.9037658863303 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1640.6214622550294, + 1745.3941221544596 + ], + "baidu/Qianfan-OCR": [ + 1505.353594633637, + 1592.798578222683 + ], + "baidu/Unlimited-OCR": [ + 1484.7547592822968, + 1573.104555028356 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1391.548644238518, + 1475.3024981210403 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1316.0459055038723, + 1420.9224328686166 + ], + "lightonai/LightOnOCR-2-1B": [ + 1671.3327838992602, + 1765.5500624633585 + ], + "numind/NuExtract3": [ + 1623.5636977281936, + 1726.5939902982832 + ], + "rednote-hilab/dots.mocr": [ + 1714.7013020248328, + 1809.1563027155514 + ], + "rednote-hilab/dots.ocr": [ + 827.3670486298138, + 1035.8191576560741 + ], + "tesseract-5": [ + 1065.9170521911703, + 1186.512209670519 + ], + "zai-org/GLM-OCR": [ + 1723.182515417619, + 1820.8804306091374 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 12.447104989232912, + "PaddlePaddle/PP-OCRv6_medium": -18.53136286048675, + "PaddlePaddle/PaddleOCR-VL-1.6": 21.414582984409208, + "allenai/olmOCR-2-7B-1025-FP8": -32.87765272252864, + "baidu/Qianfan-OCR": -20.148853018335103, + "baidu/Unlimited-OCR": -16.949269567461897, + "deepseek-ai/DeepSeek-OCR": -17.784249618928243, + "deepseek-ai/DeepSeek-OCR-2": -10.143771435058852, + "lightonai/LightOnOCR-2-1B": -23.288200658263122, + "numind/NuExtract3": -19.59026125564742, + "rednote-hilab/dots.mocr": 15.394667388384278, + "rednote-hilab/dots.ocr": 61.374681275296325, + "tesseract-5": 19.314609546403062, + "zai-org/GLM-OCR": 29.36797495298447 + }, + "final_rank_order": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "kind": "mixed-random", + "max_absolute_elo_delta": 61.374681275296325, + "max_absolute_elo_delta_model": "rednote-hilab/dots.ocr", + "median_absolute_elo_delta": 19.45243540102524, + "name": "mixed-random_2000_seed_45", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9824175824175824, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2293, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1592.7633358261285, + "PaddlePaddle/PP-OCRv6_medium": 1391.1568376958744, + "PaddlePaddle/PaddleOCR-VL-1.6": 1476.7399278109592, + "allenai/olmOCR-2-7B-1025-FP8": 1695.2128061736842, + "baidu/Qianfan-OCR": 1567.7775392389688, + "baidu/Unlimited-OCR": 1539.8106470676487, + "deepseek-ai/DeepSeek-OCR": 1444.2131492114418, + "deepseek-ai/DeepSeek-OCR-2": 1375.8677515569764, + "lightonai/LightOnOCR-2-1B": 1711.6085948995449, + "numind/NuExtract3": 1686.1783751467542, + "rednote-hilab/dots.mocr": 1786.0139731582658, + "rednote-hilab/dots.ocr": 886.2632525158735, + "tesseract-5": 1105.0174638175795, + "zai-org/GLM-OCR": 1741.3763458803003 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1549.3049418128182, + 1640.0420119127414 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1339.3139576683357, + 1437.7655261169693 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1432.7421123978856, + 1521.5127055063635 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1651.381611443974, + 1747.3820223789628 + ], + "baidu/Qianfan-OCR": [ + 1524.1602378553719, + 1611.695029619878 + ], + "baidu/Unlimited-OCR": [ + 1495.6298441425688, + 1586.9966992667776 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1400.202855044254, + 1493.6877823381708 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1320.8660722639145, + 1430.3785297344084 + ], + "lightonai/LightOnOCR-2-1B": [ + 1667.4605777117133, + 1758.0791991741712 + ], + "numind/NuExtract3": [ + 1639.4786535067883, + 1737.5595122477212 + ], + "rednote-hilab/dots.mocr": [ + 1740.294407514458, + 1841.619364913969 + ], + "rednote-hilab/dots.ocr": [ + 723.8870268960246, + 984.1705293809986 + ], + "tesseract-5": [ + 1036.6033827196957, + 1163.5900508438508 + ], + "zai-org/GLM-OCR": [ + 1702.36923859971, + 1792.6089883945833 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 35.313949141359444, + "PaddlePaddle/PP-OCRv6_medium": -5.939722232727263, + "PaddlePaddle/PaddleOCR-VL-1.6": 13.765375313569848, + "allenai/olmOCR-2-7B-1025-FP8": -24.01833185538294, + "baidu/Qianfan-OCR": -0.3169110631654348, + "baidu/Unlimited-OCR": -5.395355570904485, + "deepseek-ai/DeepSeek-OCR": -7.783874375227015, + "deepseek-ai/DeepSeek-OCR-2": -4.344796429865028, + "lightonai/LightOnOCR-2-1B": -29.643915885209708, + "numind/NuExtract3": -3.4406606432012268, + "rednote-hilab/dots.mocr": 40.588713771635184, + "rednote-hilab/dots.ocr": -4.4122960419260835, + "tesseract-5": -7.569044547110025, + "zai-org/GLM-OCR": 3.1968704181549583 + }, + "final_rank_order": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5", + "rednote-hilab/dots.ocr" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "mixed-random", + "max_absolute_elo_delta": 40.588713771635184, + "max_absolute_elo_delta_model": "rednote-hilab/dots.mocr", + "median_absolute_elo_delta": 6.754383389918644, + "name": "mixed-random_2000_seed_46", + "percentage_saved": 53.41253202888423, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9912087912087912, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null } ] } diff --git a/experiments/adaptive-stopping/static-design-summary.csv b/experiments/adaptive-stopping/static-design-summary.csv new file mode 100644 index 0000000..b03b093 --- /dev/null +++ b/experiments/adaptive-stopping/static-design-summary.csv @@ -0,0 +1,7 @@ +kind,budget,percentage_saved,kendall_tau_median,kendall_tau_min,kendall_tau_max,spearman_rho_median,top3_membership_matches,top3_order_matches,median_absolute_elo_delta_median,max_absolute_elo_delta_median,max_absolute_elo_delta_worst,minimum_pair_evidence,gate_passes,runs +mixed-random,700,83.69438621010949,0.8681318681318682,0.8681318681318682,0.8901098901098902,0.9604395604395607,5,1,48.24390901086883,146.68479511326132,191.33905916128492,7,0,5 +mixed-random,1200,72.04751921733053,0.9340659340659341,0.9340659340659341,0.9560439560439562,0.9868131868131869,5,1,28.682233615777477,79.82020330682758,122.84802081339058,13,0,5 +mixed-random,2000,53.41253202888423,0.9560439560439562,0.9340659340659341,0.9780219780219781,0.9912087912087912,5,2,13.683594358663413,46.27017754458029,61.374681275296325,21,0,5 +pair-balanced,700,83.69438621010949,0.8681318681318682,0.7582417582417583,0.9780219780219781,0.956043956043956,1,1,34.88736020560384,75.17162456695951,116.38438099675818,7,0,5 +pair-balanced,1200,72.04751921733053,0.8901098901098902,0.7802197802197802,0.9560439560439562,0.9692307692307693,2,0,18.849640840560028,49.12059412915846,88.99502230491316,13,0,5 +pair-balanced,2000,53.41253202888423,0.8681318681318682,0.8461538461538461,0.9560439560439562,0.9516483516483516,1,1,17.57609905517836,37.70822520261572,61.219755276211686,21,0,5 diff --git a/experiments/adaptive-stopping/strategy-summary.csv b/experiments/adaptive-stopping/strategy-summary.csv index 5507d0c..40812ce 100644 --- a/experiments/adaptive-stopping/strategy-summary.csv +++ b/experiments/adaptive-stopping/strategy-summary.csv @@ -13,3 +13,33 @@ targeted_v2_explore_every_3,2023,2270,52.8767761472164,10,sample_batches_exhaust targeted_v2_warmup_5_explore_3_annotate_3x,1946,2347,54.67039366410435,10,sample_batches_exhausted,0.9340659340659341,0.9868131868131869,True,False,29.7766934028906,88.15292448408707,False,True targeted_v2_warmup_10_explore_3_annotate_3x,2313,1980,46.121593291404615,10,sample_batches_exhausted,0.9560439560439562,0.9912087912087912,True,True,37.08905123317106,99.6396142776457,False, targeted_v2_warmup_5_explore_3_size_stop_3x,1820,2473,57.60540414628465,10,sample_batches_exhausted,0.9340659340659341,0.9868131868131869,True,False,31.76489367253805,90.77328734957064,False, +pair-balanced_700_seed_42,700,3593,83.69438621010949,0,predeclared_static_budget,0.8681318681318682,0.9692307692307693,False,False,35.38612554748215,64.21191215428712,False, +pair-balanced_700_seed_43,700,3593,83.69438621010949,0,predeclared_static_budget,0.7582417582417583,0.9252747252747253,False,False,36.85152517560209,116.38438099675818,False, +pair-balanced_700_seed_44,700,3593,83.69438621010949,0,predeclared_static_budget,0.9780219780219781,0.9956043956043955,True,True,15.782769046150293,72.57611807296644,False, +pair-balanced_700_seed_45,700,3593,83.69438621010949,0,predeclared_static_budget,0.8681318681318682,0.956043956043956,False,False,34.88736020560384,112.7705037139533,False, +pair-balanced_700_seed_46,700,3593,83.69438621010949,0,predeclared_static_budget,0.8241758241758242,0.9472527472527472,False,False,32.64165559093419,75.17162456695951,False, +pair-balanced_1200_seed_42,1200,3093,72.04751921733053,0,predeclared_static_budget,0.8681318681318682,0.9692307692307693,False,False,23.661987500904274,88.99502230491316,False, +pair-balanced_1200_seed_43,1200,3093,72.04751921733053,0,predeclared_static_budget,0.9120879120879122,0.9780219780219781,True,False,20.098080942588638,49.12059412915846,False, +pair-balanced_1200_seed_44,1200,3093,72.04751921733053,0,predeclared_static_budget,0.9560439560439562,0.9912087912087912,True,False,16.07126099562788,43.79809501398472,False, +pair-balanced_1200_seed_45,1200,3093,72.04751921733053,0,predeclared_static_budget,0.8901098901098902,0.9648351648351648,False,False,13.634785443899432,87.90073344403459,False, +pair-balanced_1200_seed_46,1200,3093,72.04751921733053,0,predeclared_static_budget,0.7802197802197802,0.9164835164835166,False,False,18.849640840560028,46.61779392949961,False, +pair-balanced_2000_seed_42,2000,2293,53.41253202888423,0,predeclared_static_budget,0.9340659340659341,0.9824175824175824,True,True,17.57609905517836,41.62760631920446,False, +pair-balanced_2000_seed_43,2000,2293,53.41253202888423,0,predeclared_static_budget,0.8461538461538461,0.9516483516483516,False,False,18.691147917723015,37.08440189863222,False, +pair-balanced_2000_seed_44,2000,2293,53.41253202888423,0,predeclared_static_budget,0.9560439560439562,0.9868131868131869,False,False,13.90921446886614,61.219755276211686,False, +pair-balanced_2000_seed_45,2000,2293,53.41253202888423,0,predeclared_static_budget,0.8681318681318682,0.9472527472527472,False,False,24.14393210497235,37.70822520261572,False, +pair-balanced_2000_seed_46,2000,2293,53.41253202888423,0,predeclared_static_budget,0.8461538461538461,0.9516483516483516,False,False,14.628295146603932,35.87546953284016,False, +mixed-random_700_seed_42,700,3593,83.69438621010949,0,predeclared_static_budget,0.8901098901098902,0.9736263736263736,True,False,48.24390901086883,191.33905916128492,False, +mixed-random_700_seed_43,700,3593,83.69438621010949,0,predeclared_static_budget,0.8681318681318682,0.9604395604395607,True,False,43.749006105437616,146.68479511326132,False, +mixed-random_700_seed_44,700,3593,83.69438621010949,0,predeclared_static_budget,0.8681318681318682,0.9516483516483516,True,True,47.09400014020298,141.64191457708193,False, +mixed-random_700_seed_45,700,3593,83.69438621010949,0,predeclared_static_budget,0.8681318681318682,0.9604395604395607,True,False,62.267349407023744,158.1395931362688,False, +mixed-random_700_seed_46,700,3593,83.69438621010949,0,predeclared_static_budget,0.8681318681318682,0.9604395604395607,True,False,66.90465758601249,137.55968108957904,False, +mixed-random_1200_seed_42,1200,3093,72.04751921733053,0,predeclared_static_budget,0.9340659340659341,0.9868131868131869,True,False,28.682233615777477,122.84802081339058,False, +mixed-random_1200_seed_43,1200,3093,72.04751921733053,0,predeclared_static_budget,0.9340659340659341,0.9824175824175824,True,True,25.867291902381794,74.11905072140735,False, +mixed-random_1200_seed_44,1200,3093,72.04751921733053,0,predeclared_static_budget,0.9560439560439562,0.9912087912087912,True,False,17.758862450584843,79.82020330682758,False, +mixed-random_1200_seed_45,1200,3093,72.04751921733053,0,predeclared_static_budget,0.9340659340659341,0.9824175824175824,True,False,37.41866907799374,106.16791732706633,False, +mixed-random_1200_seed_46,1200,3093,72.04751921733053,0,predeclared_static_budget,0.9560439560439562,0.9912087912087912,True,False,36.46776810638539,69.51597998502939,False, +mixed-random_2000_seed_42,2000,2293,53.41253202888423,0,predeclared_static_budget,0.9560439560439562,0.9912087912087912,True,False,15.440279834245189,46.27017754458029,False, +mixed-random_2000_seed_43,2000,2293,53.41253202888423,0,predeclared_static_budget,0.9560439560439562,0.9912087912087912,True,True,13.683594358663413,53.06399917384374,False, +mixed-random_2000_seed_44,2000,2293,53.41253202888423,0,predeclared_static_budget,0.9780219780219781,0.9956043956043955,True,True,11.886058722045277,39.050742104270284,False, +mixed-random_2000_seed_45,2000,2293,53.41253202888423,0,predeclared_static_budget,0.9340659340659341,0.9824175824175824,True,False,19.45243540102524,61.374681275296325,False, +mixed-random_2000_seed_46,2000,2293,53.41253202888423,0,predeclared_static_budget,0.9560439560439562,0.9912087912087912,True,False,6.754383389918644,40.588713771635184,False, diff --git a/tests/test_adaptive_replay_experiment.py b/tests/test_adaptive_replay_experiment.py index 67ad9a2..7df218d 100644 --- a/tests/test_adaptive_replay_experiment.py +++ b/tests/test_adaptive_replay_experiment.py @@ -134,6 +134,42 @@ def test_size_rule_can_annotate_without_controlling_sampling(): assert replay._classify(board, comparisons, config, for_sampling=True)[0].status == "unresolved" +def test_fixed_pair_balanced_sample_is_exact_balanced_and_seeded(): + stored = _stored_grid(8) + + first = replay._fixed_pair_balanced_sample(stored, budget=14, seed=42) + repeated = replay._fixed_pair_balanced_sample(stored, budget=14, seed=42) + other_seed = replay._fixed_pair_balanced_sample(stored, budget=14, seed=43) + counts = replay.comparison_pair_counts(first) + + assert len(first) == 14 + assert sorted(counts.values()) == [4, 5, 5] + assert [replay._comparison_key(row) for row in first] == [ + replay._comparison_key(row) for row in repeated + ] + assert [replay._comparison_key(row) for row in first] != [ + replay._comparison_key(row) for row in other_seed + ] + + +def test_mixed_sample_preserves_required_warmup_and_rebalances(): + stored = _stored_grid(8) + required = stored[:3] + + selected = replay._fixed_pair_balanced_sample( + stored, + budget=14, + seed=42, + required=required, + ) + keys = [replay._comparison_key(row) for row in selected] + counts = replay.comparison_pair_counts(selected) + + assert selected[:3] == required + assert len(keys) == len(set(keys)) == 14 + assert max(counts.values()) - min(counts.values()) == 1 + + def test_graph_metrics_report_coverage_and_connectivity(): complete = replay.graph_metrics(_stored_grid(1), ["a", "b", "c"]) disconnected = replay.graph_metrics( From dcd711c213e9d33398cc45ca3de1195e6fb64004 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Wed, 15 Jul 2026 23:33:35 +0100 Subject: [PATCH 4/5] Validate fixed designs across published boards --- experiments/adaptive-stopping/README.md | 137 +- .../multi-board-results.json | 11115 ++++++++++++++++ .../adaptive-stopping/multi-board-summary.csv | 37 + experiments/adaptive-stopping/multi_board.py | 522 + experiments/adaptive-stopping/replay.py | 17 +- tests/test_adaptive_replay_experiment.py | 67 + 6 files changed, 11870 insertions(+), 25 deletions(-) create mode 100644 experiments/adaptive-stopping/multi-board-results.json create mode 100644 experiments/adaptive-stopping/multi-board-summary.csv create mode 100644 experiments/adaptive-stopping/multi_board.py diff --git a/experiments/adaptive-stopping/README.md b/experiments/adaptive-stopping/README.md index 006bbdc..162719d 100644 --- a/experiments/adaptive-stopping/README.md +++ b/experiments/adaptive-stopping/README.md @@ -1,8 +1,9 @@ # Adaptive stopping counterfactual replay **Recommendation: keep targeted allocation experimental and do not run a live trial.** Follow-up -replays tested stronger adaptive warm-ups/exploration and fixed outcome-independent designs over -five allocation seeds. No design passed the predeclared fidelity/savings gate robustly. +replays tested stronger adaptive allocation, fixed outcome-independent designs, and six published +boards spanning four collections and three Rubenstein judges. There is no collection-robust basis +for a production change, especially for large model grids. The opt-in targeted allocator delivers the expected call reduction, but not by stopping early: it uses all 10 five-sample rounds while selecting fewer pairs per round. Against the full stored @@ -242,6 +243,85 @@ but does not make a small board equivalent to the full one. The fixed designs al pair-count balance alone is insufficient; preserving a common balanced warm-up made top-3 membership much more stable than independently sampling each pair. +## Cross-collection, page-clustered replay + +The fixed designs were then replayed on six suitable published boards without new judge calls. +Revisions are pinned in [`multi_board.py`](multi_board.py). + +| Board | Collection/role | Outcomes | Models | Samples | +|---|---|---:|---:|---:| +| Rubenstein 30B | independent manuscript-card collection | 299 | 4 | 50 | +| Rubenstein jury | same collection, judge sensitivity | 300 | 4 | 50 | +| Rubenstein Kimi | same collection, judge sensitivity | 300 | 4 | 50 | +| UFO 30B | independent collection | 294 | 4 | 49 | +| BPL | independent, incomplete board | 147 | 4 | 41 | +| Britannica Qwen35 | same corpus, earlier 6-model/judge sensitivity | 720 | 6 | 50 | + +Rubenstein's three judges are not counted as three independent collections. This gives four +collection groups, but only the original Britannica board has a 14-model graph. + +### Uncertainty method + +For each board, design, budget fraction (25%, 40%, 60%), and allocation seed (42–46), the replay: + +1. resamples complete `sample_idx` page clusters 200 times; +2. refits the full board inside each page bootstrap; +3. reruns the fixed allocation inside that same replicate; and +4. compares the selected board with its paired bootstrap full board. + +This is 1,000 design-aware page-bootstrap evaluations per board/design/budget. A full-board pair is +called robust only when its direction repeats in at least 95% of full-board page bootstraps. The +cross-board criteria were predeclared as: + +- clustered top-3 membership agreement ≥90%; +- agreement on robust full-board pairs ≥95%; and +- median clustered Spearman ρ ≥0.90. + +Exact ordering among unresolved models and absolute ELO drift are still reported, but are not used +as pass criteria after the first replay showed that they can be unstable even for the full board. + +### Full-board stability ceiling + +| Board | Robust pairs | Possible | Full-board top-3 set stability | Full-board top-3 order stability | +|---|---:|---:|---:|---:| +| Rubenstein 30B | 4 | 6 | 100.0% | 65.5% | +| Rubenstein jury | 3 | 6 | 83.0% | 46.5% | +| Rubenstein Kimi | 5 | 6 | 99.5% | 89.0% | +| UFO 30B | 5 | 6 | 100.0% | 84.0% | +| BPL | 4 | 6 | 52.0% | 28.5% | +| Britannica Qwen35 | 13 | 15 | 100.0% | 65.0% | + +BPL and the Rubenstein jury board cannot meet a 90% top-3 stability target reliably even when all +stored outcomes are used. Their failures below are therefore evidence that the reference board is +under-resolved, not simply that subsampling is bad. + +### Results at a 40% outcome budget + +| Board | Pair-balanced | Mixed-random | Clustered top-3 rate (pair/mixed) | Robust-pair agreement (pair/mixed) | +|---|---|---|---:|---:| +| Rubenstein 30B | pass | pass | 98.4% / 98.6% | 97.6% / 97.7% | +| Rubenstein jury | fail | fail | 79.4% / 77.2% | 97.8% / 97.4% | +| Rubenstein Kimi | pass | pass | 97.6% / 98.0% | 97.5% / 97.4% | +| UFO 30B | pass | pass | 99.9% / 99.6% | 100.0% / 99.9% | +| BPL | fail | fail | 65.4% / 62.2% | 86.9% / 85.9% | +| Britannica Qwen35 | pass | pass | 98.8% / 98.9% | 98.3% / 98.3% | + +At 40%, both fixed designs pass on Rubenstein 30B, Rubenstein Kimi, UFO, and the 6-model +Britannica board; both fail on the intrinsically unstable Rubenstein jury and BPL boards. At 25%, +results are more collection-dependent. Increasing to 60% still cannot rescue unstable reference +boards. + +There is no consistent winner between pair-balanced and mixed-random allocation on these small +model grids. Board/judge stability matters more than the choice between the two fixed selectors. +Small samples also produce heavy ELO tails—Britannica Qwen35 has extreme page-bootstrap ELO +outliers despite stable rank decisions—so rank/pairwise summaries are safer than treating ELO +error as approximately Gaussian. + +This broadens the evidence, but does not validate large-grid behavior: five of the six additional +boards have only four models, and the sixth has six. See +[`multi-board-summary.csv`](multi-board-summary.csv) and +[`multi-board-results.json`](multi-board-results.json). + ## Current sentinel-policy robustness check The primary result must use the requested 4,293-row stored board. However, current production @@ -287,38 +367,45 @@ collection. 2. **Outcome-conditioned sampling.** Later targeted pairs depend on interim outcomes and ranks. Ordinary percentile bootstrap CIs computed after selection do not account for that adaptive, optional-stopping process and may be optimistic. -3. **The bootstrap does not replay the policy.** Production resamples selected comparisons as if - the selected set were fixed. It does not re-run allocation inside each bootstrap replicate. -4. **Comparison-level resampling ignores page clustering.** Many pair outcomes share one page and - are correlated. A page/round-clustered bootstrap would better match the sampling unit. +3. **Production uncertainty does not replay the policy.** Production resamples selected + comparisons as if the selected set were fixed. The cross-board experiment reruns fixed + allocation inside page bootstraps, but this is not yet a production CI method for targeted + optional stopping. +4. **Production comparison-level resampling ignores page clustering.** Many pair outcomes share + one page and are correlated. The cross-board follow-up addresses this experimentally; ordinary + published CIs still do not. 5. **Bradley–Terry fit under sparse, nonuniform allocation.** The graph is connected, so a fit is identifiable, but adjacency-driven edge counts can amplify model misspecification and shift the global ELO scale relative to a balanced grid. -6. **One collection and one stored judge run.** Britannica cannot establish general behavior for - manuscripts, tables, noisy scans, or other judges. +6. **Large-grid evidence remains one collection.** The cross-board follow-up adds collections and + judges, but its additional grids have only four or six models. It cannot establish sparse + allocation behavior for another 10+ model board. 7. **Unobserved failed judge calls.** Six attempted comparisons have no published valid verdict. The replay cannot know how a different allocation would have distributed those failures, so savings are relative to 4,293 valid stored outcomes rather than the 4,299 attempted calls. 8. **Historical sentinel rows.** The primary board predates the current exclusion policy; the robustness replay is closer to current input integrity semantics but has only 13 rankable models. +9. **Legacy cross-board schemas.** Several older boards do not store OCR text, so historical + sentinel rows cannot be rechecked. Five of the six additional boards have only four models, + limiting what they say about sparse large-grid allocation. +10. **Judge variants are not independent data.** The three Rubenstein boards measure sensitivity + to judge choice on the same pages; they do not triple the collection-level evidence. ## Recommendation and next experiment **Keep `targeted` opt-in, retain `balanced` as the default, and do not implement the tested v2 or -fixed designs in production.** Neither threshold tuning nor the tested outcome-independent designs -passed the predeclared gate robustly. - -Do not tune further on Britannica alone. The next useful work is methodological validation: +fixed designs in production.** The cross-collection replay shows that fixed 40% subsampling can +preserve robust decisions on already-stable 4–6 model boards, but cannot repair an unstable full +board and does not test a large independent model grid. -1. replay the mixed-random design on several independent published boards; -2. predeclare a success criterion based on top-k membership and pairwise decisions, separately - from exact ordering among statistically unresolved models; -3. use page-clustered/design-aware uncertainty and report allocation-seed sensitivity; and -4. consider one live trial only if a design passes across collections and seeds. +The evidence is now sufficient to stop retrospective threshold tuning. Before any live adaptive +trial, obtain at least one additional independent, fully stored board with roughly 10+ models and +50+ pages, then apply the same page-clustered/design-aware protocol. If that passes across seeds, +a budget-capped live trial becomes defensible. -Keep the 3× rule as a post-hoc deployment annotation only. This board does not support letting it -influence sampling, making it a default, or treating it as statistical resolution. +Keep the 3× rule as a post-hoc deployment annotation only. Current evidence does not support +letting it influence sampling, making it a default, or treating it as statistical resolution. ## Reproduce @@ -334,6 +421,12 @@ machine used for this report. The pinned dataset revision is downloaded read-only. The script contains no judge backend construction and no Hub push/upload call. +Cross-collection page-clustered replay: + +```bash +uv run python experiments/adaptive-stopping/multi_board.py +``` + Optional current-sentinel-policy replay: ```bash @@ -350,12 +443,14 @@ Generated artifacts: - [`elo-deltas.csv`](elo-deltas.csv): per-model ELO and deltas - [`round-history.csv`](round-history.csv): batch-by-batch allocation and decisions - [`static-design-summary.csv`](static-design-summary.csv): five-seed fixed-design aggregates +- [`multi-board-summary.csv`](multi-board-summary.csv): cross-board design-aware aggregates +- [`multi-board-results.json`](multi-board-results.json): pinned board and page-bootstrap details - [`results-sentinels-excluded.json`](results-sentinels-excluded.json): robustness replay Validation commands: ```bash uv run pytest tests/ -q -uv run ruff check src/ tests/ experiments/adaptive-stopping/replay.py -uv run ty check src/ experiments/adaptive-stopping/replay.py +uv run ruff check src/ tests/ experiments/adaptive-stopping/*.py +uv run ty check src/ experiments/adaptive-stopping/*.py ``` diff --git a/experiments/adaptive-stopping/multi-board-results.json b/experiments/adaptive-stopping/multi-board-results.json new file mode 100644 index 0000000..5350d29 --- /dev/null +++ b/experiments/adaptive-stopping/multi-board-results.json @@ -0,0 +1,11115 @@ +{ + "aggregates": [ + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 50.96302445946719, + "clustered_max_absolute_elo_delta_p95": 103.04616386353707, + "clustered_robust_pair_agreement": 0.95625, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.958, + "clustered_topk_order_rate": 0.465, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 74.91638795986623, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.3333333333333334, + "point_max_absolute_elo_delta_median": 74.9963598224906, + "point_max_absolute_elo_delta_worst": 88.74240676353293, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 36.602588143273124, + "clustered_max_absolute_elo_delta_p95": 68.9428597271016, + "clustered_robust_pair_agreement": 0.97575, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.984, + "clustered_topk_order_rate": 0.588, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 20, + "percentage_saved": 59.86622073578595, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 35.36914634787831, + "point_max_absolute_elo_delta_worst": 48.86025867979356, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 24.451492199466998, + "clustered_max_absolute_elo_delta_p95": 45.8823914245234, + "clustered_robust_pair_agreement": 0.989, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.999, + "clustered_topk_order_rate": 0.711, + "collection": "rubenstein", + "comparisons_consumed": 179, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 29, + "percentage_saved": 40.13377926421405, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 18.229069125843807, + "point_max_absolute_elo_delta_worst": 33.695085439944705, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 55.30934564765391, + "clustered_max_absolute_elo_delta_p95": 106.11305957921516, + "clustered_robust_pair_agreement": 0.95, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.942, + "clustered_topk_order_rate": 0.454, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 74.91638795986623, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.3333333333333334, + "point_max_absolute_elo_delta_median": 66.87711490012998, + "point_max_absolute_elo_delta_worst": 79.71119499145493, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 35.961371098299765, + "clustered_max_absolute_elo_delta_p95": 71.70864575790696, + "clustered_robust_pair_agreement": 0.9765, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.986, + "clustered_topk_order_rate": 0.62, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 20, + "percentage_saved": 59.86622073578595, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 23.566703087771884, + "point_max_absolute_elo_delta_worst": 30.636759160081056, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 24.407175723937144, + "clustered_max_absolute_elo_delta_p95": 45.83763748606696, + "clustered_robust_pair_agreement": 0.9885, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.997, + "clustered_topk_order_rate": 0.733, + "collection": "rubenstein", + "comparisons_consumed": 179, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 29, + "percentage_saved": 40.13377926421405, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 20.67613087664313, + "point_max_absolute_elo_delta_worst": 37.80833728586413, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 51.30851618298459, + "clustered_max_absolute_elo_delta_p95": 99.7740401556695, + "clustered_robust_pair_agreement": 0.9546666666666667, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.686, + "clustered_topk_order_rate": 0.392, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.0, + "point_max_absolute_elo_delta_median": 55.61182715990776, + "point_max_absolute_elo_delta_worst": 109.97593752586045, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 35.47060310551785, + "clustered_max_absolute_elo_delta_p95": 70.45446369737978, + "clustered_robust_pair_agreement": 0.9783333333333333, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.794, + "clustered_topk_order_rate": 0.54, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 20, + "percentage_saved": 60.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 38.62623546994405, + "point_max_absolute_elo_delta_worst": 46.13277438995215, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 3, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 24.519517490073326, + "clustered_max_absolute_elo_delta_p95": 46.29281326901969, + "clustered_robust_pair_agreement": 0.989, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.862, + "clustered_topk_order_rate": 0.635, + "collection": "rubenstein", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 30, + "percentage_saved": 40.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 19.091514331410053, + "point_max_absolute_elo_delta_worst": 30.219231877873426, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 53.2623958108918, + "clustered_max_absolute_elo_delta_p95": 105.56940869125386, + "clustered_robust_pair_agreement": 0.9543333333333334, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.682, + "clustered_topk_order_rate": 0.39, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.0, + "point_max_absolute_elo_delta_median": 75.99679890013772, + "point_max_absolute_elo_delta_worst": 122.69516744138991, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 4, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.8333333333333335, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 37.201976556154364, + "clustered_max_absolute_elo_delta_p95": 72.11933690476818, + "clustered_robust_pair_agreement": 0.9743333333333334, + "clustered_spearman_rho_median": 0.8999999999999999, + "clustered_topk_membership_rate": 0.772, + "clustered_topk_order_rate": 0.5, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 20, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 36.97416065070661, + "point_max_absolute_elo_delta_worst": 54.29620329625482, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 24.640067470120698, + "clustered_max_absolute_elo_delta_p95": 46.53445768533318, + "clustered_robust_pair_agreement": 0.9906666666666667, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.847, + "clustered_topk_order_rate": 0.617, + "collection": "rubenstein", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 30, + "percentage_saved": 40.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 28.88916155443235, + "point_max_absolute_elo_delta_worst": 31.404853527182922, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 48.71634774955521, + "clustered_max_absolute_elo_delta_p95": 93.4917081131644, + "clustered_robust_pair_agreement": 0.954, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.941, + "clustered_topk_order_rate": 0.643, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 50.11338786679789, + "point_max_absolute_elo_delta_worst": 84.65205601294406, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 34.11168897934829, + "clustered_max_absolute_elo_delta_p95": 64.60510410890352, + "clustered_robust_pair_agreement": 0.9748, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.976, + "clustered_topk_order_rate": 0.747, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 20, + "percentage_saved": 60.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 35.743531490613805, + "point_max_absolute_elo_delta_worst": 42.450819019925575, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 22.381277927089627, + "clustered_max_absolute_elo_delta_p95": 43.52128846166924, + "clustered_robust_pair_agreement": 0.9848, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.988, + "clustered_topk_order_rate": 0.852, + "collection": "rubenstein", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 30, + "percentage_saved": 40.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 21.729922410336485, + "point_max_absolute_elo_delta_worst": 33.72959539668955, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 51.562024031424585, + "clustered_max_absolute_elo_delta_p95": 101.30568013207362, + "clustered_robust_pair_agreement": 0.9522, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.942, + "clustered_topk_order_rate": 0.619, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 34.54019650818395, + "point_max_absolute_elo_delta_worst": 46.73325502405123, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 35.019524268701616, + "clustered_max_absolute_elo_delta_p95": 69.11168669013173, + "clustered_robust_pair_agreement": 0.9744, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.98, + "clustered_topk_order_rate": 0.742, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 20, + "percentage_saved": 60.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 26.616486977659633, + "point_max_absolute_elo_delta_worst": 61.182980320161505, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 22.555386461412695, + "clustered_max_absolute_elo_delta_p95": 42.960206326319735, + "clustered_robust_pair_agreement": 0.9866, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.993, + "clustered_topk_order_rate": 0.851, + "collection": "rubenstein", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 30, + "percentage_saved": 40.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 17.935093566857404, + "point_max_absolute_elo_delta_worst": 41.47187936807018, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 66.36513865100414, + "clustered_max_absolute_elo_delta_p95": 170.6679118820609, + "clustered_robust_pair_agreement": 0.9982, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.992, + "clustered_topk_order_rate": 0.775, + "collection": "ufo", + "comparisons_consumed": 74, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 74.82993197278911, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 101.71601111147652, + "point_max_absolute_elo_delta_worst": 119.56988418080346, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 46.51917839657665, + "clustered_max_absolute_elo_delta_p95": 101.50165752723015, + "clustered_robust_pair_agreement": 0.9998, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.999, + "clustered_topk_order_rate": 0.83, + "collection": "ufo", + "comparisons_consumed": 118, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 19, + "percentage_saved": 59.863945578231295, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 47.13600047692353, + "point_max_absolute_elo_delta_worst": 83.49166477060635, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 31.45359082816958, + "clustered_max_absolute_elo_delta_p95": 62.10565127428381, + "clustered_robust_pair_agreement": 1, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 1, + "clustered_topk_order_rate": 0.869, + "collection": "ufo", + "comparisons_consumed": 176, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 29, + "percentage_saved": 40.136054421768705, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 29.467658868230956, + "point_max_absolute_elo_delta_worst": 34.09709501093266, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 69.87977884241309, + "clustered_max_absolute_elo_delta_p95": 166.35690358458058, + "clustered_robust_pair_agreement": 0.9968, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.985, + "clustered_topk_order_rate": 0.737, + "collection": "ufo", + "comparisons_consumed": 74, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 74.82993197278911, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 78.64119881236093, + "point_max_absolute_elo_delta_worst": 120.47604998638326, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 48.52463011461816, + "clustered_max_absolute_elo_delta_p95": 110.365475846818, + "clustered_robust_pair_agreement": 0.9992, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.996, + "clustered_topk_order_rate": 0.788, + "collection": "ufo", + "comparisons_consumed": 118, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 19, + "percentage_saved": 59.863945578231295, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 51.53419474381826, + "point_max_absolute_elo_delta_worst": 77.55657146009776, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 31.05830195373278, + "clustered_max_absolute_elo_delta_p95": 65.8193329542637, + "clustered_robust_pair_agreement": 1, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 1, + "clustered_topk_order_rate": 0.863, + "collection": "ufo", + "comparisons_consumed": 176, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 29, + "percentage_saved": 40.136054421768705, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 42.07911792124605, + "point_max_absolute_elo_delta_worst": 51.958382413868776, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 73.5964697110536, + "clustered_max_absolute_elo_delta_p95": 153.5851477572523, + "clustered_robust_pair_agreement": 0.81725, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.578, + "clustered_topk_order_rate": 0.22, + "collection": "bpl", + "comparisons_consumed": 37, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 6, + "percentage_saved": 74.82993197278911, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.3333333333333334, + "point_kendall_tau_min": 0.0, + "point_max_absolute_elo_delta_median": 70.72228974130076, + "point_max_absolute_elo_delta_worst": 99.62849649893337, + "point_spearman_rho_median": 0.6000000000000001, + "point_topk_membership_matches": 0, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 49.828543699435045, + "clustered_max_absolute_elo_delta_p95": 102.8327524945587, + "clustered_robust_pair_agreement": 0.869, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.654, + "clustered_topk_order_rate": 0.349, + "collection": "bpl", + "comparisons_consumed": 59, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 9, + "percentage_saved": 59.863945578231295, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.0, + "point_max_absolute_elo_delta_median": 45.03767776309405, + "point_max_absolute_elo_delta_worst": 84.52832892210927, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 0, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 33.19894047796083, + "clustered_max_absolute_elo_delta_p95": 62.93854394091554, + "clustered_robust_pair_agreement": 0.9105, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.737, + "clustered_topk_order_rate": 0.477, + "collection": "bpl", + "comparisons_consumed": 88, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 14, + "percentage_saved": 40.136054421768705, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.3333333333333334, + "point_max_absolute_elo_delta_median": 24.463256848546962, + "point_max_absolute_elo_delta_worst": 83.92125755287248, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 1, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": -0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 83.57982190290431, + "clustered_max_absolute_elo_delta_p95": 172.00152817247883, + "clustered_robust_pair_agreement": 0.8, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.549, + "clustered_topk_order_rate": 0.226, + "collection": "bpl", + "comparisons_consumed": 37, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 6, + "percentage_saved": 74.82993197278911, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 64.81689426010917, + "point_max_absolute_elo_delta_worst": 89.13457286674588, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 55.26643688568504, + "clustered_max_absolute_elo_delta_p95": 105.22972942394816, + "clustered_robust_pair_agreement": 0.85925, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.622, + "clustered_topk_order_rate": 0.32, + "collection": "bpl", + "comparisons_consumed": 59, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 9, + "percentage_saved": 59.863945578231295, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 39.9479134020994, + "point_max_absolute_elo_delta_worst": 93.26201785241119, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 3, + "point_topk_order_matches": 2, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 34.53789864225064, + "clustered_max_absolute_elo_delta_p95": 67.80927819317637, + "clustered_robust_pair_agreement": 0.90975, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.752, + "clustered_topk_order_rate": 0.489, + "collection": "bpl", + "comparisons_consumed": 88, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 14, + "percentage_saved": 40.136054421768705, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 29.05276625236911, + "point_max_absolute_elo_delta_worst": 72.53660288857373, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 3, + "point_topk_order_matches": 3, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.8666666666666666, + "clustered_kendall_tau_p05": 0.7333333333333333, + "clustered_max_absolute_elo_delta_median": 81.15453200095465, + "clustered_max_absolute_elo_delta_p95": 2306.972897111449, + "clustered_robust_pair_agreement": 0.9755384615384616, + "clustered_spearman_rho_median": 0.942857142857143, + "clustered_topk_membership_rate": 0.971, + "clustered_topk_order_rate": 0.488, + "collection": "britannica", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.9999999999999999, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 61.599642638423575, + "point_max_absolute_elo_delta_worst": 86.35632964392971, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 3, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.9999999999999999, + "clustered_kendall_tau_p05": 0.7333333333333333, + "clustered_max_absolute_elo_delta_median": 58.05154309002893, + "clustered_max_absolute_elo_delta_p95": 197.08019997153377, + "clustered_robust_pair_agreement": 0.9830769230769231, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.988, + "clustered_topk_order_rate": 0.565, + "collection": "britannica", + "comparisons_consumed": 288, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 19, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.9999999999999999, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 52.83123987349245, + "point_max_absolute_elo_delta_worst": 69.15859061734136, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 3, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.9999999999999999, + "clustered_kendall_tau_p05": 0.8666666666666666, + "clustered_max_absolute_elo_delta_median": 37.81826912701297, + "clustered_max_absolute_elo_delta_p95": 97.80114615896461, + "clustered_robust_pair_agreement": 0.9895384615384616, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.993, + "clustered_topk_order_rate": 0.713, + "collection": "britannica", + "comparisons_consumed": 432, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 28, + "percentage_saved": 40.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.9999999999999999, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 32.11192697586705, + "point_max_absolute_elo_delta_worst": 60.004263291922484, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.8666666666666666, + "clustered_kendall_tau_p05": 0.6, + "clustered_max_absolute_elo_delta_median": 91.67612246683814, + "clustered_max_absolute_elo_delta_p95": 2348.1679073592472, + "clustered_robust_pair_agreement": 0.9699230769230769, + "clustered_spearman_rho_median": 0.942857142857143, + "clustered_topk_membership_rate": 0.956, + "clustered_topk_order_rate": 0.418, + "collection": "britannica", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.8666666666666666, + "point_kendall_tau_min": 0.7333333333333333, + "point_max_absolute_elo_delta_median": 116.18768538959603, + "point_max_absolute_elo_delta_worst": 139.56132398511545, + "point_spearman_rho_median": 0.942857142857143, + "point_topk_membership_matches": 4, + "point_topk_order_matches": 1, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.9999999999999999, + "clustered_kendall_tau_p05": 0.7333333333333333, + "clustered_max_absolute_elo_delta_median": 60.51017695201898, + "clustered_max_absolute_elo_delta_p95": 197.6989721651328, + "clustered_robust_pair_agreement": 0.9826923076923078, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.989, + "clustered_topk_order_rate": 0.551, + "collection": "britannica", + "comparisons_consumed": 288, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 19, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.8666666666666666, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 57.354151366056385, + "point_max_absolute_elo_delta_worst": 82.20896779061877, + "point_spearman_rho_median": 0.942857142857143, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 2, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.9999999999999999, + "clustered_kendall_tau_p05": 0.8666666666666666, + "clustered_max_absolute_elo_delta_median": 39.35275643631411, + "clustered_max_absolute_elo_delta_p95": 105.36616237876325, + "clustered_robust_pair_agreement": 0.9906153846153847, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.999, + "clustered_topk_order_rate": 0.722, + "collection": "britannica", + "comparisons_consumed": 432, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 28, + "percentage_saved": 40.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.8666666666666666, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 56.88351052438202, + "point_max_absolute_elo_delta_worst": 67.21927176824147, + "point_spearman_rho_median": 0.942857142857143, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 2, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + } + ], + "boards": [ + { + "aggregates": [ + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 50.96302445946719, + "clustered_max_absolute_elo_delta_p95": 103.04616386353707, + "clustered_robust_pair_agreement": 0.95625, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.958, + "clustered_topk_order_rate": 0.465, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 74.91638795986623, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.3333333333333334, + "point_max_absolute_elo_delta_median": 74.9963598224906, + "point_max_absolute_elo_delta_worst": 88.74240676353293, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 36.602588143273124, + "clustered_max_absolute_elo_delta_p95": 68.9428597271016, + "clustered_robust_pair_agreement": 0.97575, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.984, + "clustered_topk_order_rate": 0.588, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 20, + "percentage_saved": 59.86622073578595, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 35.36914634787831, + "point_max_absolute_elo_delta_worst": 48.86025867979356, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 24.451492199466998, + "clustered_max_absolute_elo_delta_p95": 45.8823914245234, + "clustered_robust_pair_agreement": 0.989, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.999, + "clustered_topk_order_rate": 0.711, + "collection": "rubenstein", + "comparisons_consumed": 179, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 29, + "percentage_saved": 40.13377926421405, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 18.229069125843807, + "point_max_absolute_elo_delta_worst": 33.695085439944705, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 55.30934564765391, + "clustered_max_absolute_elo_delta_p95": 106.11305957921516, + "clustered_robust_pair_agreement": 0.95, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.942, + "clustered_topk_order_rate": 0.454, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 74.91638795986623, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.3333333333333334, + "point_max_absolute_elo_delta_median": 66.87711490012998, + "point_max_absolute_elo_delta_worst": 79.71119499145493, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 35.961371098299765, + "clustered_max_absolute_elo_delta_p95": 71.70864575790696, + "clustered_robust_pair_agreement": 0.9765, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.986, + "clustered_topk_order_rate": 0.62, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 20, + "percentage_saved": 59.86622073578595, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 23.566703087771884, + "point_max_absolute_elo_delta_worst": 30.636759160081056, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-30b", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 24.407175723937144, + "clustered_max_absolute_elo_delta_p95": 45.83763748606696, + "clustered_robust_pair_agreement": 0.9885, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.997, + "clustered_topk_order_rate": 0.733, + "collection": "rubenstein", + "comparisons_consumed": 179, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 29, + "percentage_saved": 40.13377926421405, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 20.67613087664313, + "point_max_absolute_elo_delta_worst": 37.80833728586413, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection" + } + ], + "collection": "rubenstein", + "full_board_page_bootstrap_stability": { + "kendall_tau_median": 1.0, + "kendall_tau_p05": 0.6666666666666669, + "topk_membership_rate": 1, + "topk_order_rate": 0.655 + }, + "full_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "models": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "name": "rubenstein-30b", + "repo": "davanstrien/ocr-bench-rubenstein-judge-30b", + "revision": "8559f10e71d77d65d56169d4261b7c9ac25bebb3", + "robust_pairs": [ + [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr" + ], + [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + [ + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + [ + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "role": "independent collection", + "sample_count": 50, + "seed_results": [ + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.96, + "topk_membership_rate": 0.97 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 42.12007625458705, + "median_absolute_elo_delta": 34.41249433421467, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.95375, + "topk_membership_rate": 0.955 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 77.40046310361913, + "median_absolute_elo_delta": 63.49709171452503, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.955, + "topk_membership_rate": 0.955 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 88.74240676353293, + "median_absolute_elo_delta": 44.55391694088553, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.95625, + "topk_membership_rate": 0.965 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 74.9963598224906, + "median_absolute_elo_delta": 41.41423474057069, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "rednote-hilab/dots.ocr", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.39999999999999997, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.95625, + "topk_membership_rate": 0.945 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 47.39994204015807, + "median_absolute_elo_delta": 31.356516650440994, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "rednote-hilab/dots.ocr", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.39999999999999997, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.985, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 21.985972827338628, + "median_absolute_elo_delta": 18.875147748953395, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9725, + "topk_membership_rate": 0.98 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 35.80750912138046, + "median_absolute_elo_delta": 20.5702141908356, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.97375, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 48.86025867979356, + "median_absolute_elo_delta": 22.97177734957438, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.97, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 35.36914634787831, + "median_absolute_elo_delta": 14.69420453928899, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9775, + "topk_membership_rate": 0.98 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 27.08475495205471, + "median_absolute_elo_delta": 19.012395236939824, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.99625, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 179, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 9.173230799901376, + "median_absolute_elo_delta": 6.618613434107601, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.98625, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 179, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 16.724711686402998, + "median_absolute_elo_delta": 6.367457624195595, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.99, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 179, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 18.229069125843807, + "median_absolute_elo_delta": 8.192319748855425, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.985, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 179, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 31.64918392192112, + "median_absolute_elo_delta": 17.188708097585277, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9875, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 179, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 33.695085439944705, + "median_absolute_elo_delta": 11.972614101726322, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9525, + "topk_membership_rate": 0.935 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 66.87711490012998, + "median_absolute_elo_delta": 38.625601029998506, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.95125, + "topk_membership_rate": 0.945 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 75.66027152604283, + "median_absolute_elo_delta": 33.76961888807523, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "rednote-hilab/dots.ocr", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.39999999999999997, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.95625, + "topk_membership_rate": 0.95 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 33.267511918450964, + "median_absolute_elo_delta": 24.004311213437177, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9375, + "topk_membership_rate": 0.925 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 24.3459192100554, + "median_absolute_elo_delta": 11.637639030224705, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9525, + "topk_membership_rate": 0.955 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 79.71119499145493, + "median_absolute_elo_delta": 31.44495406704675, + "percentage_saved": 74.91638795986623, + "rank_order": [ + "rednote-hilab/dots.ocr", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.39999999999999997, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.97875, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 16.74331957555478, + "median_absolute_elo_delta": 8.105839102645518, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.98, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 30.636759160081056, + "median_absolute_elo_delta": 15.858040646640461, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9825, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 20.654597900792623, + "median_absolute_elo_delta": 15.970740693349967, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.97125, + "topk_membership_rate": 0.975 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 24.209276755999326, + "median_absolute_elo_delta": 16.03619440058708, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.97, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 23.566703087771884, + "median_absolute_elo_delta": 11.164057599178818, + "percentage_saved": 59.86622073578595, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.98875, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 179, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 32.28776591470432, + "median_absolute_elo_delta": 20.116895232991624, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.98875, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 179, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 7.209449496398975, + "median_absolute_elo_delta": 5.018667765858709, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.98875, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 179, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 20.67613087664313, + "median_absolute_elo_delta": 10.914752173141096, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.99, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 179, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 37.80833728586413, + "median_absolute_elo_delta": 17.366362855080297, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.98625, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 179, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 4.43249273288302, + "median_absolute_elo_delta": 3.4461921789153394, + "percentage_saved": 40.13377926421405, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + } + ], + "sentinel_comparisons_detectable": 0, + "stored_comparisons": 299 + }, + { + "aggregates": [ + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 51.30851618298459, + "clustered_max_absolute_elo_delta_p95": 99.7740401556695, + "clustered_robust_pair_agreement": 0.9546666666666667, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.686, + "clustered_topk_order_rate": 0.392, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.0, + "point_max_absolute_elo_delta_median": 55.61182715990776, + "point_max_absolute_elo_delta_worst": 109.97593752586045, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 35.47060310551785, + "clustered_max_absolute_elo_delta_p95": 70.45446369737978, + "clustered_robust_pair_agreement": 0.9783333333333333, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.794, + "clustered_topk_order_rate": 0.54, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 20, + "percentage_saved": 60.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 38.62623546994405, + "point_max_absolute_elo_delta_worst": 46.13277438995215, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 3, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 24.519517490073326, + "clustered_max_absolute_elo_delta_p95": 46.29281326901969, + "clustered_robust_pair_agreement": 0.989, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.862, + "clustered_topk_order_rate": 0.635, + "collection": "rubenstein", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 30, + "percentage_saved": 40.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 19.091514331410053, + "point_max_absolute_elo_delta_worst": 30.219231877873426, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 53.2623958108918, + "clustered_max_absolute_elo_delta_p95": 105.56940869125386, + "clustered_robust_pair_agreement": 0.9543333333333334, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.682, + "clustered_topk_order_rate": 0.39, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.0, + "point_max_absolute_elo_delta_median": 75.99679890013772, + "point_max_absolute_elo_delta_worst": 122.69516744138991, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 4, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.8333333333333335, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 37.201976556154364, + "clustered_max_absolute_elo_delta_p95": 72.11933690476818, + "clustered_robust_pair_agreement": 0.9743333333333334, + "clustered_spearman_rho_median": 0.8999999999999999, + "clustered_topk_membership_rate": 0.772, + "clustered_topk_order_rate": 0.5, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 20, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 36.97416065070661, + "point_max_absolute_elo_delta_worst": 54.29620329625482, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-jury", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 24.640067470120698, + "clustered_max_absolute_elo_delta_p95": 46.53445768533318, + "clustered_robust_pair_agreement": 0.9906666666666667, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.847, + "clustered_topk_order_rate": 0.617, + "collection": "rubenstein", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 30, + "percentage_saved": 40.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 28.88916155443235, + "point_max_absolute_elo_delta_worst": 31.404853527182922, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 3, + "role": "judge sensitivity" + } + ], + "collection": "rubenstein", + "full_board_page_bootstrap_stability": { + "kendall_tau_median": 0.6666666666666669, + "kendall_tau_p05": 0.3333333333333334, + "topk_membership_rate": 0.83, + "topk_order_rate": 0.465 + }, + "full_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "models": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "name": "rubenstein-jury", + "repo": "davanstrien/ocr-bench-rubenstein-judge", + "revision": "62e579c82f2713a58156b8e4248fb54a070ae110", + "robust_pairs": [ + [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr" + ], + [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ] + ], + "role": "judge sensitivity", + "sample_count": 50, + "seed_results": [ + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.965, + "topk_membership_rate": 0.695 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.0, + "max_absolute_elo_delta": 109.97593752586045, + "median_absolute_elo_delta": 70.7785818667287, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.19999999999999998, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.945, + "topk_membership_rate": 0.67 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 53.0723297743757, + "median_absolute_elo_delta": 26.79907949259143, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9583333333333334, + "topk_membership_rate": 0.705 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 55.61182715990776, + "median_absolute_elo_delta": 25.876334738868195, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9533333333333334, + "topk_membership_rate": 0.685 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 47.661187856316246, + "median_absolute_elo_delta": 28.193222497089664, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9516666666666667, + "topk_membership_rate": 0.675 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 89.51364074556886, + "median_absolute_elo_delta": 37.908546257392004, + "percentage_saved": 75.0, + "rank_order": [ + "rednote-hilab/dots.ocr", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 0.6666666666666666, + "spearman_rho": 0.39999999999999997, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9883333333333333, + "topk_membership_rate": 0.805 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 27.755650233788174, + "median_absolute_elo_delta": 15.022699030953277, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9783333333333333, + "topk_membership_rate": 0.8 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 38.62623546994405, + "median_absolute_elo_delta": 22.977681377225963, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9716666666666667, + "topk_membership_rate": 0.775 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 46.13277438995215, + "median_absolute_elo_delta": 24.198122647804553, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9816666666666667, + "topk_membership_rate": 0.81 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 36.46205559133455, + "median_absolute_elo_delta": 17.49636667786615, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9716666666666667, + "topk_membership_rate": 0.78 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 45.26733539337215, + "median_absolute_elo_delta": 18.37219097340062, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9933333333333333, + "topk_membership_rate": 0.83 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 10.905725049991133, + "median_absolute_elo_delta": 8.533384091335051, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.99, + "topk_membership_rate": 0.825 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 19.091514331410053, + "median_absolute_elo_delta": 9.679462459947786, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9866666666666667, + "topk_membership_rate": 0.885 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 24.1653770599969, + "median_absolute_elo_delta": 11.581824494528746, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9883333333333333, + "topk_membership_rate": 0.895 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 13.166366459681967, + "median_absolute_elo_delta": 6.732461558812588, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9866666666666667, + "topk_membership_rate": 0.875 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 30.219231877873426, + "median_absolute_elo_delta": 15.8937646385682, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.955, + "topk_membership_rate": 0.73 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 122.69516744138991, + "median_absolute_elo_delta": 49.33031876466282, + "percentage_saved": 75.0, + "rank_order": [ + "rednote-hilab/dots.ocr", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 0.6666666666666666, + "spearman_rho": 0.39999999999999997, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9566666666666667, + "topk_membership_rate": 0.68 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 54.82612493313468, + "median_absolute_elo_delta": 23.008260733888505, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9433333333333334, + "topk_membership_rate": 0.64 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.0, + "max_absolute_elo_delta": 60.83911537193217, + "median_absolute_elo_delta": 45.3869256533859, + "percentage_saved": 75.0, + "rank_order": [ + "rednote-hilab/dots.ocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 0.6666666666666666, + "spearman_rho": 0.0, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9516666666666667, + "topk_membership_rate": 0.685 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 101.10269599499316, + "median_absolute_elo_delta": 55.24998981728959, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.965, + "topk_membership_rate": 0.675 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 75.99679890013772, + "median_absolute_elo_delta": 26.854786600916896, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.97, + "topk_membership_rate": 0.82 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 36.123876840815, + "median_absolute_elo_delta": 21.882129396438472, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9833333333333333, + "topk_membership_rate": 0.795 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 36.97416065070661, + "median_absolute_elo_delta": 26.355940955344977, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9716666666666667, + "topk_membership_rate": 0.745 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 35.84936661097504, + "median_absolute_elo_delta": 19.260178365601632, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9716666666666667, + "topk_membership_rate": 0.74 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 45.47137265037486, + "median_absolute_elo_delta": 28.58927603148038, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.975, + "topk_membership_rate": 0.76 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 54.29620329625482, + "median_absolute_elo_delta": 26.560435949146154, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9883333333333333, + "topk_membership_rate": 0.88 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 31.404853527182922, + "median_absolute_elo_delta": 15.453290915450452, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9916666666666667, + "topk_membership_rate": 0.85 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 19.168701268640916, + "median_absolute_elo_delta": 11.809792452890292, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9916666666666667, + "topk_membership_rate": 0.825 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 30.125013718043874, + "median_absolute_elo_delta": 18.851356965473656, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9866666666666667, + "topk_membership_rate": 0.845 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 23.858391396940533, + "median_absolute_elo_delta": 15.466399876797936, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.995, + "topk_membership_rate": 0.835 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 28.88916155443235, + "median_absolute_elo_delta": 15.255512195438655, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + } + ], + "sentinel_comparisons_detectable": 0, + "stored_comparisons": 300 + }, + { + "aggregates": [ + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 48.71634774955521, + "clustered_max_absolute_elo_delta_p95": 93.4917081131644, + "clustered_robust_pair_agreement": 0.954, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.941, + "clustered_topk_order_rate": 0.643, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 50.11338786679789, + "point_max_absolute_elo_delta_worst": 84.65205601294406, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 34.11168897934829, + "clustered_max_absolute_elo_delta_p95": 64.60510410890352, + "clustered_robust_pair_agreement": 0.9748, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.976, + "clustered_topk_order_rate": 0.747, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 20, + "percentage_saved": 60.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 35.743531490613805, + "point_max_absolute_elo_delta_worst": 42.450819019925575, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 22.381277927089627, + "clustered_max_absolute_elo_delta_p95": 43.52128846166924, + "clustered_robust_pair_agreement": 0.9848, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.988, + "clustered_topk_order_rate": 0.852, + "collection": "rubenstein", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 30, + "percentage_saved": 40.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 21.729922410336485, + "point_max_absolute_elo_delta_worst": 33.72959539668955, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 51.562024031424585, + "clustered_max_absolute_elo_delta_p95": 101.30568013207362, + "clustered_robust_pair_agreement": 0.9522, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.942, + "clustered_topk_order_rate": 0.619, + "collection": "rubenstein", + "comparisons_consumed": 75, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 34.54019650818395, + "point_max_absolute_elo_delta_worst": 46.73325502405123, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 35.019524268701616, + "clustered_max_absolute_elo_delta_p95": 69.11168669013173, + "clustered_robust_pair_agreement": 0.9744, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.98, + "clustered_topk_order_rate": 0.742, + "collection": "rubenstein", + "comparisons_consumed": 120, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 20, + "percentage_saved": 60.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 26.616486977659633, + "point_max_absolute_elo_delta_worst": 61.182980320161505, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "rubenstein-kimi", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 22.555386461412695, + "clustered_max_absolute_elo_delta_p95": 42.960206326319735, + "clustered_robust_pair_agreement": 0.9866, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.993, + "clustered_topk_order_rate": 0.851, + "collection": "rubenstein", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 30, + "percentage_saved": 40.0, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 17.935093566857404, + "point_max_absolute_elo_delta_worst": 41.47187936807018, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "judge sensitivity" + } + ], + "collection": "rubenstein", + "full_board_page_bootstrap_stability": { + "kendall_tau_median": 1.0, + "kendall_tau_p05": 0.6666666666666669, + "topk_membership_rate": 0.995, + "topk_order_rate": 0.89 + }, + "full_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "models": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "name": "rubenstein-kimi", + "repo": "davanstrien/ocr-bench-rubenstein-judge-kimi-k25", + "revision": "1e76f23a91480a5ec741064ad9deb19004db64d3", + "robust_pairs": [ + [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr" + ], + [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + [ + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + [ + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + [ + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "role": "judge sensitivity", + "sample_count": 50, + "seed_results": [ + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.966, + "topk_membership_rate": 0.945 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 53.89091040408698, + "median_absolute_elo_delta": 27.12846894778056, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9450000000000001, + "topk_membership_rate": 0.955 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 22.138400994200992, + "median_absolute_elo_delta": 10.727634168751138, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.95, + "topk_membership_rate": 0.935 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 40.43175356830284, + "median_absolute_elo_delta": 21.6128219149424, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.953, + "topk_membership_rate": 0.945 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 84.65205601294406, + "median_absolute_elo_delta": 36.8642621825403, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 0.8, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.956, + "topk_membership_rate": 0.925 + }, + "comparisons_consumed": 75, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 50.11338786679789, + "median_absolute_elo_delta": 26.205659636631253, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.981, + "topk_membership_rate": 0.975 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 35.743531490613805, + "median_absolute_elo_delta": 16.647797527010198, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.975, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 42.450819019925575, + "median_absolute_elo_delta": 26.890400123049858, + "percentage_saved": 60.0, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.971, + "topk_membership_rate": 0.975 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 13.933625286263577, + "median_absolute_elo_delta": 8.542868411717109, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.97, + "topk_membership_rate": 0.965 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 36.363234817072, + "median_absolute_elo_delta": 30.964637792118083, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.977, + "topk_membership_rate": 0.98 + }, + "comparisons_consumed": 120, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 34.53192810111909, + "median_absolute_elo_delta": 14.312297886341412, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.986, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 26.83356447099777, + "median_absolute_elo_delta": 14.650359234637676, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.985, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 21.28030828672081, + "median_absolute_elo_delta": 8.995698321244163, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.978, + "topk_membership_rate": 0.97 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 33.72959539668955, + "median_absolute_elo_delta": 15.052256363484844, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.987, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 17.486904695128715, + "median_absolute_elo_delta": 9.235698730393551, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.988, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 21.729922410336485, + "median_absolute_elo_delta": 9.977440066811596, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.953, + "topk_membership_rate": 0.94 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 32.903357507824694, + "median_absolute_elo_delta": 16.476256414493264, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.9470000000000001, + "topk_membership_rate": 0.935 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 46.73325502405123, + "median_absolute_elo_delta": 22.779398442005117, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.952, + "topk_membership_rate": 0.955 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 44.37427539196074, + "median_absolute_elo_delta": 25.415909271224677, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 0.8, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.949, + "topk_membership_rate": 0.93 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 34.54019650818395, + "median_absolute_elo_delta": 12.566904274256672, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.96, + "topk_membership_rate": 0.95 + }, + "comparisons_consumed": 75, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.5, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 24.59652295509477, + "median_absolute_elo_delta": 21.354371376762288, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.979, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 3.9071916176724244, + "median_absolute_elo_delta": 2.953982768507103, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.973, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 61.182980320161505, + "median_absolute_elo_delta": 31.874370854579865, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.976, + "topk_membership_rate": 0.975 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 22.53122987898746, + "median_absolute_elo_delta": 14.282914106260705, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.967, + "topk_membership_rate": 0.965 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 26.616486977659633, + "median_absolute_elo_delta": 12.357136157500918, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.977, + "topk_membership_rate": 0.98 + }, + "comparisons_consumed": 120, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 20, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 47.6197972098912, + "median_absolute_elo_delta": 21.166557288245485, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.986, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 10.512556410064917, + "median_absolute_elo_delta": 6.301254469210903, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.991, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 17.935093566857404, + "median_absolute_elo_delta": 11.19371283207579, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.988, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 11.411012159932397, + "median_absolute_elo_delta": 5.468854321115373, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.985, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 18.717572970040237, + "median_absolute_elo_delta": 10.211945275357152, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.983, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30.0, + "min_direct_comparisons": 30, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 41.47187936807018, + "median_absolute_elo_delta": 21.937396177067285, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + } + ], + "sentinel_comparisons_detectable": 0, + "stored_comparisons": 300 + }, + { + "aggregates": [ + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 66.36513865100414, + "clustered_max_absolute_elo_delta_p95": 170.6679118820609, + "clustered_robust_pair_agreement": 0.9982, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.992, + "clustered_topk_order_rate": 0.775, + "collection": "ufo", + "comparisons_consumed": 74, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 74.82993197278911, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 101.71601111147652, + "point_max_absolute_elo_delta_worst": 119.56988418080346, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 46.51917839657665, + "clustered_max_absolute_elo_delta_p95": 101.50165752723015, + "clustered_robust_pair_agreement": 0.9998, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.999, + "clustered_topk_order_rate": 0.83, + "collection": "ufo", + "comparisons_consumed": 118, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 19, + "percentage_saved": 59.863945578231295, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 47.13600047692353, + "point_max_absolute_elo_delta_worst": 83.49166477060635, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 31.45359082816958, + "clustered_max_absolute_elo_delta_p95": 62.10565127428381, + "clustered_robust_pair_agreement": 1, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 1, + "clustered_topk_order_rate": 0.869, + "collection": "ufo", + "comparisons_consumed": 176, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 29, + "percentage_saved": 40.136054421768705, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 29.467658868230956, + "point_max_absolute_elo_delta_worst": 34.09709501093266, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 69.87977884241309, + "clustered_max_absolute_elo_delta_p95": 166.35690358458058, + "clustered_robust_pair_agreement": 0.9968, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.985, + "clustered_topk_order_rate": 0.737, + "collection": "ufo", + "comparisons_consumed": 74, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 74.82993197278911, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 78.64119881236093, + "point_max_absolute_elo_delta_worst": 120.47604998638326, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 48.52463011461816, + "clustered_max_absolute_elo_delta_p95": 110.365475846818, + "clustered_robust_pair_agreement": 0.9992, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.996, + "clustered_topk_order_rate": 0.788, + "collection": "ufo", + "comparisons_consumed": 118, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 19, + "percentage_saved": 59.863945578231295, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 51.53419474381826, + "point_max_absolute_elo_delta_worst": 77.55657146009776, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "ufo-30b", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 1.0, + "clustered_kendall_tau_p05": 0.6666666666666669, + "clustered_max_absolute_elo_delta_median": 31.05830195373278, + "clustered_max_absolute_elo_delta_p95": 65.8193329542637, + "clustered_robust_pair_agreement": 1, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 1, + "clustered_topk_order_rate": 0.863, + "collection": "ufo", + "comparisons_consumed": 176, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 29, + "percentage_saved": 40.136054421768705, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 1.0, + "point_max_absolute_elo_delta_median": 42.07911792124605, + "point_max_absolute_elo_delta_worst": 51.958382413868776, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 5, + "possible_pairs": 6, + "robust_full_board_pairs": 5, + "role": "independent collection" + } + ], + "collection": "ufo", + "full_board_page_bootstrap_stability": { + "kendall_tau_median": 1.0, + "kendall_tau_p05": 0.6666666666666669, + "topk_membership_rate": 1, + "topk_order_rate": 0.84 + }, + "full_rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "models": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "name": "ufo-30b", + "repo": "davanstrien/ocr-bench-ufo-judge-30b", + "revision": "97623b511a0510e13b2f9e849f58d4002663a51b", + "robust_pairs": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B" + ], + [ + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + [ + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR" + ], + [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + [ + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "role": "independent collection", + "sample_count": 49, + "seed_results": [ + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.999, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 74, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 119.56988418080346, + "median_absolute_elo_delta": 70.91005547106886, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.997, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 74, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 109.3038822891483, + "median_absolute_elo_delta": 48.5341243123346, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.997, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 74, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 50.503951963345116, + "median_absolute_elo_delta": 22.994270879470037, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr", + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.999, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 74, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 101.71601111147652, + "median_absolute_elo_delta": 34.45332811247431, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.999, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 74, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 79.83861035452696, + "median_absolute_elo_delta": 53.81539589657518, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 118, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 68.51066019609402, + "median_absolute_elo_delta": 31.424392467178905, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 118, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 83.49166477060635, + "median_absolute_elo_delta": 31.796643032191355, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.999, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 118, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 21.023487280965355, + "median_absolute_elo_delta": 15.42433682024489, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 118, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 47.13600047692353, + "median_absolute_elo_delta": 29.257657655281378, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 118, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 29.329016179227438, + "median_absolute_elo_delta": 14.42390152604139, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 29.529807735151053, + "median_absolute_elo_delta": 13.613499977652964, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 17.119585215628376, + "median_absolute_elo_delta": 9.246357760980459, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 34.09709501093266, + "median_absolute_elo_delta": 14.04087857170316, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 29.467658868230956, + "median_absolute_elo_delta": 15.451884602746532, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 27.505985320694663, + "median_absolute_elo_delta": 12.091651855988289, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.996, + "topk_membership_rate": 0.98 + }, + "comparisons_consumed": 74, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 66.87725155533622, + "median_absolute_elo_delta": 40.04737726080077, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.997, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 74, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 96.23550640795952, + "median_absolute_elo_delta": 47.31353994129756, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.997, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 74, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 28.553987332682482, + "median_absolute_elo_delta": 18.10732850210195, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.996, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 74, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 78.64119881236093, + "median_absolute_elo_delta": 54.111995617953085, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.998, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 74, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12.0, + "min_direct_comparisons": 12, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 120.47604998638326, + "median_absolute_elo_delta": 46.99062954735177, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.997, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 118, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 44.498251778764825, + "median_absolute_elo_delta": 35.91870555561627, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.999, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 118, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 51.53419474381826, + "median_absolute_elo_delta": 34.479425977853225, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 118, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 37.55990616193503, + "median_absolute_elo_delta": 30.47472108890645, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 118, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 77.55657146009776, + "median_absolute_elo_delta": 33.17670665870901, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 118, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20.0, + "min_direct_comparisons": 19, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 77.09392664568236, + "median_absolute_elo_delta": 31.034120870038464, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 18.26375334434283, + "median_absolute_elo_delta": 8.8403833336572, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 51.958382413868776, + "median_absolute_elo_delta": 27.246302753481814, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 25.2790563218241, + "median_absolute_elo_delta": 15.167556634248399, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 46.36076416378751, + "median_absolute_elo_delta": 23.260138784993273, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 1, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 176, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 29.0, + "min_direct_comparisons": 29, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 42.07911792124605, + "median_absolute_elo_delta": 25.068895578387583, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + } + ], + "sentinel_comparisons_detectable": 0, + "stored_comparisons": 294 + }, + { + "aggregates": [ + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 73.5964697110536, + "clustered_max_absolute_elo_delta_p95": 153.5851477572523, + "clustered_robust_pair_agreement": 0.81725, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.578, + "clustered_topk_order_rate": 0.22, + "collection": "bpl", + "comparisons_consumed": 37, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 6, + "percentage_saved": 74.82993197278911, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.3333333333333334, + "point_kendall_tau_min": 0.0, + "point_max_absolute_elo_delta_median": 70.72228974130076, + "point_max_absolute_elo_delta_worst": 99.62849649893337, + "point_spearman_rho_median": 0.6000000000000001, + "point_topk_membership_matches": 0, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 49.828543699435045, + "clustered_max_absolute_elo_delta_p95": 102.8327524945587, + "clustered_robust_pair_agreement": 0.869, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.654, + "clustered_topk_order_rate": 0.349, + "collection": "bpl", + "comparisons_consumed": 59, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 9, + "percentage_saved": 59.863945578231295, + "point_kendall_tau_max": 0.6666666666666669, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.0, + "point_max_absolute_elo_delta_median": 45.03767776309405, + "point_max_absolute_elo_delta_worst": 84.52832892210927, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 0, + "point_topk_order_matches": 0, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 33.19894047796083, + "clustered_max_absolute_elo_delta_p95": 62.93854394091554, + "clustered_robust_pair_agreement": 0.9105, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.737, + "clustered_topk_order_rate": 0.477, + "collection": "bpl", + "comparisons_consumed": 88, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 14, + "percentage_saved": 40.136054421768705, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.3333333333333334, + "point_max_absolute_elo_delta_median": 24.463256848546962, + "point_max_absolute_elo_delta_worst": 83.92125755287248, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 1, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": -0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 83.57982190290431, + "clustered_max_absolute_elo_delta_p95": 172.00152817247883, + "clustered_robust_pair_agreement": 0.8, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.549, + "clustered_topk_order_rate": 0.226, + "collection": "bpl", + "comparisons_consumed": 37, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 6, + "percentage_saved": 74.82993197278911, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 64.81689426010917, + "point_max_absolute_elo_delta_worst": 89.13457286674588, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 1, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.0, + "clustered_max_absolute_elo_delta_median": 55.26643688568504, + "clustered_max_absolute_elo_delta_p95": 105.22972942394816, + "clustered_robust_pair_agreement": 0.85925, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.622, + "clustered_topk_order_rate": 0.32, + "collection": "bpl", + "comparisons_consumed": 59, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 9, + "percentage_saved": 59.863945578231295, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 0.6666666666666669, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 39.9479134020994, + "point_max_absolute_elo_delta_worst": 93.26201785241119, + "point_spearman_rho_median": 0.7999999999999999, + "point_topk_membership_matches": 3, + "point_topk_order_matches": 2, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "bpl", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.6666666666666669, + "clustered_kendall_tau_p05": 0.3333333333333334, + "clustered_max_absolute_elo_delta_median": 34.53789864225064, + "clustered_max_absolute_elo_delta_p95": 67.80927819317637, + "clustered_robust_pair_agreement": 0.90975, + "clustered_spearman_rho_median": 0.7999999999999999, + "clustered_topk_membership_rate": 0.752, + "clustered_topk_order_rate": 0.489, + "collection": "bpl", + "comparisons_consumed": 88, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": false, + "spearman_rho_median_at_least_0_90": false, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 14, + "percentage_saved": 40.136054421768705, + "point_kendall_tau_max": 1.0, + "point_kendall_tau_median": 1.0, + "point_kendall_tau_min": 0.6666666666666669, + "point_max_absolute_elo_delta_median": 29.05276625236911, + "point_max_absolute_elo_delta_worst": 72.53660288857373, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 3, + "point_topk_order_matches": 3, + "possible_pairs": 6, + "robust_full_board_pairs": 4, + "role": "independent collection; incomplete board" + } + ], + "collection": "bpl", + "full_board_page_bootstrap_stability": { + "kendall_tau_median": 0.6666666666666669, + "kendall_tau_p05": 0.3333333333333334, + "topk_membership_rate": 0.52, + "topk_order_rate": 0.285 + }, + "full_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "models": [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "name": "bpl", + "repo": "davanstrien/bpl-ocr-bench-results", + "revision": "fa9e7a032b7b245d5dd69fd40e87646613af6401", + "robust_pairs": [ + [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr" + ], + [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr" + ], + [ + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR" + ] + ], + "role": "independent collection; incomplete board", + "sample_count": 41, + "seed_results": [ + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.83, + "topk_membership_rate": 0.585 + }, + "comparisons_consumed": 37, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 35.590224662717674, + "median_absolute_elo_delta": 31.103118395500246, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.8125, + "topk_membership_rate": 0.605 + }, + "comparisons_consumed": 37, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 70.72228974130076, + "median_absolute_elo_delta": 26.84935059661086, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.6000000000000001, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.795, + "topk_membership_rate": 0.545 + }, + "comparisons_consumed": 37, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 99.62849649893337, + "median_absolute_elo_delta": 42.198095252386906, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.6000000000000001, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.81375, + "topk_membership_rate": 0.605 + }, + "comparisons_consumed": 37, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.0, + "max_absolute_elo_delta": 78.35138251720514, + "median_absolute_elo_delta": 41.33869006654743, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.0, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.835, + "topk_membership_rate": 0.55 + }, + "comparisons_consumed": 37, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 46.827266547393265, + "median_absolute_elo_delta": 17.661859862163055, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.8725, + "topk_membership_rate": 0.63 + }, + "comparisons_consumed": 59, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 30.36941489290939, + "median_absolute_elo_delta": 15.298071935324629, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.86875, + "topk_membership_rate": 0.62 + }, + "comparisons_consumed": 59, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.0, + "max_absolute_elo_delta": 67.98484442007975, + "median_absolute_elo_delta": 34.187674259964524, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.0, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.85125, + "topk_membership_rate": 0.635 + }, + "comparisons_consumed": 59, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 45.03767776309405, + "median_absolute_elo_delta": 26.825037560948886, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.8775, + "topk_membership_rate": 0.7 + }, + "comparisons_consumed": 59, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 84.52832892210927, + "median_absolute_elo_delta": 59.01254151461421, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.39999999999999997, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.875, + "topk_membership_rate": 0.685 + }, + "comparisons_consumed": 59, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 21.516876278136806, + "median_absolute_elo_delta": 10.831745030310913, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.91, + "topk_membership_rate": 0.735 + }, + "comparisons_consumed": 88, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 22.774124714008167, + "median_absolute_elo_delta": 10.826613117054421, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9, + "topk_membership_rate": 0.74 + }, + "comparisons_consumed": 88, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 24.463256848546962, + "median_absolute_elo_delta": 11.369169017714853, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.91375, + "topk_membership_rate": 0.695 + }, + "comparisons_consumed": 88, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 40.29545738941306, + "median_absolute_elo_delta": 22.855776913372438, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.92125, + "topk_membership_rate": 0.79 + }, + "comparisons_consumed": 88, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.3333333333333334, + "max_absolute_elo_delta": 83.92125755287248, + "median_absolute_elo_delta": 56.697231307037555, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.39999999999999997, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9075, + "topk_membership_rate": 0.725 + }, + "comparisons_consumed": 88, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 5.916180648651789, + "median_absolute_elo_delta": 3.349670017822177, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.815, + "topk_membership_rate": 0.57 + }, + "comparisons_consumed": 37, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 81.15511625447198, + "median_absolute_elo_delta": 58.43228037787151, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.79875, + "topk_membership_rate": 0.515 + }, + "comparisons_consumed": 37, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 64.81689426010917, + "median_absolute_elo_delta": 32.4621753671056, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 0.75, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.78125, + "topk_membership_rate": 0.56 + }, + "comparisons_consumed": 37, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 18.45028774552361, + "median_absolute_elo_delta": 13.776781270081528, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.79, + "topk_membership_rate": 0.565 + }, + "comparisons_consumed": 37, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 89.13457286674588, + "median_absolute_elo_delta": 42.23947265653521, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.815, + "topk_membership_rate": 0.535 + }, + "comparisons_consumed": 37, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 7, + "median_direct_comparisons": 6.0, + "min_direct_comparisons": 6, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 35.72998868202103, + "median_absolute_elo_delta": 17.096062125351295, + "percentage_saved": 74.82993197278911, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.86625, + "topk_membership_rate": 0.66 + }, + "comparisons_consumed": 59, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 93.26201785241119, + "median_absolute_elo_delta": 37.57454555418235, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.8425, + "topk_membership_rate": 0.595 + }, + "comparisons_consumed": 59, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 39.9479134020994, + "median_absolute_elo_delta": 29.273394412053108, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.8525, + "topk_membership_rate": 0.595 + }, + "comparisons_consumed": 59, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 25.537522280292023, + "median_absolute_elo_delta": 10.60543816672191, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.85875, + "topk_membership_rate": 0.63 + }, + "comparisons_consumed": 59, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 57.7260193567397, + "median_absolute_elo_delta": 23.59211226488344, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.87625, + "topk_membership_rate": 0.63 + }, + "comparisons_consumed": 59, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 10, + "median_direct_comparisons": 10.0, + "min_direct_comparisons": 9, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 35.559307290019206, + "median_absolute_elo_delta": 27.270817933764192, + "percentage_saved": 59.863945578231295, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.92, + "topk_membership_rate": 0.77 + }, + "comparisons_consumed": 88, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 72.53660288857373, + "median_absolute_elo_delta": 38.185318610162994, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.9, + "topk_membership_rate": 0.74 + }, + "comparisons_consumed": 88, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 29.05276625236911, + "median_absolute_elo_delta": 11.575058942913415, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.8333333333333335, + "robust_pair_agreement_rate": 0.9225, + "topk_membership_rate": 0.75 + }, + "comparisons_consumed": 88, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 21.697563026253647, + "median_absolute_elo_delta": 9.778249070929519, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 1.0, + "robust_pair_agreement_rate": 0.915, + "topk_membership_rate": 0.765 + }, + "comparisons_consumed": 88, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 0.6666666666666669, + "max_absolute_elo_delta": 31.95279024401998, + "median_absolute_elo_delta": 25.282541550557198, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.7999999999999999, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.6666666666666669, + "robust_pair_agreement_rate": 0.89125, + "topk_membership_rate": 0.735 + }, + "comparisons_consumed": 88, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 15, + "median_direct_comparisons": 15.0, + "min_direct_comparisons": 14, + "observed_pairs": 6, + "pair_coverage_pct": 100.0, + "possible_pairs": 6 + }, + "kendall_tau": 1.0, + "max_absolute_elo_delta": 13.40644476851935, + "median_absolute_elo_delta": 8.193081952509601, + "percentage_saved": 40.136054421768705, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + } + ], + "sentinel_comparisons_detectable": 0, + "stored_comparisons": 147 + }, + { + "aggregates": [ + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.8666666666666666, + "clustered_kendall_tau_p05": 0.7333333333333333, + "clustered_max_absolute_elo_delta_median": 81.15453200095465, + "clustered_max_absolute_elo_delta_p95": 2306.972897111449, + "clustered_robust_pair_agreement": 0.9755384615384616, + "clustered_spearman_rho_median": 0.942857142857143, + "clustered_topk_membership_rate": 0.971, + "clustered_topk_order_rate": 0.488, + "collection": "britannica", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.9999999999999999, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 61.599642638423575, + "point_max_absolute_elo_delta_worst": 86.35632964392971, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 3, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.9999999999999999, + "clustered_kendall_tau_p05": 0.7333333333333333, + "clustered_max_absolute_elo_delta_median": 58.05154309002893, + "clustered_max_absolute_elo_delta_p95": 197.08019997153377, + "clustered_robust_pair_agreement": 0.9830769230769231, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.988, + "clustered_topk_order_rate": 0.565, + "collection": "britannica", + "comparisons_consumed": 288, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 19, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.9999999999999999, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 52.83123987349245, + "point_max_absolute_elo_delta_worst": 69.15859061734136, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 3, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.9999999999999999, + "clustered_kendall_tau_p05": 0.8666666666666666, + "clustered_max_absolute_elo_delta_median": 37.81826912701297, + "clustered_max_absolute_elo_delta_p95": 97.80114615896461, + "clustered_robust_pair_agreement": 0.9895384615384616, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.993, + "clustered_topk_order_rate": 0.713, + "collection": "britannica", + "comparisons_consumed": 432, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "pair-balanced", + "minimum_pair_evidence": 28, + "percentage_saved": 40.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.9999999999999999, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 32.11192697586705, + "point_max_absolute_elo_delta_worst": 60.004263291922484, + "point_spearman_rho_median": 1.0, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 4, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.8666666666666666, + "clustered_kendall_tau_p05": 0.6, + "clustered_max_absolute_elo_delta_median": 91.67612246683814, + "clustered_max_absolute_elo_delta_p95": 2348.1679073592472, + "clustered_robust_pair_agreement": 0.9699230769230769, + "clustered_spearman_rho_median": 0.942857142857143, + "clustered_topk_membership_rate": 0.956, + "clustered_topk_order_rate": 0.418, + "collection": "britannica", + "comparisons_consumed": 180, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 75.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.8666666666666666, + "point_kendall_tau_min": 0.7333333333333333, + "point_max_absolute_elo_delta_median": 116.18768538959603, + "point_max_absolute_elo_delta_worst": 139.56132398511545, + "point_spearman_rho_median": 0.942857142857143, + "point_topk_membership_matches": 4, + "point_topk_order_matches": 1, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.9999999999999999, + "clustered_kendall_tau_p05": 0.7333333333333333, + "clustered_max_absolute_elo_delta_median": 60.51017695201898, + "clustered_max_absolute_elo_delta_p95": 197.6989721651328, + "clustered_robust_pair_agreement": 0.9826923076923078, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.989, + "clustered_topk_order_rate": 0.551, + "collection": "britannica", + "comparisons_consumed": 288, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 19, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.8666666666666666, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 57.354151366056385, + "point_max_absolute_elo_delta_worst": 82.20896779061877, + "point_spearman_rho_median": 0.942857142857143, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 2, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "britannica-qwen35", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.9999999999999999, + "clustered_kendall_tau_p05": 0.8666666666666666, + "clustered_max_absolute_elo_delta_median": 39.35275643631411, + "clustered_max_absolute_elo_delta_p95": 105.36616237876325, + "clustered_robust_pair_agreement": 0.9906153846153847, + "clustered_spearman_rho_median": 1.0, + "clustered_topk_membership_rate": 0.999, + "clustered_topk_order_rate": 0.722, + "collection": "britannica", + "comparisons_consumed": 432, + "cross_board_criteria": { + "passed": true, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": true + }, + "design": "mixed-random", + "minimum_pair_evidence": 28, + "percentage_saved": 40.0, + "point_kendall_tau_max": 0.9999999999999999, + "point_kendall_tau_median": 0.8666666666666666, + "point_kendall_tau_min": 0.8666666666666666, + "point_max_absolute_elo_delta_median": 56.88351052438202, + "point_max_absolute_elo_delta_worst": 67.21927176824147, + "point_spearman_rho_median": 0.942857142857143, + "point_topk_membership_matches": 5, + "point_topk_order_matches": 2, + "possible_pairs": 15, + "robust_full_board_pairs": 13, + "role": "same-corpus judge/model-grid sensitivity" + } + ], + "collection": "britannica", + "full_board_page_bootstrap_stability": { + "kendall_tau_median": 0.9999999999999999, + "kendall_tau_p05": 0.8666666666666666, + "topk_membership_rate": 1, + "topk_order_rate": 0.65 + }, + "full_rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "models": [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ], + "name": "britannica-qwen35", + "repo": "davanstrien/ocr-bench-britannica-results-qwen35", + "revision": "8b6602826e2b71c615ed229d388221a7aa7c69b9", + "robust_pairs": [ + [ + "zai-org/GLM-OCR", + "numind/NuExtract3" + ], + [ + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR" + ], + [ + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr" + ], + [ + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR" + ], + [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr" + ], + [ + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR" + ], + [ + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "numind/NuExtract3", + "rednote-hilab/dots.ocr" + ], + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr" + ], + [ + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ] + ], + "role": "same-corpus judge/model-grid sensitivity", + "sample_count": 50, + "seed_results": [ + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9765384615384616, + "topk_membership_rate": 0.97 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 38.51884725450918, + "median_absolute_elo_delta": 21.759911161862192, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9784615384615385, + "topk_membership_rate": 0.965 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 86.35632964392971, + "median_absolute_elo_delta": 10.170028958754187, + "percentage_saved": 75.0, + "rank_order": [ + "zai-org/GLM-OCR", + "numind/NuExtract3", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.975, + "topk_membership_rate": 0.965 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 61.599642638423575, + "median_absolute_elo_delta": 39.45964172304048, + "percentage_saved": 75.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9742307692307692, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 59.168055571960394, + "median_absolute_elo_delta": 19.326417358068284, + "percentage_saved": 75.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9734615384615385, + "topk_membership_rate": 0.97 + }, + "comparisons_consumed": 180, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 71.58970505486195, + "median_absolute_elo_delta": 24.64502156219214, + "percentage_saved": 75.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9826923076923078, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 288, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 43.69900336784508, + "median_absolute_elo_delta": 15.141281995695863, + "percentage_saved": 60.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9853846153846154, + "topk_membership_rate": 0.98 + }, + "comparisons_consumed": 288, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 69.15859061734136, + "median_absolute_elo_delta": 15.395500130154915, + "percentage_saved": 60.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9830769230769231, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 288, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 52.83123987349245, + "median_absolute_elo_delta": 24.727177735196506, + "percentage_saved": 60.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9784615384615385, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 288, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 36.14726057336202, + "median_absolute_elo_delta": 20.96994597320429, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9857692307692307, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 288, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 64.58076505811346, + "median_absolute_elo_delta": 20.34029581262314, + "percentage_saved": 60.0, + "rank_order": [ + "zai-org/GLM-OCR", + "numind/NuExtract3", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.99, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 432, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 33.76243498840722, + "median_absolute_elo_delta": 10.605644513464313, + "percentage_saved": 40.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9915384615384616, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 432, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 27.735651481426657, + "median_absolute_elo_delta": 23.609146409225104, + "percentage_saved": 40.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9884615384615385, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 432, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 60.004263291922484, + "median_absolute_elo_delta": 24.008955464991118, + "percentage_saved": 40.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9865384615384616, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 432, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 27.577624653221164, + "median_absolute_elo_delta": 5.481682830148088, + "percentage_saved": 40.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9911538461538462, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 432, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 32.11192697586705, + "median_absolute_elo_delta": 21.20105316010188, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9680769230769231, + "topk_membership_rate": 0.95 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 132.4505073193293, + "median_absolute_elo_delta": 32.35601155166012, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9703846153846154, + "topk_membership_rate": 0.955 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 139.56132398511545, + "median_absolute_elo_delta": 66.0684313133919, + "percentage_saved": 75.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 0.9230769230769231, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9684615384615385, + "topk_membership_rate": 0.985 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 116.18768538959603, + "median_absolute_elo_delta": 53.3184597235894, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9692307692307692, + "topk_membership_rate": 0.945 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 56.17214884720943, + "median_absolute_elo_delta": 26.92837806468617, + "percentage_saved": 75.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9734615384615385, + "topk_membership_rate": 0.945 + }, + "comparisons_consumed": 180, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 12, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.7333333333333333, + "max_absolute_elo_delta": 63.686885741541346, + "median_absolute_elo_delta": 40.34533996411767, + "percentage_saved": 75.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "zai-org/GLM-OCR", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 0.9230769230769231, + "spearman_rho": 0.8285714285714287, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9819230769230769, + "topk_membership_rate": 0.99 + }, + "comparisons_consumed": 288, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 71.49304047289615, + "median_absolute_elo_delta": 48.181860247487634, + "percentage_saved": 60.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9853846153846154, + "topk_membership_rate": 0.98 + }, + "comparisons_consumed": 288, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 82.20896779061877, + "median_absolute_elo_delta": 66.20663592491053, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.98, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 288, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 57.354151366056385, + "median_absolute_elo_delta": 25.907715452136927, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8666666666666666, + "robust_pair_agreement_rate": 0.9819230769230769, + "topk_membership_rate": 0.98 + }, + "comparisons_consumed": 288, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 44.450266647589615, + "median_absolute_elo_delta": 18.169338521310124, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9842307692307692, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 288, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 19, + "min_direct_comparisons": 19, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 33.201223203498785, + "median_absolute_elo_delta": 11.778221940493381, + "percentage_saved": 60.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9919230769230769, + "topk_membership_rate": 0.995 + }, + "comparisons_consumed": 432, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 67.21927176824147, + "median_absolute_elo_delta": 24.9339679639138, + "percentage_saved": 40.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9915384615384616, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 432, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.9999999999999999, + "max_absolute_elo_delta": 52.89432035328559, + "median_absolute_elo_delta": 23.97143290244378, + "percentage_saved": 40.0, + "rank_order": [ + "zai-org/GLM-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 1.0, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9873076923076923, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 432, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 56.88351052438202, + "median_absolute_elo_delta": 11.389662284604924, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9907692307692307, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 432, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 61.47036150386566, + "median_absolute_elo_delta": 20.236194352485086, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9999999999999999, + "robust_pair_agreement_rate": 0.9915384615384616, + "topk_membership_rate": 1 + }, + "comparisons_consumed": 432, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 29, + "median_direct_comparisons": 29, + "min_direct_comparisons": 28, + "observed_pairs": 15, + "pair_coverage_pct": 100.0, + "possible_pairs": 15 + }, + "kendall_tau": 0.8666666666666666, + "max_absolute_elo_delta": 31.062692580040903, + "median_absolute_elo_delta": 21.81787474016619, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR", + "rednote-hilab/dots.ocr" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.942857142857143, + "topk_membership_matches": true, + "topk_order_matches": false + } + ], + "sentinel_comparisons_detectable": 244, + "stored_comparisons": 720 + } + ], + "collection_design_pass_counts": { + "bpl:mixed-random@25%": { + "passed": 0, + "total": 1 + }, + "bpl:mixed-random@40%": { + "passed": 0, + "total": 1 + }, + "bpl:mixed-random@60%": { + "passed": 0, + "total": 1 + }, + "bpl:pair-balanced@25%": { + "passed": 0, + "total": 1 + }, + "bpl:pair-balanced@40%": { + "passed": 0, + "total": 1 + }, + "bpl:pair-balanced@60%": { + "passed": 0, + "total": 1 + }, + "britannica:mixed-random@25%": { + "passed": 1, + "total": 1 + }, + "britannica:mixed-random@40%": { + "passed": 1, + "total": 1 + }, + "britannica:mixed-random@60%": { + "passed": 1, + "total": 1 + }, + "britannica:pair-balanced@25%": { + "passed": 1, + "total": 1 + }, + "britannica:pair-balanced@40%": { + "passed": 1, + "total": 1 + }, + "britannica:pair-balanced@60%": { + "passed": 1, + "total": 1 + }, + "rubenstein:mixed-random@25%": { + "passed": 1, + "total": 3 + }, + "rubenstein:mixed-random@40%": { + "passed": 2, + "total": 3 + }, + "rubenstein:mixed-random@60%": { + "passed": 2, + "total": 3 + }, + "rubenstein:pair-balanced@25%": { + "passed": 1, + "total": 3 + }, + "rubenstein:pair-balanced@40%": { + "passed": 2, + "total": 3 + }, + "rubenstein:pair-balanced@60%": { + "passed": 2, + "total": 3 + }, + "ufo:mixed-random@25%": { + "passed": 1, + "total": 1 + }, + "ufo:mixed-random@40%": { + "passed": 1, + "total": 1 + }, + "ufo:mixed-random@60%": { + "passed": 1, + "total": 1 + }, + "ufo:pair-balanced@25%": { + "passed": 1, + "total": 1 + }, + "ufo:pair-balanced@40%": { + "passed": 1, + "total": 1 + }, + "ufo:pair-balanced@60%": { + "passed": 1, + "total": 1 + } + }, + "experiment": { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "boards": 6, + "budget_fractions": [ + 0.25, + 0.4, + 0.6 + ], + "cluster_replicates": 200, + "cross_board_criteria": { + "clustered_robust_pair_agreement_minimum": 0.95, + "clustered_spearman_rho_median_minimum": 0.9, + "clustered_topk_membership_rate_minimum": 0.9 + }, + "independent_collections": [ + "bpl", + "britannica", + "rubenstein", + "ufo" + ], + "notes": [ + "Rubenstein judge variants are judge sensitivity, not independent collections.", + "Allocation uses only published valid rows and never inspects winner values.", + "Older comparison schemas without OCR text cannot be rechecked for sentinels.", + "Page bootstrap resamples sample_idx clusters and reruns allocation per replicate." + ], + "robust_pair_definition": "full-board rank direction repeats in at least 95% of page-cluster bootstraps", + "top_k": 3 + } +} diff --git a/experiments/adaptive-stopping/multi-board-summary.csv b/experiments/adaptive-stopping/multi-board-summary.csv new file mode 100644 index 0000000..c01f18d --- /dev/null +++ b/experiments/adaptive-stopping/multi-board-summary.csv @@ -0,0 +1,37 @@ +board,collection,design,budget_fraction,comparisons_consumed,percentage_saved,robust_full_board_pairs,possible_pairs,point_spearman_rho_median,point_topk_membership_matches,clustered_spearman_rho_median,clustered_topk_membership_rate,clustered_robust_pair_agreement,clustered_max_absolute_elo_delta_median,clustered_max_absolute_elo_delta_p95,minimum_pair_evidence,passed +rubenstein-30b,rubenstein,pair-balanced,0.25,75,74.91638795986623,4,6,0.7999999999999999,5,0.7999999999999999,0.958,0.95625,50.96302445946719,103.04616386353707,12,False +rubenstein-30b,rubenstein,pair-balanced,0.4,120,59.86622073578595,4,6,1.0,5,1.0,0.984,0.97575,36.602588143273124,68.9428597271016,20,True +rubenstein-30b,rubenstein,pair-balanced,0.6,179,40.13377926421405,4,6,1.0,5,1.0,0.999,0.989,24.451492199466998,45.8823914245234,29,True +rubenstein-30b,rubenstein,mixed-random,0.25,75,74.91638795986623,4,6,0.7999999999999999,5,0.7999999999999999,0.942,0.95,55.30934564765391,106.11305957921516,12,False +rubenstein-30b,rubenstein,mixed-random,0.4,120,59.86622073578595,4,6,1.0,5,1.0,0.986,0.9765,35.961371098299765,71.70864575790696,20,True +rubenstein-30b,rubenstein,mixed-random,0.6,179,40.13377926421405,4,6,1.0,5,1.0,0.997,0.9885,24.407175723937144,45.83763748606696,29,True +rubenstein-jury,rubenstein,pair-balanced,0.25,75,75.0,3,6,0.7999999999999999,2,0.7999999999999999,0.686,0.9546666666666667,51.30851618298459,99.7740401556695,12,False +rubenstein-jury,rubenstein,pair-balanced,0.4,120,60.0,3,6,0.7999999999999999,3,1.0,0.794,0.9783333333333333,35.47060310551785,70.45446369737978,20,False +rubenstein-jury,rubenstein,pair-balanced,0.6,180,40.0,3,6,1.0,5,1.0,0.862,0.989,24.519517490073326,46.29281326901969,30,False +rubenstein-jury,rubenstein,mixed-random,0.25,75,75.0,3,6,0.7999999999999999,4,0.7999999999999999,0.682,0.9543333333333334,53.2623958108918,105.56940869125386,12,False +rubenstein-jury,rubenstein,mixed-random,0.4,120,60.0,3,6,0.7999999999999999,5,0.8999999999999999,0.772,0.9743333333333334,37.201976556154364,72.11933690476818,20,False +rubenstein-jury,rubenstein,mixed-random,0.6,180,40.0,3,6,0.7999999999999999,5,1.0,0.847,0.9906666666666667,24.640067470120698,46.53445768533318,30,False +rubenstein-kimi,rubenstein,pair-balanced,0.25,75,75.0,5,6,1.0,5,1.0,0.941,0.954,48.71634774955521,93.4917081131644,12,True +rubenstein-kimi,rubenstein,pair-balanced,0.4,120,60.0,5,6,1.0,5,1.0,0.976,0.9748,34.11168897934829,64.60510410890352,20,True +rubenstein-kimi,rubenstein,pair-balanced,0.6,180,40.0,5,6,1.0,5,1.0,0.988,0.9848,22.381277927089627,43.52128846166924,30,True +rubenstein-kimi,rubenstein,mixed-random,0.25,75,75.0,5,6,1.0,5,1.0,0.942,0.9522,51.562024031424585,101.30568013207362,12,True +rubenstein-kimi,rubenstein,mixed-random,0.4,120,60.0,5,6,1.0,5,1.0,0.98,0.9744,35.019524268701616,69.11168669013173,20,True +rubenstein-kimi,rubenstein,mixed-random,0.6,180,40.0,5,6,1.0,5,1.0,0.993,0.9866,22.555386461412695,42.960206326319735,30,True +ufo-30b,ufo,pair-balanced,0.25,74,74.82993197278911,5,6,1.0,5,1.0,0.992,0.9982,66.36513865100414,170.6679118820609,12,True +ufo-30b,ufo,pair-balanced,0.4,118,59.863945578231295,5,6,1.0,5,1.0,0.999,0.9998,46.51917839657665,101.50165752723015,19,True +ufo-30b,ufo,pair-balanced,0.6,176,40.136054421768705,5,6,1.0,5,1.0,1,1,31.45359082816958,62.10565127428381,29,True +ufo-30b,ufo,mixed-random,0.25,74,74.82993197278911,5,6,1.0,5,1.0,0.985,0.9968,69.87977884241309,166.35690358458058,12,True +ufo-30b,ufo,mixed-random,0.4,118,59.863945578231295,5,6,1.0,5,1.0,0.996,0.9992,48.52463011461816,110.365475846818,19,True +ufo-30b,ufo,mixed-random,0.6,176,40.136054421768705,5,6,1.0,5,1.0,1,1,31.05830195373278,65.8193329542637,29,True +bpl,bpl,pair-balanced,0.25,37,74.82993197278911,4,6,0.6000000000000001,0,0.7999999999999999,0.578,0.81725,73.5964697110536,153.5851477572523,6,False +bpl,bpl,pair-balanced,0.4,59,59.863945578231295,4,6,0.7999999999999999,0,0.7999999999999999,0.654,0.869,49.828543699435045,102.8327524945587,9,False +bpl,bpl,pair-balanced,0.6,88,40.136054421768705,4,6,0.7999999999999999,1,0.7999999999999999,0.737,0.9105,33.19894047796083,62.93854394091554,14,False +bpl,bpl,mixed-random,0.25,37,74.82993197278911,4,6,0.7999999999999999,2,0.7999999999999999,0.549,0.8,83.57982190290431,172.00152817247883,6,False +bpl,bpl,mixed-random,0.4,59,59.863945578231295,4,6,0.7999999999999999,3,0.7999999999999999,0.622,0.85925,55.26643688568504,105.22972942394816,9,False +bpl,bpl,mixed-random,0.6,88,40.136054421768705,4,6,1.0,3,0.7999999999999999,0.752,0.90975,34.53789864225064,67.80927819317637,14,False +britannica-qwen35,britannica,pair-balanced,0.25,180,75.0,13,15,1.0,5,0.942857142857143,0.971,0.9755384615384616,81.15453200095465,2306.972897111449,12,True +britannica-qwen35,britannica,pair-balanced,0.4,288,60.0,13,15,1.0,5,1.0,0.988,0.9830769230769231,58.05154309002893,197.08019997153377,19,True +britannica-qwen35,britannica,pair-balanced,0.6,432,40.0,13,15,1.0,5,1.0,0.993,0.9895384615384616,37.81826912701297,97.80114615896461,28,True +britannica-qwen35,britannica,mixed-random,0.25,180,75.0,13,15,0.942857142857143,4,0.942857142857143,0.956,0.9699230769230769,91.67612246683814,2348.1679073592472,12,True +britannica-qwen35,britannica,mixed-random,0.4,288,60.0,13,15,0.942857142857143,5,1.0,0.989,0.9826923076923078,60.51017695201898,197.6989721651328,19,True +britannica-qwen35,britannica,mixed-random,0.6,432,40.0,13,15,0.942857142857143,5,1.0,0.999,0.9906153846153847,39.35275643631411,105.36616237876325,28,True diff --git a/experiments/adaptive-stopping/multi_board.py b/experiments/adaptive-stopping/multi_board.py new file mode 100644 index 0000000..b89762f --- /dev/null +++ b/experiments/adaptive-stopping/multi_board.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +"""Cross-collection replay of fixed OCR comparison allocation designs. + +The script is read-only. It downloads pinned published comparison configs, selects +stored outcomes without inspecting winner values, and writes local experiment files. +It makes no judge API calls and performs no Hugging Face Hub writes. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import random +import statistics +from collections import defaultdict +from collections.abc import Sequence +from dataclasses import dataclass, replace +from itertools import combinations +from pathlib import Path +from typing import Any, Literal + +import numpy as np +from replay import _fixed_pair_balanced_sample, graph_metrics, load_stored_comparisons +from scipy.stats import kendalltau, spearmanr + +from ocr_bench.elo import ComparisonResult, Leaderboard, compute_elo + +DesignKind = Literal["pair-balanced", "mixed-random"] + + +@dataclass(frozen=True) +class BoardSpec: + """Pinned published comparison board.""" + + name: str + collection: str + repo: str + revision: str + role: str + + +BOARDS = ( + BoardSpec( + "rubenstein-30b", + "rubenstein", + "davanstrien/ocr-bench-rubenstein-judge-30b", + "8559f10e71d77d65d56169d4261b7c9ac25bebb3", + "independent collection", + ), + BoardSpec( + "rubenstein-jury", + "rubenstein", + "davanstrien/ocr-bench-rubenstein-judge", + "62e579c82f2713a58156b8e4248fb54a070ae110", + "judge sensitivity", + ), + BoardSpec( + "rubenstein-kimi", + "rubenstein", + "davanstrien/ocr-bench-rubenstein-judge-kimi-k25", + "1e76f23a91480a5ec741064ad9deb19004db64d3", + "judge sensitivity", + ), + BoardSpec( + "ufo-30b", + "ufo", + "davanstrien/ocr-bench-ufo-judge-30b", + "97623b511a0510e13b2f9e849f58d4002663a51b", + "independent collection", + ), + BoardSpec( + "bpl", + "bpl", + "davanstrien/bpl-ocr-bench-results", + "fa9e7a032b7b245d5dd69fd40e87646613af6401", + "independent collection; incomplete board", + ), + BoardSpec( + "britannica-qwen35", + "britannica", + "davanstrien/ocr-bench-britannica-results-qwen35", + "8b6602826e2b71c615ed229d388221a7aa7c69b9", + "same-corpus judge/model-grid sensitivity", + ), +) + +BUDGET_FRACTIONS = (0.25, 0.40, 0.60) +ALLOCATION_SEEDS = (42, 43, 44, 45, 46) +DESIGNS: tuple[DesignKind, ...] = ("pair-balanced", "mixed-random") +TOP_K = 3 + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cluster-replicates", + type=int, + default=200, + help="Page-cluster bootstrap replicates per board/design/seed (default: 200).", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(__file__).resolve().parent, + ) + return parser.parse_args() + + +def _rank_order(board: Leaderboard) -> list[str]: + return [model for model, _ in board.ranked] + + +def _rank_metrics( + board: Leaderboard, + reference: Leaderboard, + model_names: Sequence[str], +) -> dict[str, Any]: + reference_rank = _rank_order(reference) + rank = _rank_order(board) + reference_position = {model: i for i, model in enumerate(reference_rank, start=1)} + position = {model: i for i, model in enumerate(rank, start=1)} + ordered_models = sorted(model_names) + ref_positions = [reference_position[model] for model in ordered_models] + positions = [position[model] for model in ordered_models] + elo_deltas = [abs(board.elo[model] - reference.elo[model]) for model in ordered_models] + top_k = min(TOP_K, len(model_names)) + return { + "kendall_tau": float(kendalltau(ref_positions, positions).statistic), + "spearman_rho": float(spearmanr(ref_positions, positions).statistic), + "topk_membership_matches": set(rank[:top_k]) == set(reference_rank[:top_k]), + "topk_order_matches": rank[:top_k] == reference_rank[:top_k], + "median_absolute_elo_delta": statistics.median(elo_deltas), + "max_absolute_elo_delta": max(elo_deltas), + "rank_order": rank, + } + + +def _cluster_resample( + comparisons: Sequence[ComparisonResult], + *, + seed: int, +) -> list[ComparisonResult]: + """Resample page clusters and assign unique synthetic sample indices.""" + grouped: dict[int, list[ComparisonResult]] = defaultdict(list) + for comparison in comparisons: + grouped[comparison.sample_idx].append(comparison) + sample_ids = sorted(grouped) + rng = random.Random(seed) + drawn = rng.choices(sample_ids, k=len(sample_ids)) + return [ + replace(comparison, sample_idx=occurrence) + for occurrence, source_idx in enumerate(drawn) + for comparison in grouped[source_idx] + ] + + +def _mixed_warmup(comparisons: Sequence[ComparisonResult]) -> list[ComparisonResult]: + sample_ids = sorted({comparison.sample_idx for comparison in comparisons}) + warmup_ids = set(sample_ids[:5]) + return [comparison for comparison in comparisons if comparison.sample_idx in warmup_ids] + + +def _select_design( + comparisons: Sequence[ComparisonResult], + *, + design: DesignKind, + budget_fraction: float, + allocation_seed: int, +) -> list[ComparisonResult]: + required = _mixed_warmup(comparisons) if design == "mixed-random" else [] + budget = max(len(required), round(len(comparisons) * budget_fraction)) + return _fixed_pair_balanced_sample( + comparisons, + budget=budget, + seed=allocation_seed, + required=required, + ) + + +def _robust_pairs( + reference: Leaderboard, + bootstrap_boards: Sequence[Leaderboard], +) -> list[tuple[str, str]]: + """Pairs whose full-board direction repeats in at least 95% of page bootstraps.""" + reference_position = {model: i for i, (model, _) in enumerate(reference.ranked, start=1)} + robust: list[tuple[str, str]] = [] + for model_a, model_b in combinations(reference_position, 2): + higher, lower = ( + (model_a, model_b) + if reference_position[model_a] < reference_position[model_b] + else (model_b, model_a) + ) + agreements = 0 + for board in bootstrap_boards: + position = {model: i for i, (model, _) in enumerate(board.ranked, start=1)} + agreements += position[higher] < position[lower] + if agreements / len(bootstrap_boards) >= 0.95: + robust.append((higher, lower)) + return robust + + +def _robust_pair_agreement( + board: Leaderboard, + robust_pairs: Sequence[tuple[str, str]], +) -> float | None: + if not robust_pairs: + return None + position = {model: i for i, (model, _) in enumerate(board.ranked, start=1)} + return statistics.mean(position[higher] < position[lower] for higher, lower in robust_pairs) + + +def _quantile(values: Sequence[float], q: float) -> float: + return float(np.quantile(np.asarray(values), q)) + + +def _aggregate_design( + *, + spec: BoardSpec, + comparisons: Sequence[ComparisonResult], + model_names: Sequence[str], + reference: Leaderboard, + bootstrap_rows: Sequence[Sequence[ComparisonResult]], + bootstrap_boards: Sequence[Leaderboard], + robust_pairs: Sequence[tuple[str, str]], + design: DesignKind, + budget_fraction: float, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + seed_rows: list[dict[str, Any]] = [] + clustered_metrics: list[dict[str, Any]] = [] + + for allocation_seed in ALLOCATION_SEEDS: + selected = _select_design( + comparisons, + design=design, + budget_fraction=budget_fraction, + allocation_seed=allocation_seed, + ) + board = compute_elo(selected, list(model_names), n_bootstrap=0) + point = _rank_metrics(board, reference, model_names) + point.update( + { + "allocation_seed": allocation_seed, + "comparisons_consumed": len(selected), + "percentage_saved": 100.0 * (len(comparisons) - len(selected)) / len(comparisons), + "robust_pair_agreement": _robust_pair_agreement(board, robust_pairs), + "graph": graph_metrics(selected, model_names), + } + ) + + per_seed_clustered: list[dict[str, Any]] = [] + for resampled, bootstrap_reference in zip(bootstrap_rows, bootstrap_boards, strict=True): + bootstrap_selected = _select_design( + resampled, + design=design, + budget_fraction=budget_fraction, + allocation_seed=allocation_seed, + ) + bootstrap_board = compute_elo( + bootstrap_selected, + list(model_names), + n_bootstrap=0, + ) + metric = _rank_metrics(bootstrap_board, bootstrap_reference, model_names) + metric["robust_pair_agreement"] = _robust_pair_agreement( + bootstrap_board, + robust_pairs, + ) + per_seed_clustered.append(metric) + clustered_metrics.append(metric) + + point["clustered"] = { + "kendall_tau_median": statistics.median( + metric["kendall_tau"] for metric in per_seed_clustered + ), + "topk_membership_rate": statistics.mean( + metric["topk_membership_matches"] for metric in per_seed_clustered + ), + "robust_pair_agreement_rate": statistics.mean( + metric["robust_pair_agreement"] + for metric in per_seed_clustered + if metric["robust_pair_agreement"] is not None + ) + if robust_pairs + else None, + } + seed_rows.append(point) + + point_taus = [row["kendall_tau"] for row in seed_rows] + point_rhos = [row["spearman_rho"] for row in seed_rows] + point_max_deltas = [row["max_absolute_elo_delta"] for row in seed_rows] + clustered_taus = [metric["kendall_tau"] for metric in clustered_metrics] + clustered_rhos = [metric["spearman_rho"] for metric in clustered_metrics] + clustered_max_deltas = [metric["max_absolute_elo_delta"] for metric in clustered_metrics] + clustered_robust = [ + metric["robust_pair_agreement"] + for metric in clustered_metrics + if metric["robust_pair_agreement"] is not None + ] + + clustered_topk_rate = statistics.mean( + metric["topk_membership_matches"] for metric in clustered_metrics + ) + clustered_robust_rate = statistics.mean(clustered_robust) if clustered_robust else None + clustered_spearman_median = statistics.median(clustered_rhos) + cross_board_criteria = { + "topk_membership_rate_at_least_0_90": clustered_topk_rate >= 0.90, + "robust_pair_agreement_at_least_0_95": ( + clustered_robust_rate >= 0.95 if clustered_robust_rate is not None else False + ), + "spearman_rho_median_at_least_0_90": clustered_spearman_median >= 0.90, + } + cross_board_criteria["passed"] = all(cross_board_criteria.values()) + + aggregate = { + "board": spec.name, + "collection": spec.collection, + "role": spec.role, + "design": design, + "budget_fraction": budget_fraction, + "comparisons_consumed": seed_rows[0]["comparisons_consumed"], + "percentage_saved": seed_rows[0]["percentage_saved"], + "allocation_seeds": list(ALLOCATION_SEEDS), + "robust_full_board_pairs": len(robust_pairs), + "possible_pairs": len(model_names) * (len(model_names) - 1) // 2, + "point_kendall_tau_median": statistics.median(point_taus), + "point_kendall_tau_min": min(point_taus), + "point_kendall_tau_max": max(point_taus), + "point_spearman_rho_median": statistics.median(point_rhos), + "point_topk_membership_matches": sum(row["topk_membership_matches"] for row in seed_rows), + "point_topk_order_matches": sum(row["topk_order_matches"] for row in seed_rows), + "point_max_absolute_elo_delta_median": statistics.median(point_max_deltas), + "point_max_absolute_elo_delta_worst": max(point_max_deltas), + "clustered_kendall_tau_median": statistics.median(clustered_taus), + "clustered_kendall_tau_p05": _quantile(clustered_taus, 0.05), + "clustered_spearman_rho_median": clustered_spearman_median, + "clustered_topk_membership_rate": clustered_topk_rate, + "clustered_topk_order_rate": statistics.mean( + metric["topk_order_matches"] for metric in clustered_metrics + ), + "clustered_robust_pair_agreement": clustered_robust_rate, + "clustered_max_absolute_elo_delta_median": statistics.median(clustered_max_deltas), + "clustered_max_absolute_elo_delta_p95": _quantile( + clustered_max_deltas, + 0.95, + ), + "minimum_pair_evidence": min(row["graph"]["min_direct_comparisons"] for row in seed_rows), + "cross_board_criteria": cross_board_criteria, + } + return aggregate, seed_rows + + +def _full_board_stability( + reference: Leaderboard, + bootstrap_boards: Sequence[Leaderboard], + model_names: Sequence[str], +) -> dict[str, Any]: + metrics = [_rank_metrics(board, reference, model_names) for board in bootstrap_boards] + return { + "kendall_tau_median": statistics.median(metric["kendall_tau"] for metric in metrics), + "kendall_tau_p05": _quantile([metric["kendall_tau"] for metric in metrics], 0.05), + "topk_membership_rate": statistics.mean( + metric["topk_membership_matches"] for metric in metrics + ), + "topk_order_rate": statistics.mean(metric["topk_order_matches"] for metric in metrics), + } + + +def _write_summary(path: Path, aggregates: Sequence[dict[str, Any]]) -> None: + fields = [ + "board", + "collection", + "design", + "budget_fraction", + "comparisons_consumed", + "percentage_saved", + "robust_full_board_pairs", + "possible_pairs", + "point_spearman_rho_median", + "point_topk_membership_matches", + "clustered_spearman_rho_median", + "clustered_topk_membership_rate", + "clustered_robust_pair_agreement", + "clustered_max_absolute_elo_delta_median", + "clustered_max_absolute_elo_delta_p95", + "minimum_pair_evidence", + "passed", + ] + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") + writer.writeheader() + for aggregate in aggregates: + row = {field: aggregate.get(field) for field in fields} + row["passed"] = aggregate["cross_board_criteria"]["passed"] + writer.writerow(row) + + +def main() -> None: + args = _parse_args() + if args.cluster_replicates < 1: + raise ValueError("--cluster-replicates must be at least 1") + + board_results: list[dict[str, Any]] = [] + aggregates: list[dict[str, Any]] = [] + for spec in BOARDS: + print(f"Loading {spec.name} ...", flush=True) + comparisons, source_rows, sentinel_rows = load_stored_comparisons( + spec.repo, + spec.revision, + ) + model_names = sorted({model for row in comparisons for model in (row.model_a, row.model_b)}) + reference = compute_elo(comparisons, model_names, n_bootstrap=0) + + bootstrap_rows = [ + _cluster_resample(comparisons, seed=10_000 + replicate) + for replicate in range(args.cluster_replicates) + ] + bootstrap_boards = [ + compute_elo(rows, model_names, n_bootstrap=0) for rows in bootstrap_rows + ] + robust_pairs = _robust_pairs(reference, bootstrap_boards) + board_aggregates: list[dict[str, Any]] = [] + board_seed_rows: list[dict[str, Any]] = [] + for design in DESIGNS: + for budget_fraction in BUDGET_FRACTIONS: + print( + f" {design} at {budget_fraction:.0%} budget ...", + flush=True, + ) + aggregate, seed_rows = _aggregate_design( + spec=spec, + comparisons=comparisons, + model_names=model_names, + reference=reference, + bootstrap_rows=bootstrap_rows, + bootstrap_boards=bootstrap_boards, + robust_pairs=robust_pairs, + design=design, + budget_fraction=budget_fraction, + ) + board_aggregates.append(aggregate) + aggregates.append(aggregate) + board_seed_rows.extend( + { + "design": design, + "budget_fraction": budget_fraction, + **row, + } + for row in seed_rows + ) + + board_results.append( + { + "name": spec.name, + "collection": spec.collection, + "repo": spec.repo, + "revision": spec.revision, + "role": spec.role, + "stored_comparisons": source_rows, + "sentinel_comparisons_detectable": sentinel_rows, + "models": model_names, + "sample_count": len({row.sample_idx for row in comparisons}), + "full_rank_order": _rank_order(reference), + "robust_pairs": robust_pairs, + "full_board_page_bootstrap_stability": _full_board_stability( + reference, + bootstrap_boards, + model_names, + ), + "aggregates": board_aggregates, + "seed_results": board_seed_rows, + } + ) + + collection_passes: dict[str, dict[str, int]] = defaultdict(lambda: {"passed": 0, "total": 0}) + for aggregate in aggregates: + key = f"{aggregate['design']}@{aggregate['budget_fraction']:.0%}" + collection_key = f"{aggregate['collection']}:{key}" + collection_passes[collection_key]["total"] += 1 + collection_passes[collection_key]["passed"] += int( + aggregate["cross_board_criteria"]["passed"] + ) + + payload = { + "experiment": { + "boards": len(BOARDS), + "independent_collections": sorted({spec.collection for spec in BOARDS}), + "budget_fractions": list(BUDGET_FRACTIONS), + "allocation_seeds": list(ALLOCATION_SEEDS), + "cluster_replicates": args.cluster_replicates, + "top_k": TOP_K, + "robust_pair_definition": ( + "full-board rank direction repeats in at least 95% of page-cluster bootstraps" + ), + "cross_board_criteria": { + "clustered_topk_membership_rate_minimum": 0.90, + "clustered_robust_pair_agreement_minimum": 0.95, + "clustered_spearman_rho_median_minimum": 0.90, + }, + "notes": [ + "Rubenstein judge variants are judge sensitivity, not independent collections.", + "Allocation uses only published valid rows and never inspects winner values.", + "Older comparison schemas without OCR text cannot be rechecked for sentinels.", + "Page bootstrap resamples sample_idx clusters and reruns allocation per replicate.", + ], + }, + "boards": board_results, + "aggregates": aggregates, + "collection_design_pass_counts": dict(collection_passes), + } + + args.output_dir.mkdir(parents=True, exist_ok=True) + json_path = args.output_dir / "multi-board-results.json" + csv_path = args.output_dir / "multi-board-summary.csv" + json_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + _write_summary(csv_path, aggregates) + print(f"Wrote {json_path}") + print(f"Wrote {csv_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/adaptive-stopping/replay.py b/experiments/adaptive-stopping/replay.py index 2bc81f9..2827f1d 100644 --- a/experiments/adaptive-stopping/replay.py +++ b/experiments/adaptive-stopping/replay.py @@ -202,15 +202,13 @@ def load_stored_comparisons( sentinel_rows += int(has_sentinel) if exclude_sentinels and has_sentinel: continue - winner = row["winner"] - if winner not in {"A", "B", "tie"}: - raise ValueError(f"Unexpected stored winner: {winner!r}") + winner = _canonical_winner(row["winner"]) results.append( ComparisonResult( sample_idx=row["sample_idx"], model_a=row["model_a"], model_b=row["model_b"], - winner=cast(Winner, winner), + winner=winner, reason=row.get("reason", ""), agreement=row.get("agreement", "1/1"), # Published winners are already canonical/unswapped. @@ -226,6 +224,17 @@ def load_stored_comparisons( return results, source_rows, sentinel_rows +def _canonical_winner(value: object) -> Winner: + """Normalize current and historical stored winner spellings.""" + if isinstance(value, str): + normalized = value.strip() + if normalized in {"A", "B"}: + return cast(Winner, normalized) + if normalized.lower() == "tie": + return "tie" + raise ValueError(f"Unexpected stored winner: {value!r}") + + def _group_by_sample( comparisons: Iterable[ComparisonResult], ) -> dict[int, list[ComparisonResult]]: diff --git a/tests/test_adaptive_replay_experiment.py b/tests/test_adaptive_replay_experiment.py index 7df218d..cc8633c 100644 --- a/tests/test_adaptive_replay_experiment.py +++ b/tests/test_adaptive_replay_experiment.py @@ -17,6 +17,14 @@ replay = importlib.util.module_from_spec(_SPEC) sys.modules[_SPEC.name] = replay _SPEC.loader.exec_module(replay) +sys.modules["replay"] = replay + +_MULTI_PATH = _REPLAY_PATH.with_name("multi_board.py") +_MULTI_SPEC = importlib.util.spec_from_file_location("adaptive_stopping_multi_board", _MULTI_PATH) +assert _MULTI_SPEC is not None and _MULTI_SPEC.loader is not None +multi_board = importlib.util.module_from_spec(_MULTI_SPEC) +sys.modules[_MULTI_SPEC.name] = multi_board +_MULTI_SPEC.loader.exec_module(multi_board) def _stored_grid(n_samples: int = 8) -> list[ComparisonResult]: @@ -38,6 +46,12 @@ def _fixed_board() -> Leaderboard: ) +def test_historical_tie_spelling_is_canonicalized(): + assert replay._canonical_winner("tie") == "tie" + assert replay._canonical_winner("TIE") == "tie" + assert replay._canonical_winner("A") == "A" + + def test_targeted_replay_warms_up_balanced_then_filters_pairs(monkeypatch): stored = _stored_grid() monkeypatch.setattr(replay, "compute_elo", lambda *args, **kwargs: _fixed_board()) @@ -170,6 +184,59 @@ def test_mixed_sample_preserves_required_warmup_and_rebalances(): assert max(counts.values()) - min(counts.values()) == 1 +def test_cluster_resample_is_page_level_and_deterministic(): + stored = _stored_grid(8) + + first = multi_board._cluster_resample(stored, seed=123) + repeated = multi_board._cluster_resample(stored, seed=123) + + assert len(first) == len(stored) + assert sorted({row.sample_idx for row in first}) == list(range(8)) + assert Counter(row.sample_idx for row in first) == Counter({i: 3 for i in range(8)}) + assert first == repeated + + +def test_static_design_selection_does_not_inspect_winners(): + stored = _stored_grid(8) + flipped = [ComparisonResult(row.sample_idx, row.model_a, row.model_b, "B") for row in stored] + + selected = multi_board._select_design( + stored, + design="mixed-random", + budget_fraction=0.60, + allocation_seed=42, + ) + selected_flipped = multi_board._select_design( + flipped, + design="mixed-random", + budget_fraction=0.60, + allocation_seed=42, + ) + + assert [replay._comparison_key(row) for row in selected] == [ + replay._comparison_key(row) for row in selected_flipped + ] + + +def test_robust_pairs_require_95_percent_direction_stability(): + reference = Leaderboard(elo={"a": 1600.0, "b": 1500.0, "c": 1400.0}) + bootstrap_boards = [ + Leaderboard( + elo={ + "a": 1600.0, + "b": 1500.0 if i < 18 else 1390.0, + "c": 1400.0 if i < 18 else 1500.0, + } + ) + for i in range(20) + ] + + assert multi_board._robust_pairs(reference, bootstrap_boards) == [ + ("a", "b"), + ("a", "c"), + ] + + def test_graph_metrics_report_coverage_and_connectivity(): complete = replay.graph_metrics(_stored_grid(1), ["a", "b", "c"]) disconnected = replay.graph_metrics( From 668b021d7dfe6c17faaf676a87095d4bcb8b3b38 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Thu, 16 Jul 2026 18:19:07 +0100 Subject: [PATCH 5/5] Replay adaptive strategies on MOH board --- experiments/adaptive-stopping/README.md | 65 +- experiments/adaptive-stopping/moh/README.md | 90 + .../adaptive-stopping/moh/elo-deltas.csv | 617 + .../adaptive-stopping/moh/results.json | 13118 ++++++++++++++++ .../adaptive-stopping/moh/round-history.csv | 131 + .../moh/static-design-summary.csv | 7 + .../moh/strategy-summary.csv | 45 + .../multi-board-results.json | 2733 +++- .../adaptive-stopping/multi-board-summary.csv | 6 + experiments/adaptive-stopping/multi_board.py | 7 + 10 files changed, 16743 insertions(+), 76 deletions(-) create mode 100644 experiments/adaptive-stopping/moh/README.md create mode 100644 experiments/adaptive-stopping/moh/elo-deltas.csv create mode 100644 experiments/adaptive-stopping/moh/results.json create mode 100644 experiments/adaptive-stopping/moh/round-history.csv create mode 100644 experiments/adaptive-stopping/moh/static-design-summary.csv create mode 100644 experiments/adaptive-stopping/moh/strategy-summary.csv diff --git a/experiments/adaptive-stopping/README.md b/experiments/adaptive-stopping/README.md index 162719d..989e581 100644 --- a/experiments/adaptive-stopping/README.md +++ b/experiments/adaptive-stopping/README.md @@ -1,9 +1,10 @@ # Adaptive stopping counterfactual replay **Recommendation: keep targeted allocation experimental and do not run a live trial.** Follow-up -replays tested stronger adaptive allocation, fixed outcome-independent designs, and six published -boards spanning four collections and three Rubenstein judges. There is no collection-robust basis -for a production change, especially for large model grids. +replays tested stronger adaptive allocation, fixed outcome-independent designs, and seven +published boards spanning five collections and three Rubenstein judges. A newly collected +independent 14-model MOH board confirms there is no basis for a production change or live targeted +trial. The opt-in targeted allocator delivers the expected call reduction, but not by stopping early: it uses all 10 five-sample rounds while selecting fewer pairs per round. Against the full stored @@ -245,7 +246,7 @@ membership much more stable than independently sampling each pair. ## Cross-collection, page-clustered replay -The fixed designs were then replayed on six suitable published boards without new judge calls. +The fixed designs were then replayed on seven suitable published boards without new judge calls. Revisions are pinned in [`multi_board.py`](multi_board.py). | Board | Collection/role | Outcomes | Models | Samples | @@ -256,9 +257,10 @@ Revisions are pinned in [`multi_board.py`](multi_board.py). | UFO 30B | independent collection | 294 | 4 | 49 | | BPL | independent, incomplete board | 147 | 4 | 41 | | Britannica Qwen35 | same corpus, earlier 6-model/judge sensitivity | 720 | 6 | 50 | +| MOH table fidelity | independent large-grid collection | 4,505 | 14 | 50 | -Rubenstein's three judges are not counted as three independent collections. This gives four -collection groups, but only the original Britannica board has a 14-model graph. +Rubenstein's three judges are not counted as three independent collections. This gives five +collection groups and two independent 14-model boards: Britannica and MOH. ### Uncertainty method @@ -290,10 +292,12 @@ as pass criteria after the first replay showed that they can be unstable even fo | UFO 30B | 5 | 6 | 100.0% | 84.0% | | BPL | 4 | 6 | 52.0% | 28.5% | | Britannica Qwen35 | 13 | 15 | 100.0% | 65.0% | +| MOH table fidelity | 51 | 91 | 19.0% | 10.5% | -BPL and the Rubenstein jury board cannot meet a 90% top-3 stability target reliably even when all -stored outcomes are used. Their failures below are therefore evidence that the reference board is -under-resolved, not simply that subsampling is bad. +BPL, the Rubenstein jury board, and MOH cannot meet a 90% top-3 stability target reliably even when +all stored outcomes are used. Their failures below are therefore evidence that the reference board +is under-resolved, not simply that subsampling is bad. MOH still has 51 robust pair directions; +its instability is concentrated around exact ordering rather than every relationship. ### Results at a 40% outcome budget @@ -305,11 +309,12 @@ under-resolved, not simply that subsampling is bad. | UFO 30B | pass | pass | 99.9% / 99.6% | 100.0% / 99.9% | | BPL | fail | fail | 65.4% / 62.2% | 86.9% / 85.9% | | Britannica Qwen35 | pass | pass | 98.8% / 98.9% | 98.3% / 98.3% | +| MOH table fidelity | fail | fail | 47.4% / 37.2% | 98.4% / 98.3% | At 40%, both fixed designs pass on Rubenstein 30B, Rubenstein Kimi, UFO, and the 6-model -Britannica board; both fail on the intrinsically unstable Rubenstein jury and BPL boards. At 25%, -results are more collection-dependent. Increasing to 60% still cannot rescue unstable reference -boards. +Britannica board; both fail on the intrinsically unstable Rubenstein jury, BPL, and MOH boards. +At 25%, results are more collection-dependent. Increasing to 60% still cannot rescue unstable +reference boards. There is no consistent winner between pair-balanced and mixed-random allocation on these small model grids. Board/judge stability matters more than the choice between the two fixed selectors. @@ -317,9 +322,13 @@ Small samples also produce heavy ELO tails—Britannica Qwen35 has extreme page- outliers despite stable rank decisions—so rank/pairwise summaries are safer than treating ELO error as approximately Gaussian. -This broadens the evidence, but does not validate large-grid behavior: five of the six additional -boards have only four models, and the sixth has six. See -[`multi-board-summary.csv`](multi-board-summary.csv) and +MOH supplies the previously missing independent large grid. Its plain targeted replay uses 996 +outcomes (77.9% saved) but moves the full order to Kendall τ 0.846 and swaps ranks 2–3; size-aware +targeting uses 803 outcomes and falls to τ 0.780. Neither stops early. Full MOH details are in +[`moh/README.md`](moh/README.md). + +Across the two 14-model boards, targeted allocation consistently saves roughly 80% while moving +rankings and ELOs materially. See [`multi-board-summary.csv`](multi-board-summary.csv) and [`multi-board-results.json`](multi-board-results.json). ## Current sentinel-policy robustness check @@ -377,9 +386,9 @@ collection. 5. **Bradley–Terry fit under sparse, nonuniform allocation.** The graph is connected, so a fit is identifiable, but adjacency-driven edge counts can amplify model misspecification and shift the global ELO scale relative to a balanced grid. -6. **Large-grid evidence remains one collection.** The cross-board follow-up adds collections and - judges, but its additional grids have only four or six models. It cannot establish sparse - allocation behavior for another 10+ model board. +6. **Large-grid evidence is still limited.** Britannica and MOH provide two independent 14-model + boards under different criteria, and both warn against targeted allocation. More collections + would be needed before claiming a universal effect size. 7. **Unobserved failed judge calls.** Six attempted comparisons have no published valid verdict. The replay cannot know how a different allocation would have distributed those failures, so savings are relative to 4,293 valid stored outcomes rather than the 4,299 attempted calls. @@ -387,22 +396,22 @@ collection. robustness replay is closer to current input integrity semantics but has only 13 rankable models. 9. **Legacy cross-board schemas.** Several older boards do not store OCR text, so historical - sentinel rows cannot be rechecked. Five of the six additional boards have only four models, - limiting what they say about sparse large-grid allocation. + sentinel rows cannot be rechecked. Five additional boards have only four models, limiting their + contribution to sparse large-grid allocation evidence. 10. **Judge variants are not independent data.** The three Rubenstein boards measure sensitivity to judge choice on the same pages; they do not triple the collection-level evidence. ## Recommendation and next experiment **Keep `targeted` opt-in, retain `balanced` as the default, and do not implement the tested v2 or -fixed designs in production.** The cross-collection replay shows that fixed 40% subsampling can -preserve robust decisions on already-stable 4–6 model boards, but cannot repair an unstable full -board and does not test a large independent model grid. +fixed designs in production.** MOH provides the missing independent 14-model test and fails the +fidelity criteria: targeted allocation does not preserve the full board closely, while the full +board's own leading order is highly page-sensitive. -The evidence is now sufficient to stop retrospective threshold tuning. Before any live adaptive -trial, obtain at least one additional independent, fully stored board with roughly 10+ models and -50+ pages, then apply the same page-clustered/design-aware protocol. If that passes across seeds, -a budget-capped live trial becomes defensible. +The evidence is now sufficient to stop allocation tuning and reject a live targeted trial for this +strategy. The next product-level question is different: whether leaderboards should report +page-clustered rank tiers or robust pairwise relations rather than a single exact order. That work +requires a separate design issue and must not be smuggled into this experiment PR. Keep the 3× rule as a post-hoc deployment annotation only. Current evidence does not support letting it influence sampling, making it a default, or treating it as statistical resolution. @@ -445,6 +454,8 @@ Generated artifacts: - [`static-design-summary.csv`](static-design-summary.csv): five-seed fixed-design aggregates - [`multi-board-summary.csv`](multi-board-summary.csv): cross-board design-aware aggregates - [`multi-board-results.json`](multi-board-results.json): pinned board and page-bootstrap details +- [`moh/README.md`](moh/README.md): independent 14-model replay conclusions +- [`moh/results.json`](moh/results.json): complete pinned MOH replay metrics - [`results-sentinels-excluded.json`](results-sentinels-excluded.json): robustness replay Validation commands: diff --git a/experiments/adaptive-stopping/moh/README.md b/experiments/adaptive-stopping/moh/README.md new file mode 100644 index 0000000..c33987a --- /dev/null +++ b/experiments/adaptive-stopping/moh/README.md @@ -0,0 +1,90 @@ +# MOH independent 14-model replay + +This follow-up applies the adaptive-stopping experiment to a second independent 14-model board: +`davanstrien/ocr-bench-moh-results` at revision +`05d24695f131432c38984dd2cb55f5a18a3c2580`. + +The reference was collected non-adaptively over 50 Medical Officer of Health report pages using +the table-fidelity prompt, a 5,000-character normalized OCR cap, and 1,536px images. It contains +4,505 valid outcomes, 79 auto-ties, three parse failures, and no detectable sentinel comparisons. +No judge calls were made during this replay. + +## Result + +All adaptive variants exhausted all ten sample batches. As on Britannica, savings came from +judging fewer pairs per batch rather than satisfying the stopping rule early. + +| Strategy | Used | Saved | Kendall τ | Spearman ρ | Top-3 members/order | Median/max abs(ΔELO) | Final stat/practical/unresolved | +|---|---:|---:|---:|---:|---|---:|---:| +| Full / balanced | 4,505 | 0.0% | 1.000 | 1.000 | 3/3, yes | 0.0 / 0.0 | 2/0/11 | +| Targeted | 996 | 77.9% | 0.846 | 0.960 | 3/3, no | 46.2 / 115.7 | 1/0/12 | +| Targeted + 3×, min 10 | 803 | 82.2% | 0.780 | 0.925 | 3/3, no | 45.9 / 114.8 | 1/5/7 | +| Warm-up 10 | 1,777 | 60.6% | 0.868 | 0.965 | 2/3, no | 31.9 / 73.0 | 2/0/11 | +| Explore every 3 | 2,167 | 51.9% | 0.824 | 0.916 | 3/3, yes | 15.6 / 70.8 | 2/0/11 | + +No strategy passed the original gate. Size-aware stopping saved another 193 outcomes relative to +plain targeted but further reduced full-order agreement. The size annotations remain practical +deployment preferences, not statistical resolutions. + +### Rank movement + +The full top three are: + +1. `lightonai/LightOnOCR-2-1B` +2. `deepseek-ai/DeepSeek-OCR-2` +3. `baidu/Qianfan-OCR` + +Plain targeted preserves membership but swaps ranks 2 and 3. The 10-model warm-up loses one +full-board top-3 member. Periodic exploration restores the exact top-three order at the cost of +more than doubling targeted's outcome count, while full-order Kendall τ remains only 0.824. + +Per-model ELO changes are in [`elo-deltas.csv`](elo-deltas.csv). + +## Page-clustered stability + +The cross-board script resampled complete pages 200 times and reran each fixed allocation over five +seeds. The full 4,505-outcome board itself has: + +- median page-bootstrap Kendall τ: **0.780**; +- top-3 membership stability: **19.0%**; +- exact top-3 order stability: **10.5%**; and +- 51 robust pair directions out of 91 (direction repeated in ≥95% of page bootstraps). + +The leaders are therefore not stably ordered across page samples even when every stored comparison +is used. This is not a problem that a stopping threshold can solve. + +At a 40% fixed budget (1,802 outcomes): + +| Design | Clustered Spearman ρ | Top-3 membership agreement | Robust-pair agreement | Median/p95 max abs(ΔELO) | +|---|---:|---:|---:|---:| +| Pair-balanced | 0.960 | 47.4% | 98.4% | 34.3 / 55.8 | +| Mixed-random | 0.949 | 37.2% | 98.3% | 41.0 / 68.2 | + +Both designs preserve the board's robust pairwise directions well, but neither can reproduce an +unstable top-three set. At 60%, robust-pair agreement rises to roughly 98.9%, while top-3 agreement +remains only 61.5%/56.1%. + +## Conclusion + +This independent large-grid board confirms the Britannica warning more strongly: + +- targeted allocation is deterministic and cheap, but does not preserve the full board closely; +- ordinary adjacent marginal CIs do not describe page-sampling instability; +- practical size preferences should not control statistical sampling; and +- exact ranks among overlapping leaders are the wrong estimand. + +**Recommendation:** do not proceed to a live targeted trial or change the balanced default. Stop +allocation tuning until the product can report uncertainty-aware rank tiers or robust pairwise +relations using page-clustered uncertainty. Any such production change needs its own design and +validation work; this experiment should not directly change behavior. + +## Reproduce + +```bash +uv run python experiments/adaptive-stopping/replay.py \ + --repo davanstrien/ocr-bench-moh-results \ + --revision 05d24695f131432c38984dd2cb55f5a18a3c2580 \ + --output-dir experiments/adaptive-stopping/moh + +uv run python experiments/adaptive-stopping/multi_board.py +``` diff --git a/experiments/adaptive-stopping/moh/elo-deltas.csv b/experiments/adaptive-stopping/moh/elo-deltas.csv new file mode 100644 index 0000000..7c1095a --- /dev/null +++ b/experiments/adaptive-stopping/moh/elo-deltas.csv @@ -0,0 +1,617 @@ +strategy,model,elo,full_elo,delta,absolute_delta +full_stored_board,FireRedTeam/FireRed-OCR,1556.3777567649604,1556.3777567649604,0.0,0.0 +full_stored_board,PaddlePaddle/PP-OCRv6_medium,1354.3332993861072,1354.3332993861072,0.0,0.0 +full_stored_board,PaddlePaddle/PaddleOCR-VL-1.6,1503.691822219892,1503.691822219892,0.0,0.0 +full_stored_board,allenai/olmOCR-2-7B-1025-FP8,1497.230656288293,1497.230656288293,0.0,0.0 +full_stored_board,baidu/Qianfan-OCR,1576.6608094768258,1576.6608094768258,0.0,0.0 +full_stored_board,baidu/Unlimited-OCR,1563.2800425238524,1563.2800425238524,0.0,0.0 +full_stored_board,deepseek-ai/DeepSeek-OCR,1451.5246523995515,1451.5246523995515,0.0,0.0 +full_stored_board,deepseek-ai/DeepSeek-OCR-2,1579.3558964368967,1579.3558964368967,0.0,0.0 +full_stored_board,lightonai/LightOnOCR-2-1B,1617.5188506558277,1617.5188506558277,0.0,0.0 +full_stored_board,numind/NuExtract3,1571.572188853421,1571.572188853421,0.0,0.0 +full_stored_board,rednote-hilab/dots.mocr,1556.2216675111229,1556.2216675111229,0.0,0.0 +full_stored_board,rednote-hilab/dots.ocr,1529.2527243518712,1529.2527243518712,0.0,0.0 +full_stored_board,tesseract-5,1151.0531271625612,1151.0531271625612,0.0,0.0 +full_stored_board,zai-org/GLM-OCR,1491.926505968817,1491.926505968817,0.0,0.0 +balanced_adaptive,FireRedTeam/FireRed-OCR,1556.3777567649604,1556.3777567649604,0.0,0.0 +balanced_adaptive,PaddlePaddle/PP-OCRv6_medium,1354.3332993861072,1354.3332993861072,0.0,0.0 +balanced_adaptive,PaddlePaddle/PaddleOCR-VL-1.6,1503.691822219892,1503.691822219892,0.0,0.0 +balanced_adaptive,allenai/olmOCR-2-7B-1025-FP8,1497.230656288293,1497.230656288293,0.0,0.0 +balanced_adaptive,baidu/Qianfan-OCR,1576.6608094768258,1576.6608094768258,0.0,0.0 +balanced_adaptive,baidu/Unlimited-OCR,1563.2800425238524,1563.2800425238524,0.0,0.0 +balanced_adaptive,deepseek-ai/DeepSeek-OCR,1451.5246523995515,1451.5246523995515,0.0,0.0 +balanced_adaptive,deepseek-ai/DeepSeek-OCR-2,1579.3558964368967,1579.3558964368967,0.0,0.0 +balanced_adaptive,lightonai/LightOnOCR-2-1B,1617.5188506558277,1617.5188506558277,0.0,0.0 +balanced_adaptive,numind/NuExtract3,1571.572188853421,1571.572188853421,0.0,0.0 +balanced_adaptive,rednote-hilab/dots.mocr,1556.2216675111229,1556.2216675111229,0.0,0.0 +balanced_adaptive,rednote-hilab/dots.ocr,1529.2527243518712,1529.2527243518712,0.0,0.0 +balanced_adaptive,tesseract-5,1151.0531271625612,1151.0531271625612,0.0,0.0 +balanced_adaptive,zai-org/GLM-OCR,1491.926505968817,1491.926505968817,0.0,0.0 +targeted,FireRedTeam/FireRed-OCR,1634.4644819496004,1556.3777567649604,78.08672518464004,78.08672518464004 +targeted,PaddlePaddle/PP-OCRv6_medium,1298.3199433004734,1354.3332993861072,-56.01335608563386,56.01335608563386 +targeted,PaddlePaddle/PaddleOCR-VL-1.6,1419.9438300011045,1503.691822219892,-83.74799221878743,83.74799221878743 +targeted,allenai/olmOCR-2-7B-1025-FP8,1468.4599875082165,1497.230656288293,-28.77066878007645,28.77066878007645 +targeted,baidu/Qianfan-OCR,1692.33029306944,1576.6608094768258,115.66948359261414,115.66948359261414 +targeted,baidu/Unlimited-OCR,1569.642762356888,1563.2800425238524,6.362719833035499,6.362719833035499 +targeted,deepseek-ai/DeepSeek-OCR,1419.0954970052717,1451.5246523995515,-32.42915539427986,32.42915539427986 +targeted,deepseek-ai/DeepSeek-OCR-2,1656.234457507528,1579.3558964368967,76.87856107063135,76.87856107063135 +targeted,lightonai/LightOnOCR-2-1B,1723.1332227144233,1617.5188506558277,105.61437205859556,105.61437205859556 +targeted,numind/NuExtract3,1535.1101798239197,1571.572188853421,-36.46200902950136,36.46200902950136 +targeted,rednote-hilab/dots.mocr,1523.7737354890974,1556.2216675111229,-32.4479320220255,32.4479320220255 +targeted,rednote-hilab/dots.ocr,1530.7460734188971,1529.2527243518712,1.4933490670259744,1.4933490670259744 +targeted,tesseract-5,1067.0163609784408,1151.0531271625612,-84.03676618412032,84.03676618412032 +targeted,zai-org/GLM-OCR,1461.729174876699,1491.926505968817,-30.19733109211802,30.19733109211802 +targeted_size_3_min_10,FireRedTeam/FireRed-OCR,1636.6508950725304,1556.3777567649604,80.27313830756998,80.27313830756998 +targeted_size_3_min_10,PaddlePaddle/PP-OCRv6_medium,1313.406645594515,1354.3332993861072,-40.926653791592116,40.926653791592116 +targeted_size_3_min_10,PaddlePaddle/PaddleOCR-VL-1.6,1400.4352191082205,1503.691822219892,-103.25660311167144,103.25660311167144 +targeted_size_3_min_10,allenai/olmOCR-2-7B-1025-FP8,1454.5858248562365,1497.230656288293,-42.644831432056435,42.644831432056435 +targeted_size_3_min_10,baidu/Qianfan-OCR,1691.4810497148387,1576.6608094768258,114.82024023801296,114.82024023801296 +targeted_size_3_min_10,baidu/Unlimited-OCR,1577.5206071828436,1563.2800425238524,14.240564658991161,14.240564658991161 +targeted_size_3_min_10,deepseek-ai/DeepSeek-OCR,1419.982994012063,1451.5246523995515,-31.541658387488496,31.541658387488496 +targeted_size_3_min_10,deepseek-ai/DeepSeek-OCR-2,1658.3905650892693,1579.3558964368967,79.0346686523726,79.0346686523726 +targeted_size_3_min_10,lightonai/LightOnOCR-2-1B,1722.9467755513317,1617.5188506558277,105.42792489550402,105.42792489550402 +targeted_size_3_min_10,numind/NuExtract3,1522.4224783594152,1571.572188853421,-49.14971049400583,49.14971049400583 +targeted_size_3_min_10,rednote-hilab/dots.mocr,1517.0352764844622,1556.2216675111229,-39.18639102666066,39.18639102666066 +targeted_size_3_min_10,rednote-hilab/dots.ocr,1528.0978067177562,1529.2527243518712,-1.1549176341150087,1.1549176341150087 +targeted_size_3_min_10,tesseract-5,1069.2927279364471,1151.0531271625612,-81.76039922611403,81.76039922611403 +targeted_size_3_min_10,zai-org/GLM-OCR,1487.7511343200704,1491.926505968817,-4.1753716487467045,4.1753716487467045 +targeted_size_2_min_10,FireRedTeam/FireRed-OCR,1642.0704925711239,1556.3777567649604,85.69273580616345,85.69273580616345 +targeted_size_2_min_10,PaddlePaddle/PP-OCRv6_medium,1313.8310412401265,1354.3332993861072,-40.502258145980704,40.502258145980704 +targeted_size_2_min_10,PaddlePaddle/PaddleOCR-VL-1.6,1400.797992322702,1503.691822219892,-102.89382989719002,102.89382989719002 +targeted_size_2_min_10,allenai/olmOCR-2-7B-1025-FP8,1454.7497016233788,1497.230656288293,-42.48095466491418,42.48095466491418 +targeted_size_2_min_10,baidu/Qianfan-OCR,1680.9416300315613,1576.6608094768258,104.28082055473556,104.28082055473556 +targeted_size_2_min_10,baidu/Unlimited-OCR,1572.1693233029544,1563.2800425238524,8.88928077910191,8.88928077910191 +targeted_size_2_min_10,deepseek-ai/DeepSeek-OCR,1420.4502447225152,1451.5246523995515,-31.074407677036334,31.074407677036334 +targeted_size_2_min_10,deepseek-ai/DeepSeek-OCR-2,1655.9516154765304,1579.3558964368967,76.59571903963365,76.59571903963365 +targeted_size_2_min_10,lightonai/LightOnOCR-2-1B,1721.5263460984947,1617.5188506558277,104.00749544266705,104.00749544266705 +targeted_size_2_min_10,numind/NuExtract3,1521.3675017119538,1571.572188853421,-50.204687141467275,50.204687141467275 +targeted_size_2_min_10,rednote-hilab/dots.mocr,1516.8883838987947,1556.2216675111229,-39.33328361232816,39.33328361232816 +targeted_size_2_min_10,rednote-hilab/dots.ocr,1540.700126069093,1529.2527243518712,11.447401717221737,11.447401717221737 +targeted_size_2_min_10,tesseract-5,1069.802125503712,1151.0531271625612,-81.25100165884919,81.25100165884919 +targeted_size_2_min_10,zai-org/GLM-OCR,1488.75347542706,1491.926505968817,-3.1730305417570435,3.1730305417570435 +targeted_size_3_min_5,FireRedTeam/FireRed-OCR,1641.9181041792333,1556.3777567649604,85.5403474142729,85.5403474142729 +targeted_size_3_min_5,PaddlePaddle/PP-OCRv6_medium,1311.1365633965083,1354.3332993861072,-43.19673598959889,43.19673598959889 +targeted_size_3_min_5,PaddlePaddle/PaddleOCR-VL-1.6,1393.444130489657,1503.691822219892,-110.24769173023492,110.24769173023492 +targeted_size_3_min_5,allenai/olmOCR-2-7B-1025-FP8,1454.600116632278,1497.230656288293,-42.63053965601489,42.63053965601489 +targeted_size_3_min_5,baidu/Qianfan-OCR,1690.526646918854,1576.6608094768258,113.86583744202835,113.86583744202835 +targeted_size_3_min_5,baidu/Unlimited-OCR,1574.0785452974121,1563.2800425238524,10.798502773559676,10.798502773559676 +targeted_size_3_min_5,deepseek-ai/DeepSeek-OCR,1424.2509337258196,1451.5246523995515,-27.27371867373199,27.27371867373199 +targeted_size_3_min_5,deepseek-ai/DeepSeek-OCR-2,1651.3336379186167,1579.3558964368967,71.97774148171993,71.97774148171993 +targeted_size_3_min_5,lightonai/LightOnOCR-2-1B,1733.4271571063932,1617.5188506558277,115.90830645056553,115.90830645056553 +targeted_size_3_min_5,numind/NuExtract3,1521.2565641223305,1571.572188853421,-50.315624731090566,50.315624731090566 +targeted_size_3_min_5,rednote-hilab/dots.mocr,1514.6330233206916,1556.2216675111229,-41.588644190431296,41.588644190431296 +targeted_size_3_min_5,rednote-hilab/dots.ocr,1527.1124510512539,1529.2527243518712,-2.1402733006173094,2.1402733006173094 +targeted_size_3_min_5,tesseract-5,1069.0458107324857,1151.0531271625612,-82.00731643007543,82.00731643007543 +targeted_size_3_min_5,zai-org/GLM-OCR,1493.2363151084658,1491.926505968817,1.309809139648678,1.309809139648678 +targeted_size_3_min_15,FireRedTeam/FireRed-OCR,1640.6310230547772,1556.3777567649604,84.25326628981679,84.25326628981679 +targeted_size_3_min_15,PaddlePaddle/PP-OCRv6_medium,1315.9885719921695,1354.3332993861072,-38.344727393937774,38.344727393937774 +targeted_size_3_min_15,PaddlePaddle/PaddleOCR-VL-1.6,1415.8469878397334,1503.691822219892,-87.84483438015855,87.84483438015855 +targeted_size_3_min_15,allenai/olmOCR-2-7B-1025-FP8,1453.7681525506882,1497.230656288293,-43.46250373760472,43.46250373760472 +targeted_size_3_min_15,baidu/Qianfan-OCR,1698.5567961420863,1576.6608094768258,121.89598666526058,121.89598666526058 +targeted_size_3_min_15,baidu/Unlimited-OCR,1580.008338888557,1563.2800425238524,16.728296364704647,16.728296364704647 +targeted_size_3_min_15,deepseek-ai/DeepSeek-OCR,1417.8457641117898,1451.5246523995515,-33.678888287761765,33.678888287761765 +targeted_size_3_min_15,deepseek-ai/DeepSeek-OCR-2,1666.2453105478435,1579.3558964368967,86.88941411094675,86.88941411094675 +targeted_size_3_min_15,lightonai/LightOnOCR-2-1B,1708.646974850841,1617.5188506558277,91.12812419501324,91.12812419501324 +targeted_size_3_min_15,numind/NuExtract3,1523.6684667385953,1571.572188853421,-47.90372211482577,47.90372211482577 +targeted_size_3_min_15,rednote-hilab/dots.mocr,1514.9566118392627,1556.2216675111229,-41.26505567186018,41.26505567186018 +targeted_size_3_min_15,rednote-hilab/dots.ocr,1529.7231145830472,1529.2527243518712,0.47039023117599754,0.47039023117599754 +targeted_size_3_min_15,tesseract-5,1037.2744013590816,1151.0531271625612,-113.77872580347957,113.77872580347957 +targeted_size_3_min_15,zai-org/GLM-OCR,1496.8394855015276,1491.926505968817,4.912979532710551,4.912979532710551 +targeted_size_5_min_10,FireRedTeam/FireRed-OCR,1639.410750341219,1556.3777567649604,83.0329935762586,83.0329935762586 +targeted_size_5_min_10,PaddlePaddle/PP-OCRv6_medium,1303.4247635958586,1354.3332993861072,-50.90853579024861,50.90853579024861 +targeted_size_5_min_10,PaddlePaddle/PaddleOCR-VL-1.6,1420.7266637684627,1503.691822219892,-82.96515845142926,82.96515845142926 +targeted_size_5_min_10,allenai/olmOCR-2-7B-1025-FP8,1462.6672924959607,1497.230656288293,-34.56336379233221,34.56336379233221 +targeted_size_5_min_10,baidu/Qianfan-OCR,1695.758725320333,1576.6608094768258,119.09791584350728,119.09791584350728 +targeted_size_5_min_10,baidu/Unlimited-OCR,1580.5507936743256,1563.2800425238524,17.27075115047319,17.27075115047319 +targeted_size_5_min_10,deepseek-ai/DeepSeek-OCR,1413.7574993659675,1451.5246523995515,-37.76715303358401,37.76715303358401 +targeted_size_5_min_10,deepseek-ai/DeepSeek-OCR-2,1659.3956299503259,1579.3558964368967,80.03973351342916,80.03973351342916 +targeted_size_5_min_10,lightonai/LightOnOCR-2-1B,1726.0180068524983,1617.5188506558277,108.49915619667058,108.49915619667058 +targeted_size_5_min_10,numind/NuExtract3,1526.8834874726354,1571.572188853421,-44.68870138078569,44.68870138078569 +targeted_size_5_min_10,rednote-hilab/dots.mocr,1529.4121354121596,1556.2216675111229,-26.809532098963246,26.809532098963246 +targeted_size_5_min_10,rednote-hilab/dots.ocr,1531.7086234567364,1529.2527243518712,2.4558991048652388,2.4558991048652388 +targeted_size_5_min_10,tesseract-5,1033.9051505314112,1151.0531271625612,-117.14797663114996,117.14797663114996 +targeted_size_5_min_10,zai-org/GLM-OCR,1476.3804777621058,1491.926505968817,-15.546028206711298,15.546028206711298 +targeted_v2_warmup_5,FireRedTeam/FireRed-OCR,1634.4644819496004,1556.3777567649604,78.08672518464004,78.08672518464004 +targeted_v2_warmup_5,PaddlePaddle/PP-OCRv6_medium,1298.3199433004734,1354.3332993861072,-56.01335608563386,56.01335608563386 +targeted_v2_warmup_5,PaddlePaddle/PaddleOCR-VL-1.6,1419.9438300011045,1503.691822219892,-83.74799221878743,83.74799221878743 +targeted_v2_warmup_5,allenai/olmOCR-2-7B-1025-FP8,1468.4599875082165,1497.230656288293,-28.77066878007645,28.77066878007645 +targeted_v2_warmup_5,baidu/Qianfan-OCR,1692.33029306944,1576.6608094768258,115.66948359261414,115.66948359261414 +targeted_v2_warmup_5,baidu/Unlimited-OCR,1569.642762356888,1563.2800425238524,6.362719833035499,6.362719833035499 +targeted_v2_warmup_5,deepseek-ai/DeepSeek-OCR,1419.0954970052717,1451.5246523995515,-32.42915539427986,32.42915539427986 +targeted_v2_warmup_5,deepseek-ai/DeepSeek-OCR-2,1656.234457507528,1579.3558964368967,76.87856107063135,76.87856107063135 +targeted_v2_warmup_5,lightonai/LightOnOCR-2-1B,1723.1332227144233,1617.5188506558277,105.61437205859556,105.61437205859556 +targeted_v2_warmup_5,numind/NuExtract3,1535.1101798239197,1571.572188853421,-36.46200902950136,36.46200902950136 +targeted_v2_warmup_5,rednote-hilab/dots.mocr,1523.7737354890974,1556.2216675111229,-32.4479320220255,32.4479320220255 +targeted_v2_warmup_5,rednote-hilab/dots.ocr,1530.7460734188971,1529.2527243518712,1.4933490670259744,1.4933490670259744 +targeted_v2_warmup_5,tesseract-5,1067.0163609784408,1151.0531271625612,-84.03676618412032,84.03676618412032 +targeted_v2_warmup_5,zai-org/GLM-OCR,1461.729174876699,1491.926505968817,-30.19733109211802,30.19733109211802 +targeted_v2_warmup_10,FireRedTeam/FireRed-OCR,1568.722241102218,1556.3777567649604,12.344484337257654,12.344484337257654 +targeted_v2_warmup_10,PaddlePaddle/PP-OCRv6_medium,1312.1837194170303,1354.3332993861072,-42.14957996907697,42.14957996907697 +targeted_v2_warmup_10,PaddlePaddle/PaddleOCR-VL-1.6,1473.2092356039213,1503.691822219892,-30.482586615970604,30.482586615970604 +targeted_v2_warmup_10,allenai/olmOCR-2-7B-1025-FP8,1468.553310120653,1497.230656288293,-28.677346167640053,28.677346167640053 +targeted_v2_warmup_10,baidu/Qianfan-OCR,1615.9765833705583,1576.6608094768258,39.31577389373251,39.31577389373251 +targeted_v2_warmup_10,baidu/Unlimited-OCR,1620.327206191052,1563.2800425238524,57.047163667199584,57.047163667199584 +targeted_v2_warmup_10,deepseek-ai/DeepSeek-OCR,1410.843527149632,1451.5246523995515,-40.68112524991943,40.68112524991943 +targeted_v2_warmup_10,deepseek-ai/DeepSeek-OCR-2,1652.3727276354937,1579.3558964368967,73.01683119859695,73.01683119859695 +targeted_v2_warmup_10,lightonai/LightOnOCR-2-1B,1650.8084138434176,1617.5188506558277,33.28956318758992,33.28956318758992 +targeted_v2_warmup_10,numind/NuExtract3,1556.150855104141,1571.572188853421,-15.421333749279938,15.421333749279938 +targeted_v2_warmup_10,rednote-hilab/dots.mocr,1504.3510309194155,1556.2216675111229,-51.87063659170735,51.87063659170735 +targeted_v2_warmup_10,rednote-hilab/dots.ocr,1522.6633875676125,1529.2527243518712,-6.5893367842586485,6.5893367842586485 +targeted_v2_warmup_10,tesseract-5,1172.3480433624047,1151.0531271625612,21.294916199843556,21.294916199843556 +targeted_v2_warmup_10,zai-org/GLM-OCR,1471.4897186124501,1491.926505968817,-20.43678735636695,20.43678735636695 +targeted_v2_explore_every_3,FireRedTeam/FireRed-OCR,1545.815410558876,1556.3777567649604,-10.562346206084385,10.562346206084385 +targeted_v2_explore_every_3,PaddlePaddle/PP-OCRv6_medium,1369.9519849650007,1354.3332993861072,15.618685578893519,15.618685578893519 +targeted_v2_explore_every_3,PaddlePaddle/PaddleOCR-VL-1.6,1509.5780822093675,1503.691822219892,5.886259989475548,5.886259989475548 +targeted_v2_explore_every_3,allenai/olmOCR-2-7B-1025-FP8,1506.8824721365972,1497.230656288293,9.651815848304295,9.651815848304295 +targeted_v2_explore_every_3,baidu/Qianfan-OCR,1580.0381538536838,1576.6608094768258,3.377344376857991,3.377344376857991 +targeted_v2_explore_every_3,baidu/Unlimited-OCR,1514.9961885430473,1563.2800425238524,-48.283853980805134,48.283853980805134 +targeted_v2_explore_every_3,deepseek-ai/DeepSeek-OCR,1468.233442075427,1451.5246523995515,16.70878967587555,16.70878967587555 +targeted_v2_explore_every_3,deepseek-ai/DeepSeek-OCR-2,1603.075284692148,1579.3558964368967,23.71938825525126,23.71938825525126 +targeted_v2_explore_every_3,lightonai/LightOnOCR-2-1B,1637.946154262932,1617.5188506558277,20.42730360710425,20.42730360710425 +targeted_v2_explore_every_3,numind/NuExtract3,1562.9140412754496,1571.572188853421,-8.658147577971476,8.658147577971476 +targeted_v2_explore_every_3,rednote-hilab/dots.mocr,1571.765282146598,1556.2216675111229,15.543614635475251,15.543614635475251 +targeted_v2_explore_every_3,rednote-hilab/dots.ocr,1532.8370744381737,1529.2527243518712,3.584350086302493,3.584350086302493 +targeted_v2_explore_every_3,tesseract-5,1080.2926058353735,1151.0531271625612,-70.76052132718769,70.76052132718769 +targeted_v2_explore_every_3,zai-org/GLM-OCR,1515.6738230073256,1491.926505968817,23.747317038508527,23.747317038508527 +targeted_v2_warmup_5_explore_3_annotate_3x,FireRedTeam/FireRed-OCR,1545.815410558876,1556.3777567649604,-10.562346206084385,10.562346206084385 +targeted_v2_warmup_5_explore_3_annotate_3x,PaddlePaddle/PP-OCRv6_medium,1369.9519849650007,1354.3332993861072,15.618685578893519,15.618685578893519 +targeted_v2_warmup_5_explore_3_annotate_3x,PaddlePaddle/PaddleOCR-VL-1.6,1509.5780822093675,1503.691822219892,5.886259989475548,5.886259989475548 +targeted_v2_warmup_5_explore_3_annotate_3x,allenai/olmOCR-2-7B-1025-FP8,1506.8824721365972,1497.230656288293,9.651815848304295,9.651815848304295 +targeted_v2_warmup_5_explore_3_annotate_3x,baidu/Qianfan-OCR,1580.0381538536838,1576.6608094768258,3.377344376857991,3.377344376857991 +targeted_v2_warmup_5_explore_3_annotate_3x,baidu/Unlimited-OCR,1514.9961885430473,1563.2800425238524,-48.283853980805134,48.283853980805134 +targeted_v2_warmup_5_explore_3_annotate_3x,deepseek-ai/DeepSeek-OCR,1468.233442075427,1451.5246523995515,16.70878967587555,16.70878967587555 +targeted_v2_warmup_5_explore_3_annotate_3x,deepseek-ai/DeepSeek-OCR-2,1603.075284692148,1579.3558964368967,23.71938825525126,23.71938825525126 +targeted_v2_warmup_5_explore_3_annotate_3x,lightonai/LightOnOCR-2-1B,1637.946154262932,1617.5188506558277,20.42730360710425,20.42730360710425 +targeted_v2_warmup_5_explore_3_annotate_3x,numind/NuExtract3,1562.9140412754496,1571.572188853421,-8.658147577971476,8.658147577971476 +targeted_v2_warmup_5_explore_3_annotate_3x,rednote-hilab/dots.mocr,1571.765282146598,1556.2216675111229,15.543614635475251,15.543614635475251 +targeted_v2_warmup_5_explore_3_annotate_3x,rednote-hilab/dots.ocr,1532.8370744381737,1529.2527243518712,3.584350086302493,3.584350086302493 +targeted_v2_warmup_5_explore_3_annotate_3x,tesseract-5,1080.2926058353735,1151.0531271625612,-70.76052132718769,70.76052132718769 +targeted_v2_warmup_5_explore_3_annotate_3x,zai-org/GLM-OCR,1515.6738230073256,1491.926505968817,23.747317038508527,23.747317038508527 +targeted_v2_warmup_10_explore_3_annotate_3x,FireRedTeam/FireRed-OCR,1537.996225060517,1556.3777567649604,-18.381531704443432,18.381531704443432 +targeted_v2_warmup_10_explore_3_annotate_3x,PaddlePaddle/PP-OCRv6_medium,1309.0487571037,1354.3332993861072,-45.2845422824073,45.2845422824073 +targeted_v2_warmup_10_explore_3_annotate_3x,PaddlePaddle/PaddleOCR-VL-1.6,1498.377656828886,1503.691822219892,-5.314165391005872,5.314165391005872 +targeted_v2_warmup_10_explore_3_annotate_3x,allenai/olmOCR-2-7B-1025-FP8,1493.5465299268135,1497.230656288293,-3.684126361479457,3.684126361479457 +targeted_v2_warmup_10_explore_3_annotate_3x,baidu/Qianfan-OCR,1584.2588280224645,1576.6608094768258,7.598018545638752,7.598018545638752 +targeted_v2_warmup_10_explore_3_annotate_3x,baidu/Unlimited-OCR,1621.1862678017374,1563.2800425238524,57.906225277884914,57.906225277884914 +targeted_v2_warmup_10_explore_3_annotate_3x,deepseek-ai/DeepSeek-OCR,1457.2257557515416,1451.5246523995515,5.701103351990014,5.701103351990014 +targeted_v2_warmup_10_explore_3_annotate_3x,deepseek-ai/DeepSeek-OCR-2,1609.6430928052614,1579.3558964368967,30.287196368364675,30.287196368364675 +targeted_v2_warmup_10_explore_3_annotate_3x,lightonai/LightOnOCR-2-1B,1620.723096200397,1617.5188506558277,3.204245544569403,3.204245544569403 +targeted_v2_warmup_10_explore_3_annotate_3x,numind/NuExtract3,1562.9043531474144,1571.572188853421,-8.6678357060066,8.6678357060066 +targeted_v2_warmup_10_explore_3_annotate_3x,rednote-hilab/dots.mocr,1528.5864254602425,1556.2216675111229,-27.63524205088038,27.63524205088038 +targeted_v2_warmup_10_explore_3_annotate_3x,rednote-hilab/dots.ocr,1532.5608224809787,1529.2527243518712,3.3080981291075204,3.3080981291075204 +targeted_v2_warmup_10_explore_3_annotate_3x,tesseract-5,1140.262547488169,1151.0531271625612,-10.79057967439212,10.79057967439212 +targeted_v2_warmup_10_explore_3_annotate_3x,zai-org/GLM-OCR,1503.6796419218767,1491.926505968817,11.753135953059655,11.753135953059655 +targeted_v2_warmup_5_explore_3_size_stop_3x,FireRedTeam/FireRed-OCR,1545.4047185750896,1556.3777567649604,-10.97303818987075,10.97303818987075 +targeted_v2_warmup_5_explore_3_size_stop_3x,PaddlePaddle/PP-OCRv6_medium,1375.8585174192635,1354.3332993861072,21.525218033156307,21.525218033156307 +targeted_v2_warmup_5_explore_3_size_stop_3x,PaddlePaddle/PaddleOCR-VL-1.6,1508.0305981942972,1503.691822219892,4.33877597440528,4.33877597440528 +targeted_v2_warmup_5_explore_3_size_stop_3x,allenai/olmOCR-2-7B-1025-FP8,1505.1502583318468,1497.230656288293,7.919602043553823,7.919602043553823 +targeted_v2_warmup_5_explore_3_size_stop_3x,baidu/Qianfan-OCR,1577.014532947124,1576.6608094768258,0.35372347029829143,0.35372347029829143 +targeted_v2_warmup_5_explore_3_size_stop_3x,baidu/Unlimited-OCR,1506.9474328102363,1563.2800425238524,-56.33260971361619,56.33260971361619 +targeted_v2_warmup_5_explore_3_size_stop_3x,deepseek-ai/DeepSeek-OCR,1462.9418704656837,1451.5246523995515,11.417218066132136,11.417218066132136 +targeted_v2_warmup_5_explore_3_size_stop_3x,deepseek-ai/DeepSeek-OCR-2,1592.8473406766884,1579.3558964368967,13.49144423979169,13.49144423979169 +targeted_v2_warmup_5_explore_3_size_stop_3x,lightonai/LightOnOCR-2-1B,1644.6710642007395,1617.5188506558277,27.152213544911774,27.152213544911774 +targeted_v2_warmup_5_explore_3_size_stop_3x,numind/NuExtract3,1568.4357083685052,1571.572188853421,-3.1364804849158645,3.1364804849158645 +targeted_v2_warmup_5_explore_3_size_stop_3x,rednote-hilab/dots.mocr,1576.3012975274455,1556.2216675111229,20.079630016322653,20.079630016322653 +targeted_v2_warmup_5_explore_3_size_stop_3x,rednote-hilab/dots.ocr,1532.7277199959258,1529.2527243518712,3.4749956440546157,3.4749956440546157 +targeted_v2_warmup_5_explore_3_size_stop_3x,tesseract-5,1080.454570248567,1151.0531271625612,-70.59855691399412,70.59855691399412 +targeted_v2_warmup_5_explore_3_size_stop_3x,zai-org/GLM-OCR,1523.2143702385874,1491.926505968817,31.287864269770353,31.287864269770353 +pair-balanced_700_seed_42,FireRedTeam/FireRed-OCR,1573.982835294599,1556.3777567649604,17.605078529638604,17.605078529638604 +pair-balanced_700_seed_42,PaddlePaddle/PP-OCRv6_medium,1357.5509310796788,1354.3332993861072,3.2176316935715477,3.2176316935715477 +pair-balanced_700_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1530.0064351725828,1503.691822219892,26.314612952690823,26.314612952690823 +pair-balanced_700_seed_42,allenai/olmOCR-2-7B-1025-FP8,1471.1968180060758,1497.230656288293,-26.033838282217175,26.033838282217175 +pair-balanced_700_seed_42,baidu/Qianfan-OCR,1603.0802291962627,1576.6608094768258,26.419419719436974,26.419419719436974 +pair-balanced_700_seed_42,baidu/Unlimited-OCR,1514.0060940937908,1563.2800425238524,-49.27394843006164,49.27394843006164 +pair-balanced_700_seed_42,deepseek-ai/DeepSeek-OCR,1442.3316646750677,1451.5246523995515,-9.192987724483828,9.192987724483828 +pair-balanced_700_seed_42,deepseek-ai/DeepSeek-OCR-2,1590.3119212899653,1579.3558964368967,10.956024853068584,10.956024853068584 +pair-balanced_700_seed_42,lightonai/LightOnOCR-2-1B,1619.6290885657727,1617.5188506558277,2.110237909944999,2.110237909944999 +pair-balanced_700_seed_42,numind/NuExtract3,1631.6114663312278,1571.572188853421,60.03927747780676,60.03927747780676 +pair-balanced_700_seed_42,rednote-hilab/dots.mocr,1549.4149805020652,1556.2216675111229,-6.8066870090576685,6.8066870090576685 +pair-balanced_700_seed_42,rednote-hilab/dots.ocr,1534.1398922397602,1529.2527243518712,4.887167887889063,4.887167887889063 +pair-balanced_700_seed_42,tesseract-5,1065.556262003309,1151.0531271625612,-85.49686515925214,85.49686515925214 +pair-balanced_700_seed_42,zai-org/GLM-OCR,1517.181381549842,1491.926505968817,25.254875581024862,25.254875581024862 +pair-balanced_700_seed_43,FireRedTeam/FireRed-OCR,1560.1992800222502,1556.3777567649604,3.821523257289755,3.821523257289755 +pair-balanced_700_seed_43,PaddlePaddle/PP-OCRv6_medium,1353.5944584982378,1354.3332993861072,-0.7388408878694008,0.7388408878694008 +pair-balanced_700_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1523.8670557313837,1503.691822219892,20.175233511491797,20.175233511491797 +pair-balanced_700_seed_43,allenai/olmOCR-2-7B-1025-FP8,1516.913085608723,1497.230656288293,19.682429320429947,19.682429320429947 +pair-balanced_700_seed_43,baidu/Qianfan-OCR,1675.310893142534,1576.6608094768258,98.65008366570828,98.65008366570828 +pair-balanced_700_seed_43,baidu/Unlimited-OCR,1524.4919418693005,1563.2800425238524,-38.788100654551954,38.788100654551954 +pair-balanced_700_seed_43,deepseek-ai/DeepSeek-OCR,1425.5194439479358,1451.5246523995515,-26.005208451615772,26.005208451615772 +pair-balanced_700_seed_43,deepseek-ai/DeepSeek-OCR-2,1560.06824806695,1579.3558964368967,-19.287648369946737,19.287648369946737 +pair-balanced_700_seed_43,lightonai/LightOnOCR-2-1B,1554.7946811684756,1617.5188506558277,-62.72416948735213,62.72416948735213 +pair-balanced_700_seed_43,numind/NuExtract3,1592.206244601023,1571.572188853421,20.634055747601906,20.634055747601906 +pair-balanced_700_seed_43,rednote-hilab/dots.mocr,1539.1372691464655,1556.2216675111229,-17.084398364657318,17.084398364657318 +pair-balanced_700_seed_43,rednote-hilab/dots.ocr,1533.9480682358856,1529.2527243518712,4.695343884014392,4.695343884014392 +pair-balanced_700_seed_43,tesseract-5,1161.6522544732761,1151.0531271625612,10.599127310714948,10.599127310714948 +pair-balanced_700_seed_43,zai-org/GLM-OCR,1478.2970754875594,1491.926505968817,-13.629430481257714,13.629430481257714 +pair-balanced_700_seed_44,FireRedTeam/FireRed-OCR,1531.4711595033516,1556.3777567649604,-24.906597261608795,24.906597261608795 +pair-balanced_700_seed_44,PaddlePaddle/PP-OCRv6_medium,1346.0471769056799,1354.3332993861072,-8.286122480427366,8.286122480427366 +pair-balanced_700_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1514.5208697159937,1503.691822219892,10.829047496101794,10.829047496101794 +pair-balanced_700_seed_44,allenai/olmOCR-2-7B-1025-FP8,1482.4173659177045,1497.230656288293,-14.813290370588447,14.813290370588447 +pair-balanced_700_seed_44,baidu/Qianfan-OCR,1567.9057103694615,1576.6608094768258,-8.75509910736423,8.75509910736423 +pair-balanced_700_seed_44,baidu/Unlimited-OCR,1594.2855683934006,1563.2800425238524,31.005525869548137,31.005525869548137 +pair-balanced_700_seed_44,deepseek-ai/DeepSeek-OCR,1428.2005655530627,1451.5246523995515,-23.324086846488854,23.324086846488854 +pair-balanced_700_seed_44,deepseek-ai/DeepSeek-OCR-2,1568.8286703433039,1579.3558964368967,-10.527226093592844,10.527226093592844 +pair-balanced_700_seed_44,lightonai/LightOnOCR-2-1B,1662.449374961442,1617.5188506558277,44.9305243056142,44.9305243056142 +pair-balanced_700_seed_44,numind/NuExtract3,1570.1613186589232,1571.572188853421,-1.4108701944978748,1.4108701944978748 +pair-balanced_700_seed_44,rednote-hilab/dots.mocr,1582.490125957941,1556.2216675111229,26.26845844681816,26.26845844681816 +pair-balanced_700_seed_44,rednote-hilab/dots.ocr,1493.0538204626152,1529.2527243518712,-36.19890388925592,36.19890388925592 +pair-balanced_700_seed_44,tesseract-5,1131.173795327273,1151.0531271625612,-19.879331835288212,19.879331835288212 +pair-balanced_700_seed_44,zai-org/GLM-OCR,1526.994477929847,1491.926505968817,35.067971961030025,35.067971961030025 +pair-balanced_700_seed_45,FireRedTeam/FireRed-OCR,1594.526849157187,1556.3777567649604,38.14909239222652,38.14909239222652 +pair-balanced_700_seed_45,PaddlePaddle/PP-OCRv6_medium,1289.6391390893948,1354.3332993861072,-64.69416029671243,64.69416029671243 +pair-balanced_700_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1464.4648691206246,1503.691822219892,-39.22695309926735,39.22695309926735 +pair-balanced_700_seed_45,allenai/olmOCR-2-7B-1025-FP8,1519.6215858879607,1497.230656288293,22.390929599667743,22.390929599667743 +pair-balanced_700_seed_45,baidu/Qianfan-OCR,1581.6932596425293,1576.6608094768258,5.032450165703494,5.032450165703494 +pair-balanced_700_seed_45,baidu/Unlimited-OCR,1597.514161595467,1563.2800425238524,34.23411907161449,34.23411907161449 +pair-balanced_700_seed_45,deepseek-ai/DeepSeek-OCR,1446.7255673626232,1451.5246523995515,-4.799085036928318,4.799085036928318 +pair-balanced_700_seed_45,deepseek-ai/DeepSeek-OCR-2,1565.8283467213232,1579.3558964368967,-13.527549715573514,13.527549715573514 +pair-balanced_700_seed_45,lightonai/LightOnOCR-2-1B,1622.219189567636,1617.5188506558277,4.7003389118083305,4.7003389118083305 +pair-balanced_700_seed_45,numind/NuExtract3,1585.0799509046547,1571.572188853421,13.507762051233613,13.507762051233613 +pair-balanced_700_seed_45,rednote-hilab/dots.mocr,1570.6534844748194,1556.2216675111229,14.43181696369652,14.43181696369652 +pair-balanced_700_seed_45,rednote-hilab/dots.ocr,1527.5530799301696,1529.2527243518712,-1.6996444217015778,1.6996444217015778 +pair-balanced_700_seed_45,tesseract-5,1151.3630861841089,1151.0531271625612,0.30995902154768373,0.30995902154768373 +pair-balanced_700_seed_45,zai-org/GLM-OCR,1483.1174303615019,1491.926505968817,-8.809075607315208,8.809075607315208 +pair-balanced_700_seed_46,FireRedTeam/FireRed-OCR,1567.088202442937,1556.3777567649604,10.710445677976622,10.710445677976622 +pair-balanced_700_seed_46,PaddlePaddle/PP-OCRv6_medium,1296.0058747802757,1354.3332993861072,-58.32742460583154,58.32742460583154 +pair-balanced_700_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1533.3558035508788,1503.691822219892,29.66398133098687,29.66398133098687 +pair-balanced_700_seed_46,allenai/olmOCR-2-7B-1025-FP8,1450.5528339960401,1497.230656288293,-46.67782229225281,46.67782229225281 +pair-balanced_700_seed_46,baidu/Qianfan-OCR,1575.8852929524037,1576.6608094768258,-0.7755165244220734,0.7755165244220734 +pair-balanced_700_seed_46,baidu/Unlimited-OCR,1544.297359008243,1563.2800425238524,-18.98268351560955,18.98268351560955 +pair-balanced_700_seed_46,deepseek-ai/DeepSeek-OCR,1434.0849405009258,1451.5246523995515,-17.439711898625774,17.439711898625774 +pair-balanced_700_seed_46,deepseek-ai/DeepSeek-OCR-2,1593.3427003277786,1579.3558964368967,13.986803890881902,13.986803890881902 +pair-balanced_700_seed_46,lightonai/LightOnOCR-2-1B,1642.7835315574166,1617.5188506558277,25.264680901588918,25.264680901588918 +pair-balanced_700_seed_46,numind/NuExtract3,1562.1427762195021,1571.572188853421,-9.4294126339189,9.4294126339189 +pair-balanced_700_seed_46,rednote-hilab/dots.mocr,1622.5768356844173,1556.2216675111229,66.35516817329449,66.35516817329449 +pair-balanced_700_seed_46,rednote-hilab/dots.ocr,1536.794825260003,1529.2527243518712,7.542100908131943,7.542100908131943 +pair-balanced_700_seed_46,tesseract-5,1185.0923468797534,1151.0531271625612,34.03921971719228,34.03921971719228 +pair-balanced_700_seed_46,zai-org/GLM-OCR,1455.9966768394245,1491.926505968817,-35.9298291293926,35.9298291293926 +pair-balanced_1200_seed_42,FireRedTeam/FireRed-OCR,1545.878132560636,1556.3777567649604,-10.499624204324391,10.499624204324391 +pair-balanced_1200_seed_42,PaddlePaddle/PP-OCRv6_medium,1384.8643383634874,1354.3332993861072,30.53103897738015,30.53103897738015 +pair-balanced_1200_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1513.3367979875577,1503.691822219892,9.644975767665755,9.644975767665755 +pair-balanced_1200_seed_42,allenai/olmOCR-2-7B-1025-FP8,1499.785427616002,1497.230656288293,2.554771327708977,2.554771327708977 +pair-balanced_1200_seed_42,baidu/Qianfan-OCR,1572.5778571879964,1576.6608094768258,-4.082952288829347,4.082952288829347 +pair-balanced_1200_seed_42,baidu/Unlimited-OCR,1553.7590619231871,1563.2800425238524,-9.520980600665325,9.520980600665325 +pair-balanced_1200_seed_42,deepseek-ai/DeepSeek-OCR,1433.768989444524,1451.5246523995515,-17.755662955027447,17.755662955027447 +pair-balanced_1200_seed_42,deepseek-ai/DeepSeek-OCR-2,1571.356224103035,1579.3558964368967,-7.999672333861781,7.999672333861781 +pair-balanced_1200_seed_42,lightonai/LightOnOCR-2-1B,1636.0515571501478,1617.5188506558277,18.53270649432011,18.53270649432011 +pair-balanced_1200_seed_42,numind/NuExtract3,1589.737311612596,1571.572188853421,18.165122759174892,18.165122759174892 +pair-balanced_1200_seed_42,rednote-hilab/dots.mocr,1580.0386751370065,1556.2216675111229,23.81700762588366,23.81700762588366 +pair-balanced_1200_seed_42,rednote-hilab/dots.ocr,1526.4550480020846,1529.2527243518712,-2.7976763497865704,2.7976763497865704 +pair-balanced_1200_seed_42,tesseract-5,1086.0009072748323,1151.0531271625612,-65.05221988772882,65.05221988772882 +pair-balanced_1200_seed_42,zai-org/GLM-OCR,1506.3896716369075,1491.926505968817,14.463165668090369,14.463165668090369 +pair-balanced_1200_seed_43,FireRedTeam/FireRed-OCR,1539.607306260055,1556.3777567649604,-16.770450504905284,16.770450504905284 +pair-balanced_1200_seed_43,PaddlePaddle/PP-OCRv6_medium,1358.160393540098,1354.3332993861072,3.8270941539908563,3.8270941539908563 +pair-balanced_1200_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1499.9706394807906,1503.691822219892,-3.721182739101323,3.721182739101323 +pair-balanced_1200_seed_43,allenai/olmOCR-2-7B-1025-FP8,1496.8564927393788,1497.230656288293,-0.37416354891411174,0.37416354891411174 +pair-balanced_1200_seed_43,baidu/Qianfan-OCR,1619.2404273697982,1576.6608094768258,42.579617892972465,42.579617892972465 +pair-balanced_1200_seed_43,baidu/Unlimited-OCR,1546.3262451315748,1563.2800425238524,-16.953797392277693,16.953797392277693 +pair-balanced_1200_seed_43,deepseek-ai/DeepSeek-OCR,1468.828014924821,1451.5246523995515,17.30336252526945,17.30336252526945 +pair-balanced_1200_seed_43,deepseek-ai/DeepSeek-OCR-2,1577.9004124467074,1579.3558964368967,-1.4554839901893502,1.4554839901893502 +pair-balanced_1200_seed_43,lightonai/LightOnOCR-2-1B,1590.0972410258867,1617.5188506558277,-27.42160962994103,27.42160962994103 +pair-balanced_1200_seed_43,numind/NuExtract3,1565.2416797691344,1571.572188853421,-6.330509084286632,6.330509084286632 +pair-balanced_1200_seed_43,rednote-hilab/dots.mocr,1541.8030054564763,1556.2216675111229,-14.418662054646575,14.418662054646575 +pair-balanced_1200_seed_43,rednote-hilab/dots.ocr,1552.497522616654,1529.2527243518712,23.24479826478273,23.24479826478273 +pair-balanced_1200_seed_43,tesseract-5,1134.1104140835798,1151.0531271625612,-16.942713078981342,16.942713078981342 +pair-balanced_1200_seed_43,zai-org/GLM-OCR,1509.360205155045,1491.926505968817,17.43369918622784,17.43369918622784 +pair-balanced_1200_seed_44,FireRedTeam/FireRed-OCR,1559.9439070357234,1556.3777567649604,3.566150270763046,3.566150270763046 +pair-balanced_1200_seed_44,PaddlePaddle/PP-OCRv6_medium,1346.439860590207,1354.3332993861072,-7.893438795900238,7.893438795900238 +pair-balanced_1200_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1504.8985264434998,1503.691822219892,1.2067042236078578,1.2067042236078578 +pair-balanced_1200_seed_44,allenai/olmOCR-2-7B-1025-FP8,1499.5796881000992,1497.230656288293,2.3490318118062987,2.3490318118062987 +pair-balanced_1200_seed_44,baidu/Qianfan-OCR,1540.0352987241208,1576.6608094768258,-36.62551075270494,36.62551075270494 +pair-balanced_1200_seed_44,baidu/Unlimited-OCR,1576.3048114351172,1563.2800425238524,13.0247689112648,13.0247689112648 +pair-balanced_1200_seed_44,deepseek-ai/DeepSeek-OCR,1421.2530138702846,1451.5246523995515,-30.271638529266966,30.271638529266966 +pair-balanced_1200_seed_44,deepseek-ai/DeepSeek-OCR-2,1585.6500885739306,1579.3558964368967,6.2941921370338605,6.2941921370338605 +pair-balanced_1200_seed_44,lightonai/LightOnOCR-2-1B,1644.696543297451,1617.5188506558277,27.177692641623253,27.177692641623253 +pair-balanced_1200_seed_44,numind/NuExtract3,1572.0422078100323,1571.572188853421,0.470018956611284,0.470018956611284 +pair-balanced_1200_seed_44,rednote-hilab/dots.mocr,1536.0188767077175,1556.2216675111229,-20.202790803405378,20.202790803405378 +pair-balanced_1200_seed_44,rednote-hilab/dots.ocr,1526.7596056062018,1529.2527243518712,-2.493118745669335,2.493118745669335 +pair-balanced_1200_seed_44,tesseract-5,1180.9700382216424,1151.0531271625612,29.916911059081258,29.916911059081258 +pair-balanced_1200_seed_44,zai-org/GLM-OCR,1505.407533583972,1491.926505968817,13.481027615154971,13.481027615154971 +pair-balanced_1200_seed_45,FireRedTeam/FireRed-OCR,1572.1386057089649,1556.3777567649604,15.760848944004465,15.760848944004465 +pair-balanced_1200_seed_45,PaddlePaddle/PP-OCRv6_medium,1331.298457646523,1354.3332993861072,-23.03484173958418,23.03484173958418 +pair-balanced_1200_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1482.8357522778276,1503.691822219892,-20.856069942064323,20.856069942064323 +pair-balanced_1200_seed_45,allenai/olmOCR-2-7B-1025-FP8,1533.7055763825083,1497.230656288293,36.47492009421535,36.47492009421535 +pair-balanced_1200_seed_45,baidu/Qianfan-OCR,1581.4817977385949,1576.6608094768258,4.820988261769116,4.820988261769116 +pair-balanced_1200_seed_45,baidu/Unlimited-OCR,1576.2684447626,1563.2800425238524,12.98840223874754,12.98840223874754 +pair-balanced_1200_seed_45,deepseek-ai/DeepSeek-OCR,1430.962135126057,1451.5246523995515,-20.56251727349445,20.56251727349445 +pair-balanced_1200_seed_45,deepseek-ai/DeepSeek-OCR-2,1544.4619450968623,1579.3558964368967,-34.89395134003439,34.89395134003439 +pair-balanced_1200_seed_45,lightonai/LightOnOCR-2-1B,1614.0483231099577,1617.5188506558277,-3.470527545870027,3.470527545870027 +pair-balanced_1200_seed_45,numind/NuExtract3,1594.3022759303524,1571.572188853421,22.73008707693134,22.73008707693134 +pair-balanced_1200_seed_45,rednote-hilab/dots.mocr,1575.7543502996234,1556.2216675111229,19.53268278850055,19.53268278850055 +pair-balanced_1200_seed_45,rednote-hilab/dots.ocr,1537.7674025924164,1529.2527243518712,8.514678240545209,8.514678240545209 +pair-balanced_1200_seed_45,tesseract-5,1142.304536340088,1151.0531271625612,-8.748590822473261,8.748590822473261 +pair-balanced_1200_seed_45,zai-org/GLM-OCR,1482.6703969876241,1491.926505968817,-9.256108981192938,9.256108981192938 +pair-balanced_1200_seed_46,FireRedTeam/FireRed-OCR,1556.152279191893,1556.3777567649604,-0.2254775730673373,0.2254775730673373 +pair-balanced_1200_seed_46,PaddlePaddle/PP-OCRv6_medium,1351.606529176169,1354.3332993861072,-2.726770209938195,2.726770209938195 +pair-balanced_1200_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1522.3269288649906,1503.691822219892,18.63510664509863,18.63510664509863 +pair-balanced_1200_seed_46,allenai/olmOCR-2-7B-1025-FP8,1474.1722235401328,1497.230656288293,-23.058432748160158,23.058432748160158 +pair-balanced_1200_seed_46,baidu/Qianfan-OCR,1570.59747445459,1576.6608094768258,-6.063335022235833,6.063335022235833 +pair-balanced_1200_seed_46,baidu/Unlimited-OCR,1528.4699024283605,1563.2800425238524,-34.81014009549199,34.81014009549199 +pair-balanced_1200_seed_46,deepseek-ai/DeepSeek-OCR,1436.9566686609003,1451.5246523995515,-14.567983738651265,14.567983738651265 +pair-balanced_1200_seed_46,deepseek-ai/DeepSeek-OCR-2,1596.6972549022357,1579.3558964368967,17.341358465339,17.341358465339 +pair-balanced_1200_seed_46,lightonai/LightOnOCR-2-1B,1637.2984685761928,1617.5188506558277,19.779617920365126,19.779617920365126 +pair-balanced_1200_seed_46,numind/NuExtract3,1574.0310277708513,1571.572188853421,2.4588389174302847,2.4588389174302847 +pair-balanced_1200_seed_46,rednote-hilab/dots.mocr,1574.7915477894535,1556.2216675111229,18.56988027833063,18.56988027833063 +pair-balanced_1200_seed_46,rednote-hilab/dots.ocr,1533.9432510044749,1529.2527243518712,4.690526652603694,4.690526652603694 +pair-balanced_1200_seed_46,tesseract-5,1180.2072284627986,1151.0531271625612,29.15410130023747,29.15410130023747 +pair-balanced_1200_seed_46,zai-org/GLM-OCR,1462.7492151769566,1491.926505968817,-29.177290791860514,29.177290791860514 +pair-balanced_2000_seed_42,FireRedTeam/FireRed-OCR,1556.3198631042937,1556.3777567649604,-0.05789366066665025,0.05789366066665025 +pair-balanced_2000_seed_42,PaddlePaddle/PP-OCRv6_medium,1350.3931854278414,1354.3332993861072,-3.9401139582657834,3.9401139582657834 +pair-balanced_2000_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1513.8109379155633,1503.691822219892,10.119115695671326,10.119115695671326 +pair-balanced_2000_seed_42,allenai/olmOCR-2-7B-1025-FP8,1493.076293548267,1497.230656288293,-4.154362740026045,4.154362740026045 +pair-balanced_2000_seed_42,baidu/Qianfan-OCR,1565.8691631789404,1576.6608094768258,-10.791646297885336,10.791646297885336 +pair-balanced_2000_seed_42,baidu/Unlimited-OCR,1570.8544447940942,1563.2800425238524,7.574402270241762,7.574402270241762 +pair-balanced_2000_seed_42,deepseek-ai/DeepSeek-OCR,1446.1586060206175,1451.5246523995515,-5.366046378934016,5.366046378934016 +pair-balanced_2000_seed_42,deepseek-ai/DeepSeek-OCR-2,1565.8691631789404,1579.3558964368967,-13.486733257956303,13.486733257956303 +pair-balanced_2000_seed_42,lightonai/LightOnOCR-2-1B,1637.9055353640174,1617.5188506558277,20.386684708189705,20.386684708189705 +pair-balanced_2000_seed_42,numind/NuExtract3,1583.6430871343907,1571.572188853421,12.070898280969686,12.070898280969686 +pair-balanced_2000_seed_42,rednote-hilab/dots.mocr,1567.1140576943537,1556.2216675111229,10.892390183230873,10.892390183230873 +pair-balanced_2000_seed_42,rednote-hilab/dots.ocr,1536.1972365728448,1529.2527243518712,6.944512220973593,6.944512220973593 +pair-balanced_2000_seed_42,tesseract-5,1117.2363984872097,1151.0531271625612,-33.81672867535144,33.81672867535144 +pair-balanced_2000_seed_42,zai-org/GLM-OCR,1495.5520275786257,1491.926505968817,3.6255216098086294,3.6255216098086294 +pair-balanced_2000_seed_43,FireRedTeam/FireRed-OCR,1561.7520143757624,1556.3777567649604,5.374257610802033,5.374257610802033 +pair-balanced_2000_seed_43,PaddlePaddle/PP-OCRv6_medium,1360.1111904073277,1354.3332993861072,5.777891021220512,5.777891021220512 +pair-balanced_2000_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1507.621750438433,1503.691822219892,3.9299282185411357,3.9299282185411357 +pair-balanced_2000_seed_43,allenai/olmOCR-2-7B-1025-FP8,1501.694444858008,1497.230656288293,4.463788569715007,4.463788569715007 +pair-balanced_2000_seed_43,baidu/Qianfan-OCR,1599.3018969795046,1576.6608094768258,22.64108750267883,22.64108750267883 +pair-balanced_2000_seed_43,baidu/Unlimited-OCR,1549.4397904485882,1563.2800425238524,-13.840252075264289,13.840252075264289 +pair-balanced_2000_seed_43,deepseek-ai/DeepSeek-OCR,1459.1908811144028,1451.5246523995515,7.666228714851286,7.666228714851286 +pair-balanced_2000_seed_43,deepseek-ai/DeepSeek-OCR-2,1565.665132385358,1579.3558964368967,-13.690764051538736,13.690764051538736 +pair-balanced_2000_seed_43,lightonai/LightOnOCR-2-1B,1614.716795849111,1617.5188506558277,-2.8020548067167965,2.8020548067167965 +pair-balanced_2000_seed_43,numind/NuExtract3,1569.1770033195064,1571.572188853421,-2.3951855339146277,2.3951855339146277 +pair-balanced_2000_seed_43,rednote-hilab/dots.mocr,1550.9852458473056,1556.2216675111229,-5.236421663817282,5.236421663817282 +pair-balanced_2000_seed_43,rednote-hilab/dots.ocr,1540.8551618069512,1529.2527243518712,11.602437455080008,11.602437455080008 +pair-balanced_2000_seed_43,tesseract-5,1119.0215803741876,1151.0531271625612,-32.03154678837359,32.03154678837359 +pair-balanced_2000_seed_43,zai-org/GLM-OCR,1500.4671117955538,1491.926505968817,8.540605826736737,8.540605826736737 +pair-balanced_2000_seed_44,FireRedTeam/FireRed-OCR,1566.0382779204729,1556.3777567649604,9.66052115551247,9.66052115551247 +pair-balanced_2000_seed_44,PaddlePaddle/PP-OCRv6_medium,1353.3349830891023,1354.3332993861072,-0.9983162970049761,0.9983162970049761 +pair-balanced_2000_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1504.6088454017972,1503.691822219892,0.9170231819052788,0.9170231819052788 +pair-balanced_2000_seed_44,allenai/olmOCR-2-7B-1025-FP8,1516.843571724278,1497.230656288293,19.612915435985087,19.612915435985087 +pair-balanced_2000_seed_44,baidu/Qianfan-OCR,1553.5359614442066,1576.6608094768258,-23.124848032619184,23.124848032619184 +pair-balanced_2000_seed_44,baidu/Unlimited-OCR,1575.8198966399332,1563.2800425238524,12.539854116080733,12.539854116080733 +pair-balanced_2000_seed_44,deepseek-ai/DeepSeek-OCR,1433.6491106806277,1451.5246523995515,-17.87554171892384,17.87554171892384 +pair-balanced_2000_seed_44,deepseek-ai/DeepSeek-OCR-2,1578.3176327559609,1579.3558964368967,-1.03826368093587,1.03826368093587 +pair-balanced_2000_seed_44,lightonai/LightOnOCR-2-1B,1637.7529273390214,1617.5188506558277,20.234076683193734,20.234076683193734 +pair-balanced_2000_seed_44,numind/NuExtract3,1541.268240466551,1571.572188853421,-30.303948386869934,30.303948386869934 +pair-balanced_2000_seed_44,rednote-hilab/dots.mocr,1544.9425543402667,1556.2216675111229,-11.279113170856135,11.279113170856135 +pair-balanced_2000_seed_44,rednote-hilab/dots.ocr,1520.3443584148463,1529.2527243518712,-8.908365937024882,8.908365937024882 +pair-balanced_2000_seed_44,tesseract-5,1161.5555046613165,1151.0531271625612,10.502377498755322,10.502377498755322 +pair-balanced_2000_seed_44,zai-org/GLM-OCR,1511.9881351216195,1491.926505968817,20.061629152802425,20.061629152802425 +pair-balanced_2000_seed_45,FireRedTeam/FireRed-OCR,1567.9738562515588,1556.3777567649604,11.596099486598405,11.596099486598405 +pair-balanced_2000_seed_45,PaddlePaddle/PP-OCRv6_medium,1344.1725100261235,1354.3332993861072,-10.16078935998371,10.16078935998371 +pair-balanced_2000_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1514.0300181515338,1503.691822219892,10.338195931641849,10.338195931641849 +pair-balanced_2000_seed_45,allenai/olmOCR-2-7B-1025-FP8,1509.1233022080776,1497.230656288293,11.892645919784627,11.892645919784627 +pair-balanced_2000_seed_45,baidu/Qianfan-OCR,1558.2655547549132,1576.6608094768258,-18.39525472191258,18.39525472191258 +pair-balanced_2000_seed_45,baidu/Unlimited-OCR,1584.4585328149224,1563.2800425238524,21.17849029106992,21.17849029106992 +pair-balanced_2000_seed_45,deepseek-ai/DeepSeek-OCR,1444.3216129333432,1451.5246523995515,-7.203039466208338,7.203039466208338 +pair-balanced_2000_seed_45,deepseek-ai/DeepSeek-OCR-2,1570.6789821488717,1579.3558964368967,-8.676914288025046,8.676914288025046 +pair-balanced_2000_seed_45,lightonai/LightOnOCR-2-1B,1628.5558124657518,1617.5188506558277,11.03696180992415,11.03696180992415 +pair-balanced_2000_seed_45,numind/NuExtract3,1573.173560680442,1571.572188853421,1.601371827021012,1.601371827021012 +pair-balanced_2000_seed_45,rednote-hilab/dots.mocr,1552.1732191066562,1556.2216675111229,-4.048448404466626,4.048448404466626 +pair-balanced_2000_seed_45,rednote-hilab/dots.ocr,1528.657413230858,1529.2527243518712,-0.5953111210130828,0.5953111210130828 +pair-balanced_2000_seed_45,tesseract-5,1146.124743502976,1151.0531271625612,-4.928383659585279,4.928383659585279 +pair-balanced_2000_seed_45,zai-org/GLM-OCR,1478.2908817239716,1491.926505968817,-13.63562424484553,13.63562424484553 +pair-balanced_2000_seed_46,FireRedTeam/FireRed-OCR,1572.8495576364064,1556.3777567649604,16.47180087144602,16.47180087144602 +pair-balanced_2000_seed_46,PaddlePaddle/PP-OCRv6_medium,1324.463970061903,1354.3332993861072,-29.869329324204273,29.869329324204273 +pair-balanced_2000_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1511.7834281660416,1503.691822219892,8.091605946149684,8.091605946149684 +pair-balanced_2000_seed_46,allenai/olmOCR-2-7B-1025-FP8,1467.303753606735,1497.230656288293,-29.926902681557976,29.926902681557976 +pair-balanced_2000_seed_46,baidu/Qianfan-OCR,1576.4953022307723,1576.6608094768258,-0.16550724605349387,0.16550724605349387 +pair-balanced_2000_seed_46,baidu/Unlimited-OCR,1562.7121970030844,1563.2800425238524,-0.5678455207680599,0.5678455207680599 +pair-balanced_2000_seed_46,deepseek-ai/DeepSeek-OCR,1447.0417408142685,1451.5246523995515,-4.482911585283091,4.482911585283091 +pair-balanced_2000_seed_46,deepseek-ai/DeepSeek-OCR-2,1607.1398806842576,1579.3558964368967,27.783984247360877,27.783984247360877 +pair-balanced_2000_seed_46,lightonai/LightOnOCR-2-1B,1621.5456099833466,1617.5188506558277,4.026759327518903,4.026759327518903 +pair-balanced_2000_seed_46,numind/NuExtract3,1576.7279541495727,1571.572188853421,5.155765296151685,5.155765296151685 +pair-balanced_2000_seed_46,rednote-hilab/dots.mocr,1561.4651721700325,1556.2216675111229,5.243504658909615,5.243504658909615 +pair-balanced_2000_seed_46,rednote-hilab/dots.ocr,1537.7818744765173,1529.2527243518712,8.529150124646094,8.529150124646094 +pair-balanced_2000_seed_46,tesseract-5,1169.1596975342054,1151.0531271625612,18.10657037164424,18.10657037164424 +pair-balanced_2000_seed_46,zai-org/GLM-OCR,1463.5298614828566,1491.926505968817,-28.39664448596045,28.39664448596045 +mixed-random_700_seed_42,FireRedTeam/FireRed-OCR,1607.8901087705,1556.3777567649604,51.5123520055397,51.5123520055397 +mixed-random_700_seed_42,PaddlePaddle/PP-OCRv6_medium,1324.1322577119297,1354.3332993861072,-30.20104167417753,30.20104167417753 +mixed-random_700_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1432.6828904222969,1503.691822219892,-71.0089317975951,71.0089317975951 +mixed-random_700_seed_42,allenai/olmOCR-2-7B-1025-FP8,1506.8102751994074,1497.230656288293,9.579618911114494,9.579618911114494 +mixed-random_700_seed_42,baidu/Qianfan-OCR,1577.824105502385,1576.6608094768258,1.163296025559248,1.163296025559248 +mixed-random_700_seed_42,baidu/Unlimited-OCR,1591.545258008199,1563.2800425238524,28.265215484346527,28.265215484346527 +mixed-random_700_seed_42,deepseek-ai/DeepSeek-OCR,1406.0347450044267,1451.5246523995515,-45.489907395124874,45.489907395124874 +mixed-random_700_seed_42,deepseek-ai/DeepSeek-OCR-2,1680.9014764916626,1579.3558964368967,101.54558005476588,101.54558005476588 +mixed-random_700_seed_42,lightonai/LightOnOCR-2-1B,1711.7301329953527,1617.5188506558277,94.21128233952504,94.21128233952504 +mixed-random_700_seed_42,numind/NuExtract3,1544.8028614561783,1571.572188853421,-26.769327397242705,26.769327397242705 +mixed-random_700_seed_42,rednote-hilab/dots.mocr,1510.1326575577705,1556.2216675111229,-46.08900995335239,46.08900995335239 +mixed-random_700_seed_42,rednote-hilab/dots.ocr,1517.1655294546413,1529.2527243518712,-12.087194897229892,12.087194897229892 +mixed-random_700_seed_42,tesseract-5,1083.546911033531,1151.0531271625612,-67.50621612903024,67.50621612903024 +mixed-random_700_seed_42,zai-org/GLM-OCR,1504.8007903917191,1491.926505968817,12.874284422902065,12.874284422902065 +mixed-random_700_seed_43,FireRedTeam/FireRed-OCR,1627.1741473193943,1556.3777567649604,70.79639055443386,70.79639055443386 +mixed-random_700_seed_43,PaddlePaddle/PP-OCRv6_medium,1304.9844851467524,1354.3332993861072,-49.34881423935485,49.34881423935485 +mixed-random_700_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1462.7150288511957,1503.691822219892,-40.976793368696235,40.976793368696235 +mixed-random_700_seed_43,allenai/olmOCR-2-7B-1025-FP8,1441.296941484788,1497.230656288293,-55.93371480350493,55.93371480350493 +mixed-random_700_seed_43,baidu/Qianfan-OCR,1621.9516901182708,1576.6608094768258,45.29088064144503,45.29088064144503 +mixed-random_700_seed_43,baidu/Unlimited-OCR,1549.911884471927,1563.2800425238524,-13.368158051925548,13.368158051925548 +mixed-random_700_seed_43,deepseek-ai/DeepSeek-OCR,1442.471670650833,1451.5246523995515,-9.052981748718594,9.052981748718594 +mixed-random_700_seed_43,deepseek-ai/DeepSeek-OCR-2,1647.719724252598,1579.3558964368967,68.36382781570137,68.36382781570137 +mixed-random_700_seed_43,lightonai/LightOnOCR-2-1B,1666.6517651370589,1617.5188506558277,49.13291448123118,49.13291448123118 +mixed-random_700_seed_43,numind/NuExtract3,1525.1733891908048,1571.572188853421,-46.398799662616284,46.398799662616284 +mixed-random_700_seed_43,rednote-hilab/dots.mocr,1554.088250095837,1556.2216675111229,-2.1334174152857486,2.1334174152857486 +mixed-random_700_seed_43,rednote-hilab/dots.ocr,1501.480133057811,1529.2527243518712,-27.77259129406025,27.77259129406025 +mixed-random_700_seed_43,tesseract-5,1146.7924255255678,1151.0531271625612,-4.260701636993417,4.260701636993417 +mixed-random_700_seed_43,zai-org/GLM-OCR,1507.5884646971617,1491.926505968817,15.661958728344644,15.661958728344644 +mixed-random_700_seed_44,FireRedTeam/FireRed-OCR,1673.188282934586,1556.3777567649604,116.81052616962552,116.81052616962552 +mixed-random_700_seed_44,PaddlePaddle/PP-OCRv6_medium,1281.6639118530522,1354.3332993861072,-72.66938753305499,72.66938753305499 +mixed-random_700_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1434.778032305935,1503.691822219892,-68.91378991395686,68.91378991395686 +mixed-random_700_seed_44,allenai/olmOCR-2-7B-1025-FP8,1520.5836447664847,1497.230656288293,23.35298847819172,23.35298847819172 +mixed-random_700_seed_44,baidu/Qianfan-OCR,1661.446223453312,1576.6608094768258,84.78541397648632,84.78541397648632 +mixed-random_700_seed_44,baidu/Unlimited-OCR,1572.3165298586125,1563.2800425238524,9.036487334760068,9.036487334760068 +mixed-random_700_seed_44,deepseek-ai/DeepSeek-OCR,1431.7235728805313,1451.5246523995515,-19.801079519020277,19.801079519020277 +mixed-random_700_seed_44,deepseek-ai/DeepSeek-OCR-2,1632.4475423609897,1579.3558964368967,53.09164592409297,53.09164592409297 +mixed-random_700_seed_44,lightonai/LightOnOCR-2-1B,1688.0311735938176,1617.5188506558277,70.5123229379899,70.5123229379899 +mixed-random_700_seed_44,numind/NuExtract3,1517.7510356076311,1571.572188853421,-53.8211532457899,53.8211532457899 +mixed-random_700_seed_44,rednote-hilab/dots.mocr,1497.914748263184,1556.2216675111229,-58.30691924793882,58.30691924793882 +mixed-random_700_seed_44,rednote-hilab/dots.ocr,1514.9465196472356,1529.2527243518712,-14.306204704635547,14.306204704635547 +mixed-random_700_seed_44,tesseract-5,1074.9628403858978,1151.0531271625612,-76.09028677666333,76.09028677666333 +mixed-random_700_seed_44,zai-org/GLM-OCR,1498.2459420887308,1491.926505968817,6.319436119913689,6.319436119913689 +mixed-random_700_seed_45,FireRedTeam/FireRed-OCR,1607.6555491548213,1556.3777567649604,51.277792389860906,51.277792389860906 +mixed-random_700_seed_45,PaddlePaddle/PP-OCRv6_medium,1352.1414508836476,1354.3332993861072,-2.1918485024596066,2.1918485024596066 +mixed-random_700_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1437.8284775709838,1503.691822219892,-65.8633446489082,65.8633446489082 +mixed-random_700_seed_45,allenai/olmOCR-2-7B-1025-FP8,1489.6664396303022,1497.230656288293,-7.564216657990755,7.564216657990755 +mixed-random_700_seed_45,baidu/Qianfan-OCR,1597.1412419693654,1576.6608094768258,20.48043249253965,20.48043249253965 +mixed-random_700_seed_45,baidu/Unlimited-OCR,1586.5221157596163,1563.2800425238524,23.242073235763883,23.242073235763883 +mixed-random_700_seed_45,deepseek-ai/DeepSeek-OCR,1431.342811092415,1451.5246523995515,-20.181841307136438,20.181841307136438 +mixed-random_700_seed_45,deepseek-ai/DeepSeek-OCR-2,1631.150345413672,1579.3558964368967,51.79444897677536,51.79444897677536 +mixed-random_700_seed_45,lightonai/LightOnOCR-2-1B,1720.296774041724,1617.5188506558277,102.7779233858962,102.7779233858962 +mixed-random_700_seed_45,numind/NuExtract3,1551.9239772540213,1571.572188853421,-19.6482115993997,19.6482115993997 +mixed-random_700_seed_45,rednote-hilab/dots.mocr,1516.9674433402572,1556.2216675111229,-39.254224170865655,39.254224170865655 +mixed-random_700_seed_45,rednote-hilab/dots.ocr,1508.048931365834,1529.2527243518712,-21.203792986037115,21.203792986037115 +mixed-random_700_seed_45,tesseract-5,1063.6567478026761,1151.0531271625612,-87.39637935988503,87.39637935988503 +mixed-random_700_seed_45,zai-org/GLM-OCR,1505.6576947206636,1491.926505968817,13.731188751846503,13.731188751846503 +mixed-random_700_seed_46,FireRedTeam/FireRed-OCR,1652.650075828782,1556.3777567649604,96.27231906382167,96.27231906382167 +mixed-random_700_seed_46,PaddlePaddle/PP-OCRv6_medium,1340.3972595669645,1354.3332993861072,-13.936039819142707,13.936039819142707 +mixed-random_700_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1437.818732129255,1503.691822219892,-65.87309009063688,65.87309009063688 +mixed-random_700_seed_46,allenai/olmOCR-2-7B-1025-FP8,1466.0479919755378,1497.230656288293,-31.18266431275515,31.18266431275515 +mixed-random_700_seed_46,baidu/Qianfan-OCR,1607.2737598059603,1576.6608094768258,30.61295032913449,30.61295032913449 +mixed-random_700_seed_46,baidu/Unlimited-OCR,1609.125735268791,1563.2800425238524,45.84569274493856,45.84569274493856 +mixed-random_700_seed_46,deepseek-ai/DeepSeek-OCR,1415.544306236939,1451.5246523995515,-35.98034616261248,35.98034616261248 +mixed-random_700_seed_46,deepseek-ai/DeepSeek-OCR-2,1661.533523408763,1579.3558964368967,82.17762697186618,82.17762697186618 +mixed-random_700_seed_46,lightonai/LightOnOCR-2-1B,1682.959703545581,1617.5188506558277,65.44085288975339,65.44085288975339 +mixed-random_700_seed_46,numind/NuExtract3,1551.674958751879,1571.572188853421,-19.897230101541936,19.897230101541936 +mixed-random_700_seed_46,rednote-hilab/dots.mocr,1486.1642370411985,1556.2216675111229,-70.05743046992438,70.05743046992438 +mixed-random_700_seed_46,rednote-hilab/dots.ocr,1537.2047617337425,1529.2527243518712,7.952037381871378,7.952037381871378 +mixed-random_700_seed_46,tesseract-5,1061.1815330960849,1151.0531271625612,-89.87159406647629,89.87159406647629 +mixed-random_700_seed_46,zai-org/GLM-OCR,1490.423421610521,1491.926505968817,-1.5030843582960642,1.5030843582960642 +mixed-random_1200_seed_42,FireRedTeam/FireRed-OCR,1582.6452065080616,1556.3777567649604,26.267449743101224,26.267449743101224 +mixed-random_1200_seed_42,PaddlePaddle/PP-OCRv6_medium,1364.9558984081502,1354.3332993861072,10.622599022043005,10.622599022043005 +mixed-random_1200_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1477.8915337701383,1503.691822219892,-25.80028844975368,25.80028844975368 +mixed-random_1200_seed_42,allenai/olmOCR-2-7B-1025-FP8,1523.2322260922804,1497.230656288293,26.001569803987422,26.001569803987422 +mixed-random_1200_seed_42,baidu/Qianfan-OCR,1548.586005881871,1576.6608094768258,-28.0748035949548,28.0748035949548 +mixed-random_1200_seed_42,baidu/Unlimited-OCR,1570.1080110130335,1563.2800425238524,6.827968489181103,6.827968489181103 +mixed-random_1200_seed_42,deepseek-ai/DeepSeek-OCR,1405.3038002095473,1451.5246523995515,-46.22085219000428,46.22085219000428 +mixed-random_1200_seed_42,deepseek-ai/DeepSeek-OCR-2,1658.8810948721898,1579.3558964368967,79.52519843529308,79.52519843529308 +mixed-random_1200_seed_42,lightonai/LightOnOCR-2-1B,1687.7393640094738,1617.5188506558277,70.22051335364608,70.22051335364608 +mixed-random_1200_seed_42,numind/NuExtract3,1550.9783726906985,1571.572188853421,-20.593816162722533,20.593816162722533 +mixed-random_1200_seed_42,rednote-hilab/dots.mocr,1538.7355061384053,1556.2216675111229,-17.48616137271756,17.48616137271756 +mixed-random_1200_seed_42,rednote-hilab/dots.ocr,1478.5602992631827,1529.2527243518712,-50.692425088688424,50.692425088688424 +mixed-random_1200_seed_42,tesseract-5,1113.8461646271267,1151.0531271625612,-37.20696253543451,37.20696253543451 +mixed-random_1200_seed_42,zai-org/GLM-OCR,1498.5365165158412,1491.926505968817,6.610010547024103,6.610010547024103 +mixed-random_1200_seed_43,FireRedTeam/FireRed-OCR,1620.0914993638414,1556.3777567649604,63.713742598881026,63.713742598881026 +mixed-random_1200_seed_43,PaddlePaddle/PP-OCRv6_medium,1330.619162306705,1354.3332993861072,-23.714137079402235,23.714137079402235 +mixed-random_1200_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1490.8200972437673,1503.691822219892,-12.871724976124597,12.871724976124597 +mixed-random_1200_seed_43,allenai/olmOCR-2-7B-1025-FP8,1451.9459466186784,1497.230656288293,-45.284709669614585,45.284709669614585 +mixed-random_1200_seed_43,baidu/Qianfan-OCR,1581.38425373927,1576.6608094768258,4.723444262444218,4.723444262444218 +mixed-random_1200_seed_43,baidu/Unlimited-OCR,1583.5020067780258,1563.2800425238524,20.221964254173372,20.221964254173372 +mixed-random_1200_seed_43,deepseek-ai/DeepSeek-OCR,1448.319432190444,1451.5246523995515,-3.2052202091074378,3.2052202091074378 +mixed-random_1200_seed_43,deepseek-ai/DeepSeek-OCR-2,1661.835679161675,1579.3558964368967,82.47978272477826,82.47978272477826 +mixed-random_1200_seed_43,lightonai/LightOnOCR-2-1B,1640.0129578701667,1617.5188506558277,22.49410721433901,22.49410721433901 +mixed-random_1200_seed_43,numind/NuExtract3,1537.0827287700786,1571.572188853421,-34.489460083342465,34.489460083342465 +mixed-random_1200_seed_43,rednote-hilab/dots.mocr,1528.2044087002694,1556.2216675111229,-28.01725881085349,28.01725881085349 +mixed-random_1200_seed_43,rednote-hilab/dots.ocr,1494.0277477963368,1529.2527243518712,-35.22497655553434,35.22497655553434 +mixed-random_1200_seed_43,tesseract-5,1127.096097306829,1151.0531271625612,-23.957029855732117,23.957029855732117 +mixed-random_1200_seed_43,zai-org/GLM-OCR,1505.0579821539131,1491.926505968817,13.131476185096062,13.131476185096062 +mixed-random_1200_seed_44,FireRedTeam/FireRed-OCR,1614.8782412199141,1556.3777567649604,58.50048445495372,58.50048445495372 +mixed-random_1200_seed_44,PaddlePaddle/PP-OCRv6_medium,1336.3677139611366,1354.3332993861072,-17.96558542497064,17.96558542497064 +mixed-random_1200_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1476.747087824031,1503.691822219892,-26.944734395860905,26.944734395860905 +mixed-random_1200_seed_44,allenai/olmOCR-2-7B-1025-FP8,1494.424848606755,1497.230656288293,-2.805807681537999,2.805807681537999 +mixed-random_1200_seed_44,baidu/Qianfan-OCR,1608.026483913197,1576.6608094768258,31.365674436371137,31.365674436371137 +mixed-random_1200_seed_44,baidu/Unlimited-OCR,1575.6047075569065,1563.2800425238524,12.324665033054089,12.324665033054089 +mixed-random_1200_seed_44,deepseek-ai/DeepSeek-OCR,1414.358620538918,1451.5246523995515,-37.16603186063344,37.16603186063344 +mixed-random_1200_seed_44,deepseek-ai/DeepSeek-OCR-2,1597.6043232088473,1579.3558964368967,18.248426771950562,18.248426771950562 +mixed-random_1200_seed_44,lightonai/LightOnOCR-2-1B,1633.2568398341439,1617.5188506558277,15.737989178316184,15.737989178316184 +mixed-random_1200_seed_44,numind/NuExtract3,1553.7823519030087,1571.572188853421,-17.789836950412337,17.789836950412337 +mixed-random_1200_seed_44,rednote-hilab/dots.mocr,1532.8940016822075,1556.2216675111229,-23.327665828915315,23.327665828915315 +mixed-random_1200_seed_44,rednote-hilab/dots.ocr,1515.535232214481,1529.2527243518712,-13.717492137390082,13.717492137390082 +mixed-random_1200_seed_44,tesseract-5,1150.1899539253086,1151.0531271625612,-0.8631732372525676,0.8631732372525676 +mixed-random_1200_seed_44,zai-org/GLM-OCR,1496.3295936111451,1491.926505968817,4.403087642328046,4.403087642328046 +mixed-random_1200_seed_45,FireRedTeam/FireRed-OCR,1587.41846281403,1556.3777567649604,31.040706049069513,31.040706049069513 +mixed-random_1200_seed_45,PaddlePaddle/PP-OCRv6_medium,1366.2017247029062,1354.3332993861072,11.868425316798948,11.868425316798948 +mixed-random_1200_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1454.8180238916289,1503.691822219892,-48.87379832826309,48.87379832826309 +mixed-random_1200_seed_45,allenai/olmOCR-2-7B-1025-FP8,1480.0504595839022,1497.230656288293,-17.18019670439071,17.18019670439071 +mixed-random_1200_seed_45,baidu/Qianfan-OCR,1593.4162604871158,1576.6608094768258,16.755451010290017,16.755451010290017 +mixed-random_1200_seed_45,baidu/Unlimited-OCR,1611.119318783886,1563.2800425238524,47.83927626003356,47.83927626003356 +mixed-random_1200_seed_45,deepseek-ai/DeepSeek-OCR,1421.2041862742628,1451.5246523995515,-30.320466125288704,30.320466125288704 +mixed-random_1200_seed_45,deepseek-ai/DeepSeek-OCR-2,1598.365769580556,1579.3558964368967,19.009873143659206,19.009873143659206 +mixed-random_1200_seed_45,lightonai/LightOnOCR-2-1B,1646.082758603662,1617.5188506558277,28.563907947834196,28.563907947834196 +mixed-random_1200_seed_45,numind/NuExtract3,1590.9942151002767,1571.572188853421,19.422026246855694,19.422026246855694 +mixed-random_1200_seed_45,rednote-hilab/dots.mocr,1552.3188611309392,1556.2216675111229,-3.9028063801836197,3.9028063801836197 +mixed-random_1200_seed_45,rednote-hilab/dots.ocr,1542.6388406063425,1529.2527243518712,13.38611625447129,13.38611625447129 +mixed-random_1200_seed_45,tesseract-5,1066.9644494778336,1151.0531271625612,-84.0886776847276,84.0886776847276 +mixed-random_1200_seed_45,zai-org/GLM-OCR,1488.4066689626586,1491.926505968817,-3.5198370061584683,3.5198370061584683 +mixed-random_1200_seed_46,FireRedTeam/FireRed-OCR,1597.4821947072717,1556.3777567649604,41.10443794231128,41.10443794231128 +mixed-random_1200_seed_46,PaddlePaddle/PP-OCRv6_medium,1310.1797508380314,1354.3332993861072,-44.15354854807583,44.15354854807583 +mixed-random_1200_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1516.8156415707144,1503.691822219892,13.123819350822487,13.123819350822487 +mixed-random_1200_seed_46,allenai/olmOCR-2-7B-1025-FP8,1502.8727469434361,1497.230656288293,5.642090655143193,5.642090655143193 +mixed-random_1200_seed_46,baidu/Qianfan-OCR,1568.257340067952,1576.6608094768258,-8.403469408873661,8.403469408873661 +mixed-random_1200_seed_46,baidu/Unlimited-OCR,1598.1665076184513,1563.2800425238524,34.88646509459886,34.88646509459886 +mixed-random_1200_seed_46,deepseek-ai/DeepSeek-OCR,1432.1829257942113,1451.5246523995515,-19.341726605340227,19.341726605340227 +mixed-random_1200_seed_46,deepseek-ai/DeepSeek-OCR-2,1624.501345837839,1579.3558964368967,45.1454494009422,45.1454494009422 +mixed-random_1200_seed_46,lightonai/LightOnOCR-2-1B,1638.1368066217763,1617.5188506558277,20.617955965948568,20.617955965948568 +mixed-random_1200_seed_46,numind/NuExtract3,1540.4416332959775,1571.572188853421,-31.130555557443586,31.130555557443586 +mixed-random_1200_seed_46,rednote-hilab/dots.mocr,1512.138457240629,1556.2216675111229,-44.083210270493964,44.083210270493964 +mixed-random_1200_seed_46,rednote-hilab/dots.ocr,1546.8241404181824,1529.2527243518712,17.571416066311258,17.571416066311258 +mixed-random_1200_seed_46,tesseract-5,1105.7196907415798,1151.0531271625612,-45.33343642098134,45.33343642098134 +mixed-random_1200_seed_46,zai-org/GLM-OCR,1506.2808183039479,1491.926505968817,14.354312335130771,14.354312335130771 +mixed-random_2000_seed_42,FireRedTeam/FireRed-OCR,1560.7484823591735,1556.3777567649604,4.370725594213127,4.370725594213127 +mixed-random_2000_seed_42,PaddlePaddle/PP-OCRv6_medium,1359.8917394471987,1354.3332993861072,5.55844006109146,5.55844006109146 +mixed-random_2000_seed_42,PaddlePaddle/PaddleOCR-VL-1.6,1496.5982637161483,1503.691822219892,-7.0935585037436795,7.0935585037436795 +mixed-random_2000_seed_42,allenai/olmOCR-2-7B-1025-FP8,1496.5982637161483,1497.230656288293,-0.6323925721446813,0.6323925721446813 +mixed-random_2000_seed_42,baidu/Qianfan-OCR,1547.111024551955,1576.6608094768258,-29.549784924870664,29.549784924870664 +mixed-random_2000_seed_42,baidu/Unlimited-OCR,1578.2620916139485,1563.2800425238524,14.982049090096098,14.982049090096098 +mixed-random_2000_seed_42,deepseek-ai/DeepSeek-OCR,1443.9392504974448,1451.5246523995515,-7.585401902106696,7.585401902106696 +mixed-random_2000_seed_42,deepseek-ai/DeepSeek-OCR-2,1615.5487357737957,1579.3558964368967,36.19283933689894,36.19283933689894 +mixed-random_2000_seed_42,lightonai/LightOnOCR-2-1B,1657.3509253098446,1617.5188506558277,39.83207465401688,39.83207465401688 +mixed-random_2000_seed_42,numind/NuExtract3,1544.3903403748022,1571.572188853421,-27.181848478618804,27.181848478618804 +mixed-random_2000_seed_42,rednote-hilab/dots.mocr,1548.3474402353147,1556.2216675111229,-7.874227275808153,7.874227275808153 +mixed-random_2000_seed_42,rednote-hilab/dots.ocr,1523.6953233540257,1529.2527243518712,-5.557400997845434,5.557400997845434 +mixed-random_2000_seed_42,tesseract-5,1137.1026931962745,1151.0531271625612,-13.95043396628671,13.95043396628671 +mixed-random_2000_seed_42,zai-org/GLM-OCR,1490.4154258539252,1491.926505968817,-1.5110801148919109,1.5110801148919109 +mixed-random_2000_seed_43,FireRedTeam/FireRed-OCR,1591.1042633777524,1556.3777567649604,34.726506612792036,34.726506612792036 +mixed-random_2000_seed_43,PaddlePaddle/PP-OCRv6_medium,1349.6397340472583,1354.3332993861072,-4.6935653388488845,4.6935653388488845 +mixed-random_2000_seed_43,PaddlePaddle/PaddleOCR-VL-1.6,1505.2970915991664,1503.691822219892,1.6052693792744321,1.6052693792744321 +mixed-random_2000_seed_43,allenai/olmOCR-2-7B-1025-FP8,1474.3240727580157,1497.230656288293,-22.906583530277203,22.906583530277203 +mixed-random_2000_seed_43,baidu/Qianfan-OCR,1583.3256686649402,1576.6608094768258,6.664859188114406,6.664859188114406 +mixed-random_2000_seed_43,baidu/Unlimited-OCR,1548.39259919778,1563.2800425238524,-14.887443326072344,14.887443326072344 +mixed-random_2000_seed_43,deepseek-ai/DeepSeek-OCR,1448.9087082331641,1451.5246523995515,-2.6159441663874077,2.6159441663874077 +mixed-random_2000_seed_43,deepseek-ai/DeepSeek-OCR-2,1619.3570654291432,1579.3558964368967,40.00116899224645,40.00116899224645 +mixed-random_2000_seed_43,lightonai/LightOnOCR-2-1B,1625.96354664103,1617.5188506558277,8.444695985202316,8.444695985202316 +mixed-random_2000_seed_43,numind/NuExtract3,1543.2844060525185,1571.572188853421,-28.287782800902505,28.287782800902505 +mixed-random_2000_seed_43,rednote-hilab/dots.mocr,1549.6288214449187,1556.2216675111229,-6.592846066204174,6.592846066204174 +mixed-random_2000_seed_43,rednote-hilab/dots.ocr,1505.4959633665599,1529.2527243518712,-23.756760985311303,23.756760985311303 +mixed-random_2000_seed_43,tesseract-5,1142.5943021784717,1151.0531271625612,-8.45882498408946,8.45882498408946 +mixed-random_2000_seed_43,zai-org/GLM-OCR,1512.6837570092805,1491.926505968817,20.757251040463416,20.757251040463416 +mixed-random_2000_seed_44,FireRedTeam/FireRed-OCR,1567.8218939493238,1556.3777567649604,11.444137184363399,11.444137184363399 +mixed-random_2000_seed_44,PaddlePaddle/PP-OCRv6_medium,1344.2451255324154,1354.3332993861072,-10.088173853691842,10.088173853691842 +mixed-random_2000_seed_44,PaddlePaddle/PaddleOCR-VL-1.6,1507.3967994261914,1503.691822219892,3.704977206299418,3.704977206299418 +mixed-random_2000_seed_44,allenai/olmOCR-2-7B-1025-FP8,1480.2191151777952,1497.230656288293,-17.011541110497774,17.011541110497774 +mixed-random_2000_seed_44,baidu/Qianfan-OCR,1596.7730465391403,1576.6608094768258,20.112237062314534,20.112237062314534 +mixed-random_2000_seed_44,baidu/Unlimited-OCR,1560.3681088421688,1563.2800425238524,-2.9119336816836494,2.9119336816836494 +mixed-random_2000_seed_44,deepseek-ai/DeepSeek-OCR,1446.1735599967656,1451.5246523995515,-5.35109240278598,5.35109240278598 +mixed-random_2000_seed_44,deepseek-ai/DeepSeek-OCR-2,1592.9547187573637,1579.3558964368967,13.59882232046698,13.59882232046698 +mixed-random_2000_seed_44,lightonai/LightOnOCR-2-1B,1612.207584474081,1617.5188506558277,-5.31126618174676,5.31126618174676 +mixed-random_2000_seed_44,numind/NuExtract3,1569.152219774452,1571.572188853421,-2.419969078968961,2.419969078968961 +mixed-random_2000_seed_44,rednote-hilab/dots.mocr,1545.4559040219756,1556.2216675111229,-10.765763489147275,10.765763489147275 +mixed-random_2000_seed_44,rednote-hilab/dots.ocr,1530.7775551744517,1529.2527243518712,1.5248308225804976,1.5248308225804976 +mixed-random_2000_seed_44,tesseract-5,1130.3882127068746,1151.0531271625612,-20.664914455686585,20.664914455686585 +mixed-random_2000_seed_44,zai-org/GLM-OCR,1516.0661556270013,1491.926505968817,24.139649658184226,24.139649658184226 +mixed-random_2000_seed_45,FireRedTeam/FireRed-OCR,1568.289879761484,1556.3777567649604,11.912122996523522,11.912122996523522 +mixed-random_2000_seed_45,PaddlePaddle/PP-OCRv6_medium,1360.5080496215521,1354.3332993861072,6.17475023544489,6.17475023544489 +mixed-random_2000_seed_45,PaddlePaddle/PaddleOCR-VL-1.6,1482.9446249148502,1503.691822219892,-20.74719730504171,20.74719730504171 +mixed-random_2000_seed_45,allenai/olmOCR-2-7B-1025-FP8,1494.1292208466746,1497.230656288293,-3.101435441618378,3.101435441618378 +mixed-random_2000_seed_45,baidu/Qianfan-OCR,1606.3165592146804,1576.6608094768258,29.65574973785465,29.65574973785465 +mixed-random_2000_seed_45,baidu/Unlimited-OCR,1597.321448817019,1563.2800425238524,34.041406293166574,34.041406293166574 +mixed-random_2000_seed_45,deepseek-ai/DeepSeek-OCR,1448.4484957575162,1451.5246523995515,-3.0761566420353574,3.0761566420353574 +mixed-random_2000_seed_45,deepseek-ai/DeepSeek-OCR-2,1570.0725916421022,1579.3558964368967,-9.283304794794503,9.283304794794503 +mixed-random_2000_seed_45,lightonai/LightOnOCR-2-1B,1618.0230590150675,1617.5188506558277,0.504208359239783,0.504208359239783 +mixed-random_2000_seed_45,numind/NuExtract3,1563.3034939904162,1571.572188853421,-8.26869486300484,8.26869486300484 +mixed-random_2000_seed_45,rednote-hilab/dots.mocr,1554.6089637057007,1556.2216675111229,-1.612703805422143,1.612703805422143 +mixed-random_2000_seed_45,rednote-hilab/dots.ocr,1538.4277730888525,1529.2527243518712,9.175048736981353,9.175048736981353 +mixed-random_2000_seed_45,tesseract-5,1117.1556947380154,1151.0531271625612,-33.89743242454574,33.89743242454574 +mixed-random_2000_seed_45,zai-org/GLM-OCR,1480.4501448860694,1491.926505968817,-11.476361082747644,11.476361082747644 +mixed-random_2000_seed_46,FireRedTeam/FireRed-OCR,1586.3302874791902,1556.3777567649604,29.95253071422985,29.95253071422985 +mixed-random_2000_seed_46,PaddlePaddle/PP-OCRv6_medium,1334.066100400177,1354.3332993861072,-20.267198985930236,20.267198985930236 +mixed-random_2000_seed_46,PaddlePaddle/PaddleOCR-VL-1.6,1500.3611306840182,1503.691822219892,-3.3306915358737115,3.3306915358737115 +mixed-random_2000_seed_46,allenai/olmOCR-2-7B-1025-FP8,1510.3048489316286,1497.230656288293,13.074192643335664,13.074192643335664 +mixed-random_2000_seed_46,baidu/Qianfan-OCR,1576.1865507823966,1576.6608094768258,-0.47425869442918156,0.47425869442918156 +mixed-random_2000_seed_46,baidu/Unlimited-OCR,1578.6803044221558,1563.2800425238524,15.40026189830337,15.40026189830337 +mixed-random_2000_seed_46,deepseek-ai/DeepSeek-OCR,1430.4761399378813,1451.5246523995515,-21.04851246167027,21.04851246167027 +mixed-random_2000_seed_46,deepseek-ai/DeepSeek-OCR-2,1619.8779291438252,1579.3558964368967,40.522032706928485,40.522032706928485 +mixed-random_2000_seed_46,lightonai/LightOnOCR-2-1B,1625.1650995307086,1617.5188506558277,7.646248874880939,7.646248874880939 +mixed-random_2000_seed_46,numind/NuExtract3,1547.4540722735708,1571.572188853421,-24.118116579850266,24.118116579850266 +mixed-random_2000_seed_46,rednote-hilab/dots.mocr,1533.8214638515783,1556.2216675111229,-22.40020365954456,22.40020365954456 +mixed-random_2000_seed_46,rednote-hilab/dots.ocr,1530.108675625945,1529.2527243518712,0.8559512740737318,0.8559512740737318 +mixed-random_2000_seed_46,tesseract-5,1121.804248552401,1151.0531271625612,-29.24887861016009,29.24887861016009 +mixed-random_2000_seed_46,zai-org/GLM-OCR,1505.3631483845236,1491.926505968817,13.436642415706501,13.436642415706501 diff --git a/experiments/adaptive-stopping/moh/results.json b/experiments/adaptive-stopping/moh/results.json new file mode 100644 index 0000000..6c57d5b --- /dev/null +++ b/experiments/adaptive-stopping/moh/results.json @@ -0,0 +1,13118 @@ +{ + "experiment": { + "batch_samples": 5, + "bootstrap_replicates": 1000, + "bootstrap_seed": 42, + "comparisons_replayed": 4505, + "followup_acceptance_gate": { + "kendall_tau_minimum": 0.95, + "max_absolute_elo_delta_maximum": 50.0, + "percentage_saved_minimum": 60.0, + "top3_order_exact": true + }, + "historical_sentinel_comparisons": 0, + "min_before_check": 273, + "model_count": 14, + "models": [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ], + "production_helpers": [ + "compute_elo", + "classify_adjacent_pairs", + "comparison_pair_counts", + "model_parameter_counts", + "normalize_model_pair", + "practical_preferences", + "unresolved_pairs" + ], + "repeated_strategy_names": [ + "balanced_adaptive", + "full_stored_board", + "targeted", + "targeted_size_3_min_10", + "targeted_v2_warmup_5_explore_3_annotate_3x" + ], + "repeats": 2, + "sample_count": 50, + "sentinel_comparisons_excluded": false, + "source_repo": "davanstrien/ocr-bench-moh-results", + "source_revision": "05d24695f131432c38984dd2cb55f5a18a3c2580", + "static_design": { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "budgets": [ + 700, + 1200, + 2000 + ], + "mixed_random": "first five-sample balanced batch, then seeded outcome-independent pair-balanced exploration", + "pair_balanced": "fixed equal per-pair quotas across all stored samples" + }, + "stored_comparisons_at_revision": 4505 + }, + "static_design_aggregates": [ + { + "budget": 700, + "gate_passes": 0, + "kendall_tau_max": 0.8461538461538461, + "kendall_tau_median": 0.7802197802197802, + "kendall_tau_min": 0.7142857142857144, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 101.54558005476588, + "max_absolute_elo_delta_worst": 116.81052616962552, + "median_absolute_elo_delta_median": 40.91301945377552, + "minimum_pair_evidence": 7, + "percentage_saved": 84.46170921198669, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9516483516483516, + "spearman_rho_median": 0.9208791208791209, + "spearman_rho_min": 0.8637362637362638, + "top3_membership_matches": 0, + "top3_order_matches": 0 + }, + { + "budget": 1200, + "gate_passes": 0, + "kendall_tau_max": 0.8681318681318682, + "kendall_tau_median": 0.8021978021978022, + "kendall_tau_min": 0.7802197802197802, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 79.52519843529308, + "max_absolute_elo_delta_worst": 84.0886776847276, + "median_absolute_elo_delta_median": 23.835583467567176, + "minimum_pair_evidence": 13, + "percentage_saved": 73.36293007769146, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.956043956043956, + "spearman_rho_median": 0.9296703296703297, + "spearman_rho_min": 0.9208791208791209, + "top3_membership_matches": 0, + "top3_order_matches": 0 + }, + { + "budget": 2000, + "gate_passes": 0, + "kendall_tau_max": 0.9120879120879122, + "kendall_tau_median": 0.8681318681318682, + "kendall_tau_min": 0.8021978021978022, + "kind": "mixed-random", + "max_absolute_elo_delta_median": 39.83207465401688, + "max_absolute_elo_delta_worst": 40.522032706928485, + "median_absolute_elo_delta_median": 10.426968671419559, + "minimum_pair_evidence": 21, + "percentage_saved": 55.60488346281909, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9780219780219781, + "spearman_rho_median": 0.9472527472527472, + "spearman_rho_min": 0.9208791208791209, + "top3_membership_matches": 1, + "top3_order_matches": 0 + }, + { + "budget": 700, + "gate_passes": 0, + "kendall_tau_max": 0.8241758241758242, + "kendall_tau_median": 0.7802197802197802, + "kendall_tau_min": 0.7142857142857144, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 66.35516817329449, + "max_absolute_elo_delta_worst": 98.65008366570828, + "median_absolute_elo_delta_median": 21.429977055331733, + "minimum_pair_evidence": 7, + "percentage_saved": 84.46170921198669, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9164835164835166, + "spearman_rho_median": 0.8945054945054945, + "spearman_rho_min": 0.8725274725274725, + "top3_membership_matches": 0, + "top3_order_matches": 0 + }, + { + "budget": 1200, + "gate_passes": 0, + "kendall_tau_max": 0.8681318681318682, + "kendall_tau_median": 0.8241758241758242, + "kendall_tau_min": 0.8241758241758242, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 36.62551075270494, + "max_absolute_elo_delta_worst": 65.05221988772882, + "median_absolute_elo_delta_median": 16.856581791943313, + "minimum_pair_evidence": 13, + "percentage_saved": 73.36293007769146, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.956043956043956, + "spearman_rho_median": 0.9340659340659341, + "spearman_rho_min": 0.9208791208791209, + "top3_membership_matches": 1, + "top3_order_matches": 0 + }, + { + "budget": 2000, + "gate_passes": 0, + "kendall_tau_max": 0.9560439560439562, + "kendall_tau_median": 0.8681318681318682, + "kendall_tau_min": 0.8021978021978022, + "kind": "pair-balanced", + "max_absolute_elo_delta_median": 30.303948386869934, + "max_absolute_elo_delta_worst": 33.81672867535144, + "median_absolute_elo_delta_median": 8.846758982956544, + "minimum_pair_evidence": 21, + "percentage_saved": 55.60488346281909, + "runs": 5, + "seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "spearman_rho_max": 0.9912087912087912, + "spearman_rho_median": 0.9472527472527472, + "spearman_rho_min": 0.9120879120879121, + "top3_membership_matches": 0, + "top3_order_matches": 0 + } + ], + "strategies": [ + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 4505, + "comparisons_saved": 0, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1556.3777567649604, + "PaddlePaddle/PP-OCRv6_medium": 1354.3332993861072, + "PaddlePaddle/PaddleOCR-VL-1.6": 1503.691822219892, + "allenai/olmOCR-2-7B-1025-FP8": 1497.230656288293, + "baidu/Qianfan-OCR": 1576.6608094768258, + "baidu/Unlimited-OCR": 1563.2800425238524, + "deepseek-ai/DeepSeek-OCR": 1451.5246523995515, + "deepseek-ai/DeepSeek-OCR-2": 1579.3558964368967, + "lightonai/LightOnOCR-2-1B": 1617.5188506558277, + "numind/NuExtract3": 1571.572188853421, + "rednote-hilab/dots.mocr": 1556.2216675111229, + "rednote-hilab/dots.ocr": 1529.2527243518712, + "tesseract-5": 1151.0531271625612, + "zai-org/GLM-OCR": 1491.926505968817 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1532.9827744640256, + 1581.4686643608454 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1326.1763067307545, + 1382.1557540875185 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1478.3279488062924, + 1529.6656702363468 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1471.6223530887685, + 1521.8903426255604 + ], + "baidu/Qianfan-OCR": [ + 1550.1806294291996, + 1604.2099849302886 + ], + "baidu/Unlimited-OCR": [ + 1539.100912227264, + 1593.2751232986484 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1425.5061220961072, + 1477.1741644986862 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1554.8407193672601, + 1604.127358260131 + ], + "lightonai/LightOnOCR-2-1B": [ + 1591.6139972558697, + 1643.1658920595753 + ], + "numind/NuExtract3": [ + 1549.1770192655326, + 1597.523500534782 + ], + "rednote-hilab/dots.mocr": [ + 1531.108216193696, + 1581.6075388070872 + ], + "rednote-hilab/dots.ocr": [ + 1504.3745991709281, + 1554.3737791541428 + ], + "tesseract-5": [ + 1107.0114115905271, + 1192.6250802568054 + ], + "zai-org/GLM-OCR": [ + 1465.6490071574, + 1517.0332505607557 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 0.0, + "PaddlePaddle/PP-OCRv6_medium": 0.0, + "PaddlePaddle/PaddleOCR-VL-1.6": 0.0, + "allenai/olmOCR-2-7B-1025-FP8": 0.0, + "baidu/Qianfan-OCR": 0.0, + "baidu/Unlimited-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR-2": 0.0, + "lightonai/LightOnOCR-2-1B": 0.0, + "numind/NuExtract3": 0.0, + "rednote-hilab/dots.mocr": 0.0, + "rednote-hilab/dots.ocr": 0.0, + "tesseract-5": 0.0, + "zai-org/GLM-OCR": 0.0 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 50, + "min_direct_comparisons": 47, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 1.0, + "kind": "full", + "max_absolute_elo_delta": 0.0, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 0.0, + "name": "full_stored_board", + "percentage_saved": 0.0, + "practical_preferences": [], + "resolution_summary": "reference only; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 1.0, + "static_budget": null, + "stopping_reason": "reference_full_stored_board", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 4505, + "comparisons_saved": 0, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1556.3777567649604, + "PaddlePaddle/PP-OCRv6_medium": 1354.3332993861072, + "PaddlePaddle/PaddleOCR-VL-1.6": 1503.691822219892, + "allenai/olmOCR-2-7B-1025-FP8": 1497.230656288293, + "baidu/Qianfan-OCR": 1576.6608094768258, + "baidu/Unlimited-OCR": 1563.2800425238524, + "deepseek-ai/DeepSeek-OCR": 1451.5246523995515, + "deepseek-ai/DeepSeek-OCR-2": 1579.3558964368967, + "lightonai/LightOnOCR-2-1B": 1617.5188506558277, + "numind/NuExtract3": 1571.572188853421, + "rednote-hilab/dots.mocr": 1556.2216675111229, + "rednote-hilab/dots.ocr": 1529.2527243518712, + "tesseract-5": 1151.0531271625612, + "zai-org/GLM-OCR": 1491.926505968817 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1532.9827744640256, + 1581.4686643608454 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1326.1763067307545, + 1382.1557540875185 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1478.3279488062924, + 1529.6656702363468 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1471.6223530887685, + 1521.8903426255604 + ], + "baidu/Qianfan-OCR": [ + 1550.1806294291996, + 1604.2099849302886 + ], + "baidu/Unlimited-OCR": [ + 1539.100912227264, + 1593.2751232986484 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1425.5061220961072, + 1477.1741644986862 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1554.8407193672601, + 1604.127358260131 + ], + "lightonai/LightOnOCR-2-1B": [ + 1591.6139972558697, + 1643.1658920595753 + ], + "numind/NuExtract3": [ + 1549.1770192655326, + 1597.523500534782 + ], + "rednote-hilab/dots.mocr": [ + 1531.108216193696, + 1581.6075388070872 + ], + "rednote-hilab/dots.ocr": [ + 1504.3745991709281, + 1554.3737791541428 + ], + "tesseract-5": [ + 1107.0114115905271, + 1192.6250802568054 + ], + "zai-org/GLM-OCR": [ + 1465.6490071574, + 1517.0332505607557 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 0.0, + "PaddlePaddle/PP-OCRv6_medium": 0.0, + "PaddlePaddle/PaddleOCR-VL-1.6": 0.0, + "allenai/olmOCR-2-7B-1025-FP8": 0.0, + "baidu/Qianfan-OCR": 0.0, + "baidu/Unlimited-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR": 0.0, + "deepseek-ai/DeepSeek-OCR-2": 0.0, + "lightonai/LightOnOCR-2-1B": 0.0, + "numind/NuExtract3": 0.0, + "rednote-hilab/dots.mocr": 0.0, + "rednote-hilab/dots.ocr": 0.0, + "tesseract-5": 0.0, + "zai-org/GLM-OCR": 0.0 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 50, + "min_direct_comparisons": 47, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 1.0, + "kind": "balanced", + "max_absolute_elo_delta": 0.0, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 0.0, + "name": "balanced_adaptive", + "percentage_saved": 0.0, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 454, + "cumulative_comparisons": 909, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 454, + "cumulative_comparisons": 1363, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 455, + "cumulative_comparisons": 1818, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 455, + "cumulative_comparisons": 2273, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 455, + "cumulative_comparisons": 2728, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 455, + "cumulative_comparisons": 3183, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 418, + "cumulative_comparisons": 3601, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 449, + "cumulative_comparisons": 4050, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced", + "batch_comparisons": 455, + "cumulative_comparisons": 4505, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 1.0, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 996, + "comparisons_saved": 3509, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1634.4644819496004, + "PaddlePaddle/PP-OCRv6_medium": 1298.3199433004734, + "PaddlePaddle/PaddleOCR-VL-1.6": 1419.9438300011045, + "allenai/olmOCR-2-7B-1025-FP8": 1468.4599875082165, + "baidu/Qianfan-OCR": 1692.33029306944, + "baidu/Unlimited-OCR": 1569.642762356888, + "deepseek-ai/DeepSeek-OCR": 1419.0954970052717, + "deepseek-ai/DeepSeek-OCR-2": 1656.234457507528, + "lightonai/LightOnOCR-2-1B": 1723.1332227144233, + "numind/NuExtract3": 1535.1101798239197, + "rednote-hilab/dots.mocr": 1523.7737354890974, + "rednote-hilab/dots.ocr": 1530.7460734188971, + "tesseract-5": 1067.0163609784408, + "zai-org/GLM-OCR": 1461.729174876699 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1576.5350755164786, + 1703.97236858941 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1213.4059266217419, + 1372.7222551826442 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1349.4651137580386, + 1482.581999409025 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1411.359191545635, + 1526.2409475792035 + ], + "baidu/Qianfan-OCR": [ + 1633.3320489902512, + 1764.566162461044 + ], + "baidu/Unlimited-OCR": [ + 1511.21854063291, + 1632.123489618203 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1358.3331775250128, + 1477.2946197068798 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1597.9538150584044, + 1722.3336185602025 + ], + "lightonai/LightOnOCR-2-1B": [ + 1657.224421665378, + 1802.5264801876574 + ], + "numind/NuExtract3": [ + 1478.3231852431256, + 1598.3213337008717 + ], + "rednote-hilab/dots.mocr": [ + 1471.9829974668153, + 1579.9318446303453 + ], + "rednote-hilab/dots.ocr": [ + 1474.077337909754, + 1594.2550043909198 + ], + "tesseract-5": [ + 866.3749703339511, + 1180.0396858032805 + ], + "zai-org/GLM-OCR": [ + 1404.9098799470337, + 1517.788988222921 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 78.08672518464004, + "PaddlePaddle/PP-OCRv6_medium": -56.01335608563386, + "PaddlePaddle/PaddleOCR-VL-1.6": -83.74799221878743, + "allenai/olmOCR-2-7B-1025-FP8": -28.77066878007645, + "baidu/Qianfan-OCR": 115.66948359261414, + "baidu/Unlimited-OCR": 6.362719833035499, + "deepseek-ai/DeepSeek-OCR": -32.42915539427986, + "deepseek-ai/DeepSeek-OCR-2": 76.87856107063135, + "lightonai/LightOnOCR-2-1B": 105.61437205859556, + "numind/NuExtract3": -36.46200902950136, + "rednote-hilab/dots.mocr": -32.4479320220255, + "rednote-hilab/dots.ocr": 1.4933490670259744, + "tesseract-5": -84.03676618412032, + "zai-org/GLM-OCR": -30.19733109211802 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 5, + "min_direct_comparisons": 5, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "targeted", + "max_absolute_elo_delta": 115.66948359261414, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 46.23768255756761, + "name": "targeted", + "percentage_saved": 77.89123196448391, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 12 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 575, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 635, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 695, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 755, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 815, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 57, + "cumulative_comparisons": 872, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 13, + "allocation_mode": "targeted", + "batch_comparisons": 64, + "cumulative_comparisons": 936, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "baidu/Qianfan-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 996, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9604395604395607, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 803, + "comparisons_saved": 3702, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1636.6508950725304, + "PaddlePaddle/PP-OCRv6_medium": 1313.406645594515, + "PaddlePaddle/PaddleOCR-VL-1.6": 1400.4352191082205, + "allenai/olmOCR-2-7B-1025-FP8": 1454.5858248562365, + "baidu/Qianfan-OCR": 1691.4810497148387, + "baidu/Unlimited-OCR": 1577.5206071828436, + "deepseek-ai/DeepSeek-OCR": 1419.982994012063, + "deepseek-ai/DeepSeek-OCR-2": 1658.3905650892693, + "lightonai/LightOnOCR-2-1B": 1722.9467755513317, + "numind/NuExtract3": 1522.4224783594152, + "rednote-hilab/dots.mocr": 1517.0352764844622, + "rednote-hilab/dots.ocr": 1528.0978067177562, + "tesseract-5": 1069.2927279364471, + "zai-org/GLM-OCR": 1487.7511343200704 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1580.092141473442, + 1702.8771998597847 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1205.637317287868, + 1409.7151996957798 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1304.848612933805, + 1488.75308859053 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1392.1409321192202, + 1530.8169344472085 + ], + "baidu/Qianfan-OCR": [ + 1633.42146663266, + 1769.1717737472368 + ], + "baidu/Unlimited-OCR": [ + 1524.7214588336894, + 1643.9903947622965 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1346.439548036567, + 1495.9986926190245 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1592.8797251608648, + 1734.117739843473 + ], + "lightonai/LightOnOCR-2-1B": [ + 1645.745656069409, + 1838.0156456225652 + ], + "numind/NuExtract3": [ + 1466.822352076729, + 1586.3234588101993 + ], + "rednote-hilab/dots.mocr": [ + 1459.0891310676122, + 1586.6598776511155 + ], + "rednote-hilab/dots.ocr": [ + 1466.3334943242735, + 1594.0795714799963 + ], + "tesseract-5": [ + 804.7376852413589, + 1185.4101173999475 + ], + "zai-org/GLM-OCR": [ + 1415.8179283105544, + 1572.7870369857276 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 80.27313830756998, + "PaddlePaddle/PP-OCRv6_medium": -40.926653791592116, + "PaddlePaddle/PaddleOCR-VL-1.6": -103.25660311167144, + "allenai/olmOCR-2-7B-1025-FP8": -42.644831432056435, + "baidu/Qianfan-OCR": 114.82024023801296, + "baidu/Unlimited-OCR": 14.240564658991161, + "deepseek-ai/DeepSeek-OCR": -31.541658387488496, + "deepseek-ai/DeepSeek-OCR-2": 79.0346686523726, + "lightonai/LightOnOCR-2-1B": 105.42792489550402, + "numind/NuExtract3": -49.14971049400583, + "rednote-hilab/dots.mocr": -39.18639102666066, + "rednote-hilab/dots.ocr": -1.1549176341150087, + "tesseract-5": -81.76039922611403, + "zai-org/GLM-OCR": -4.1753716487467045 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 5, + "min_direct_comparisons": 5, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "targeted", + "max_absolute_elo_delta": 114.82024023801296, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 45.89727096303113, + "name": "targeted_size_3_min_10", + "percentage_saved": 82.17536071032187, + "practical_preferences": [ + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "size_ratio": 26.08695652173913, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "baidu/Qianfan-OCR", + "size_ratio": 4.7, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 7 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 555, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 590, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 625, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 665, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 700, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 34, + "cumulative_comparisons": 734, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 34, + "cumulative_comparisons": 768, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 803, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9252747252747253, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 7, + "resolved": 1, + "unresolved": 5 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 703, + "comparisons_saved": 3802, + "decision_statuses": { + "prefer-smaller": 7, + "resolved": 1, + "unresolved": 5 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1642.0704925711239, + "PaddlePaddle/PP-OCRv6_medium": 1313.8310412401265, + "PaddlePaddle/PaddleOCR-VL-1.6": 1400.797992322702, + "allenai/olmOCR-2-7B-1025-FP8": 1454.7497016233788, + "baidu/Qianfan-OCR": 1680.9416300315613, + "baidu/Unlimited-OCR": 1572.1693233029544, + "deepseek-ai/DeepSeek-OCR": 1420.4502447225152, + "deepseek-ai/DeepSeek-OCR-2": 1655.9516154765304, + "lightonai/LightOnOCR-2-1B": 1721.5263460984947, + "numind/NuExtract3": 1521.3675017119538, + "rednote-hilab/dots.mocr": 1516.8883838987947, + "rednote-hilab/dots.ocr": 1540.700126069093, + "tesseract-5": 1069.802125503712, + "zai-org/GLM-OCR": 1488.75347542706 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1580.8257719818696, + 1714.2350328959233 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1205.8506514607152, + 1406.2598171414738 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1312.229311696012, + 1489.0016881839085 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1379.4579814891936, + 1527.5301644934511 + ], + "baidu/Qianfan-OCR": [ + 1616.650099052028, + 1759.645695090153 + ], + "baidu/Unlimited-OCR": [ + 1517.070283870577, + 1636.7249411696462 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1334.0512827598327, + 1503.4503779100526 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1591.3336926042548, + 1735.9820678273454 + ], + "lightonai/LightOnOCR-2-1B": [ + 1641.4588160760513, + 1830.896480420274 + ], + "numind/NuExtract3": [ + 1458.0715541951158, + 1592.1157863254434 + ], + "rednote-hilab/dots.mocr": [ + 1461.4800023176704, + 1582.9704463130706 + ], + "rednote-hilab/dots.ocr": [ + 1470.2013548555997, + 1611.398897173005 + ], + "tesseract-5": [ + 817.833641320224, + 1191.8306048629804 + ], + "zai-org/GLM-OCR": [ + 1404.7669507459073, + 1572.0618443489807 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 85.69273580616345, + "PaddlePaddle/PP-OCRv6_medium": -40.502258145980704, + "PaddlePaddle/PaddleOCR-VL-1.6": -102.89382989719002, + "allenai/olmOCR-2-7B-1025-FP8": -42.48095466491418, + "baidu/Qianfan-OCR": 104.28082055473556, + "baidu/Unlimited-OCR": 8.88928077910191, + "deepseek-ai/DeepSeek-OCR": -31.074407677036334, + "deepseek-ai/DeepSeek-OCR-2": 76.59571903963365, + "lightonai/LightOnOCR-2-1B": 104.00749544266705, + "numind/NuExtract3": -50.204687141467275, + "rednote-hilab/dots.mocr": -39.33328361232816, + "rednote-hilab/dots.ocr": 11.447401717221737, + "tesseract-5": -81.25100165884919, + "zai-org/GLM-OCR": -3.1730305417570435 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 45, + "median_direct_comparisons": 5, + "min_direct_comparisons": 5, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "targeted", + "max_absolute_elo_delta": 104.28082055473556, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 46.34282090319073, + "name": "targeted_size_2_min_10", + "percentage_saved": 84.39511653718091, + "practical_preferences": [ + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "size_ratio": 26.08695652173913, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "allenai/olmOCR-2-7B-1025-FP8", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 2.075, + "smaller_model": "deepseek-ai/DeepSeek-OCR" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "baidu/Qianfan-OCR", + "size_ratio": 4.7, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.ocr", + "larger_model": "numind/NuExtract3", + "size_ratio": 2.3529411764705883, + "smaller_model": "rednote-hilab/dots.ocr" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 5 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 7, + "resolved": 1, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 5, + "allocation_mode": "targeted", + "batch_comparisons": 25, + "cumulative_comparisons": 540, + "decision": { + "prefer-smaller": 8, + "resolved": 1, + "unresolved": 4 + }, + "next_active_pairs": 4, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 4, + "allocation_mode": "targeted", + "batch_comparisons": 20, + "cumulative_comparisons": 560, + "decision": { + "prefer-smaller": 8, + "resolved": 1, + "unresolved": 4 + }, + "next_active_pairs": 4, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 4, + "allocation_mode": "targeted", + "batch_comparisons": 20, + "cumulative_comparisons": 580, + "decision": { + "prefer-smaller": 6, + "resolved": 1, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 30, + "cumulative_comparisons": 610, + "decision": { + "prefer-smaller": 8, + "resolved": 1, + "unresolved": 4 + }, + "next_active_pairs": 4, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 4, + "allocation_mode": "targeted", + "batch_comparisons": 20, + "cumulative_comparisons": 630, + "decision": { + "prefer-smaller": 7, + "resolved": 1, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 5, + "allocation_mode": "targeted", + "batch_comparisons": 24, + "cumulative_comparisons": 654, + "decision": { + "prefer-smaller": 7, + "resolved": 1, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 5, + "allocation_mode": "targeted", + "batch_comparisons": 24, + "cumulative_comparisons": 678, + "decision": { + "prefer-smaller": 7, + "resolved": 1, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 5, + "allocation_mode": "targeted", + "batch_comparisons": 25, + "cumulative_comparisons": 703, + "decision": { + "prefer-smaller": 7, + "resolved": 1, + "unresolved": 5 + }, + "next_active_pairs": 5, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": 2.0, + "spearman_rho": 0.9252747252747253, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 783, + "comparisons_saved": 3722, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1641.9181041792333, + "PaddlePaddle/PP-OCRv6_medium": 1311.1365633965083, + "PaddlePaddle/PaddleOCR-VL-1.6": 1393.444130489657, + "allenai/olmOCR-2-7B-1025-FP8": 1454.600116632278, + "baidu/Qianfan-OCR": 1690.526646918854, + "baidu/Unlimited-OCR": 1574.0785452974121, + "deepseek-ai/DeepSeek-OCR": 1424.2509337258196, + "deepseek-ai/DeepSeek-OCR-2": 1651.3336379186167, + "lightonai/LightOnOCR-2-1B": 1733.4271571063932, + "numind/NuExtract3": 1521.2565641223305, + "rednote-hilab/dots.mocr": 1514.6330233206916, + "rednote-hilab/dots.ocr": 1527.1124510512539, + "tesseract-5": 1069.0458107324857, + "zai-org/GLM-OCR": 1493.2363151084658 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1586.420921847669, + 1707.6163074125852 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1206.3442655106412, + 1405.8398320585286 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1289.6401736201308, + 1486.838199675255 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1391.0123424759577, + 1524.656232098653 + ], + "baidu/Qianfan-OCR": [ + 1628.2714642171247, + 1769.6165352622188 + ], + "baidu/Unlimited-OCR": [ + 1521.3838892524784, + 1635.2874013937294 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1349.606521912436, + 1494.2711510274532 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1584.9747224269613, + 1727.5256526044075 + ], + "lightonai/LightOnOCR-2-1B": [ + 1646.6781076921982, + 1841.2200480519625 + ], + "numind/NuExtract3": [ + 1463.587300844859, + 1580.9953414858592 + ], + "rednote-hilab/dots.mocr": [ + 1454.0644167501346, + 1582.0184142525263 + ], + "rednote-hilab/dots.ocr": [ + 1465.4930902684437, + 1590.055300059446 + ], + "tesseract-5": [ + 879.5109984194168, + 1177.1306210340115 + ], + "zai-org/GLM-OCR": [ + 1411.7314408814113, + 1572.8953596518975 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 85.5403474142729, + "PaddlePaddle/PP-OCRv6_medium": -43.19673598959889, + "PaddlePaddle/PaddleOCR-VL-1.6": -110.24769173023492, + "allenai/olmOCR-2-7B-1025-FP8": -42.63053965601489, + "baidu/Qianfan-OCR": 113.86583744202835, + "baidu/Unlimited-OCR": 10.798502773559676, + "deepseek-ai/DeepSeek-OCR": -27.27371867373199, + "deepseek-ai/DeepSeek-OCR-2": 71.97774148171993, + "lightonai/LightOnOCR-2-1B": 115.90830645056553, + "numind/NuExtract3": -50.315624731090566, + "rednote-hilab/dots.mocr": -41.588644190431296, + "rednote-hilab/dots.ocr": -2.1402733006173094, + "tesseract-5": -82.00731643007543, + "zai-org/GLM-OCR": 1.309809139648678 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 5, + "min_direct_comparisons": 5, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "targeted", + "max_absolute_elo_delta": 115.90830645056553, + "max_absolute_elo_delta_model": "lightonai/LightOnOCR-2-1B", + "median_absolute_elo_delta": 46.75618036034473, + "name": "targeted_size_3_min_5", + "percentage_saved": 82.61931187569367, + "practical_preferences": [ + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "size_ratio": 26.08695652173913, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "baidu/Qianfan-OCR", + "size_ratio": 4.7, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 7 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 495, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 530, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 565, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 600, + "decision": { + "prefer-smaller": 5, + "resolved": 0, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 640, + "decision": { + "prefer-smaller": 5, + "resolved": 0, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 680, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 34, + "cumulative_comparisons": 714, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 34, + "cumulative_comparisons": 748, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 783, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 5, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9252747252747253, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 842, + "comparisons_saved": 3663, + "decision_statuses": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1640.6310230547772, + "PaddlePaddle/PP-OCRv6_medium": 1315.9885719921695, + "PaddlePaddle/PaddleOCR-VL-1.6": 1415.8469878397334, + "allenai/olmOCR-2-7B-1025-FP8": 1453.7681525506882, + "baidu/Qianfan-OCR": 1698.5567961420863, + "baidu/Unlimited-OCR": 1580.008338888557, + "deepseek-ai/DeepSeek-OCR": 1417.8457641117898, + "deepseek-ai/DeepSeek-OCR-2": 1666.2453105478435, + "lightonai/LightOnOCR-2-1B": 1708.646974850841, + "numind/NuExtract3": 1523.6684667385953, + "rednote-hilab/dots.mocr": 1514.9566118392627, + "rednote-hilab/dots.ocr": 1529.7231145830472, + "tesseract-5": 1037.2744013590816, + "zai-org/GLM-OCR": 1496.8394855015276 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1585.8985884817187, + 1708.472455104735 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1214.3810854263174, + 1397.7511242948824 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1335.6672691803135, + 1496.8596240587867 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1392.738901595417, + 1518.8612334684926 + ], + "baidu/Qianfan-OCR": [ + 1640.6590703751413, + 1774.9260469294559 + ], + "baidu/Unlimited-OCR": [ + 1529.676024509674, + 1643.9304165121723 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1349.7722200701157, + 1491.5357463609093 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1605.0434624557818, + 1744.5998212291574 + ], + "lightonai/LightOnOCR-2-1B": [ + 1633.678298127067, + 1804.6072124634024 + ], + "numind/NuExtract3": [ + 1460.0513345186528, + 1589.9582744817715 + ], + "rednote-hilab/dots.mocr": [ + 1459.8251274632216, + 1579.4101511483939 + ], + "rednote-hilab/dots.ocr": [ + 1468.2307282852114, + 1592.061162563933 + ], + "tesseract-5": [ + 764.3950135468816, + 1157.8863215885963 + ], + "zai-org/GLM-OCR": [ + 1429.5704386095213, + 1572.3966813354834 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 84.25326628981679, + "PaddlePaddle/PP-OCRv6_medium": -38.344727393937774, + "PaddlePaddle/PaddleOCR-VL-1.6": -87.84483438015855, + "allenai/olmOCR-2-7B-1025-FP8": -43.46250373760472, + "baidu/Qianfan-OCR": 121.89598666526058, + "baidu/Unlimited-OCR": 16.728296364704647, + "deepseek-ai/DeepSeek-OCR": -33.678888287761765, + "deepseek-ai/DeepSeek-OCR-2": 86.88941411094675, + "lightonai/LightOnOCR-2-1B": 91.12812419501324, + "numind/NuExtract3": -47.90372211482577, + "rednote-hilab/dots.mocr": -41.26505567186018, + "rednote-hilab/dots.ocr": 0.47039023117599754, + "tesseract-5": -113.77872580347957, + "zai-org/GLM-OCR": 4.912979532710551 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 5, + "min_direct_comparisons": 5, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "targeted", + "max_absolute_elo_delta": 121.89598666526058, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 45.683112926215244, + "name": "targeted_size_3_min_15", + "percentage_saved": 81.30965593784684, + "practical_preferences": [ + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "size_ratio": 26.08695652173913, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 4.444444444444445, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "baidu/Qianfan-OCR", + "size_ratio": 4.7, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 7 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 575, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 615, + "decision": { + "prefer-smaller": 5, + "resolved": 0, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 655, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 695, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 730, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 38, + "cumulative_comparisons": 768, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 39, + "cumulative_comparisons": 807, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "targeted", + "batch_comparisons": 35, + "cumulative_comparisons": 842, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 15, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9252747252747253, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 9 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 926, + "comparisons_saved": 3579, + "decision_statuses": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1639.410750341219, + "PaddlePaddle/PP-OCRv6_medium": 1303.4247635958586, + "PaddlePaddle/PaddleOCR-VL-1.6": 1420.7266637684627, + "allenai/olmOCR-2-7B-1025-FP8": 1462.6672924959607, + "baidu/Qianfan-OCR": 1695.758725320333, + "baidu/Unlimited-OCR": 1580.5507936743256, + "deepseek-ai/DeepSeek-OCR": 1413.7574993659675, + "deepseek-ai/DeepSeek-OCR-2": 1659.3956299503259, + "lightonai/LightOnOCR-2-1B": 1726.0180068524983, + "numind/NuExtract3": 1526.8834874726354, + "rednote-hilab/dots.mocr": 1529.4121354121596, + "rednote-hilab/dots.ocr": 1531.7086234567364, + "tesseract-5": 1033.9051505314112, + "zai-org/GLM-OCR": 1476.3804777621058 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1583.2659391743957, + 1707.498124322913 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1193.0576627964126, + 1399.1058669025908 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1339.2065292633536, + 1492.363341505442 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1399.4547228881697, + 1527.7420584702807 + ], + "baidu/Qianfan-OCR": [ + 1633.6305000666346, + 1767.8821164050235 + ], + "baidu/Unlimited-OCR": [ + 1525.857100307986, + 1644.184143951672 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1347.1611653369416, + 1479.5172496175876 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1600.6781949640474, + 1735.4620231420172 + ], + "lightonai/LightOnOCR-2-1B": [ + 1658.568536673539, + 1812.9263726398653 + ], + "numind/NuExtract3": [ + 1469.685436656376, + 1591.350202711791 + ], + "rednote-hilab/dots.mocr": [ + 1473.5453788482903, + 1591.6428245502204 + ], + "rednote-hilab/dots.ocr": [ + 1476.5175302705588, + 1599.812291424842 + ], + "tesseract-5": [ + 789.0941797260847, + 1156.961500448551 + ], + "zai-org/GLM-OCR": [ + 1409.9561417445148, + 1542.4788700032009 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 83.0329935762586, + "PaddlePaddle/PP-OCRv6_medium": -50.90853579024861, + "PaddlePaddle/PaddleOCR-VL-1.6": -82.96515845142926, + "allenai/olmOCR-2-7B-1025-FP8": -34.56336379233221, + "baidu/Qianfan-OCR": 119.09791584350728, + "baidu/Unlimited-OCR": 17.27075115047319, + "deepseek-ai/DeepSeek-OCR": -37.76715303358401, + "deepseek-ai/DeepSeek-OCR-2": 80.03973351342916, + "lightonai/LightOnOCR-2-1B": 108.49915619667058, + "numind/NuExtract3": -44.68870138078569, + "rednote-hilab/dots.mocr": -26.809532098963246, + "rednote-hilab/dots.ocr": 2.4558991048652388, + "tesseract-5": -117.14797663114996, + "zai-org/GLM-OCR": -15.546028206711298 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 5, + "min_direct_comparisons": 5, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "targeted", + "max_absolute_elo_delta": 119.09791584350728, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 47.79861858551715, + "name": "targeted_size_5_min_10", + "percentage_saved": 79.44506104328524, + "practical_preferences": [ + { + "higher_ranked_model": "deepseek-ai/DeepSeek-OCR", + "larger_model": "deepseek-ai/DeepSeek-OCR", + "size_ratio": 115.94202898550725, + "smaller_model": "PaddlePaddle/PP-OCRv6_medium" + }, + { + "higher_ranked_model": "allenai/olmOCR-2-7B-1025-FP8", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 9 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 1, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 570, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 10, + "allocation_mode": "targeted", + "batch_comparisons": 50, + "cumulative_comparisons": 620, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 10, + "allocation_mode": "targeted", + "batch_comparisons": 50, + "cumulative_comparisons": 670, + "decision": { + "prefer-smaller": 1, + "resolved": 1, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 725, + "decision": { + "prefer-smaller": 2, + "resolved": 0, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 780, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 10, + "allocation_mode": "targeted", + "batch_comparisons": 47, + "cumulative_comparisons": 827, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 10, + "allocation_mode": "targeted", + "batch_comparisons": 49, + "cumulative_comparisons": 876, + "decision": { + "prefer-smaller": 2, + "resolved": 1, + "unresolved": 10 + }, + "next_active_pairs": 10, + "rank_order": [ + "baidu/Qianfan-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 10, + "allocation_mode": "targeted", + "batch_comparisons": 50, + "cumulative_comparisons": 926, + "decision": { + "prefer-smaller": 3, + "resolved": 1, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": 5.0, + "spearman_rho": 0.9252747252747253, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 996, + "comparisons_saved": 3509, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1634.4644819496004, + "PaddlePaddle/PP-OCRv6_medium": 1298.3199433004734, + "PaddlePaddle/PaddleOCR-VL-1.6": 1419.9438300011045, + "allenai/olmOCR-2-7B-1025-FP8": 1468.4599875082165, + "baidu/Qianfan-OCR": 1692.33029306944, + "baidu/Unlimited-OCR": 1569.642762356888, + "deepseek-ai/DeepSeek-OCR": 1419.0954970052717, + "deepseek-ai/DeepSeek-OCR-2": 1656.234457507528, + "lightonai/LightOnOCR-2-1B": 1723.1332227144233, + "numind/NuExtract3": 1535.1101798239197, + "rednote-hilab/dots.mocr": 1523.7737354890974, + "rednote-hilab/dots.ocr": 1530.7460734188971, + "tesseract-5": 1067.0163609784408, + "zai-org/GLM-OCR": 1461.729174876699 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1576.5350755164786, + 1703.97236858941 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1213.4059266217419, + 1372.7222551826442 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1349.4651137580386, + 1482.581999409025 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1411.359191545635, + 1526.2409475792035 + ], + "baidu/Qianfan-OCR": [ + 1633.3320489902512, + 1764.566162461044 + ], + "baidu/Unlimited-OCR": [ + 1511.21854063291, + 1632.123489618203 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1358.3331775250128, + 1477.2946197068798 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1597.9538150584044, + 1722.3336185602025 + ], + "lightonai/LightOnOCR-2-1B": [ + 1657.224421665378, + 1802.5264801876574 + ], + "numind/NuExtract3": [ + 1478.3231852431256, + 1598.3213337008717 + ], + "rednote-hilab/dots.mocr": [ + 1471.9829974668153, + 1579.9318446303453 + ], + "rednote-hilab/dots.ocr": [ + 1474.077337909754, + 1594.2550043909198 + ], + "tesseract-5": [ + 866.3749703339511, + 1180.0396858032805 + ], + "zai-org/GLM-OCR": [ + 1404.9098799470337, + 1517.788988222921 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 78.08672518464004, + "PaddlePaddle/PP-OCRv6_medium": -56.01335608563386, + "PaddlePaddle/PaddleOCR-VL-1.6": -83.74799221878743, + "allenai/olmOCR-2-7B-1025-FP8": -28.77066878007645, + "baidu/Qianfan-OCR": 115.66948359261414, + "baidu/Unlimited-OCR": 6.362719833035499, + "deepseek-ai/DeepSeek-OCR": -32.42915539427986, + "deepseek-ai/DeepSeek-OCR-2": 76.87856107063135, + "lightonai/LightOnOCR-2-1B": 105.61437205859556, + "numind/NuExtract3": -36.46200902950136, + "rednote-hilab/dots.mocr": -32.4479320220255, + "rednote-hilab/dots.ocr": 1.4933490670259744, + "tesseract-5": -84.03676618412032, + "zai-org/GLM-OCR": -30.19733109211802 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 5, + "min_direct_comparisons": 5, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "targeted", + "max_absolute_elo_delta": 115.66948359261414, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 46.23768255756761, + "name": "targeted_v2_warmup_5", + "percentage_saved": 77.89123196448391, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 12 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 575, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 635, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 695, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 755, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 815, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 57, + "cumulative_comparisons": 872, + "decision": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "next_active_pairs": 13, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 13, + "allocation_mode": "targeted", + "batch_comparisons": 64, + "cumulative_comparisons": 936, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "baidu/Qianfan-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 996, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9604395604395607, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1777, + "comparisons_saved": 2728, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1568.722241102218, + "PaddlePaddle/PP-OCRv6_medium": 1312.1837194170303, + "PaddlePaddle/PaddleOCR-VL-1.6": 1473.2092356039213, + "allenai/olmOCR-2-7B-1025-FP8": 1468.553310120653, + "baidu/Qianfan-OCR": 1615.9765833705583, + "baidu/Unlimited-OCR": 1620.327206191052, + "deepseek-ai/DeepSeek-OCR": 1410.843527149632, + "deepseek-ai/DeepSeek-OCR-2": 1652.3727276354937, + "lightonai/LightOnOCR-2-1B": 1650.8084138434176, + "numind/NuExtract3": 1556.150855104141, + "rednote-hilab/dots.mocr": 1504.3510309194155, + "rednote-hilab/dots.ocr": 1522.6633875676125, + "tesseract-5": 1172.3480433624047, + "zai-org/GLM-OCR": 1471.4897186124501 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1527.7335939853303, + 1613.3884169430612 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1257.0497265105425, + 1358.819218421634 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1431.2453264483836, + 1516.8788684254284 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1427.240147451559, + 1505.4922939436944 + ], + "baidu/Qianfan-OCR": [ + 1574.710787342894, + 1660.6789771758004 + ], + "baidu/Unlimited-OCR": [ + 1577.0143493972596, + 1664.8008408936623 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1369.828105776937, + 1451.6621235906537 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1607.1097946393406, + 1701.2361577255288 + ], + "lightonai/LightOnOCR-2-1B": [ + 1608.4150410735385, + 1699.4019487180208 + ], + "numind/NuExtract3": [ + 1514.4393115969642, + 1598.0289367497141 + ], + "rednote-hilab/dots.mocr": [ + 1461.8978067858861, + 1547.3104260056218 + ], + "rednote-hilab/dots.ocr": [ + 1477.2855980780193, + 1563.8585476301653 + ], + "tesseract-5": [ + 1097.583763918539, + 1234.7453088326563 + ], + "zai-org/GLM-OCR": [ + 1432.8551325287046, + 1514.9447712958822 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 12.344484337257654, + "PaddlePaddle/PP-OCRv6_medium": -42.14957996907697, + "PaddlePaddle/PaddleOCR-VL-1.6": -30.482586615970604, + "allenai/olmOCR-2-7B-1025-FP8": -28.677346167640053, + "baidu/Qianfan-OCR": 39.31577389373251, + "baidu/Unlimited-OCR": 57.047163667199584, + "deepseek-ai/DeepSeek-OCR": -40.68112524991943, + "deepseek-ai/DeepSeek-OCR-2": 73.01683119859695, + "lightonai/LightOnOCR-2-1B": 33.28956318758992, + "numind/NuExtract3": -15.421333749279938, + "rednote-hilab/dots.mocr": -51.87063659170735, + "rednote-hilab/dots.ocr": -6.5893367842586485, + "tesseract-5": 21.294916199843556, + "zai-org/GLM-OCR": -20.43678735636695 + }, + "final_rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 15, + "min_direct_comparisons": 14, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "targeted", + "max_absolute_elo_delta": 73.01683119859695, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 31.886074901780262, + "name": "targeted_v2_warmup_10", + "percentage_saved": 60.554938956714764, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": "warm-up", + "min_direct_comparisons": 5, + "observed_pairs": 91, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 454, + "cumulative_comparisons": 909, + "decision": "warm-up", + "min_direct_comparisons": 9, + "observed_pairs": 91, + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 454, + "cumulative_comparisons": 1363, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1423, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1483, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1543, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1603, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1658, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 59, + "cumulative_comparisons": 1717, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1777, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9648351648351648, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": 10 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 2167, + "comparisons_saved": 2338, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1545.815410558876, + "PaddlePaddle/PP-OCRv6_medium": 1369.9519849650007, + "PaddlePaddle/PaddleOCR-VL-1.6": 1509.5780822093675, + "allenai/olmOCR-2-7B-1025-FP8": 1506.8824721365972, + "baidu/Qianfan-OCR": 1580.0381538536838, + "baidu/Unlimited-OCR": 1514.9961885430473, + "deepseek-ai/DeepSeek-OCR": 1468.233442075427, + "deepseek-ai/DeepSeek-OCR-2": 1603.075284692148, + "lightonai/LightOnOCR-2-1B": 1637.946154262932, + "numind/NuExtract3": 1562.9140412754496, + "rednote-hilab/dots.mocr": 1571.765282146598, + "rednote-hilab/dots.ocr": 1532.8370744381737, + "tesseract-5": 1080.2926058353735, + "zai-org/GLM-OCR": 1515.6738230073256 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1511.9448430815664, + 1584.1557220075406 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1325.7871813710644, + 1411.3354950247133 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1472.816474431783, + 1545.3142077630428 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1472.6716584216845, + 1539.6197279554792 + ], + "baidu/Qianfan-OCR": [ + 1542.3001232201514, + 1620.1614832199687 + ], + "baidu/Unlimited-OCR": [ + 1475.2328185451383, + 1553.187184513493 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1431.446682037785, + 1507.1819371483978 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1566.483341296381, + 1640.606753515597 + ], + "lightonai/LightOnOCR-2-1B": [ + 1597.821901214702, + 1679.0449290221313 + ], + "numind/NuExtract3": [ + 1527.9223673747854, + 1599.609399314255 + ], + "rednote-hilab/dots.mocr": [ + 1534.5121192937215, + 1608.7639449299725 + ], + "rednote-hilab/dots.ocr": [ + 1492.5030813808846, + 1568.8912805462521 + ], + "tesseract-5": [ + 992.3782830144316, + 1155.2862267320927 + ], + "zai-org/GLM-OCR": [ + 1481.368571324148, + 1551.259352374265 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -10.562346206084385, + "PaddlePaddle/PP-OCRv6_medium": 15.618685578893519, + "PaddlePaddle/PaddleOCR-VL-1.6": 5.886259989475548, + "allenai/olmOCR-2-7B-1025-FP8": 9.651815848304295, + "baidu/Qianfan-OCR": 3.377344376857991, + "baidu/Unlimited-OCR": -48.283853980805134, + "deepseek-ai/DeepSeek-OCR": 16.70878967587555, + "deepseek-ai/DeepSeek-OCR-2": 23.71938825525126, + "lightonai/LightOnOCR-2-1B": 20.42730360710425, + "numind/NuExtract3": -8.658147577971476, + "rednote-hilab/dots.mocr": 15.543614635475251, + "rednote-hilab/dots.ocr": 3.584350086302493, + "tesseract-5": -70.76052132718769, + "zai-org/GLM-OCR": 23.747317038508527 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 20, + "min_direct_comparisons": 20, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "targeted", + "max_absolute_elo_delta": 70.76052132718769, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 15.581150107184385, + "name": "targeted_v2_explore_every_3", + "percentage_saved": 51.897891231964486, + "practical_preferences": [], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 575, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1030, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1090, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1150, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1605, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 53, + "cumulative_comparisons": 1658, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 1712, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 2167, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9164835164835166, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 2167, + "comparisons_saved": 2338, + "decision_statuses": { + "prefer-smaller": 4, + "resolved": 2, + "unresolved": 7 + }, + "deterministic_across_repeats": true, + "elo": { + "FireRedTeam/FireRed-OCR": 1545.815410558876, + "PaddlePaddle/PP-OCRv6_medium": 1369.9519849650007, + "PaddlePaddle/PaddleOCR-VL-1.6": 1509.5780822093675, + "allenai/olmOCR-2-7B-1025-FP8": 1506.8824721365972, + "baidu/Qianfan-OCR": 1580.0381538536838, + "baidu/Unlimited-OCR": 1514.9961885430473, + "deepseek-ai/DeepSeek-OCR": 1468.233442075427, + "deepseek-ai/DeepSeek-OCR-2": 1603.075284692148, + "lightonai/LightOnOCR-2-1B": 1637.946154262932, + "numind/NuExtract3": 1562.9140412754496, + "rednote-hilab/dots.mocr": 1571.765282146598, + "rednote-hilab/dots.ocr": 1532.8370744381737, + "tesseract-5": 1080.2926058353735, + "zai-org/GLM-OCR": 1515.6738230073256 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1511.9448430815664, + 1584.1557220075406 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1325.7871813710644, + 1411.3354950247133 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1472.816474431783, + 1545.3142077630428 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1472.6716584216845, + 1539.6197279554792 + ], + "baidu/Qianfan-OCR": [ + 1542.3001232201514, + 1620.1614832199687 + ], + "baidu/Unlimited-OCR": [ + 1475.2328185451383, + 1553.187184513493 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1431.446682037785, + 1507.1819371483978 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1566.483341296381, + 1640.606753515597 + ], + "lightonai/LightOnOCR-2-1B": [ + 1597.821901214702, + 1679.0449290221313 + ], + "numind/NuExtract3": [ + 1527.9223673747854, + 1599.609399314255 + ], + "rednote-hilab/dots.mocr": [ + 1534.5121192937215, + 1608.7639449299725 + ], + "rednote-hilab/dots.ocr": [ + 1492.5030813808846, + 1568.8912805462521 + ], + "tesseract-5": [ + 992.3782830144316, + 1155.2862267320927 + ], + "zai-org/GLM-OCR": [ + 1481.368571324148, + 1551.259352374265 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -10.562346206084385, + "PaddlePaddle/PP-OCRv6_medium": 15.618685578893519, + "PaddlePaddle/PaddleOCR-VL-1.6": 5.886259989475548, + "allenai/olmOCR-2-7B-1025-FP8": 9.651815848304295, + "baidu/Qianfan-OCR": 3.377344376857991, + "baidu/Unlimited-OCR": -48.283853980805134, + "deepseek-ai/DeepSeek-OCR": 16.70878967587555, + "deepseek-ai/DeepSeek-OCR-2": 23.71938825525126, + "lightonai/LightOnOCR-2-1B": 20.42730360710425, + "numind/NuExtract3": -8.658147577971476, + "rednote-hilab/dots.mocr": 15.543614635475251, + "rednote-hilab/dots.ocr": 3.584350086302493, + "tesseract-5": -70.76052132718769, + "zai-org/GLM-OCR": 23.747317038508527 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 50, + "median_direct_comparisons": 20, + "min_direct_comparisons": 20, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "targeted", + "max_absolute_elo_delta": 70.76052132718769, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 15.581150107184385, + "name": "targeted_v2_warmup_5_explore_3_annotate_3x", + "percentage_saved": 51.897891231964486, + "practical_preferences": [ + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 3.4, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "zai-org/GLM-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 575, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1030, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1090, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1150, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1605, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 53, + "cumulative_comparisons": 1658, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 54, + "cumulative_comparisons": 1712, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 2167, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": false, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9164835164835166, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 2549, + "comparisons_saved": 1956, + "decision_statuses": { + "prefer-smaller": 4, + "resolved": 2, + "unresolved": 7 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1537.996225060517, + "PaddlePaddle/PP-OCRv6_medium": 1309.0487571037, + "PaddlePaddle/PaddleOCR-VL-1.6": 1498.377656828886, + "allenai/olmOCR-2-7B-1025-FP8": 1493.5465299268135, + "baidu/Qianfan-OCR": 1584.2588280224645, + "baidu/Unlimited-OCR": 1621.1862678017374, + "deepseek-ai/DeepSeek-OCR": 1457.2257557515416, + "deepseek-ai/DeepSeek-OCR-2": 1609.6430928052614, + "lightonai/LightOnOCR-2-1B": 1620.723096200397, + "numind/NuExtract3": 1562.9043531474144, + "rednote-hilab/dots.mocr": 1528.5864254602425, + "rednote-hilab/dots.ocr": 1532.5608224809787, + "tesseract-5": 1140.262547488169, + "zai-org/GLM-OCR": 1503.6796419218767 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1505.9345399514773, + 1572.314283128327 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1265.2198278822839, + 1350.396596003761 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1463.9619455106167, + 1532.7341852015131 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1461.141289818368, + 1527.9587490716026 + ], + "baidu/Qianfan-OCR": [ + 1551.110735858532, + 1620.2069729261889 + ], + "baidu/Unlimited-OCR": [ + 1589.818007194278, + 1659.034922876804 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1424.9975087117123, + 1490.5737076414234 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1575.02359125055, + 1647.4327891058028 + ], + "lightonai/LightOnOCR-2-1B": [ + 1585.2900371369328, + 1658.7834738907095 + ], + "numind/NuExtract3": [ + 1529.149159470197, + 1598.5754378766378 + ], + "rednote-hilab/dots.mocr": [ + 1496.1098937448098, + 1562.1573955273066 + ], + "rednote-hilab/dots.ocr": [ + 1495.88982030829, + 1567.539118614849 + ], + "tesseract-5": [ + 1067.4408704824268, + 1190.2966451429386 + ], + "zai-org/GLM-OCR": [ + 1468.1174314809496, + 1538.241308406339 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -18.381531704443432, + "PaddlePaddle/PP-OCRv6_medium": -45.2845422824073, + "PaddlePaddle/PaddleOCR-VL-1.6": -5.314165391005872, + "allenai/olmOCR-2-7B-1025-FP8": -3.684126361479457, + "baidu/Qianfan-OCR": 7.598018545638752, + "baidu/Unlimited-OCR": 57.906225277884914, + "deepseek-ai/DeepSeek-OCR": 5.701103351990014, + "deepseek-ai/DeepSeek-OCR-2": 30.287196368364675, + "lightonai/LightOnOCR-2-1B": 3.204245544569403, + "numind/NuExtract3": -8.6678357060066, + "rednote-hilab/dots.mocr": -27.63524205088038, + "rednote-hilab/dots.ocr": 3.3080981291075204, + "tesseract-5": -10.79057967439212, + "zai-org/GLM-OCR": 11.753135953059655 + }, + "final_rank_order": [ + "baidu/Unlimited-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 45, + "median_direct_comparisons": 25, + "min_direct_comparisons": 24, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "targeted", + "max_absolute_elo_delta": 57.906225277884914, + "max_absolute_elo_delta_model": "baidu/Unlimited-OCR", + "median_absolute_elo_delta": 9.72920769019936, + "name": "targeted_v2_warmup_10_explore_3_annotate_3x", + "percentage_saved": 43.41842397336293, + "practical_preferences": [ + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "allenai/olmOCR-2-7B-1025-FP8", + "size_ratio": 9.222222222222221, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "baidu/Unlimited-OCR", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.3, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 3.4, + "smaller_model": "lightonai/LightOnOCR-2-1B" + }, + { + "higher_ranked_model": "rednote-hilab/dots.mocr", + "larger_model": "rednote-hilab/dots.mocr", + "size_ratio": 3.3333333333333335, + "smaller_model": "zai-org/GLM-OCR" + } + ], + "resolution_summary": "sample batches exhausted: 11 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": "warm-up", + "min_direct_comparisons": 5, + "observed_pairs": 91, + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 454, + "cumulative_comparisons": 909, + "decision": "warm-up", + "min_direct_comparisons": 9, + "observed_pairs": 91, + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 454, + "cumulative_comparisons": 1363, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1423, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 1483, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1938, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 1993, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 52, + "cumulative_comparisons": 2045, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 449, + "cumulative_comparisons": 2494, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "baidu/Unlimited-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 11, + "allocation_mode": "targeted", + "batch_comparisons": 55, + "cumulative_comparisons": 2549, + "decision": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "next_active_pairs": 11, + "rank_order": [ + "baidu/Unlimited-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": false, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.9384615384615385, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "baidu/Unlimited-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": 10 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 2, + "resolved": 2, + "unresolved": 9 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": 3, + "comparisons_consumed": 2058, + "comparisons_saved": 2447, + "decision_statuses": { + "prefer-smaller": 2, + "resolved": 2, + "unresolved": 9 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1545.4047185750896, + "PaddlePaddle/PP-OCRv6_medium": 1375.8585174192635, + "PaddlePaddle/PaddleOCR-VL-1.6": 1508.0305981942972, + "allenai/olmOCR-2-7B-1025-FP8": 1505.1502583318468, + "baidu/Qianfan-OCR": 1577.014532947124, + "baidu/Unlimited-OCR": 1506.9474328102363, + "deepseek-ai/DeepSeek-OCR": 1462.9418704656837, + "deepseek-ai/DeepSeek-OCR-2": 1592.8473406766884, + "lightonai/LightOnOCR-2-1B": 1644.6710642007395, + "numind/NuExtract3": 1568.4357083685052, + "rednote-hilab/dots.mocr": 1576.3012975274455, + "rednote-hilab/dots.ocr": 1532.7277199959258, + "tesseract-5": 1080.454570248567, + "zai-org/GLM-OCR": 1523.2143702385874 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1507.7735041575188, + 1582.477761340905 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1332.6511267654007, + 1420.1390143456106 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1468.7754180249908, + 1550.8182110166972 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1466.2809498832205, + 1539.7504058356576 + ], + "baidu/Qianfan-OCR": [ + 1540.0667154242792, + 1617.441892107507 + ], + "baidu/Unlimited-OCR": [ + 1469.3184873862158, + 1544.1558420349375 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1424.8713125202332, + 1501.3498842396145 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1557.3219558091707, + 1631.7264991869856 + ], + "lightonai/LightOnOCR-2-1B": [ + 1604.6247330046872, + 1688.985717063443 + ], + "numind/NuExtract3": [ + 1532.894908965717, + 1606.0009712455528 + ], + "rednote-hilab/dots.mocr": [ + 1538.6138050180257, + 1613.561779198647 + ], + "rednote-hilab/dots.ocr": [ + 1497.8486543029433, + 1572.7421635228964 + ], + "tesseract-5": [ + 980.3743478146074, + 1149.1162549662547 + ], + "zai-org/GLM-OCR": [ + 1485.6461044310304, + 1564.2411911745505 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -10.97303818987075, + "PaddlePaddle/PP-OCRv6_medium": 21.525218033156307, + "PaddlePaddle/PaddleOCR-VL-1.6": 4.33877597440528, + "allenai/olmOCR-2-7B-1025-FP8": 7.919602043553823, + "baidu/Qianfan-OCR": 0.35372347029829143, + "baidu/Unlimited-OCR": -56.33260971361619, + "deepseek-ai/DeepSeek-OCR": 11.417218066132136, + "deepseek-ai/DeepSeek-OCR-2": 13.49144423979169, + "lightonai/LightOnOCR-2-1B": 27.152213544911774, + "numind/NuExtract3": -3.1364804849158645, + "rednote-hilab/dots.mocr": 20.079630016322653, + "rednote-hilab/dots.ocr": 3.4749956440546157, + "tesseract-5": -70.59855691399412, + "zai-org/GLM-OCR": 31.287864269770353 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": true + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 45, + "median_direct_comparisons": 20, + "min_direct_comparisons": 20, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "kind": "targeted", + "max_absolute_elo_delta": 70.59855691399412, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 12.454331152961913, + "name": "targeted_v2_warmup_5_explore_3_size_stop_3x", + "percentage_saved": 54.31742508324084, + "practical_preferences": [ + { + "higher_ranked_model": "PaddlePaddle/PaddleOCR-VL-1.6", + "larger_model": "baidu/Unlimited-OCR", + "size_ratio": 3.6666666666666665, + "smaller_model": "PaddlePaddle/PaddleOCR-VL-1.6" + }, + { + "higher_ranked_model": "lightonai/LightOnOCR-2-1B", + "larger_model": "deepseek-ai/DeepSeek-OCR-2", + "size_ratio": 3.4, + "smaller_model": "lightonai/LightOnOCR-2-1B" + } + ], + "resolution_summary": "sample batches exhausted: 9 allocation pairs unresolved", + "round_history": [ + { + "active_pairs": null, + "allocation_mode": "balanced-warmup", + "batch_comparisons": 455, + "cumulative_comparisons": 455, + "decision": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "next_active_pairs": 12, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 1, + "sample_indices": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "active_pairs": 12, + "allocation_mode": "targeted", + "batch_comparisons": 60, + "cumulative_comparisons": 515, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 2, + "sample_indices": [ + 5, + 6, + 7, + 8, + 9 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 555, + "decision": { + "prefer-smaller": 5, + "resolved": 1, + "unresolved": 7 + }, + "next_active_pairs": 7, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 3, + "sample_indices": [ + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "active_pairs": 7, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1010, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 4, + "sample_indices": [ + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 1050, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 5, + "sample_indices": [ + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "targeted", + "batch_comparisons": 40, + "cumulative_comparisons": 1090, + "decision": { + "prefer-smaller": 4, + "resolved": 1, + "unresolved": 8 + }, + "next_active_pairs": 8, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 6, + "sample_indices": [ + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "active_pairs": 8, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 1545, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 7, + "sample_indices": [ + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 29, + "cumulative_comparisons": 1574, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 8, + "sample_indices": [ + 35, + 36, + 37, + 38, + 39 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "targeted", + "batch_comparisons": 29, + "cumulative_comparisons": 1603, + "decision": { + "prefer-smaller": 5, + "resolved": 2, + "unresolved": 6 + }, + "next_active_pairs": 6, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 9, + "sample_indices": [ + 40, + 41, + 42, + 43, + 44 + ] + }, + { + "active_pairs": 6, + "allocation_mode": "balanced-exploration", + "batch_comparisons": 455, + "cumulative_comparisons": 2058, + "decision": { + "prefer-smaller": 2, + "resolved": 2, + "unresolved": 9 + }, + "next_active_pairs": 9, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "round": 10, + "sample_indices": [ + 45, + 46, + 47, + 48, + 49 + ] + } + ], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": 3.0, + "spearman_rho": 0.8989010989010989, + "static_budget": null, + "stopping_reason": "sample_batches_exhausted", + "stopping_round": 10, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": true, + "warmup_min_per_pair": 5 + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1573.982835294599, + "PaddlePaddle/PP-OCRv6_medium": 1357.5509310796788, + "PaddlePaddle/PaddleOCR-VL-1.6": 1530.0064351725828, + "allenai/olmOCR-2-7B-1025-FP8": 1471.1968180060758, + "baidu/Qianfan-OCR": 1603.0802291962627, + "baidu/Unlimited-OCR": 1514.0060940937908, + "deepseek-ai/DeepSeek-OCR": 1442.3316646750677, + "deepseek-ai/DeepSeek-OCR-2": 1590.3119212899653, + "lightonai/LightOnOCR-2-1B": 1619.6290885657727, + "numind/NuExtract3": 1631.6114663312278, + "rednote-hilab/dots.mocr": 1549.4149805020652, + "rednote-hilab/dots.ocr": 1534.1398922397602, + "tesseract-5": 1065.556262003309, + "zai-org/GLM-OCR": 1517.181381549842 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1508.181035319062, + 1656.6368848853992 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1283.3714621986674, + 1427.7665609177538 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1459.1631283496201, + 1600.282362862368 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1409.6228460528316, + 1531.7905100202024 + ], + "baidu/Qianfan-OCR": [ + 1533.224434089268, + 1676.824647250343 + ], + "baidu/Unlimited-OCR": [ + 1444.1671401322635, + 1589.1635284186564 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1365.0535569874946, + 1512.6682372766575 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1522.0111206161434, + 1661.9905254509308 + ], + "lightonai/LightOnOCR-2-1B": [ + 1556.7689428369677, + 1699.444375599162 + ], + "numind/NuExtract3": [ + 1570.210299955375, + 1711.1496911794577 + ], + "rednote-hilab/dots.mocr": [ + 1484.1091945104884, + 1622.3212025302223 + ], + "rednote-hilab/dots.ocr": [ + 1465.1142801023707, + 1600.454931424971 + ], + "tesseract-5": [ + 881.2464412704488, + 1169.3323866895435 + ], + "zai-org/GLM-OCR": [ + 1447.043564067578, + 1587.225126742539 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 17.605078529638604, + "PaddlePaddle/PP-OCRv6_medium": 3.2176316935715477, + "PaddlePaddle/PaddleOCR-VL-1.6": 26.314612952690823, + "allenai/olmOCR-2-7B-1025-FP8": -26.033838282217175, + "baidu/Qianfan-OCR": 26.419419719436974, + "baidu/Unlimited-OCR": -49.27394843006164, + "deepseek-ai/DeepSeek-OCR": -9.192987724483828, + "deepseek-ai/DeepSeek-OCR-2": 10.956024853068584, + "lightonai/LightOnOCR-2-1B": 2.110237909944999, + "numind/NuExtract3": 60.03927747780676, + "rednote-hilab/dots.mocr": -6.8066870090576685, + "rednote-hilab/dots.ocr": 4.887167887889063, + "tesseract-5": -85.49686515925214, + "zai-org/GLM-OCR": 25.254875581024862 + }, + "final_rank_order": [ + "numind/NuExtract3", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "baidu/Unlimited-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "pair-balanced", + "max_absolute_elo_delta": 85.49686515925214, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 21.429977055331733, + "name": "pair-balanced_700_seed_42", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.8945054945054945, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "numind/NuExtract3", + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1560.1992800222502, + "PaddlePaddle/PP-OCRv6_medium": 1353.5944584982378, + "PaddlePaddle/PaddleOCR-VL-1.6": 1523.8670557313837, + "allenai/olmOCR-2-7B-1025-FP8": 1516.913085608723, + "baidu/Qianfan-OCR": 1675.310893142534, + "baidu/Unlimited-OCR": 1524.4919418693005, + "deepseek-ai/DeepSeek-OCR": 1425.5194439479358, + "deepseek-ai/DeepSeek-OCR-2": 1560.06824806695, + "lightonai/LightOnOCR-2-1B": 1554.7946811684756, + "numind/NuExtract3": 1592.206244601023, + "rednote-hilab/dots.mocr": 1539.1372691464655, + "rednote-hilab/dots.ocr": 1533.9480682358856, + "tesseract-5": 1161.6522544732761, + "zai-org/GLM-OCR": 1478.2970754875594 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1497.282012352779, + 1628.753086743919 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1288.6164260292405, + 1416.9136265112788 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1456.6052247121409, + 1594.2348574359842 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1448.059388380249, + 1588.245423972934 + ], + "baidu/Qianfan-OCR": [ + 1608.3331717943638, + 1763.6580706188586 + ], + "baidu/Unlimited-OCR": [ + 1452.3269220808602, + 1592.8891337033185 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1352.9405615740993, + 1489.92274044211 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1494.9506083409556, + 1630.9015386236536 + ], + "lightonai/LightOnOCR-2-1B": [ + 1491.8177219566883, + 1626.0693247550894 + ], + "numind/NuExtract3": [ + 1537.186928530326, + 1658.8514518602062 + ], + "rednote-hilab/dots.mocr": [ + 1469.823277150718, + 1606.2600855946343 + ], + "rednote-hilab/dots.ocr": [ + 1466.3783553437029, + 1597.7088636407614 + ], + "tesseract-5": [ + 1031.7384339115658, + 1243.4990147571548 + ], + "zai-org/GLM-OCR": [ + 1410.364782289306, + 1544.6242663602898 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 3.821523257289755, + "PaddlePaddle/PP-OCRv6_medium": -0.7388408878694008, + "PaddlePaddle/PaddleOCR-VL-1.6": 20.175233511491797, + "allenai/olmOCR-2-7B-1025-FP8": 19.682429320429947, + "baidu/Qianfan-OCR": 98.65008366570828, + "baidu/Unlimited-OCR": -38.788100654551954, + "deepseek-ai/DeepSeek-OCR": -26.005208451615772, + "deepseek-ai/DeepSeek-OCR-2": -19.287648369946737, + "lightonai/LightOnOCR-2-1B": -62.72416948735213, + "numind/NuExtract3": 20.634055747601906, + "rednote-hilab/dots.mocr": -17.084398364657318, + "rednote-hilab/dots.ocr": 4.695343884014392, + "tesseract-5": 10.599127310714948, + "zai-org/GLM-OCR": -13.629430481257714 + }, + "final_rank_order": [ + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "pair-balanced", + "max_absolute_elo_delta": 98.65008366570828, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 19.485038845188342, + "name": "pair-balanced_700_seed_43", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.8945054945054945, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 1, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1531.4711595033516, + "PaddlePaddle/PP-OCRv6_medium": 1346.0471769056799, + "PaddlePaddle/PaddleOCR-VL-1.6": 1514.5208697159937, + "allenai/olmOCR-2-7B-1025-FP8": 1482.4173659177045, + "baidu/Qianfan-OCR": 1567.9057103694615, + "baidu/Unlimited-OCR": 1594.2855683934006, + "deepseek-ai/DeepSeek-OCR": 1428.2005655530627, + "deepseek-ai/DeepSeek-OCR-2": 1568.8286703433039, + "lightonai/LightOnOCR-2-1B": 1662.449374961442, + "numind/NuExtract3": 1570.1613186589232, + "rednote-hilab/dots.mocr": 1582.490125957941, + "rednote-hilab/dots.ocr": 1493.0538204626152, + "tesseract-5": 1131.173795327273, + "zai-org/GLM-OCR": 1526.994477929847 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1470.4807365602117, + 1595.1892236324186 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1249.813827299111, + 1419.0384026114057 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1444.258905051165, + 1583.2787199263776 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1412.625143292349, + 1548.707155634785 + ], + "baidu/Qianfan-OCR": [ + 1508.1915011676183, + 1639.8039019915102 + ], + "baidu/Unlimited-OCR": [ + 1523.8310358002802, + 1670.1703442368082 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1352.3254006715965, + 1497.3785141024055 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1506.5848732957802, + 1637.962555521298 + ], + "lightonai/LightOnOCR-2-1B": [ + 1596.9700058485298, + 1752.0287164040506 + ], + "numind/NuExtract3": [ + 1509.5091599715968, + 1640.1180498004937 + ], + "rednote-hilab/dots.mocr": [ + 1519.0897297383447, + 1649.968947259104 + ], + "rednote-hilab/dots.ocr": [ + 1420.8493420402303, + 1562.8770799251038 + ], + "tesseract-5": [ + 998.3763761752045, + 1214.8741594787828 + ], + "zai-org/GLM-OCR": [ + 1463.3802468594406, + 1597.1420420472386 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -24.906597261608795, + "PaddlePaddle/PP-OCRv6_medium": -8.286122480427366, + "PaddlePaddle/PaddleOCR-VL-1.6": 10.829047496101794, + "allenai/olmOCR-2-7B-1025-FP8": -14.813290370588447, + "baidu/Qianfan-OCR": -8.75509910736423, + "baidu/Unlimited-OCR": 31.005525869548137, + "deepseek-ai/DeepSeek-OCR": -23.324086846488854, + "deepseek-ai/DeepSeek-OCR-2": -10.527226093592844, + "lightonai/LightOnOCR-2-1B": 44.9305243056142, + "numind/NuExtract3": -1.4108701944978748, + "rednote-hilab/dots.mocr": 26.26845844681816, + "rednote-hilab/dots.ocr": -36.19890388925592, + "tesseract-5": -19.879331835288212, + "zai-org/GLM-OCR": 35.067971961030025 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7142857142857144, + "kind": "pair-balanced", + "max_absolute_elo_delta": 44.9305243056142, + "max_absolute_elo_delta_model": "lightonai/LightOnOCR-2-1B", + "median_absolute_elo_delta": 21.601709340888533, + "name": "pair-balanced_700_seed_44", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.8725274725274725, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 1, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1594.526849157187, + "PaddlePaddle/PP-OCRv6_medium": 1289.6391390893948, + "PaddlePaddle/PaddleOCR-VL-1.6": 1464.4648691206246, + "allenai/olmOCR-2-7B-1025-FP8": 1519.6215858879607, + "baidu/Qianfan-OCR": 1581.6932596425293, + "baidu/Unlimited-OCR": 1597.514161595467, + "deepseek-ai/DeepSeek-OCR": 1446.7255673626232, + "deepseek-ai/DeepSeek-OCR-2": 1565.8283467213232, + "lightonai/LightOnOCR-2-1B": 1622.219189567636, + "numind/NuExtract3": 1585.0799509046547, + "rednote-hilab/dots.mocr": 1570.6534844748194, + "rednote-hilab/dots.ocr": 1527.5530799301696, + "tesseract-5": 1151.3630861841089, + "zai-org/GLM-OCR": 1483.1174303615019 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1535.416805621037, + 1666.6933235851461 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1200.8119327127567, + 1360.8694496645894 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1396.589579511224, + 1533.2322381746535 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1451.8856970722704, + 1588.2398243489854 + ], + "baidu/Qianfan-OCR": [ + 1507.5609638958038, + 1664.516946788557 + ], + "baidu/Unlimited-OCR": [ + 1532.861465044839, + 1677.0076630779783 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1373.6223835379913, + 1514.6875914843633 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1505.3866668131782, + 1637.2400154547931 + ], + "lightonai/LightOnOCR-2-1B": [ + 1554.913389052774, + 1695.3414131317918 + ], + "numind/NuExtract3": [ + 1520.8368871237612, + 1657.5623404502594 + ], + "rednote-hilab/dots.mocr": [ + 1505.4707881456259, + 1639.872328508352 + ], + "rednote-hilab/dots.ocr": [ + 1465.658875511516, + 1600.9908557853953 + ], + "tesseract-5": [ + 1000.5890908381366, + 1241.0734562677376 + ], + "zai-org/GLM-OCR": [ + 1417.0390242519095, + 1553.1557617451422 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 38.14909239222652, + "PaddlePaddle/PP-OCRv6_medium": -64.69416029671243, + "PaddlePaddle/PaddleOCR-VL-1.6": -39.22695309926735, + "allenai/olmOCR-2-7B-1025-FP8": 22.390929599667743, + "baidu/Qianfan-OCR": 5.032450165703494, + "baidu/Unlimited-OCR": 34.23411907161449, + "deepseek-ai/DeepSeek-OCR": -4.799085036928318, + "deepseek-ai/DeepSeek-OCR-2": -13.527549715573514, + "lightonai/LightOnOCR-2-1B": 4.7003389118083305, + "numind/NuExtract3": 13.507762051233613, + "rednote-hilab/dots.mocr": 14.43181696369652, + "rednote-hilab/dots.ocr": -1.6996444217015778, + "tesseract-5": 0.30995902154768373, + "zai-org/GLM-OCR": -8.809075607315208 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "deepseek-ai/DeepSeek-OCR-2", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7362637362637363, + "kind": "pair-balanced", + "max_absolute_elo_delta": 64.69416029671243, + "max_absolute_elo_delta_model": "PaddlePaddle/PP-OCRv6_medium", + "median_absolute_elo_delta": 13.517655883403563, + "name": "pair-balanced_700_seed_45", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.8813186813186813, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 1, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1567.088202442937, + "PaddlePaddle/PP-OCRv6_medium": 1296.0058747802757, + "PaddlePaddle/PaddleOCR-VL-1.6": 1533.3558035508788, + "allenai/olmOCR-2-7B-1025-FP8": 1450.5528339960401, + "baidu/Qianfan-OCR": 1575.8852929524037, + "baidu/Unlimited-OCR": 1544.297359008243, + "deepseek-ai/DeepSeek-OCR": 1434.0849405009258, + "deepseek-ai/DeepSeek-OCR-2": 1593.3427003277786, + "lightonai/LightOnOCR-2-1B": 1642.7835315574166, + "numind/NuExtract3": 1562.1427762195021, + "rednote-hilab/dots.mocr": 1622.5768356844173, + "rednote-hilab/dots.ocr": 1536.794825260003, + "tesseract-5": 1185.0923468797534, + "zai-org/GLM-OCR": 1455.9966768394245 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1501.7894731975316, + 1639.124957201701 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1209.1018882743745, + 1365.4243556264053 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1460.1038531181364, + 1606.1110580442305 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1387.25769817205, + 1511.7485672679095 + ], + "baidu/Qianfan-OCR": [ + 1506.7699447114317, + 1646.1846007817053 + ], + "baidu/Unlimited-OCR": [ + 1477.5053880509652, + 1614.4520853303754 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1368.5798258070306, + 1497.106855914394 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1525.4927203080135, + 1660.7407171216541 + ], + "lightonai/LightOnOCR-2-1B": [ + 1582.590915828316, + 1720.058917338517 + ], + "numind/NuExtract3": [ + 1502.3410792070192, + 1626.3897683593848 + ], + "rednote-hilab/dots.mocr": [ + 1558.5737600419463, + 1698.6054622072336 + ], + "rednote-hilab/dots.ocr": [ + 1470.9257777794367, + 1611.4629117906693 + ], + "tesseract-5": [ + 1061.1574251938403, + 1273.2578036546806 + ], + "zai-org/GLM-OCR": [ + 1386.914156679012, + 1520.5013404693832 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 10.710445677976622, + "PaddlePaddle/PP-OCRv6_medium": -58.32742460583154, + "PaddlePaddle/PaddleOCR-VL-1.6": 29.66398133098687, + "allenai/olmOCR-2-7B-1025-FP8": -46.67782229225281, + "baidu/Qianfan-OCR": -0.7755165244220734, + "baidu/Unlimited-OCR": -18.98268351560955, + "deepseek-ai/DeepSeek-OCR": -17.439711898625774, + "deepseek-ai/DeepSeek-OCR-2": 13.986803890881902, + "lightonai/LightOnOCR-2-1B": 25.264680901588918, + "numind/NuExtract3": -9.4294126339189, + "rednote-hilab/dots.mocr": 66.35516817329449, + "rednote-hilab/dots.ocr": 7.542100908131943, + "tesseract-5": 34.03921971719228, + "zai-org/GLM-OCR": -35.9298291293926 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "pair-balanced", + "max_absolute_elo_delta": 66.35516817329449, + "max_absolute_elo_delta_model": "rednote-hilab/dots.mocr", + "median_absolute_elo_delta": 22.123682208599234, + "name": "pair-balanced_700_seed_46", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9164835164835166, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1545.878132560636, + "PaddlePaddle/PP-OCRv6_medium": 1384.8643383634874, + "PaddlePaddle/PaddleOCR-VL-1.6": 1513.3367979875577, + "allenai/olmOCR-2-7B-1025-FP8": 1499.785427616002, + "baidu/Qianfan-OCR": 1572.5778571879964, + "baidu/Unlimited-OCR": 1553.7590619231871, + "deepseek-ai/DeepSeek-OCR": 1433.768989444524, + "deepseek-ai/DeepSeek-OCR-2": 1571.356224103035, + "lightonai/LightOnOCR-2-1B": 1636.0515571501478, + "numind/NuExtract3": 1589.737311612596, + "rednote-hilab/dots.mocr": 1580.0386751370065, + "rednote-hilab/dots.ocr": 1526.4550480020846, + "tesseract-5": 1086.0009072748323, + "zai-org/GLM-OCR": 1506.3896716369075 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1495.5968539612659, + 1597.0763698968533 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1327.4591090785912, + 1437.5104565909044 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1463.0109272852262, + 1564.7638284487891 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1452.6420141925187, + 1551.1575404596558 + ], + "baidu/Qianfan-OCR": [ + 1519.8600058555155, + 1626.3188606194083 + ], + "baidu/Unlimited-OCR": [ + 1500.3151838954657, + 1609.4895408146908 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1382.0010177345405, + 1484.3860651098607 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1519.9541126643528, + 1624.050676128681 + ], + "lightonai/LightOnOCR-2-1B": [ + 1588.6396036765505, + 1692.638118626678 + ], + "numind/NuExtract3": [ + 1541.3946258197368, + 1643.1123736278996 + ], + "rednote-hilab/dots.mocr": [ + 1532.0836220171145, + 1630.8902136987597 + ], + "rednote-hilab/dots.ocr": [ + 1475.885159792247, + 1576.5525998103162 + ], + "tesseract-5": [ + 974.3227870233962, + 1165.1804971572774 + ], + "zai-org/GLM-OCR": [ + 1456.769239920302, + 1557.3711915423678 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -10.499624204324391, + "PaddlePaddle/PP-OCRv6_medium": 30.53103897738015, + "PaddlePaddle/PaddleOCR-VL-1.6": 9.644975767665755, + "allenai/olmOCR-2-7B-1025-FP8": 2.554771327708977, + "baidu/Qianfan-OCR": -4.082952288829347, + "baidu/Unlimited-OCR": -9.520980600665325, + "deepseek-ai/DeepSeek-OCR": -17.755662955027447, + "deepseek-ai/DeepSeek-OCR-2": -7.999672333861781, + "lightonai/LightOnOCR-2-1B": 18.53270649432011, + "numind/NuExtract3": 18.165122759174892, + "rednote-hilab/dots.mocr": 23.81700762588366, + "rednote-hilab/dots.ocr": -2.7976763497865704, + "tesseract-5": -65.05221988772882, + "zai-org/GLM-OCR": 14.463165668090369 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "pair-balanced", + "max_absolute_elo_delta": 65.05221988772882, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 12.48139493620738, + "name": "pair-balanced_1200_seed_42", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9252747252747253, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 1, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1539.607306260055, + "PaddlePaddle/PP-OCRv6_medium": 1358.160393540098, + "PaddlePaddle/PaddleOCR-VL-1.6": 1499.9706394807906, + "allenai/olmOCR-2-7B-1025-FP8": 1496.8564927393788, + "baidu/Qianfan-OCR": 1619.2404273697982, + "baidu/Unlimited-OCR": 1546.3262451315748, + "deepseek-ai/DeepSeek-OCR": 1468.828014924821, + "deepseek-ai/DeepSeek-OCR-2": 1577.9004124467074, + "lightonai/LightOnOCR-2-1B": 1590.0972410258867, + "numind/NuExtract3": 1565.2416797691344, + "rednote-hilab/dots.mocr": 1541.8030054564763, + "rednote-hilab/dots.ocr": 1552.497522616654, + "tesseract-5": 1134.1104140835798, + "zai-org/GLM-OCR": 1509.360205155045 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1491.1335438568722, + 1588.9773806818366 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1301.701986605773, + 1405.874784803688 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1451.3677767628596, + 1548.253368261489 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1449.0455196406806, + 1548.0223198914864 + ], + "baidu/Qianfan-OCR": [ + 1567.5802088542714, + 1673.8971159221749 + ], + "baidu/Unlimited-OCR": [ + 1498.1493362580472, + 1600.8180502219252 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1417.876621292324, + 1519.1521326089392 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1530.7045744778918, + 1632.135440465218 + ], + "lightonai/LightOnOCR-2-1B": [ + 1540.7467292820654, + 1642.077211460929 + ], + "numind/NuExtract3": [ + 1516.1574487737073, + 1616.72947622346 + ], + "rednote-hilab/dots.mocr": [ + 1490.349218309999, + 1591.471401472334 + ], + "rednote-hilab/dots.ocr": [ + 1507.6785990107485, + 1603.8192441762749 + ], + "tesseract-5": [ + 1028.3937468799477, + 1210.9060088114516 + ], + "zai-org/GLM-OCR": [ + 1456.074460441987, + 1558.3131610126193 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -16.770450504905284, + "PaddlePaddle/PP-OCRv6_medium": 3.8270941539908563, + "PaddlePaddle/PaddleOCR-VL-1.6": -3.721182739101323, + "allenai/olmOCR-2-7B-1025-FP8": -0.37416354891411174, + "baidu/Qianfan-OCR": 42.579617892972465, + "baidu/Unlimited-OCR": -16.953797392277693, + "deepseek-ai/DeepSeek-OCR": 17.30336252526945, + "deepseek-ai/DeepSeek-OCR-2": -1.4554839901893502, + "lightonai/LightOnOCR-2-1B": -27.42160962994103, + "numind/NuExtract3": -6.330509084286632, + "rednote-hilab/dots.mocr": -14.418662054646575, + "rednote-hilab/dots.ocr": 23.24479826478273, + "tesseract-5": -16.942713078981342, + "zai-org/GLM-OCR": 17.43369918622784 + }, + "final_rank_order": [ + "baidu/Qianfan-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "pair-balanced", + "max_absolute_elo_delta": 42.579617892972465, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 16.856581791943313, + "name": "pair-balanced_1200_seed_43", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9428571428571428, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "baidu/Qianfan-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1559.9439070357234, + "PaddlePaddle/PP-OCRv6_medium": 1346.439860590207, + "PaddlePaddle/PaddleOCR-VL-1.6": 1504.8985264434998, + "allenai/olmOCR-2-7B-1025-FP8": 1499.5796881000992, + "baidu/Qianfan-OCR": 1540.0352987241208, + "baidu/Unlimited-OCR": 1576.3048114351172, + "deepseek-ai/DeepSeek-OCR": 1421.2530138702846, + "deepseek-ai/DeepSeek-OCR-2": 1585.6500885739306, + "lightonai/LightOnOCR-2-1B": 1644.696543297451, + "numind/NuExtract3": 1572.0422078100323, + "rednote-hilab/dots.mocr": 1536.0188767077175, + "rednote-hilab/dots.ocr": 1526.7596056062018, + "tesseract-5": 1180.9700382216424, + "zai-org/GLM-OCR": 1505.407533583972 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1512.307361133168, + 1612.1627581652042 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1287.2544210484944, + 1402.6428172296837 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1454.6511624427483, + 1556.3972661318587 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1450.8834707554017, + 1545.0129019808862 + ], + "baidu/Qianfan-OCR": [ + 1490.4837228754714, + 1596.4312453435186 + ], + "baidu/Unlimited-OCR": [ + 1527.078652535017, + 1625.932894674195 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1364.977562021714, + 1471.540046715359 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1537.2529069127263, + 1637.8387493851774 + ], + "lightonai/LightOnOCR-2-1B": [ + 1597.0972078409204, + 1702.4400477376957 + ], + "numind/NuExtract3": [ + 1522.5748205265784, + 1623.4670840018907 + ], + "rednote-hilab/dots.mocr": [ + 1486.440737918992, + 1585.619577363463 + ], + "rednote-hilab/dots.ocr": [ + 1478.2222121307827, + 1574.1426113709924 + ], + "tesseract-5": [ + 1093.1763104624033, + 1245.8151168996826 + ], + "zai-org/GLM-OCR": [ + 1452.4805021661227, + 1552.3435684918932 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 3.566150270763046, + "PaddlePaddle/PP-OCRv6_medium": -7.893438795900238, + "PaddlePaddle/PaddleOCR-VL-1.6": 1.2067042236078578, + "allenai/olmOCR-2-7B-1025-FP8": 2.3490318118062987, + "baidu/Qianfan-OCR": -36.62551075270494, + "baidu/Unlimited-OCR": 13.0247689112648, + "deepseek-ai/DeepSeek-OCR": -30.271638529266966, + "deepseek-ai/DeepSeek-OCR-2": 6.2941921370338605, + "lightonai/LightOnOCR-2-1B": 27.177692641623253, + "numind/NuExtract3": 0.470018956611284, + "rednote-hilab/dots.mocr": -20.202790803405378, + "rednote-hilab/dots.ocr": -2.493118745669335, + "tesseract-5": 29.916911059081258, + "zai-org/GLM-OCR": 13.481027615154971 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "pair-balanced", + "max_absolute_elo_delta": 36.62551075270494, + "max_absolute_elo_delta_model": "baidu/Qianfan-OCR", + "median_absolute_elo_delta": 10.45910385358252, + "name": "pair-balanced_1200_seed_44", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.956043956043956, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1572.1386057089649, + "PaddlePaddle/PP-OCRv6_medium": 1331.298457646523, + "PaddlePaddle/PaddleOCR-VL-1.6": 1482.8357522778276, + "allenai/olmOCR-2-7B-1025-FP8": 1533.7055763825083, + "baidu/Qianfan-OCR": 1581.4817977385949, + "baidu/Unlimited-OCR": 1576.2684447626, + "deepseek-ai/DeepSeek-OCR": 1430.962135126057, + "deepseek-ai/DeepSeek-OCR-2": 1544.4619450968623, + "lightonai/LightOnOCR-2-1B": 1614.0483231099577, + "numind/NuExtract3": 1594.3022759303524, + "rednote-hilab/dots.mocr": 1575.7543502996234, + "rednote-hilab/dots.ocr": 1537.7674025924164, + "tesseract-5": 1142.304536340088, + "zai-org/GLM-OCR": 1482.6703969876241 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1529.2199634241174, + 1622.8783490337953 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1270.4182344085077, + 1385.1809371350957 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1432.293560848816, + 1536.7370754921592 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1483.0019701343308, + 1582.195980257229 + ], + "baidu/Qianfan-OCR": [ + 1532.9911361859267, + 1638.390693888486 + ], + "baidu/Unlimited-OCR": [ + 1525.390922936181, + 1631.3147135149516 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1378.825343944627, + 1486.1242685587556 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1491.192817321412, + 1597.594868448914 + ], + "lightonai/LightOnOCR-2-1B": [ + 1569.8081072311522, + 1668.335365214688 + ], + "numind/NuExtract3": [ + 1545.1848163359182, + 1648.3491966591353 + ], + "rednote-hilab/dots.mocr": [ + 1526.7242164563306, + 1633.1908376683443 + ], + "rednote-hilab/dots.ocr": [ + 1484.1553142849334, + 1586.0013329852802 + ], + "tesseract-5": [ + 1046.4778354125017, + 1211.978538286196 + ], + "zai-org/GLM-OCR": [ + 1429.5433080631274, + 1538.3320573986505 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 15.760848944004465, + "PaddlePaddle/PP-OCRv6_medium": -23.03484173958418, + "PaddlePaddle/PaddleOCR-VL-1.6": -20.856069942064323, + "allenai/olmOCR-2-7B-1025-FP8": 36.47492009421535, + "baidu/Qianfan-OCR": 4.820988261769116, + "baidu/Unlimited-OCR": 12.98840223874754, + "deepseek-ai/DeepSeek-OCR": -20.56251727349445, + "deepseek-ai/DeepSeek-OCR-2": -34.89395134003439, + "lightonai/LightOnOCR-2-1B": -3.470527545870027, + "numind/NuExtract3": 22.73008707693134, + "rednote-hilab/dots.mocr": 19.53268278850055, + "rednote-hilab/dots.ocr": 8.514678240545209, + "tesseract-5": -8.748590822473261, + "zai-org/GLM-OCR": -9.256108981192938 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "pair-balanced", + "max_absolute_elo_delta": 36.47492009421535, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 17.646765866252508, + "name": "pair-balanced_1200_seed_45", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9208791208791209, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1556.152279191893, + "PaddlePaddle/PP-OCRv6_medium": 1351.606529176169, + "PaddlePaddle/PaddleOCR-VL-1.6": 1522.3269288649906, + "allenai/olmOCR-2-7B-1025-FP8": 1474.1722235401328, + "baidu/Qianfan-OCR": 1570.59747445459, + "baidu/Unlimited-OCR": 1528.4699024283605, + "deepseek-ai/DeepSeek-OCR": 1436.9566686609003, + "deepseek-ai/DeepSeek-OCR-2": 1596.6972549022357, + "lightonai/LightOnOCR-2-1B": 1637.2984685761928, + "numind/NuExtract3": 1574.0310277708513, + "rednote-hilab/dots.mocr": 1574.7915477894535, + "rednote-hilab/dots.ocr": 1533.9432510044749, + "tesseract-5": 1180.2072284627986, + "zai-org/GLM-OCR": 1462.7492151769566 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1507.2107565244098, + 1608.6296677804246 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1295.260491384592, + 1406.7893904769767 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1470.2417865544564, + 1573.7916207594296 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1423.37771086294, + 1522.3387912526669 + ], + "baidu/Qianfan-OCR": [ + 1515.5597451385115, + 1628.2558076927548 + ], + "baidu/Unlimited-OCR": [ + 1479.162648552208, + 1582.3558445771646 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1389.6566534062179, + 1481.6981769022777 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1544.7702679868996, + 1647.5108973139565 + ], + "lightonai/LightOnOCR-2-1B": [ + 1587.851822723328, + 1690.7413179764171 + ], + "numind/NuExtract3": [ + 1523.1647125937345, + 1627.4495558954811 + ], + "rednote-hilab/dots.mocr": [ + 1524.4672261964874, + 1634.1194033231272 + ], + "rednote-hilab/dots.ocr": [ + 1485.5070841995998, + 1586.5349645148206 + ], + "tesseract-5": [ + 1092.633999255946, + 1249.886152483193 + ], + "zai-org/GLM-OCR": [ + 1410.6952162570603, + 1510.2450899465725 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -0.2254775730673373, + "PaddlePaddle/PP-OCRv6_medium": -2.726770209938195, + "PaddlePaddle/PaddleOCR-VL-1.6": 18.63510664509863, + "allenai/olmOCR-2-7B-1025-FP8": -23.058432748160158, + "baidu/Qianfan-OCR": -6.063335022235833, + "baidu/Unlimited-OCR": -34.81014009549199, + "deepseek-ai/DeepSeek-OCR": -14.567983738651265, + "deepseek-ai/DeepSeek-OCR-2": 17.341358465339, + "lightonai/LightOnOCR-2-1B": 19.779617920365126, + "numind/NuExtract3": 2.4588389174302847, + "rednote-hilab/dots.mocr": 18.56988027833063, + "rednote-hilab/dots.ocr": 4.690526652603694, + "tesseract-5": 29.15410130023747, + "zai-org/GLM-OCR": -29.177290791860514 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "pair-balanced", + "max_absolute_elo_delta": 34.81014009549199, + "max_absolute_elo_delta_model": "baidu/Unlimited-OCR", + "median_absolute_elo_delta": 17.955619371834814, + "name": "pair-balanced_1200_seed_46", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9340659340659341, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "rednote-hilab/dots.mocr" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1556.3198631042937, + "PaddlePaddle/PP-OCRv6_medium": 1350.3931854278414, + "PaddlePaddle/PaddleOCR-VL-1.6": 1513.8109379155633, + "allenai/olmOCR-2-7B-1025-FP8": 1493.076293548267, + "baidu/Qianfan-OCR": 1565.8691631789404, + "baidu/Unlimited-OCR": 1570.8544447940942, + "deepseek-ai/DeepSeek-OCR": 1446.1586060206175, + "deepseek-ai/DeepSeek-OCR-2": 1565.8691631789404, + "lightonai/LightOnOCR-2-1B": 1637.9055353640174, + "numind/NuExtract3": 1583.6430871343907, + "rednote-hilab/dots.mocr": 1567.1140576943537, + "rednote-hilab/dots.ocr": 1536.1972365728448, + "tesseract-5": 1117.2363984872097, + "zai-org/GLM-OCR": 1495.5520275786257 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1520.992647088273, + 1595.1956521555821 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1304.3457994278267, + 1392.0614165778384 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1472.887353111539, + 1557.676275563285 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1454.4062057729775, + 1532.1853252470946 + ], + "baidu/Qianfan-OCR": [ + 1527.1904909003156, + 1607.869720127911 + ], + "baidu/Unlimited-OCR": [ + 1528.0195250109307, + 1612.351350645112 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1405.7918730435022, + 1484.7966200601747 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1527.9433929412828, + 1605.4891860314722 + ], + "lightonai/LightOnOCR-2-1B": [ + 1600.8697626850887, + 1682.647222216355 + ], + "numind/NuExtract3": [ + 1546.761974616082, + 1624.1317360144214 + ], + "rednote-hilab/dots.mocr": [ + 1527.1681273679367, + 1606.793605635516 + ], + "rednote-hilab/dots.ocr": [ + 1498.6070016728288, + 1575.9008726228951 + ], + "tesseract-5": [ + 1034.4620182822373, + 1178.3319133295863 + ], + "zai-org/GLM-OCR": [ + 1459.4517955863307, + 1531.3438286815087 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": -0.05789366066665025, + "PaddlePaddle/PP-OCRv6_medium": -3.9401139582657834, + "PaddlePaddle/PaddleOCR-VL-1.6": 10.119115695671326, + "allenai/olmOCR-2-7B-1025-FP8": -4.154362740026045, + "baidu/Qianfan-OCR": -10.791646297885336, + "baidu/Unlimited-OCR": 7.574402270241762, + "deepseek-ai/DeepSeek-OCR": -5.366046378934016, + "deepseek-ai/DeepSeek-OCR-2": -13.486733257956303, + "lightonai/LightOnOCR-2-1B": 20.386684708189705, + "numind/NuExtract3": 12.070898280969686, + "rednote-hilab/dots.mocr": 10.892390183230873, + "rednote-hilab/dots.ocr": 6.944512220973593, + "tesseract-5": -33.81672867535144, + "zai-org/GLM-OCR": 3.6255216098086294 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "kind": "pair-balanced", + "max_absolute_elo_delta": 33.81672867535144, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 8.846758982956544, + "name": "pair-balanced_2000_seed_42", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9120879120879121, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Unlimited-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 1, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1561.7520143757624, + "PaddlePaddle/PP-OCRv6_medium": 1360.1111904073277, + "PaddlePaddle/PaddleOCR-VL-1.6": 1507.621750438433, + "allenai/olmOCR-2-7B-1025-FP8": 1501.694444858008, + "baidu/Qianfan-OCR": 1599.3018969795046, + "baidu/Unlimited-OCR": 1549.4397904485882, + "deepseek-ai/DeepSeek-OCR": 1459.1908811144028, + "deepseek-ai/DeepSeek-OCR-2": 1565.665132385358, + "lightonai/LightOnOCR-2-1B": 1614.716795849111, + "numind/NuExtract3": 1569.1770033195064, + "rednote-hilab/dots.mocr": 1550.9852458473056, + "rednote-hilab/dots.ocr": 1540.8551618069512, + "tesseract-5": 1119.0215803741876, + "zai-org/GLM-OCR": 1500.4671117955538 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1524.967349042977, + 1602.8651438606926 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1314.4082320807406, + 1398.5658632562083 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1467.7787486023904, + 1546.9081753586975 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1465.0550949983342, + 1540.3363663205087 + ], + "baidu/Qianfan-OCR": [ + 1557.9491710440116, + 1640.6595475451768 + ], + "baidu/Unlimited-OCR": [ + 1509.7380848935543, + 1590.5434976062131 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1420.4482264981787, + 1500.759198927008 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1525.9817377739932, + 1603.8931549808183 + ], + "lightonai/LightOnOCR-2-1B": [ + 1581.2173487290374, + 1657.309901729196 + ], + "numind/NuExtract3": [ + 1529.691478155324, + 1609.0156090654482 + ], + "rednote-hilab/dots.mocr": [ + 1512.0058700686736, + 1591.9733449856872 + ], + "rednote-hilab/dots.ocr": [ + 1502.8108935370853, + 1582.6832619806496 + ], + "tesseract-5": [ + 1039.127725339461, + 1180.284574772949 + ], + "zai-org/GLM-OCR": [ + 1461.5394928904193, + 1541.1455369213472 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 5.374257610802033, + "PaddlePaddle/PP-OCRv6_medium": 5.777891021220512, + "PaddlePaddle/PaddleOCR-VL-1.6": 3.9299282185411357, + "allenai/olmOCR-2-7B-1025-FP8": 4.463788569715007, + "baidu/Qianfan-OCR": 22.64108750267883, + "baidu/Unlimited-OCR": -13.840252075264289, + "deepseek-ai/DeepSeek-OCR": 7.666228714851286, + "deepseek-ai/DeepSeek-OCR-2": -13.690764051538736, + "lightonai/LightOnOCR-2-1B": -2.8020548067167965, + "numind/NuExtract3": -2.3951855339146277, + "rednote-hilab/dots.mocr": -5.236421663817282, + "rednote-hilab/dots.ocr": 11.602437455080008, + "tesseract-5": -32.03154678837359, + "zai-org/GLM-OCR": 8.540605826736737 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9120879120879122, + "kind": "pair-balanced", + "max_absolute_elo_delta": 32.03154678837359, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 6.722059868035899, + "name": "pair-balanced_2000_seed_43", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9736263736263736, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "numind/NuExtract3" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1566.0382779204729, + "PaddlePaddle/PP-OCRv6_medium": 1353.3349830891023, + "PaddlePaddle/PaddleOCR-VL-1.6": 1504.6088454017972, + "allenai/olmOCR-2-7B-1025-FP8": 1516.843571724278, + "baidu/Qianfan-OCR": 1553.5359614442066, + "baidu/Unlimited-OCR": 1575.8198966399332, + "deepseek-ai/DeepSeek-OCR": 1433.6491106806277, + "deepseek-ai/DeepSeek-OCR-2": 1578.3176327559609, + "lightonai/LightOnOCR-2-1B": 1637.7529273390214, + "numind/NuExtract3": 1541.268240466551, + "rednote-hilab/dots.mocr": 1544.9425543402667, + "rednote-hilab/dots.ocr": 1520.3443584148463, + "tesseract-5": 1161.5555046613165, + "zai-org/GLM-OCR": 1511.9881351216195 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1527.3864593761775, + 1604.0706877107623 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1308.2747146763209, + 1394.0576988975943 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1465.358497249507, + 1542.9227397836999 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1480.998794642672, + 1556.4731861188397 + ], + "baidu/Qianfan-OCR": [ + 1510.8731437579218, + 1594.2190852932204 + ], + "baidu/Unlimited-OCR": [ + 1538.9213426204312, + 1619.7492419275795 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1392.6271648478737, + 1472.5907941851972 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1539.3265236769032, + 1617.0273091733047 + ], + "lightonai/LightOnOCR-2-1B": [ + 1598.5577630435182, + 1684.3998409257063 + ], + "numind/NuExtract3": [ + 1500.472973134435, + 1580.4249189614613 + ], + "rednote-hilab/dots.mocr": [ + 1506.6488182842593, + 1585.5507824665312 + ], + "rednote-hilab/dots.ocr": [ + 1483.7009393087617, + 1561.834862445297 + ], + "tesseract-5": [ + 1096.1440305295155, + 1215.3971178545794 + ], + "zai-org/GLM-OCR": [ + 1475.7395591782697, + 1551.3114808071668 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 9.66052115551247, + "PaddlePaddle/PP-OCRv6_medium": -0.9983162970049761, + "PaddlePaddle/PaddleOCR-VL-1.6": 0.9170231819052788, + "allenai/olmOCR-2-7B-1025-FP8": 19.612915435985087, + "baidu/Qianfan-OCR": -23.124848032619184, + "baidu/Unlimited-OCR": 12.539854116080733, + "deepseek-ai/DeepSeek-OCR": -17.87554171892384, + "deepseek-ai/DeepSeek-OCR-2": -1.03826368093587, + "lightonai/LightOnOCR-2-1B": 20.234076683193734, + "numind/NuExtract3": -30.303948386869934, + "rednote-hilab/dots.mocr": -11.279113170856135, + "rednote-hilab/dots.ocr": -8.908365937024882, + "tesseract-5": 10.502377498755322, + "zai-org/GLM-OCR": 20.061629152802425 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "pair-balanced", + "max_absolute_elo_delta": 30.303948386869934, + "max_absolute_elo_delta_model": "numind/NuExtract3", + "median_absolute_elo_delta": 11.909483643468434, + "name": "pair-balanced_2000_seed_44", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9384615384615385, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1567.9738562515588, + "PaddlePaddle/PP-OCRv6_medium": 1344.1725100261235, + "PaddlePaddle/PaddleOCR-VL-1.6": 1514.0300181515338, + "allenai/olmOCR-2-7B-1025-FP8": 1509.1233022080776, + "baidu/Qianfan-OCR": 1558.2655547549132, + "baidu/Unlimited-OCR": 1584.4585328149224, + "deepseek-ai/DeepSeek-OCR": 1444.3216129333432, + "deepseek-ai/DeepSeek-OCR-2": 1570.6789821488717, + "lightonai/LightOnOCR-2-1B": 1628.5558124657518, + "numind/NuExtract3": 1573.173560680442, + "rednote-hilab/dots.mocr": 1552.1732191066562, + "rednote-hilab/dots.ocr": 1528.657413230858, + "tesseract-5": 1146.124743502976, + "zai-org/GLM-OCR": 1478.2908817239716 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1528.8587748823452, + 1608.6142697957448 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1295.5091147686385, + 1383.7655221209875 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1474.452565778139, + 1550.5272629862714 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1470.3271310865791, + 1546.1945328156273 + ], + "baidu/Qianfan-OCR": [ + 1516.738540729897, + 1602.871979192973 + ], + "baidu/Unlimited-OCR": [ + 1543.4054915636311, + 1625.2339364070147 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1405.5906494441952, + 1482.9650437201333 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1531.0230713082335, + 1610.1814523526657 + ], + "lightonai/LightOnOCR-2-1B": [ + 1590.5185187016327, + 1671.5491228662215 + ], + "numind/NuExtract3": [ + 1537.3952167652355, + 1615.6039278733151 + ], + "rednote-hilab/dots.mocr": [ + 1514.4215299134971, + 1593.217099013404 + ], + "rednote-hilab/dots.ocr": [ + 1492.5197218195324, + 1569.6878046153229 + ], + "tesseract-5": [ + 1079.046568645851, + 1204.4939461384442 + ], + "zai-org/GLM-OCR": [ + 1441.1148344916687, + 1516.0545594686505 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 11.596099486598405, + "PaddlePaddle/PP-OCRv6_medium": -10.16078935998371, + "PaddlePaddle/PaddleOCR-VL-1.6": 10.338195931641849, + "allenai/olmOCR-2-7B-1025-FP8": 11.892645919784627, + "baidu/Qianfan-OCR": -18.39525472191258, + "baidu/Unlimited-OCR": 21.17849029106992, + "deepseek-ai/DeepSeek-OCR": -7.203039466208338, + "deepseek-ai/DeepSeek-OCR-2": -8.676914288025046, + "lightonai/LightOnOCR-2-1B": 11.03696180992415, + "numind/NuExtract3": 1.601371827021012, + "rednote-hilab/dots.mocr": -4.048448404466626, + "rednote-hilab/dots.ocr": -0.5953111210130828, + "tesseract-5": -4.928383659585279, + "zai-org/GLM-OCR": -13.63562424484553 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "pair-balanced", + "max_absolute_elo_delta": 21.17849029106992, + "max_absolute_elo_delta_model": "baidu/Unlimited-OCR", + "median_absolute_elo_delta": 10.24949264581278, + "name": "pair-balanced_2000_seed_45", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9472527472527472, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "numind/NuExtract3" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 1, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1572.8495576364064, + "PaddlePaddle/PP-OCRv6_medium": 1324.463970061903, + "PaddlePaddle/PaddleOCR-VL-1.6": 1511.7834281660416, + "allenai/olmOCR-2-7B-1025-FP8": 1467.303753606735, + "baidu/Qianfan-OCR": 1576.4953022307723, + "baidu/Unlimited-OCR": 1562.7121970030844, + "deepseek-ai/DeepSeek-OCR": 1447.0417408142685, + "deepseek-ai/DeepSeek-OCR-2": 1607.1398806842576, + "lightonai/LightOnOCR-2-1B": 1621.5456099833466, + "numind/NuExtract3": 1576.7279541495727, + "rednote-hilab/dots.mocr": 1561.4651721700325, + "rednote-hilab/dots.ocr": 1537.7818744765173, + "tesseract-5": 1169.1596975342054, + "zai-org/GLM-OCR": 1463.5298614828566 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1536.8719469463958, + 1615.2665719605875 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1274.5900171743415, + 1367.9082535804455 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1470.283728767947, + 1551.618221153021 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1429.1463398346048, + 1504.6596113433222 + ], + "baidu/Qianfan-OCR": [ + 1532.9999632566792, + 1617.5696628352732 + ], + "baidu/Unlimited-OCR": [ + 1523.4440716780657, + 1602.845253407533 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1409.2337599978666, + 1484.9272957903236 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1568.1258636022374, + 1651.2844569638771 + ], + "lightonai/LightOnOCR-2-1B": [ + 1583.425184018907, + 1662.6689895921536 + ], + "numind/NuExtract3": [ + 1538.2882625297875, + 1617.054027353998 + ], + "rednote-hilab/dots.mocr": [ + 1524.5161365608174, + 1602.0019558365784 + ], + "rednote-hilab/dots.ocr": [ + 1500.1394039123466, + 1579.1576474438932 + ], + "tesseract-5": [ + 1108.0302292183283, + 1222.3623293885537 + ], + "zai-org/GLM-OCR": [ + 1423.2451007570178, + 1504.0386640035351 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 16.47180087144602, + "PaddlePaddle/PP-OCRv6_medium": -29.869329324204273, + "PaddlePaddle/PaddleOCR-VL-1.6": 8.091605946149684, + "allenai/olmOCR-2-7B-1025-FP8": -29.926902681557976, + "baidu/Qianfan-OCR": -0.16550724605349387, + "baidu/Unlimited-OCR": -0.5678455207680599, + "deepseek-ai/DeepSeek-OCR": -4.482911585283091, + "deepseek-ai/DeepSeek-OCR-2": 27.783984247360877, + "lightonai/LightOnOCR-2-1B": 4.026759327518903, + "numind/NuExtract3": 5.155765296151685, + "rednote-hilab/dots.mocr": 5.243504658909615, + "rednote-hilab/dots.ocr": 8.529150124646094, + "tesseract-5": 18.10657037164424, + "zai-org/GLM-OCR": -28.39664448596045 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": true, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "kind": "pair-balanced", + "max_absolute_elo_delta": 29.926902681557976, + "max_absolute_elo_delta_model": "allenai/olmOCR-2-7B-1025-FP8", + "median_absolute_elo_delta": 8.310378035397889, + "name": "pair-balanced_2000_seed_46", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9912087912087912, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1607.8901087705, + "PaddlePaddle/PP-OCRv6_medium": 1324.1322577119297, + "PaddlePaddle/PaddleOCR-VL-1.6": 1432.6828904222969, + "allenai/olmOCR-2-7B-1025-FP8": 1506.8102751994074, + "baidu/Qianfan-OCR": 1577.824105502385, + "baidu/Unlimited-OCR": 1591.545258008199, + "deepseek-ai/DeepSeek-OCR": 1406.0347450044267, + "deepseek-ai/DeepSeek-OCR-2": 1680.9014764916626, + "lightonai/LightOnOCR-2-1B": 1711.7301329953527, + "numind/NuExtract3": 1544.8028614561783, + "rednote-hilab/dots.mocr": 1510.1326575577705, + "rednote-hilab/dots.ocr": 1517.1655294546413, + "tesseract-5": 1083.546911033531, + "zai-org/GLM-OCR": 1504.8007903917191 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1537.41619104448, + 1687.1799282076845 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1239.8641970783945, + 1397.035551971108 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1360.0321307303948, + 1501.5003913341925 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1439.868484039383, + 1575.4485887564044 + ], + "baidu/Qianfan-OCR": [ + 1512.1728255302455, + 1651.7127237756008 + ], + "baidu/Unlimited-OCR": [ + 1523.1909436564972, + 1666.085106773666 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1327.8959854701307, + 1475.8329059438893 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1610.5232893976367, + 1767.1973212983062 + ], + "lightonai/LightOnOCR-2-1B": [ + 1646.0637047225325, + 1803.3676071680648 + ], + "numind/NuExtract3": [ + 1484.0899535728815, + 1617.880372529497 + ], + "rednote-hilab/dots.mocr": [ + 1442.7146948935772, + 1574.5071027070126 + ], + "rednote-hilab/dots.ocr": [ + 1450.8756443582772, + 1589.374286881702 + ], + "tesseract-5": [ + 908.5558725182159, + 1178.6745162134548 + ], + "zai-org/GLM-OCR": [ + 1436.3636153316697, + 1575.3762013860703 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 51.5123520055397, + "PaddlePaddle/PP-OCRv6_medium": -30.20104167417753, + "PaddlePaddle/PaddleOCR-VL-1.6": -71.0089317975951, + "allenai/olmOCR-2-7B-1025-FP8": 9.579618911114494, + "baidu/Qianfan-OCR": 1.163296025559248, + "baidu/Unlimited-OCR": 28.265215484346527, + "deepseek-ai/DeepSeek-OCR": -45.489907395124874, + "deepseek-ai/DeepSeek-OCR-2": 101.54558005476588, + "lightonai/LightOnOCR-2-1B": 94.21128233952504, + "numind/NuExtract3": -26.769327397242705, + "rednote-hilab/dots.mocr": -46.08900995335239, + "rednote-hilab/dots.ocr": -12.087194897229892, + "tesseract-5": -67.50621612903024, + "zai-org/GLM-OCR": 12.874284422902065 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "mixed-random", + "max_absolute_elo_delta": 101.54558005476588, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 37.8454745346512, + "name": "mixed-random_700_seed_42", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9428571428571428, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 0, + "unresolved": 13 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1627.1741473193943, + "PaddlePaddle/PP-OCRv6_medium": 1304.9844851467524, + "PaddlePaddle/PaddleOCR-VL-1.6": 1462.7150288511957, + "allenai/olmOCR-2-7B-1025-FP8": 1441.296941484788, + "baidu/Qianfan-OCR": 1621.9516901182708, + "baidu/Unlimited-OCR": 1549.911884471927, + "deepseek-ai/DeepSeek-OCR": 1442.471670650833, + "deepseek-ai/DeepSeek-OCR-2": 1647.719724252598, + "lightonai/LightOnOCR-2-1B": 1666.6517651370589, + "numind/NuExtract3": 1525.1733891908048, + "rednote-hilab/dots.mocr": 1554.088250095837, + "rednote-hilab/dots.ocr": 1501.480133057811, + "tesseract-5": 1146.7924255255678, + "zai-org/GLM-OCR": 1507.5884646971617 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1560.331581908084, + 1701.4720403977838 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1217.2575804876783, + 1381.167624184091 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1383.2751832504837, + 1527.2708486078989 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1371.7215038591187, + 1507.8905093956914 + ], + "baidu/Qianfan-OCR": [ + 1552.2762899288591, + 1699.4009139750337 + ], + "baidu/Unlimited-OCR": [ + 1479.8720112553858, + 1621.8368057717266 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1367.3416439036787, + 1516.3399118897585 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1576.3068243932532, + 1724.8516145871938 + ], + "lightonai/LightOnOCR-2-1B": [ + 1599.4065540778129, + 1744.790289660717 + ], + "numind/NuExtract3": [ + 1461.5566407624092, + 1592.7322593938345 + ], + "rednote-hilab/dots.mocr": [ + 1491.691074120144, + 1619.092829571701 + ], + "rednote-hilab/dots.ocr": [ + 1435.6639686909218, + 1568.4481808977835 + ], + "tesseract-5": [ + 1024.2627587128763, + 1233.4463621850164 + ], + "zai-org/GLM-OCR": [ + 1433.4745399025624, + 1573.0954829052937 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 70.79639055443386, + "PaddlePaddle/PP-OCRv6_medium": -49.34881423935485, + "PaddlePaddle/PaddleOCR-VL-1.6": -40.976793368696235, + "allenai/olmOCR-2-7B-1025-FP8": -55.93371480350493, + "baidu/Qianfan-OCR": 45.29088064144503, + "baidu/Unlimited-OCR": -13.368158051925548, + "deepseek-ai/DeepSeek-OCR": -9.052981748718594, + "deepseek-ai/DeepSeek-OCR-2": 68.36382781570137, + "lightonai/LightOnOCR-2-1B": 49.13291448123118, + "numind/NuExtract3": -46.398799662616284, + "rednote-hilab/dots.mocr": -2.1334174152857486, + "rednote-hilab/dots.ocr": -27.77259129406025, + "tesseract-5": -4.260701636993417, + "zai-org/GLM-OCR": 15.661958728344644 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "mixed-random", + "max_absolute_elo_delta": 70.79639055443386, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 43.13383700507063, + "name": "mixed-random_700_seed_43", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9120879120879121, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1673.188282934586, + "PaddlePaddle/PP-OCRv6_medium": 1281.6639118530522, + "PaddlePaddle/PaddleOCR-VL-1.6": 1434.778032305935, + "allenai/olmOCR-2-7B-1025-FP8": 1520.5836447664847, + "baidu/Qianfan-OCR": 1661.446223453312, + "baidu/Unlimited-OCR": 1572.3165298586125, + "deepseek-ai/DeepSeek-OCR": 1431.7235728805313, + "deepseek-ai/DeepSeek-OCR-2": 1632.4475423609897, + "lightonai/LightOnOCR-2-1B": 1688.0311735938176, + "numind/NuExtract3": 1517.7510356076311, + "rednote-hilab/dots.mocr": 1497.914748263184, + "rednote-hilab/dots.ocr": 1514.9465196472356, + "tesseract-5": 1074.9628403858978, + "zai-org/GLM-OCR": 1498.2459420887308 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1609.1227255935344, + 1755.1508592770801 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1184.3341999451477, + 1362.7725595757556 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1357.377041378551, + 1504.7276453498441 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1451.0528601341016, + 1589.0698687017193 + ], + "baidu/Qianfan-OCR": [ + 1584.056106179743, + 1752.059552912486 + ], + "baidu/Unlimited-OCR": [ + 1510.5318293021985, + 1645.3469416383532 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1355.4522504754032, + 1503.9431200309352 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1566.6065469088715, + 1711.2124597939828 + ], + "lightonai/LightOnOCR-2-1B": [ + 1618.9361445565376, + 1781.2408999691443 + ], + "numind/NuExtract3": [ + 1451.7468784250916, + 1589.2210151692518 + ], + "rednote-hilab/dots.mocr": [ + 1433.9772483713687, + 1562.9300093213396 + ], + "rednote-hilab/dots.ocr": [ + 1447.0342665517746, + 1583.268952566137 + ], + "tesseract-5": [ + 908.8573176320168, + 1171.0115883945004 + ], + "zai-org/GLM-OCR": [ + 1432.0219564351905, + 1565.9446907528334 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 116.81052616962552, + "PaddlePaddle/PP-OCRv6_medium": -72.66938753305499, + "PaddlePaddle/PaddleOCR-VL-1.6": -68.91378991395686, + "allenai/olmOCR-2-7B-1025-FP8": 23.35298847819172, + "baidu/Qianfan-OCR": 84.78541397648632, + "baidu/Unlimited-OCR": 9.036487334760068, + "deepseek-ai/DeepSeek-OCR": -19.801079519020277, + "deepseek-ai/DeepSeek-OCR-2": 53.09164592409297, + "lightonai/LightOnOCR-2-1B": 70.5123229379899, + "numind/NuExtract3": -53.8211532457899, + "rednote-hilab/dots.mocr": -58.30691924793882, + "rednote-hilab/dots.ocr": -14.306204704635547, + "tesseract-5": -76.09028677666333, + "zai-org/GLM-OCR": 6.319436119913689 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7142857142857144, + "kind": "mixed-random", + "max_absolute_elo_delta": 116.81052616962552, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 56.06403624686436, + "name": "mixed-random_700_seed_44", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.8637362637362638, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1607.6555491548213, + "PaddlePaddle/PP-OCRv6_medium": 1352.1414508836476, + "PaddlePaddle/PaddleOCR-VL-1.6": 1437.8284775709838, + "allenai/olmOCR-2-7B-1025-FP8": 1489.6664396303022, + "baidu/Qianfan-OCR": 1597.1412419693654, + "baidu/Unlimited-OCR": 1586.5221157596163, + "deepseek-ai/DeepSeek-OCR": 1431.342811092415, + "deepseek-ai/DeepSeek-OCR-2": 1631.150345413672, + "lightonai/LightOnOCR-2-1B": 1720.296774041724, + "numind/NuExtract3": 1551.9239772540213, + "rednote-hilab/dots.mocr": 1516.9674433402572, + "rednote-hilab/dots.ocr": 1508.048931365834, + "tesseract-5": 1063.6567478026761, + "zai-org/GLM-OCR": 1505.6576947206636 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1542.3305444403445, + 1681.6589503854434 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1265.4532657115885, + 1424.6293143588732 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1356.7702405622756, + 1506.8573270331915 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1416.2541888941705, + 1555.755943824818 + ], + "baidu/Qianfan-OCR": [ + 1530.7018788856071, + 1668.0324213164508 + ], + "baidu/Unlimited-OCR": [ + 1524.7381764523532, + 1660.3642938037824 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1357.2759420969953, + 1499.2227666354365 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1563.0769333934704, + 1710.3349686436177 + ], + "lightonai/LightOnOCR-2-1B": [ + 1655.2646298082745, + 1814.5512735109605 + ], + "numind/NuExtract3": [ + 1484.7670955558215, + 1628.796162338098 + ], + "rednote-hilab/dots.mocr": [ + 1451.815383529807, + 1587.0007108198142 + ], + "rednote-hilab/dots.ocr": [ + 1436.8902484561368, + 1584.3512967201027 + ], + "tesseract-5": [ + 860.8613332909522, + 1163.8436119702465 + ], + "zai-org/GLM-OCR": [ + 1436.3865681608556, + 1568.6186597509145 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 51.277792389860906, + "PaddlePaddle/PP-OCRv6_medium": -2.1918485024596066, + "PaddlePaddle/PaddleOCR-VL-1.6": -65.8633446489082, + "allenai/olmOCR-2-7B-1025-FP8": -7.564216657990755, + "baidu/Qianfan-OCR": 20.48043249253965, + "baidu/Unlimited-OCR": 23.242073235763883, + "deepseek-ai/DeepSeek-OCR": -20.181841307136438, + "deepseek-ai/DeepSeek-OCR-2": 51.79444897677536, + "lightonai/LightOnOCR-2-1B": 102.7779233858962, + "numind/NuExtract3": -19.6482115993997, + "rednote-hilab/dots.mocr": -39.254224170865655, + "rednote-hilab/dots.ocr": -21.203792986037115, + "tesseract-5": -87.39637935988503, + "zai-org/GLM-OCR": 13.731188751846503 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "mixed-random", + "max_absolute_elo_delta": 102.7779233858962, + "max_absolute_elo_delta_model": "lightonai/LightOnOCR-2-1B", + "median_absolute_elo_delta": 22.2229331109005, + "name": "mixed-random_700_seed_45", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9516483516483516, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 700, + "comparisons_saved": 3805, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1652.650075828782, + "PaddlePaddle/PP-OCRv6_medium": 1340.3972595669645, + "PaddlePaddle/PaddleOCR-VL-1.6": 1437.818732129255, + "allenai/olmOCR-2-7B-1025-FP8": 1466.0479919755378, + "baidu/Qianfan-OCR": 1607.2737598059603, + "baidu/Unlimited-OCR": 1609.125735268791, + "deepseek-ai/DeepSeek-OCR": 1415.544306236939, + "deepseek-ai/DeepSeek-OCR-2": 1661.533523408763, + "lightonai/LightOnOCR-2-1B": 1682.959703545581, + "numind/NuExtract3": 1551.674958751879, + "rednote-hilab/dots.mocr": 1486.1642370411985, + "rednote-hilab/dots.ocr": 1537.2047617337425, + "tesseract-5": 1061.1815330960849, + "zai-org/GLM-OCR": 1490.423421610521 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1588.8484655721636, + 1735.1177462019996 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1255.9446176042873, + 1415.0413060900187 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1354.6363409835938, + 1515.0000816274264 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1397.0832483072288, + 1527.5204726248967 + ], + "baidu/Qianfan-OCR": [ + 1538.7490305353715, + 1684.3592097304213 + ], + "baidu/Unlimited-OCR": [ + 1543.2545554907733, + 1683.8220596416827 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1336.3330838101779, + 1489.9908786431672 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1591.811758061037, + 1755.5855750675964 + ], + "lightonai/LightOnOCR-2-1B": [ + 1613.2377296811856, + 1768.9686807711419 + ], + "numind/NuExtract3": [ + 1487.4709284283958, + 1625.3005541054576 + ], + "rednote-hilab/dots.mocr": [ + 1421.7560227263386, + 1550.175780361418 + ], + "rednote-hilab/dots.ocr": [ + 1464.4941224916502, + 1613.8407852451069 + ], + "tesseract-5": [ + 871.1127205443988, + 1158.315696570632 + ], + "zai-org/GLM-OCR": [ + 1424.461820495788, + 1558.9219882978698 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 96.27231906382167, + "PaddlePaddle/PP-OCRv6_medium": -13.936039819142707, + "PaddlePaddle/PaddleOCR-VL-1.6": -65.87309009063688, + "allenai/olmOCR-2-7B-1025-FP8": -31.18266431275515, + "baidu/Qianfan-OCR": 30.61295032913449, + "baidu/Unlimited-OCR": 45.84569274493856, + "deepseek-ai/DeepSeek-OCR": -35.98034616261248, + "deepseek-ai/DeepSeek-OCR-2": 82.17762697186618, + "lightonai/LightOnOCR-2-1B": 65.44085288975339, + "numind/NuExtract3": -19.897230101541936, + "rednote-hilab/dots.mocr": -70.05743046992438, + "rednote-hilab/dots.ocr": 7.952037381871378, + "tesseract-5": -89.87159406647629, + "zai-org/GLM-OCR": -1.5030843582960642 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 8, + "median_direct_comparisons": 8, + "min_direct_comparisons": 7, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "mixed-random", + "max_absolute_elo_delta": 96.27231906382167, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 40.91301945377552, + "name": "mixed-random_700_seed_46", + "percentage_saved": 84.46170921198669, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9208791208791209, + "static_budget": 700, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1582.6452065080616, + "PaddlePaddle/PP-OCRv6_medium": 1364.9558984081502, + "PaddlePaddle/PaddleOCR-VL-1.6": 1477.8915337701383, + "allenai/olmOCR-2-7B-1025-FP8": 1523.2322260922804, + "baidu/Qianfan-OCR": 1548.586005881871, + "baidu/Unlimited-OCR": 1570.1080110130335, + "deepseek-ai/DeepSeek-OCR": 1405.3038002095473, + "deepseek-ai/DeepSeek-OCR-2": 1658.8810948721898, + "lightonai/LightOnOCR-2-1B": 1687.7393640094738, + "numind/NuExtract3": 1550.9783726906985, + "rednote-hilab/dots.mocr": 1538.7355061384053, + "rednote-hilab/dots.ocr": 1478.5602992631827, + "tesseract-5": 1113.8461646271267, + "zai-org/GLM-OCR": 1498.5365165158412 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1533.7954363003914, + 1635.8113604007751 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1304.051558291, + 1423.5544455264076 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1426.4396226370682, + 1528.9882447914424 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1473.953477685962, + 1573.8904047548103 + ], + "baidu/Qianfan-OCR": [ + 1499.0134799985642, + 1603.3539599575665 + ], + "baidu/Unlimited-OCR": [ + 1519.8492814542847, + 1624.214463908602 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1356.6951350657798, + 1457.7773299553219 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1605.505343591541, + 1722.567874957069 + ], + "lightonai/LightOnOCR-2-1B": [ + 1636.2932534153383, + 1748.0152255245696 + ], + "numind/NuExtract3": [ + 1501.3028607169115, + 1603.7652303915486 + ], + "rednote-hilab/dots.mocr": [ + 1492.345739008028, + 1587.6530562592716 + ], + "rednote-hilab/dots.ocr": [ + 1423.3057001123254, + 1527.2376947240177 + ], + "tesseract-5": [ + 1009.6652014765323, + 1187.3347579528752 + ], + "zai-org/GLM-OCR": [ + 1442.85823524457, + 1547.6694986841333 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 26.267449743101224, + "PaddlePaddle/PP-OCRv6_medium": 10.622599022043005, + "PaddlePaddle/PaddleOCR-VL-1.6": -25.80028844975368, + "allenai/olmOCR-2-7B-1025-FP8": 26.001569803987422, + "baidu/Qianfan-OCR": -28.0748035949548, + "baidu/Unlimited-OCR": 6.827968489181103, + "deepseek-ai/DeepSeek-OCR": -46.22085219000428, + "deepseek-ai/DeepSeek-OCR-2": 79.52519843529308, + "lightonai/LightOnOCR-2-1B": 70.22051335364608, + "numind/NuExtract3": -20.593816162722533, + "rednote-hilab/dots.mocr": -17.48616137271756, + "rednote-hilab/dots.ocr": -50.692425088688424, + "tesseract-5": -37.20696253543451, + "zai-org/GLM-OCR": 6.610010547024103 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "kind": "mixed-random", + "max_absolute_elo_delta": 79.52519843529308, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 26.134509773544323, + "name": "mixed-random_1200_seed_42", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9208791208791209, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1620.0914993638414, + "PaddlePaddle/PP-OCRv6_medium": 1330.619162306705, + "PaddlePaddle/PaddleOCR-VL-1.6": 1490.8200972437673, + "allenai/olmOCR-2-7B-1025-FP8": 1451.9459466186784, + "baidu/Qianfan-OCR": 1581.38425373927, + "baidu/Unlimited-OCR": 1583.5020067780258, + "deepseek-ai/DeepSeek-OCR": 1448.319432190444, + "deepseek-ai/DeepSeek-OCR-2": 1661.835679161675, + "lightonai/LightOnOCR-2-1B": 1640.0129578701667, + "numind/NuExtract3": 1537.0827287700786, + "rednote-hilab/dots.mocr": 1528.2044087002694, + "rednote-hilab/dots.ocr": 1494.0277477963368, + "tesseract-5": 1127.096097306829, + "zai-org/GLM-OCR": 1505.0579821539131 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1569.4570440585626, + 1670.3380901712362 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1273.8919897657595, + 1380.8972935416734 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1435.3013454869529, + 1545.7768612094842 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1404.7620648103732, + 1497.9890747838385 + ], + "baidu/Qianfan-OCR": [ + 1526.0586672337129, + 1640.4379440278597 + ], + "baidu/Unlimited-OCR": [ + 1533.7001960643138, + 1636.6420861196973 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1395.5031442533052, + 1502.8368670095813 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1609.1332569883677, + 1722.5904708354926 + ], + "lightonai/LightOnOCR-2-1B": [ + 1593.0202850930152, + 1696.7663562008674 + ], + "numind/NuExtract3": [ + 1489.8691702667827, + 1590.9704211898002 + ], + "rednote-hilab/dots.mocr": [ + 1477.7421403509156, + 1581.2073784857314 + ], + "rednote-hilab/dots.ocr": [ + 1444.7564113124472, + 1544.5366881285584 + ], + "tesseract-5": [ + 1027.361682838254, + 1199.287222405877 + ], + "zai-org/GLM-OCR": [ + 1459.7139542560053, + 1560.549018734508 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 63.713742598881026, + "PaddlePaddle/PP-OCRv6_medium": -23.714137079402235, + "PaddlePaddle/PaddleOCR-VL-1.6": -12.871724976124597, + "allenai/olmOCR-2-7B-1025-FP8": -45.284709669614585, + "baidu/Qianfan-OCR": 4.723444262444218, + "baidu/Unlimited-OCR": 20.221964254173372, + "deepseek-ai/DeepSeek-OCR": -3.2052202091074378, + "deepseek-ai/DeepSeek-OCR-2": 82.47978272477826, + "lightonai/LightOnOCR-2-1B": 22.49410721433901, + "numind/NuExtract3": -34.489460083342465, + "rednote-hilab/dots.mocr": -28.01725881085349, + "rednote-hilab/dots.ocr": -35.22497655553434, + "tesseract-5": -23.957029855732117, + "zai-org/GLM-OCR": 13.131476185096062 + }, + "final_rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "kind": "mixed-random", + "max_absolute_elo_delta": 82.47978272477826, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 23.835583467567176, + "name": "mixed-random_1200_seed_43", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9296703296703297, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1614.8782412199141, + "PaddlePaddle/PP-OCRv6_medium": 1336.3677139611366, + "PaddlePaddle/PaddleOCR-VL-1.6": 1476.747087824031, + "allenai/olmOCR-2-7B-1025-FP8": 1494.424848606755, + "baidu/Qianfan-OCR": 1608.026483913197, + "baidu/Unlimited-OCR": 1575.6047075569065, + "deepseek-ai/DeepSeek-OCR": 1414.358620538918, + "deepseek-ai/DeepSeek-OCR-2": 1597.6043232088473, + "lightonai/LightOnOCR-2-1B": 1633.2568398341439, + "numind/NuExtract3": 1553.7823519030087, + "rednote-hilab/dots.mocr": 1532.8940016822075, + "rednote-hilab/dots.ocr": 1515.535232214481, + "tesseract-5": 1150.1899539253086, + "zai-org/GLM-OCR": 1496.3295936111451 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1562.5887431426645, + 1668.8648602971814 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1279.9458315477618, + 1394.4444422903136 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1429.8377203484301, + 1530.041783108539 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1440.3162956915269, + 1541.513340621652 + ], + "baidu/Qianfan-OCR": [ + 1560.1932486181072, + 1661.8186001612457 + ], + "baidu/Unlimited-OCR": [ + 1522.1180142867547, + 1628.3866131456905 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1359.4216555891003, + 1464.5434238387807 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1548.9433270670818, + 1651.0469435700022 + ], + "lightonai/LightOnOCR-2-1B": [ + 1585.1469206351478, + 1688.5248919176342 + ], + "numind/NuExtract3": [ + 1504.4743037882092, + 1603.3886233973847 + ], + "rednote-hilab/dots.mocr": [ + 1480.620711423424, + 1581.9272002801229 + ], + "rednote-hilab/dots.ocr": [ + 1466.0422555258936, + 1564.5079123376413 + ], + "tesseract-5": [ + 1063.565238664069, + 1215.0993265240566 + ], + "zai-org/GLM-OCR": [ + 1443.8916045355093, + 1550.0817854010586 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 58.50048445495372, + "PaddlePaddle/PP-OCRv6_medium": -17.96558542497064, + "PaddlePaddle/PaddleOCR-VL-1.6": -26.944734395860905, + "allenai/olmOCR-2-7B-1025-FP8": -2.805807681537999, + "baidu/Qianfan-OCR": 31.365674436371137, + "baidu/Unlimited-OCR": 12.324665033054089, + "deepseek-ai/DeepSeek-OCR": -37.16603186063344, + "deepseek-ai/DeepSeek-OCR-2": 18.248426771950562, + "lightonai/LightOnOCR-2-1B": 15.737989178316184, + "numind/NuExtract3": -17.789836950412337, + "rednote-hilab/dots.mocr": -23.327665828915315, + "rednote-hilab/dots.ocr": -13.717492137390082, + "tesseract-5": -0.8631732372525676, + "zai-org/GLM-OCR": 4.403087642328046 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "kind": "mixed-random", + "max_absolute_elo_delta": 58.50048445495372, + "max_absolute_elo_delta_model": "FireRedTeam/FireRed-OCR", + "median_absolute_elo_delta": 17.87771118769149, + "name": "mixed-random_1200_seed_44", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9296703296703297, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 1, + "unresolved": 12 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1587.41846281403, + "PaddlePaddle/PP-OCRv6_medium": 1366.2017247029062, + "PaddlePaddle/PaddleOCR-VL-1.6": 1454.8180238916289, + "allenai/olmOCR-2-7B-1025-FP8": 1480.0504595839022, + "baidu/Qianfan-OCR": 1593.4162604871158, + "baidu/Unlimited-OCR": 1611.119318783886, + "deepseek-ai/DeepSeek-OCR": 1421.2041862742628, + "deepseek-ai/DeepSeek-OCR-2": 1598.365769580556, + "lightonai/LightOnOCR-2-1B": 1646.082758603662, + "numind/NuExtract3": 1590.9942151002767, + "rednote-hilab/dots.mocr": 1552.3188611309392, + "rednote-hilab/dots.ocr": 1542.6388406063425, + "tesseract-5": 1066.9644494778336, + "zai-org/GLM-OCR": 1488.4066689626586 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1533.6596993816268, + 1637.998897173496 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1306.4010319223796, + 1420.350268083324 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1402.451862592841, + 1508.9786527646565 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1430.072840870192, + 1526.8785909762453 + ], + "baidu/Qianfan-OCR": [ + 1542.4144730791572, + 1650.741321430447 + ], + "baidu/Unlimited-OCR": [ + 1559.156420113837, + 1668.81489062812 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1365.3860689352996, + 1475.82801014154 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1552.6311138611945, + 1647.4707031840298 + ], + "lightonai/LightOnOCR-2-1B": [ + 1600.365563867444, + 1706.8688521640258 + ], + "numind/NuExtract3": [ + 1535.8573555658318, + 1647.631397330616 + ], + "rednote-hilab/dots.mocr": [ + 1500.890110592928, + 1610.7970601033749 + ], + "rednote-hilab/dots.ocr": [ + 1490.169432273786, + 1593.7800187217147 + ], + "tesseract-5": [ + 949.4377030135701, + 1149.0232740634394 + ], + "zai-org/GLM-OCR": [ + 1438.1612701012064, + 1536.418796203984 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 31.040706049069513, + "PaddlePaddle/PP-OCRv6_medium": 11.868425316798948, + "PaddlePaddle/PaddleOCR-VL-1.6": -48.87379832826309, + "allenai/olmOCR-2-7B-1025-FP8": -17.18019670439071, + "baidu/Qianfan-OCR": 16.755451010290017, + "baidu/Unlimited-OCR": 47.83927626003356, + "deepseek-ai/DeepSeek-OCR": -30.320466125288704, + "deepseek-ai/DeepSeek-OCR-2": 19.009873143659206, + "lightonai/LightOnOCR-2-1B": 28.563907947834196, + "numind/NuExtract3": 19.422026246855694, + "rednote-hilab/dots.mocr": -3.9028063801836197, + "rednote-hilab/dots.ocr": 13.38611625447129, + "tesseract-5": -84.0886776847276, + "zai-org/GLM-OCR": -3.5198370061584683 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": false, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "mixed-random", + "max_absolute_elo_delta": 84.0886776847276, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 19.21594969525745, + "name": "mixed-random_1200_seed_45", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.956043956043956, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 1200, + "comparisons_saved": 3305, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1597.4821947072717, + "PaddlePaddle/PP-OCRv6_medium": 1310.1797508380314, + "PaddlePaddle/PaddleOCR-VL-1.6": 1516.8156415707144, + "allenai/olmOCR-2-7B-1025-FP8": 1502.8727469434361, + "baidu/Qianfan-OCR": 1568.257340067952, + "baidu/Unlimited-OCR": 1598.1665076184513, + "deepseek-ai/DeepSeek-OCR": 1432.1829257942113, + "deepseek-ai/DeepSeek-OCR-2": 1624.501345837839, + "lightonai/LightOnOCR-2-1B": 1638.1368066217763, + "numind/NuExtract3": 1540.4416332959775, + "rednote-hilab/dots.mocr": 1512.138457240629, + "rednote-hilab/dots.ocr": 1546.8241404181824, + "tesseract-5": 1105.7196907415798, + "zai-org/GLM-OCR": 1506.2808183039479 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1547.9669151921437, + 1654.170416887006 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1250.5375680695245, + 1362.4053786507397 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1466.6253265600728, + 1570.3396343757106 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1451.8795443806036, + 1555.5013972117495 + ], + "baidu/Qianfan-OCR": [ + 1522.5341970345091, + 1620.0091500150302 + ], + "baidu/Unlimited-OCR": [ + 1542.690667959922, + 1651.9194490568952 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1378.384853989813, + 1483.5063589052206 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1574.7425310081085, + 1681.2578380164305 + ], + "lightonai/LightOnOCR-2-1B": [ + 1587.2189276861802, + 1694.1088889398766 + ], + "numind/NuExtract3": [ + 1493.095843880112, + 1592.556292062013 + ], + "rednote-hilab/dots.mocr": [ + 1460.4853236947308, + 1562.7683201619873 + ], + "rednote-hilab/dots.ocr": [ + 1496.4018961205904, + 1594.3387005641505 + ], + "tesseract-5": [ + 990.4199050828915, + 1177.246038010202 + ], + "zai-org/GLM-OCR": [ + 1453.9329844413635, + 1554.2416079290854 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 41.10443794231128, + "PaddlePaddle/PP-OCRv6_medium": -44.15354854807583, + "PaddlePaddle/PaddleOCR-VL-1.6": 13.123819350822487, + "allenai/olmOCR-2-7B-1025-FP8": 5.642090655143193, + "baidu/Qianfan-OCR": -8.403469408873661, + "baidu/Unlimited-OCR": 34.88646509459886, + "deepseek-ai/DeepSeek-OCR": -19.341726605340227, + "deepseek-ai/DeepSeek-OCR-2": 45.1454494009422, + "lightonai/LightOnOCR-2-1B": 20.617955965948568, + "numind/NuExtract3": -31.130555557443586, + "rednote-hilab/dots.mocr": -44.083210270493964, + "rednote-hilab/dots.ocr": 17.571416066311258, + "tesseract-5": -45.33343642098134, + "zai-org/GLM-OCR": 14.354312335130771 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": true, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 14, + "median_direct_comparisons": 13, + "min_direct_comparisons": 13, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "kind": "mixed-random", + "max_absolute_elo_delta": 45.33343642098134, + "max_absolute_elo_delta_model": "tesseract-5", + "median_absolute_elo_delta": 25.874255761696077, + "name": "mixed-random_1200_seed_46", + "percentage_saved": 73.36293007769146, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9296703296703297, + "static_budget": 1200, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 42, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1560.7484823591735, + "PaddlePaddle/PP-OCRv6_medium": 1359.8917394471987, + "PaddlePaddle/PaddleOCR-VL-1.6": 1496.5982637161483, + "allenai/olmOCR-2-7B-1025-FP8": 1496.5982637161483, + "baidu/Qianfan-OCR": 1547.111024551955, + "baidu/Unlimited-OCR": 1578.2620916139485, + "deepseek-ai/DeepSeek-OCR": 1443.9392504974448, + "deepseek-ai/DeepSeek-OCR-2": 1615.5487357737957, + "lightonai/LightOnOCR-2-1B": 1657.3509253098446, + "numind/NuExtract3": 1544.3903403748022, + "rednote-hilab/dots.mocr": 1548.3474402353147, + "rednote-hilab/dots.ocr": 1523.6953233540257, + "tesseract-5": 1137.1026931962745, + "zai-org/GLM-OCR": 1490.4154258539252 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1523.00088870466, + 1600.8070730591523 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1317.2181012778215, + 1402.9413924969726 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1457.1159067676092, + 1533.6132244309858 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1459.1459993575997, + 1534.4036172018014 + ], + "baidu/Qianfan-OCR": [ + 1509.1531257830375, + 1585.1156784402826 + ], + "baidu/Unlimited-OCR": [ + 1538.4626844872196, + 1623.2088852377922 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1403.8559429424474, + 1484.7188677014506 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1574.3262014339198, + 1658.1860214121707 + ], + "lightonai/LightOnOCR-2-1B": [ + 1617.0631943785133, + 1703.0810563520943 + ], + "numind/NuExtract3": [ + 1506.9466679606296, + 1587.5447721122136 + ], + "rednote-hilab/dots.mocr": [ + 1508.820835636519, + 1590.6341394533108 + ], + "rednote-hilab/dots.ocr": [ + 1487.9558600117402, + 1561.8035462468401 + ], + "tesseract-5": [ + 1069.663841372689, + 1189.3646362682127 + ], + "zai-org/GLM-OCR": [ + 1452.4869856397702, + 1528.2343245514007 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 4.370725594213127, + "PaddlePaddle/PP-OCRv6_medium": 5.55844006109146, + "PaddlePaddle/PaddleOCR-VL-1.6": -7.0935585037436795, + "allenai/olmOCR-2-7B-1025-FP8": -0.6323925721446813, + "baidu/Qianfan-OCR": -29.549784924870664, + "baidu/Unlimited-OCR": 14.982049090096098, + "deepseek-ai/DeepSeek-OCR": -7.585401902106696, + "deepseek-ai/DeepSeek-OCR-2": 36.19283933689894, + "lightonai/LightOnOCR-2-1B": 39.83207465401688, + "numind/NuExtract3": -27.181848478618804, + "rednote-hilab/dots.mocr": -7.874227275808153, + "rednote-hilab/dots.ocr": -5.557400997845434, + "tesseract-5": -13.95043396628671, + "zai-org/GLM-OCR": -1.5110801148919109 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "kind": "mixed-random", + "max_absolute_elo_delta": 39.83207465401688, + "max_absolute_elo_delta_model": "lightonai/LightOnOCR-2-1B", + "median_absolute_elo_delta": 7.729814588957424, + "name": "mixed-random_2000_seed_42", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9340659340659341, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 43, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1591.1042633777524, + "PaddlePaddle/PP-OCRv6_medium": 1349.6397340472583, + "PaddlePaddle/PaddleOCR-VL-1.6": 1505.2970915991664, + "allenai/olmOCR-2-7B-1025-FP8": 1474.3240727580157, + "baidu/Qianfan-OCR": 1583.3256686649402, + "baidu/Unlimited-OCR": 1548.39259919778, + "deepseek-ai/DeepSeek-OCR": 1448.9087082331641, + "deepseek-ai/DeepSeek-OCR-2": 1619.3570654291432, + "lightonai/LightOnOCR-2-1B": 1625.96354664103, + "numind/NuExtract3": 1543.2844060525185, + "rednote-hilab/dots.mocr": 1549.6288214449187, + "rednote-hilab/dots.ocr": 1505.4959633665599, + "tesseract-5": 1142.5943021784717, + "zai-org/GLM-OCR": 1512.6837570092805 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1550.7953767199238, + 1633.2003766492144 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1305.4010979446664, + 1390.272248215836 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1465.0474849846792, + 1544.9401036534323 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1436.8198486781048, + 1513.3806505938019 + ], + "baidu/Qianfan-OCR": [ + 1547.0596106076296, + 1624.7278943177412 + ], + "baidu/Unlimited-OCR": [ + 1510.488696909561, + 1593.6421546494298 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1407.8534496456116, + 1485.8344649897042 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1579.0139928017209, + 1663.7842868513421 + ], + "lightonai/LightOnOCR-2-1B": [ + 1586.9683297984407, + 1668.109399545862 + ], + "numind/NuExtract3": [ + 1506.7621311878881, + 1580.6824746998682 + ], + "rednote-hilab/dots.mocr": [ + 1512.3155374572834, + 1590.481060528405 + ], + "rednote-hilab/dots.ocr": [ + 1467.3330457925263, + 1546.310610627695 + ], + "tesseract-5": [ + 1069.5617192936468, + 1197.538620953128 + ], + "zai-org/GLM-OCR": [ + 1474.2775973293813, + 1550.888550383451 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 34.726506612792036, + "PaddlePaddle/PP-OCRv6_medium": -4.6935653388488845, + "PaddlePaddle/PaddleOCR-VL-1.6": 1.6052693792744321, + "allenai/olmOCR-2-7B-1025-FP8": -22.906583530277203, + "baidu/Qianfan-OCR": 6.664859188114406, + "baidu/Unlimited-OCR": -14.887443326072344, + "deepseek-ai/DeepSeek-OCR": -2.6159441663874077, + "deepseek-ai/DeepSeek-OCR-2": 40.00116899224645, + "lightonai/LightOnOCR-2-1B": 8.444695985202316, + "numind/NuExtract3": -28.287782800902505, + "rednote-hilab/dots.mocr": -6.592846066204174, + "rednote-hilab/dots.ocr": -23.756760985311303, + "tesseract-5": -8.45882498408946, + "zai-org/GLM-OCR": 20.757251040463416 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "kind": "mixed-random", + "max_absolute_elo_delta": 40.00116899224645, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 11.673134155080902, + "name": "mixed-random_2000_seed_43", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9208791208791209, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 44, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1567.8218939493238, + "PaddlePaddle/PP-OCRv6_medium": 1344.2451255324154, + "PaddlePaddle/PaddleOCR-VL-1.6": 1507.3967994261914, + "allenai/olmOCR-2-7B-1025-FP8": 1480.2191151777952, + "baidu/Qianfan-OCR": 1596.7730465391403, + "baidu/Unlimited-OCR": 1560.3681088421688, + "deepseek-ai/DeepSeek-OCR": 1446.1735599967656, + "deepseek-ai/DeepSeek-OCR-2": 1592.9547187573637, + "lightonai/LightOnOCR-2-1B": 1612.207584474081, + "numind/NuExtract3": 1569.152219774452, + "rednote-hilab/dots.mocr": 1545.4559040219756, + "rednote-hilab/dots.ocr": 1530.7775551744517, + "tesseract-5": 1130.3882127068746, + "zai-org/GLM-OCR": 1516.0661556270013 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1528.2975190946834, + 1606.7988428759907 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1300.4473125297407, + 1388.3878277268034 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1466.667352590452, + 1546.487772987278 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1441.3869942241643, + 1517.3883607941912 + ], + "baidu/Qianfan-OCR": [ + 1557.4464505939097, + 1639.6583732901133 + ], + "baidu/Unlimited-OCR": [ + 1520.0003567110837, + 1602.8280229920097 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1403.7942895547594, + 1484.5198490595428 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1554.3638376971583, + 1635.512837087481 + ], + "lightonai/LightOnOCR-2-1B": [ + 1572.4036154492371, + 1652.5294248179757 + ], + "numind/NuExtract3": [ + 1531.6681215435337, + 1610.136956041936 + ], + "rednote-hilab/dots.mocr": [ + 1508.750154673534, + 1585.8625844373487 + ], + "rednote-hilab/dots.ocr": [ + 1494.6075091832085, + 1567.8964693565467 + ], + "tesseract-5": [ + 1052.755217459483, + 1187.100863061981 + ], + "zai-org/GLM-OCR": [ + 1477.3692917282374, + 1555.0233639346466 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 11.444137184363399, + "PaddlePaddle/PP-OCRv6_medium": -10.088173853691842, + "PaddlePaddle/PaddleOCR-VL-1.6": 3.704977206299418, + "allenai/olmOCR-2-7B-1025-FP8": -17.011541110497774, + "baidu/Qianfan-OCR": 20.112237062314534, + "baidu/Unlimited-OCR": -2.9119336816836494, + "deepseek-ai/DeepSeek-OCR": -5.35109240278598, + "deepseek-ai/DeepSeek-OCR-2": 13.59882232046698, + "lightonai/LightOnOCR-2-1B": -5.31126618174676, + "numind/NuExtract3": -2.419969078968961, + "rednote-hilab/dots.mocr": -10.765763489147275, + "rednote-hilab/dots.ocr": 1.5248308225804976, + "tesseract-5": -20.664914455686585, + "zai-org/GLM-OCR": 24.139649658184226 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9120879120879122, + "kind": "mixed-random", + "max_absolute_elo_delta": 24.139649658184226, + "max_absolute_elo_delta_model": "zai-org/GLM-OCR", + "median_absolute_elo_delta": 10.426968671419559, + "name": "mixed-random_2000_seed_44", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9780219780219781, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2" + ], + "top3_membership_matches": true, + "top3_membership_overlap": 3, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 45, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1568.289879761484, + "PaddlePaddle/PP-OCRv6_medium": 1360.5080496215521, + "PaddlePaddle/PaddleOCR-VL-1.6": 1482.9446249148502, + "allenai/olmOCR-2-7B-1025-FP8": 1494.1292208466746, + "baidu/Qianfan-OCR": 1606.3165592146804, + "baidu/Unlimited-OCR": 1597.321448817019, + "deepseek-ai/DeepSeek-OCR": 1448.4484957575162, + "deepseek-ai/DeepSeek-OCR-2": 1570.0725916421022, + "lightonai/LightOnOCR-2-1B": 1618.0230590150675, + "numind/NuExtract3": 1563.3034939904162, + "rednote-hilab/dots.mocr": 1554.6089637057007, + "rednote-hilab/dots.ocr": 1538.4277730888525, + "tesseract-5": 1117.1556947380154, + "zai-org/GLM-OCR": 1480.4501448860694 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1532.7505610728508, + 1608.8757843594617 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1318.9249502190346, + 1403.3128893532275 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1443.7361650300813, + 1524.9507685493595 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1454.842792540175, + 1532.9041537345765 + ], + "baidu/Qianfan-OCR": [ + 1568.7302211159742, + 1646.6740975944363 + ], + "baidu/Unlimited-OCR": [ + 1557.0598268708463, + 1640.8489238978425 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1409.577276146938, + 1483.3907464988476 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1532.163300663281, + 1611.7202689806375 + ], + "lightonai/LightOnOCR-2-1B": [ + 1579.8351546369536, + 1660.8547385993797 + ], + "numind/NuExtract3": [ + 1527.0241001799805, + 1603.3450847053605 + ], + "rednote-hilab/dots.mocr": [ + 1516.2938278950476, + 1597.5802681682044 + ], + "rednote-hilab/dots.ocr": [ + 1502.65475035624, + 1579.261776211167 + ], + "tesseract-5": [ + 1039.9960359736756, + 1174.79969514313 + ], + "zai-org/GLM-OCR": [ + 1441.0556829379098, + 1519.9484349068712 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 11.912122996523522, + "PaddlePaddle/PP-OCRv6_medium": 6.17475023544489, + "PaddlePaddle/PaddleOCR-VL-1.6": -20.74719730504171, + "allenai/olmOCR-2-7B-1025-FP8": -3.101435441618378, + "baidu/Qianfan-OCR": 29.65574973785465, + "baidu/Unlimited-OCR": 34.041406293166574, + "deepseek-ai/DeepSeek-OCR": -3.0761566420353574, + "deepseek-ai/DeepSeek-OCR-2": -9.283304794794503, + "lightonai/LightOnOCR-2-1B": 0.504208359239783, + "numind/NuExtract3": -8.26869486300484, + "rednote-hilab/dots.mocr": -1.612703805422143, + "rednote-hilab/dots.ocr": 9.175048736981353, + "tesseract-5": -33.89743242454574, + "zai-org/GLM-OCR": -11.476361082747644 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8901098901098902, + "kind": "mixed-random", + "max_absolute_elo_delta": 34.041406293166574, + "max_absolute_elo_delta_model": "baidu/Unlimited-OCR", + "median_absolute_elo_delta": 9.229176765887928, + "name": "mixed-random_2000_seed_45", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9648351648351648, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + }, + { + "allocation_decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "allocation_seed": 46, + "balanced_every_n_post_warmup_batches": null, + "comparisons_consumed": 2000, + "comparisons_saved": 2505, + "decision_statuses": { + "prefer-smaller": 0, + "resolved": 2, + "unresolved": 11 + }, + "deterministic_across_repeats": null, + "elo": { + "FireRedTeam/FireRed-OCR": 1586.3302874791902, + "PaddlePaddle/PP-OCRv6_medium": 1334.066100400177, + "PaddlePaddle/PaddleOCR-VL-1.6": 1500.3611306840182, + "allenai/olmOCR-2-7B-1025-FP8": 1510.3048489316286, + "baidu/Qianfan-OCR": 1576.1865507823966, + "baidu/Unlimited-OCR": 1578.6803044221558, + "deepseek-ai/DeepSeek-OCR": 1430.4761399378813, + "deepseek-ai/DeepSeek-OCR-2": 1619.8779291438252, + "lightonai/LightOnOCR-2-1B": 1625.1650995307086, + "numind/NuExtract3": 1547.4540722735708, + "rednote-hilab/dots.mocr": 1533.8214638515783, + "rednote-hilab/dots.ocr": 1530.108675625945, + "tesseract-5": 1121.804248552401, + "zai-org/GLM-OCR": 1505.3631483845236 + }, + "elo_ci": { + "FireRedTeam/FireRed-OCR": [ + 1550.0435665319062, + 1626.0829877905921 + ], + "PaddlePaddle/PP-OCRv6_medium": [ + 1288.7612350624386, + 1377.1888808130768 + ], + "PaddlePaddle/PaddleOCR-VL-1.6": [ + 1461.520808782099, + 1540.64907471849 + ], + "allenai/olmOCR-2-7B-1025-FP8": [ + 1471.903652287295, + 1548.4289369946655 + ], + "baidu/Qianfan-OCR": [ + 1537.098168696444, + 1620.292412594707 + ], + "baidu/Unlimited-OCR": [ + 1542.1334241271202, + 1623.6171091528163 + ], + "deepseek-ai/DeepSeek-OCR": [ + 1389.0615146575417, + 1467.5854852242603 + ], + "deepseek-ai/DeepSeek-OCR-2": [ + 1578.469504106214, + 1662.0775831244546 + ], + "lightonai/LightOnOCR-2-1B": [ + 1585.2711904824343, + 1664.7280269209343 + ], + "numind/NuExtract3": [ + 1510.2412340781607, + 1583.591879887805 + ], + "rednote-hilab/dots.mocr": [ + 1498.5223115680697, + 1574.0270429548034 + ], + "rednote-hilab/dots.ocr": [ + 1493.649549488542, + 1564.485460382485 + ], + "tesseract-5": [ + 1049.4121839468871, + 1177.0443491335118 + ], + "zai-org/GLM-OCR": [ + 1466.473215000129, + 1544.944538421565 + ] + }, + "elo_delta_from_full": { + "FireRedTeam/FireRed-OCR": 29.95253071422985, + "PaddlePaddle/PP-OCRv6_medium": -20.267198985930236, + "PaddlePaddle/PaddleOCR-VL-1.6": -3.3306915358737115, + "allenai/olmOCR-2-7B-1025-FP8": 13.074192643335664, + "baidu/Qianfan-OCR": -0.47425869442918156, + "baidu/Unlimited-OCR": 15.40026189830337, + "deepseek-ai/DeepSeek-OCR": -21.04851246167027, + "deepseek-ai/DeepSeek-OCR-2": 40.522032706928485, + "lightonai/LightOnOCR-2-1B": 7.646248874880939, + "numind/NuExtract3": -24.118116579850266, + "rednote-hilab/dots.mocr": -22.40020365954456, + "rednote-hilab/dots.ocr": 0.8559512740737318, + "tesseract-5": -29.24887861016009, + "zai-org/GLM-OCR": 13.436642415706501 + }, + "final_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "followup_acceptance_gate": { + "kendall_tau_at_least_0_95": false, + "max_absolute_elo_delta_at_most_50": true, + "passed": false, + "percentage_saved_at_least_60": false, + "top3_order_exact": false + }, + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 22, + "median_direct_comparisons": 22, + "min_direct_comparisons": 21, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "kind": "mixed-random", + "max_absolute_elo_delta": 40.522032706928485, + "max_absolute_elo_delta_model": "deepseek-ai/DeepSeek-OCR-2", + "median_absolute_elo_delta": 17.833730442116803, + "name": "mixed-random_2000_seed_46", + "percentage_saved": 55.60488346281909, + "practical_preferences": [], + "resolution_summary": "predeclared outcome-independent budget; no adaptive stop", + "round_history": [], + "size_rule_controls_sampling": true, + "size_tie_min_samples": 10, + "size_tie_ratio": null, + "spearman_rho": 0.9472527472527472, + "static_budget": 2000, + "stopping_reason": "predeclared_static_budget", + "stopping_round": 0, + "top3": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR" + ], + "top3_membership_matches": false, + "top3_membership_overlap": 2, + "top3_order_matches": false, + "warmup_min_per_pair": null + } + ] +} diff --git a/experiments/adaptive-stopping/moh/round-history.csv b/experiments/adaptive-stopping/moh/round-history.csv new file mode 100644 index 0000000..1f311ca --- /dev/null +++ b/experiments/adaptive-stopping/moh/round-history.csv @@ -0,0 +1,131 @@ +strategy,round,sample_indices,batch_comparisons,cumulative_comparisons,allocation_mode,active_pairs,next_active_pairs,min_direct_comparisons,resolved,prefer_smaller,unresolved +balanced_adaptive,1,"[0, 1, 2, 3, 4]",455,455,balanced,,12,,1,0,12 +balanced_adaptive,2,"[5, 6, 7, 8, 9]",454,909,balanced,,12,,1,0,12 +balanced_adaptive,3,"[10, 11, 12, 13, 14]",454,1363,balanced,,12,,1,0,12 +balanced_adaptive,4,"[15, 16, 17, 18, 19]",455,1818,balanced,,12,,1,0,12 +balanced_adaptive,5,"[20, 21, 22, 23, 24]",455,2273,balanced,,12,,1,0,12 +balanced_adaptive,6,"[25, 26, 27, 28, 29]",455,2728,balanced,,11,,2,0,11 +balanced_adaptive,7,"[30, 31, 32, 33, 34]",455,3183,balanced,,11,,2,0,11 +balanced_adaptive,8,"[35, 36, 37, 38, 39]",418,3601,balanced,,11,,2,0,11 +balanced_adaptive,9,"[40, 41, 42, 43, 44]",449,4050,balanced,,11,,2,0,11 +balanced_adaptive,10,"[45, 46, 47, 48, 49]",455,4505,balanced,,11,,2,0,11 +targeted,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,12,,1,0,12 +targeted,3,"[10, 11, 12, 13, 14]",60,575,targeted,12,12,,1,0,12 +targeted,4,"[15, 16, 17, 18, 19]",60,635,targeted,12,12,,1,0,12 +targeted,5,"[20, 21, 22, 23, 24]",60,695,targeted,12,12,,1,0,12 +targeted,6,"[25, 26, 27, 28, 29]",60,755,targeted,12,12,,1,0,12 +targeted,7,"[30, 31, 32, 33, 34]",60,815,targeted,12,12,,1,0,12 +targeted,8,"[35, 36, 37, 38, 39]",57,872,targeted,12,13,,0,0,13 +targeted,9,"[40, 41, 42, 43, 44]",64,936,targeted,13,12,,1,0,12 +targeted,10,"[45, 46, 47, 48, 49]",60,996,targeted,12,12,,1,0,12 +targeted_size_3_min_10,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted_size_3_min_10,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,8,,1,4,8 +targeted_size_3_min_10,3,"[10, 11, 12, 13, 14]",40,555,targeted,8,7,,1,5,7 +targeted_size_3_min_10,4,"[15, 16, 17, 18, 19]",35,590,targeted,7,7,,1,5,7 +targeted_size_3_min_10,5,"[20, 21, 22, 23, 24]",35,625,targeted,7,8,,1,4,8 +targeted_size_3_min_10,6,"[25, 26, 27, 28, 29]",40,665,targeted,8,7,,1,5,7 +targeted_size_3_min_10,7,"[30, 31, 32, 33, 34]",35,700,targeted,7,7,,1,5,7 +targeted_size_3_min_10,8,"[35, 36, 37, 38, 39]",34,734,targeted,7,7,,1,5,7 +targeted_size_3_min_10,9,"[40, 41, 42, 43, 44]",34,768,targeted,7,7,,1,5,7 +targeted_size_3_min_10,10,"[45, 46, 47, 48, 49]",35,803,targeted,7,7,,1,5,7 +targeted_size_2_min_10,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted_size_2_min_10,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,5,,1,7,5 +targeted_size_2_min_10,3,"[10, 11, 12, 13, 14]",25,540,targeted,5,4,,1,8,4 +targeted_size_2_min_10,4,"[15, 16, 17, 18, 19]",20,560,targeted,4,4,,1,8,4 +targeted_size_2_min_10,5,"[20, 21, 22, 23, 24]",20,580,targeted,4,6,,1,6,6 +targeted_size_2_min_10,6,"[25, 26, 27, 28, 29]",30,610,targeted,6,4,,1,8,4 +targeted_size_2_min_10,7,"[30, 31, 32, 33, 34]",20,630,targeted,4,5,,1,7,5 +targeted_size_2_min_10,8,"[35, 36, 37, 38, 39]",24,654,targeted,5,5,,1,7,5 +targeted_size_2_min_10,9,"[40, 41, 42, 43, 44]",24,678,targeted,5,5,,1,7,5 +targeted_size_2_min_10,10,"[45, 46, 47, 48, 49]",25,703,targeted,5,5,,1,7,5 +targeted_size_3_min_5,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,8,,1,4,8 +targeted_size_3_min_5,2,"[5, 6, 7, 8, 9]",40,495,targeted,8,7,,1,5,7 +targeted_size_3_min_5,3,"[10, 11, 12, 13, 14]",35,530,targeted,7,7,,1,5,7 +targeted_size_3_min_5,4,"[15, 16, 17, 18, 19]",35,565,targeted,7,7,,1,5,7 +targeted_size_3_min_5,5,"[20, 21, 22, 23, 24]",35,600,targeted,7,8,,0,5,8 +targeted_size_3_min_5,6,"[25, 26, 27, 28, 29]",40,640,targeted,8,8,,0,5,8 +targeted_size_3_min_5,7,"[30, 31, 32, 33, 34]",40,680,targeted,8,7,,1,5,7 +targeted_size_3_min_5,8,"[35, 36, 37, 38, 39]",34,714,targeted,7,7,,1,5,7 +targeted_size_3_min_5,9,"[40, 41, 42, 43, 44]",34,748,targeted,7,7,,1,5,7 +targeted_size_3_min_5,10,"[45, 46, 47, 48, 49]",35,783,targeted,7,7,,1,5,7 +targeted_size_3_min_15,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted_size_3_min_15,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,12,,1,0,12 +targeted_size_3_min_15,3,"[10, 11, 12, 13, 14]",60,575,targeted,12,8,,1,4,8 +targeted_size_3_min_15,4,"[15, 16, 17, 18, 19]",40,615,targeted,8,8,,0,5,8 +targeted_size_3_min_15,5,"[20, 21, 22, 23, 24]",40,655,targeted,8,8,,1,4,8 +targeted_size_3_min_15,6,"[25, 26, 27, 28, 29]",40,695,targeted,8,7,,1,5,7 +targeted_size_3_min_15,7,"[30, 31, 32, 33, 34]",35,730,targeted,7,8,,1,4,8 +targeted_size_3_min_15,8,"[35, 36, 37, 38, 39]",38,768,targeted,8,8,,1,4,8 +targeted_size_3_min_15,9,"[40, 41, 42, 43, 44]",39,807,targeted,8,7,,1,5,7 +targeted_size_3_min_15,10,"[45, 46, 47, 48, 49]",35,842,targeted,7,7,,1,5,7 +targeted_size_5_min_10,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted_size_5_min_10,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,11,,1,1,11 +targeted_size_5_min_10,3,"[10, 11, 12, 13, 14]",55,570,targeted,11,10,,1,2,10 +targeted_size_5_min_10,4,"[15, 16, 17, 18, 19]",50,620,targeted,10,10,,1,2,10 +targeted_size_5_min_10,5,"[20, 21, 22, 23, 24]",50,670,targeted,10,11,,1,1,11 +targeted_size_5_min_10,6,"[25, 26, 27, 28, 29]",55,725,targeted,11,11,,0,2,11 +targeted_size_5_min_10,7,"[30, 31, 32, 33, 34]",55,780,targeted,11,10,,1,2,10 +targeted_size_5_min_10,8,"[35, 36, 37, 38, 39]",47,827,targeted,10,10,,1,2,10 +targeted_size_5_min_10,9,"[40, 41, 42, 43, 44]",49,876,targeted,10,10,,1,2,10 +targeted_size_5_min_10,10,"[45, 46, 47, 48, 49]",50,926,targeted,10,9,,1,3,9 +targeted_v2_warmup_5,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_5,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,3,"[10, 11, 12, 13, 14]",60,575,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,4,"[15, 16, 17, 18, 19]",60,635,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,5,"[20, 21, 22, 23, 24]",60,695,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,6,"[25, 26, 27, 28, 29]",60,755,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,7,"[30, 31, 32, 33, 34]",60,815,targeted,12,12,,1,0,12 +targeted_v2_warmup_5,8,"[35, 36, 37, 38, 39]",57,872,targeted,12,13,,0,0,13 +targeted_v2_warmup_5,9,"[40, 41, 42, 43, 44]",64,936,targeted,13,12,,1,0,12 +targeted_v2_warmup_5,10,"[45, 46, 47, 48, 49]",60,996,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,,5,,, +targeted_v2_warmup_10,2,"[5, 6, 7, 8, 9]",454,909,balanced-warmup,,,9,,, +targeted_v2_warmup_10,3,"[10, 11, 12, 13, 14]",454,1363,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_10,4,"[15, 16, 17, 18, 19]",60,1423,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,5,"[20, 21, 22, 23, 24]",60,1483,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,6,"[25, 26, 27, 28, 29]",60,1543,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,7,"[30, 31, 32, 33, 34]",60,1603,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,8,"[35, 36, 37, 38, 39]",55,1658,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,9,"[40, 41, 42, 43, 44]",59,1717,targeted,12,12,,1,0,12 +targeted_v2_warmup_10,10,"[45, 46, 47, 48, 49]",60,1777,targeted,12,11,,2,0,11 +targeted_v2_explore_every_3,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted_v2_explore_every_3,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,12,,1,0,12 +targeted_v2_explore_every_3,3,"[10, 11, 12, 13, 14]",60,575,targeted,12,12,,1,0,12 +targeted_v2_explore_every_3,4,"[15, 16, 17, 18, 19]",455,1030,balanced-exploration,12,12,,1,0,12 +targeted_v2_explore_every_3,5,"[20, 21, 22, 23, 24]",60,1090,targeted,12,12,,1,0,12 +targeted_v2_explore_every_3,6,"[25, 26, 27, 28, 29]",60,1150,targeted,12,12,,1,0,12 +targeted_v2_explore_every_3,7,"[30, 31, 32, 33, 34]",455,1605,balanced-exploration,12,11,,2,0,11 +targeted_v2_explore_every_3,8,"[35, 36, 37, 38, 39]",53,1658,targeted,11,11,,2,0,11 +targeted_v2_explore_every_3,9,"[40, 41, 42, 43, 44]",54,1712,targeted,11,11,,2,0,11 +targeted_v2_explore_every_3,10,"[45, 46, 47, 48, 49]",455,2167,balanced-exploration,11,11,,2,0,11 +targeted_v2_warmup_5_explore_3_annotate_3x,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,3,"[10, 11, 12, 13, 14]",60,575,targeted,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,4,"[15, 16, 17, 18, 19]",455,1030,balanced-exploration,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,5,"[20, 21, 22, 23, 24]",60,1090,targeted,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,6,"[25, 26, 27, 28, 29]",60,1150,targeted,12,12,,1,0,12 +targeted_v2_warmup_5_explore_3_annotate_3x,7,"[30, 31, 32, 33, 34]",455,1605,balanced-exploration,12,11,,2,0,11 +targeted_v2_warmup_5_explore_3_annotate_3x,8,"[35, 36, 37, 38, 39]",53,1658,targeted,11,11,,2,0,11 +targeted_v2_warmup_5_explore_3_annotate_3x,9,"[40, 41, 42, 43, 44]",54,1712,targeted,11,11,,2,0,11 +targeted_v2_warmup_5_explore_3_annotate_3x,10,"[45, 46, 47, 48, 49]",455,2167,balanced-exploration,11,11,,2,0,11 +targeted_v2_warmup_10_explore_3_annotate_3x,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,,5,,, +targeted_v2_warmup_10_explore_3_annotate_3x,2,"[5, 6, 7, 8, 9]",454,909,balanced-warmup,,,9,,, +targeted_v2_warmup_10_explore_3_annotate_3x,3,"[10, 11, 12, 13, 14]",454,1363,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_10_explore_3_annotate_3x,4,"[15, 16, 17, 18, 19]",60,1423,targeted,12,12,,1,0,12 +targeted_v2_warmup_10_explore_3_annotate_3x,5,"[20, 21, 22, 23, 24]",60,1483,targeted,12,12,,1,0,12 +targeted_v2_warmup_10_explore_3_annotate_3x,6,"[25, 26, 27, 28, 29]",455,1938,balanced-exploration,12,11,,2,0,11 +targeted_v2_warmup_10_explore_3_annotate_3x,7,"[30, 31, 32, 33, 34]",55,1993,targeted,11,11,,2,0,11 +targeted_v2_warmup_10_explore_3_annotate_3x,8,"[35, 36, 37, 38, 39]",52,2045,targeted,11,11,,2,0,11 +targeted_v2_warmup_10_explore_3_annotate_3x,9,"[40, 41, 42, 43, 44]",449,2494,balanced-exploration,11,11,,2,0,11 +targeted_v2_warmup_10_explore_3_annotate_3x,10,"[45, 46, 47, 48, 49]",55,2549,targeted,11,11,,2,0,11 +targeted_v2_warmup_5_explore_3_size_stop_3x,1,"[0, 1, 2, 3, 4]",455,455,balanced-warmup,,12,,1,0,12 +targeted_v2_warmup_5_explore_3_size_stop_3x,2,"[5, 6, 7, 8, 9]",60,515,targeted,12,8,,1,4,8 +targeted_v2_warmup_5_explore_3_size_stop_3x,3,"[10, 11, 12, 13, 14]",40,555,targeted,8,7,,1,5,7 +targeted_v2_warmup_5_explore_3_size_stop_3x,4,"[15, 16, 17, 18, 19]",455,1010,balanced-exploration,7,8,,1,4,8 +targeted_v2_warmup_5_explore_3_size_stop_3x,5,"[20, 21, 22, 23, 24]",40,1050,targeted,8,8,,1,4,8 +targeted_v2_warmup_5_explore_3_size_stop_3x,6,"[25, 26, 27, 28, 29]",40,1090,targeted,8,8,,1,4,8 +targeted_v2_warmup_5_explore_3_size_stop_3x,7,"[30, 31, 32, 33, 34]",455,1545,balanced-exploration,8,6,,2,5,6 +targeted_v2_warmup_5_explore_3_size_stop_3x,8,"[35, 36, 37, 38, 39]",29,1574,targeted,6,6,,2,5,6 +targeted_v2_warmup_5_explore_3_size_stop_3x,9,"[40, 41, 42, 43, 44]",29,1603,targeted,6,6,,2,5,6 +targeted_v2_warmup_5_explore_3_size_stop_3x,10,"[45, 46, 47, 48, 49]",455,2058,balanced-exploration,6,9,,2,2,9 diff --git a/experiments/adaptive-stopping/moh/static-design-summary.csv b/experiments/adaptive-stopping/moh/static-design-summary.csv new file mode 100644 index 0000000..536d7f2 --- /dev/null +++ b/experiments/adaptive-stopping/moh/static-design-summary.csv @@ -0,0 +1,7 @@ +kind,budget,percentage_saved,kendall_tau_median,kendall_tau_min,kendall_tau_max,spearman_rho_median,top3_membership_matches,top3_order_matches,median_absolute_elo_delta_median,max_absolute_elo_delta_median,max_absolute_elo_delta_worst,minimum_pair_evidence,gate_passes,runs +mixed-random,700,84.46170921198669,0.7802197802197802,0.7142857142857144,0.8461538461538461,0.9208791208791209,0,0,40.91301945377552,101.54558005476588,116.81052616962552,7,0,5 +mixed-random,1200,73.36293007769146,0.8021978021978022,0.7802197802197802,0.8681318681318682,0.9296703296703297,0,0,23.835583467567176,79.52519843529308,84.0886776847276,13,0,5 +mixed-random,2000,55.60488346281909,0.8681318681318682,0.8021978021978022,0.9120879120879122,0.9472527472527472,1,0,10.426968671419559,39.83207465401688,40.522032706928485,21,0,5 +pair-balanced,700,84.46170921198669,0.7802197802197802,0.7142857142857144,0.8241758241758242,0.8945054945054945,0,0,21.429977055331733,66.35516817329449,98.65008366570828,7,0,5 +pair-balanced,1200,73.36293007769146,0.8241758241758242,0.8241758241758242,0.8681318681318682,0.9340659340659341,1,0,16.856581791943313,36.62551075270494,65.05221988772882,13,0,5 +pair-balanced,2000,55.60488346281909,0.8681318681318682,0.8021978021978022,0.9560439560439562,0.9472527472527472,0,0,8.846758982956544,30.303948386869934,33.81672867535144,21,0,5 diff --git a/experiments/adaptive-stopping/moh/strategy-summary.csv b/experiments/adaptive-stopping/moh/strategy-summary.csv new file mode 100644 index 0000000..426f4c8 --- /dev/null +++ b/experiments/adaptive-stopping/moh/strategy-summary.csv @@ -0,0 +1,45 @@ +name,comparisons_consumed,comparisons_saved,percentage_saved,stopping_round,stopping_reason,kendall_tau,spearman_rho,top3_membership_matches,top3_order_matches,median_absolute_elo_delta,max_absolute_elo_delta,followup_gate_passed,deterministic_across_repeats +full_stored_board,4505,0,0.0,10,reference_full_stored_board,1.0,1.0,True,True,0.0,0.0,False,True +balanced_adaptive,4505,0,0.0,10,sample_batches_exhausted,1.0,1.0,True,True,0.0,0.0,False,True +targeted,996,3509,77.89123196448391,10,sample_batches_exhausted,0.8461538461538461,0.9604395604395607,True,False,46.23768255756761,115.66948359261414,False,True +targeted_size_3_min_10,803,3702,82.17536071032187,10,sample_batches_exhausted,0.7802197802197802,0.9252747252747253,True,False,45.89727096303113,114.82024023801296,False,True +targeted_size_2_min_10,703,3802,84.39511653718091,10,sample_batches_exhausted,0.7802197802197802,0.9252747252747253,True,False,46.34282090319073,104.28082055473556,False, +targeted_size_3_min_5,783,3722,82.61931187569367,10,sample_batches_exhausted,0.7802197802197802,0.9252747252747253,True,False,46.75618036034473,115.90830645056553,False, +targeted_size_3_min_15,842,3663,81.30965593784684,10,sample_batches_exhausted,0.7802197802197802,0.9252747252747253,True,False,45.683112926215244,121.89598666526058,False, +targeted_size_5_min_10,926,3579,79.44506104328524,10,sample_batches_exhausted,0.7802197802197802,0.9252747252747253,True,False,47.79861858551715,119.09791584350728,False, +targeted_v2_warmup_5,996,3509,77.89123196448391,10,sample_batches_exhausted,0.8461538461538461,0.9604395604395607,True,False,46.23768255756761,115.66948359261414,False, +targeted_v2_warmup_10,1777,2728,60.554938956714764,10,sample_batches_exhausted,0.8681318681318682,0.9648351648351648,False,False,31.886074901780262,73.01683119859695,False, +targeted_v2_explore_every_3,2167,2338,51.897891231964486,10,sample_batches_exhausted,0.8241758241758242,0.9164835164835166,True,True,15.581150107184385,70.76052132718769,False, +targeted_v2_warmup_5_explore_3_annotate_3x,2167,2338,51.897891231964486,10,sample_batches_exhausted,0.8241758241758242,0.9164835164835166,True,True,15.581150107184385,70.76052132718769,False,True +targeted_v2_warmup_10_explore_3_annotate_3x,2549,1956,43.41842397336293,10,sample_batches_exhausted,0.8461538461538461,0.9384615384615385,False,False,9.72920769019936,57.906225277884914,False, +targeted_v2_warmup_5_explore_3_size_stop_3x,2058,2447,54.31742508324084,10,sample_batches_exhausted,0.8021978021978022,0.8989010989010989,True,True,12.454331152961913,70.59855691399412,False, +pair-balanced_700_seed_42,700,3805,84.46170921198669,0,predeclared_static_budget,0.7802197802197802,0.8945054945054945,False,False,21.429977055331733,85.49686515925214,False, +pair-balanced_700_seed_43,700,3805,84.46170921198669,0,predeclared_static_budget,0.7802197802197802,0.8945054945054945,False,False,19.485038845188342,98.65008366570828,False, +pair-balanced_700_seed_44,700,3805,84.46170921198669,0,predeclared_static_budget,0.7142857142857144,0.8725274725274725,False,False,21.601709340888533,44.9305243056142,False, +pair-balanced_700_seed_45,700,3805,84.46170921198669,0,predeclared_static_budget,0.7362637362637363,0.8813186813186813,False,False,13.517655883403563,64.69416029671243,False, +pair-balanced_700_seed_46,700,3805,84.46170921198669,0,predeclared_static_budget,0.8241758241758242,0.9164835164835166,False,False,22.123682208599234,66.35516817329449,False, +pair-balanced_1200_seed_42,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8241758241758242,0.9252747252747253,False,False,12.48139493620738,65.05221988772882,False, +pair-balanced_1200_seed_43,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8241758241758242,0.9428571428571428,True,False,16.856581791943313,42.579617892972465,False, +pair-balanced_1200_seed_44,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8681318681318682,0.956043956043956,False,False,10.45910385358252,36.62551075270494,False, +pair-balanced_1200_seed_45,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8241758241758242,0.9208791208791209,False,False,17.646765866252508,36.47492009421535,False, +pair-balanced_1200_seed_46,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8461538461538461,0.9340659340659341,False,False,17.955619371834814,34.81014009549199,False, +pair-balanced_2000_seed_42,2000,2505,55.60488346281909,0,predeclared_static_budget,0.8021978021978022,0.9120879120879121,False,False,8.846758982956544,33.81672867535144,False, +pair-balanced_2000_seed_43,2000,2505,55.60488346281909,0,predeclared_static_budget,0.9120879120879122,0.9736263736263736,False,False,6.722059868035899,32.03154678837359,False, +pair-balanced_2000_seed_44,2000,2505,55.60488346281909,0,predeclared_static_budget,0.8461538461538461,0.9384615384615385,False,False,11.909483643468434,30.303948386869934,False, +pair-balanced_2000_seed_45,2000,2505,55.60488346281909,0,predeclared_static_budget,0.8681318681318682,0.9472527472527472,False,False,10.24949264581278,21.17849029106992,False, +pair-balanced_2000_seed_46,2000,2505,55.60488346281909,0,predeclared_static_budget,0.9560439560439562,0.9912087912087912,False,False,8.310378035397889,29.926902681557976,False, +mixed-random_700_seed_42,700,3805,84.46170921198669,0,predeclared_static_budget,0.8241758241758242,0.9428571428571428,False,False,37.8454745346512,101.54558005476588,False, +mixed-random_700_seed_43,700,3805,84.46170921198669,0,predeclared_static_budget,0.7802197802197802,0.9120879120879121,False,False,43.13383700507063,70.79639055443386,False, +mixed-random_700_seed_44,700,3805,84.46170921198669,0,predeclared_static_budget,0.7142857142857144,0.8637362637362638,False,False,56.06403624686436,116.81052616962552,False, +mixed-random_700_seed_45,700,3805,84.46170921198669,0,predeclared_static_budget,0.8461538461538461,0.9516483516483516,False,False,22.2229331109005,102.7779233858962,False, +mixed-random_700_seed_46,700,3805,84.46170921198669,0,predeclared_static_budget,0.7802197802197802,0.9208791208791209,False,False,40.91301945377552,96.27231906382167,False, +mixed-random_1200_seed_42,1200,3305,73.36293007769146,0,predeclared_static_budget,0.7802197802197802,0.9208791208791209,False,False,26.134509773544323,79.52519843529308,False, +mixed-random_1200_seed_43,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8021978021978022,0.9296703296703297,False,False,23.835583467567176,82.47978272477826,False, +mixed-random_1200_seed_44,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8021978021978022,0.9296703296703297,False,False,17.87771118769149,58.50048445495372,False, +mixed-random_1200_seed_45,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8681318681318682,0.956043956043956,False,False,19.21594969525745,84.0886776847276,False, +mixed-random_1200_seed_46,1200,3305,73.36293007769146,0,predeclared_static_budget,0.8241758241758242,0.9296703296703297,False,False,25.874255761696077,45.33343642098134,False, +mixed-random_2000_seed_42,2000,2505,55.60488346281909,0,predeclared_static_budget,0.8681318681318682,0.9340659340659341,False,False,7.729814588957424,39.83207465401688,False, +mixed-random_2000_seed_43,2000,2505,55.60488346281909,0,predeclared_static_budget,0.8021978021978022,0.9208791208791209,False,False,11.673134155080902,40.00116899224645,False, +mixed-random_2000_seed_44,2000,2505,55.60488346281909,0,predeclared_static_budget,0.9120879120879122,0.9780219780219781,True,False,10.426968671419559,24.139649658184226,False, +mixed-random_2000_seed_45,2000,2505,55.60488346281909,0,predeclared_static_budget,0.8901098901098902,0.9648351648351648,False,False,9.229176765887928,34.041406293166574,False, +mixed-random_2000_seed_46,2000,2505,55.60488346281909,0,predeclared_static_budget,0.8461538461538461,0.9472527472527472,False,False,17.833730442116803,40.522032706928485,False, diff --git a/experiments/adaptive-stopping/multi-board-results.json b/experiments/adaptive-stopping/multi-board-results.json index 5350d29..589a3cf 100644 --- a/experiments/adaptive-stopping/multi-board-results.json +++ b/experiments/adaptive-stopping/multi-board-results.json @@ -1475,6 +1475,252 @@ "possible_pairs": 15, "robust_full_board_pairs": 13, "role": "same-corpus judge/model-grid sensitivity" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.8241758241758242, + "clustered_kendall_tau_p05": 0.7142857142857144, + "clustered_max_absolute_elo_delta_median": 48.939915803189024, + "clustered_max_absolute_elo_delta_p95": 84.31332593481861, + "clustered_robust_pair_agreement": 0.9773725490196078, + "clustered_spearman_rho_median": 0.9384615384615385, + "clustered_topk_membership_rate": 0.34, + "clustered_topk_order_rate": 0.132, + "collection": "moh", + "comparisons_consumed": 1126, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 75.00554938956715, + "point_kendall_tau_max": 0.8461538461538461, + "point_kendall_tau_median": 0.8241758241758242, + "point_kendall_tau_min": 0.7582417582417583, + "point_max_absolute_elo_delta_median": 43.531941958760854, + "point_max_absolute_elo_delta_worst": 84.60404452855505, + "point_spearman_rho_median": 0.9296703296703297, + "point_topk_membership_matches": 1, + "point_topk_order_matches": 0, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.8681318681318682, + "clustered_kendall_tau_p05": 0.7582417582417583, + "clustered_max_absolute_elo_delta_median": 34.285550887453155, + "clustered_max_absolute_elo_delta_p95": 55.81184566831936, + "clustered_robust_pair_agreement": 0.9844313725490196, + "clustered_spearman_rho_median": 0.9604395604395607, + "clustered_topk_membership_rate": 0.474, + "clustered_topk_order_rate": 0.224, + "collection": "moh", + "comparisons_consumed": 1802, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 19, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.9340659340659341, + "point_kendall_tau_median": 0.8241758241758242, + "point_kendall_tau_min": 0.8021978021978022, + "point_max_absolute_elo_delta_median": 30.757182086236526, + "point_max_absolute_elo_delta_worst": 37.53302303767623, + "point_spearman_rho_median": 0.9252747252747253, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 1, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.9120879120879122, + "clustered_kendall_tau_p05": 0.8241758241758242, + "clustered_max_absolute_elo_delta_median": 22.887958838301074, + "clustered_max_absolute_elo_delta_p95": 36.55624739703296, + "clustered_robust_pair_agreement": 0.9885294117647059, + "clustered_spearman_rho_median": 0.9780219780219781, + "clustered_topk_membership_rate": 0.615, + "clustered_topk_order_rate": 0.347, + "collection": "moh", + "comparisons_consumed": 2703, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 29, + "percentage_saved": 40.0, + "point_kendall_tau_max": 0.9560439560439562, + "point_kendall_tau_median": 0.9120879120879122, + "point_kendall_tau_min": 0.8461538461538461, + "point_max_absolute_elo_delta_median": 20.111822912160733, + "point_max_absolute_elo_delta_worst": 25.11475250155536, + "point_spearman_rho_median": 0.9736263736263736, + "point_topk_membership_matches": 3, + "point_topk_order_matches": 2, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.7802197802197802, + "clustered_kendall_tau_p05": 0.6043956043956044, + "clustered_max_absolute_elo_delta_median": 67.13236783675984, + "clustered_max_absolute_elo_delta_p95": 113.83754428615104, + "clustered_robust_pair_agreement": 0.9658627450980392, + "clustered_spearman_rho_median": 0.9032967032967033, + "clustered_topk_membership_rate": 0.228, + "clustered_topk_order_rate": 0.074, + "collection": "moh", + "comparisons_consumed": 1126, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 75.00554938956715, + "point_kendall_tau_max": 0.8021978021978022, + "point_kendall_tau_median": 0.8021978021978022, + "point_kendall_tau_min": 0.7802197802197802, + "point_max_absolute_elo_delta_median": 75.07628080257223, + "point_max_absolute_elo_delta_worst": 103.77096208907892, + "point_spearman_rho_median": 0.9296703296703297, + "point_topk_membership_matches": 0, + "point_topk_order_matches": 0, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.8461538461538461, + "clustered_kendall_tau_p05": 0.7362637362637363, + "clustered_max_absolute_elo_delta_median": 40.96385137541381, + "clustered_max_absolute_elo_delta_p95": 68.24805202727987, + "clustered_robust_pair_agreement": 0.9825294117647059, + "clustered_spearman_rho_median": 0.9494505494505494, + "clustered_topk_membership_rate": 0.372, + "clustered_topk_order_rate": 0.169, + "collection": "moh", + "comparisons_consumed": 1802, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 19, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.8901098901098902, + "point_kendall_tau_median": 0.8461538461538461, + "point_kendall_tau_min": 0.8461538461538461, + "point_max_absolute_elo_delta_median": 43.66100877748113, + "point_max_absolute_elo_delta_worst": 51.03398034125166, + "point_spearman_rho_median": 0.9472527472527472, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 1, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.9120879120879122, + "clustered_kendall_tau_p05": 0.8241758241758242, + "clustered_max_absolute_elo_delta_median": 24.994304320712672, + "clustered_max_absolute_elo_delta_p95": 40.05965087285978, + "clustered_robust_pair_agreement": 0.987921568627451, + "clustered_spearman_rho_median": 0.9736263736263736, + "clustered_topk_membership_rate": 0.561, + "clustered_topk_order_rate": 0.308, + "collection": "moh", + "comparisons_consumed": 2703, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 29, + "percentage_saved": 40.0, + "point_kendall_tau_max": 0.9120879120879122, + "point_kendall_tau_median": 0.8901098901098902, + "point_kendall_tau_min": 0.8241758241758242, + "point_max_absolute_elo_delta_median": 29.58597113580481, + "point_max_absolute_elo_delta_worst": 35.15264403241281, + "point_spearman_rho_median": 0.9648351648351648, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 1, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" } ], "boards": [ @@ -10977,55 +11223,2443 @@ ], "sentinel_comparisons_detectable": 244, "stored_comparisons": 720 - } - ], - "collection_design_pass_counts": { - "bpl:mixed-random@25%": { - "passed": 0, - "total": 1 - }, - "bpl:mixed-random@40%": { - "passed": 0, - "total": 1 - }, - "bpl:mixed-random@60%": { - "passed": 0, - "total": 1 - }, - "bpl:pair-balanced@25%": { - "passed": 0, - "total": 1 - }, - "bpl:pair-balanced@40%": { - "passed": 0, - "total": 1 - }, - "bpl:pair-balanced@60%": { - "passed": 0, - "total": 1 - }, - "britannica:mixed-random@25%": { - "passed": 1, - "total": 1 - }, - "britannica:mixed-random@40%": { - "passed": 1, - "total": 1 - }, - "britannica:mixed-random@60%": { - "passed": 1, - "total": 1 - }, - "britannica:pair-balanced@25%": { - "passed": 1, - "total": 1 - }, - "britannica:pair-balanced@40%": { - "passed": 1, - "total": 1 }, - "britannica:pair-balanced@60%": { - "passed": 1, + { + "aggregates": [ + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.8241758241758242, + "clustered_kendall_tau_p05": 0.7142857142857144, + "clustered_max_absolute_elo_delta_median": 48.939915803189024, + "clustered_max_absolute_elo_delta_p95": 84.31332593481861, + "clustered_robust_pair_agreement": 0.9773725490196078, + "clustered_spearman_rho_median": 0.9384615384615385, + "clustered_topk_membership_rate": 0.34, + "clustered_topk_order_rate": 0.132, + "collection": "moh", + "comparisons_consumed": 1126, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 12, + "percentage_saved": 75.00554938956715, + "point_kendall_tau_max": 0.8461538461538461, + "point_kendall_tau_median": 0.8241758241758242, + "point_kendall_tau_min": 0.7582417582417583, + "point_max_absolute_elo_delta_median": 43.531941958760854, + "point_max_absolute_elo_delta_worst": 84.60404452855505, + "point_spearman_rho_median": 0.9296703296703297, + "point_topk_membership_matches": 1, + "point_topk_order_matches": 0, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.8681318681318682, + "clustered_kendall_tau_p05": 0.7582417582417583, + "clustered_max_absolute_elo_delta_median": 34.285550887453155, + "clustered_max_absolute_elo_delta_p95": 55.81184566831936, + "clustered_robust_pair_agreement": 0.9844313725490196, + "clustered_spearman_rho_median": 0.9604395604395607, + "clustered_topk_membership_rate": 0.474, + "clustered_topk_order_rate": 0.224, + "collection": "moh", + "comparisons_consumed": 1802, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 19, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.9340659340659341, + "point_kendall_tau_median": 0.8241758241758242, + "point_kendall_tau_min": 0.8021978021978022, + "point_max_absolute_elo_delta_median": 30.757182086236526, + "point_max_absolute_elo_delta_worst": 37.53302303767623, + "point_spearman_rho_median": 0.9252747252747253, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 1, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.9120879120879122, + "clustered_kendall_tau_p05": 0.8241758241758242, + "clustered_max_absolute_elo_delta_median": 22.887958838301074, + "clustered_max_absolute_elo_delta_p95": 36.55624739703296, + "clustered_robust_pair_agreement": 0.9885294117647059, + "clustered_spearman_rho_median": 0.9780219780219781, + "clustered_topk_membership_rate": 0.615, + "clustered_topk_order_rate": 0.347, + "collection": "moh", + "comparisons_consumed": 2703, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "pair-balanced", + "minimum_pair_evidence": 29, + "percentage_saved": 40.0, + "point_kendall_tau_max": 0.9560439560439562, + "point_kendall_tau_median": 0.9120879120879122, + "point_kendall_tau_min": 0.8461538461538461, + "point_max_absolute_elo_delta_median": 20.111822912160733, + "point_max_absolute_elo_delta_worst": 25.11475250155536, + "point_spearman_rho_median": 0.9736263736263736, + "point_topk_membership_matches": 3, + "point_topk_order_matches": 2, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.25, + "clustered_kendall_tau_median": 0.7802197802197802, + "clustered_kendall_tau_p05": 0.6043956043956044, + "clustered_max_absolute_elo_delta_median": 67.13236783675984, + "clustered_max_absolute_elo_delta_p95": 113.83754428615104, + "clustered_robust_pair_agreement": 0.9658627450980392, + "clustered_spearman_rho_median": 0.9032967032967033, + "clustered_topk_membership_rate": 0.228, + "clustered_topk_order_rate": 0.074, + "collection": "moh", + "comparisons_consumed": 1126, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 12, + "percentage_saved": 75.00554938956715, + "point_kendall_tau_max": 0.8021978021978022, + "point_kendall_tau_median": 0.8021978021978022, + "point_kendall_tau_min": 0.7802197802197802, + "point_max_absolute_elo_delta_median": 75.07628080257223, + "point_max_absolute_elo_delta_worst": 103.77096208907892, + "point_spearman_rho_median": 0.9296703296703297, + "point_topk_membership_matches": 0, + "point_topk_order_matches": 0, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.4, + "clustered_kendall_tau_median": 0.8461538461538461, + "clustered_kendall_tau_p05": 0.7362637362637363, + "clustered_max_absolute_elo_delta_median": 40.96385137541381, + "clustered_max_absolute_elo_delta_p95": 68.24805202727987, + "clustered_robust_pair_agreement": 0.9825294117647059, + "clustered_spearman_rho_median": 0.9494505494505494, + "clustered_topk_membership_rate": 0.372, + "clustered_topk_order_rate": 0.169, + "collection": "moh", + "comparisons_consumed": 1802, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 19, + "percentage_saved": 60.0, + "point_kendall_tau_max": 0.8901098901098902, + "point_kendall_tau_median": 0.8461538461538461, + "point_kendall_tau_min": 0.8461538461538461, + "point_max_absolute_elo_delta_median": 43.66100877748113, + "point_max_absolute_elo_delta_worst": 51.03398034125166, + "point_spearman_rho_median": 0.9472527472527472, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 1, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + }, + { + "allocation_seeds": [ + 42, + 43, + 44, + 45, + 46 + ], + "board": "moh-table-fidelity", + "budget_fraction": 0.6, + "clustered_kendall_tau_median": 0.9120879120879122, + "clustered_kendall_tau_p05": 0.8241758241758242, + "clustered_max_absolute_elo_delta_median": 24.994304320712672, + "clustered_max_absolute_elo_delta_p95": 40.05965087285978, + "clustered_robust_pair_agreement": 0.987921568627451, + "clustered_spearman_rho_median": 0.9736263736263736, + "clustered_topk_membership_rate": 0.561, + "clustered_topk_order_rate": 0.308, + "collection": "moh", + "comparisons_consumed": 2703, + "cross_board_criteria": { + "passed": false, + "robust_pair_agreement_at_least_0_95": true, + "spearman_rho_median_at_least_0_90": true, + "topk_membership_rate_at_least_0_90": false + }, + "design": "mixed-random", + "minimum_pair_evidence": 29, + "percentage_saved": 40.0, + "point_kendall_tau_max": 0.9120879120879122, + "point_kendall_tau_median": 0.8901098901098902, + "point_kendall_tau_min": 0.8241758241758242, + "point_max_absolute_elo_delta_median": 29.58597113580481, + "point_max_absolute_elo_delta_worst": 35.15264403241281, + "point_spearman_rho_median": 0.9648351648351648, + "point_topk_membership_matches": 2, + "point_topk_order_matches": 1, + "possible_pairs": 91, + "robust_full_board_pairs": 51, + "role": "independent 14-model collection" + } + ], + "collection": "moh", + "full_board_page_bootstrap_stability": { + "kendall_tau_median": 0.7802197802197802, + "kendall_tau_p05": 0.6263736263736264, + "topk_membership_rate": 0.19, + "topk_order_rate": 0.105 + }, + "full_rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "models": [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ], + "name": "moh-table-fidelity", + "repo": "davanstrien/ocr-bench-moh-results", + "revision": "05d24695f131432c38984dd2cb55f5a18a3c2580", + "robust_pairs": [ + [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.mocr" + ], + [ + "lightonai/LightOnOCR-2-1B", + "rednote-hilab/dots.ocr" + ], + [ + "lightonai/LightOnOCR-2-1B", + "PaddlePaddle/PaddleOCR-VL-1.6" + ], + [ + "lightonai/LightOnOCR-2-1B", + "allenai/olmOCR-2-7B-1025-FP8" + ], + [ + "lightonai/LightOnOCR-2-1B", + "zai-org/GLM-OCR" + ], + [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "lightonai/LightOnOCR-2-1B", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "lightonai/LightOnOCR-2-1B", + "tesseract-5" + ], + [ + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PaddleOCR-VL-1.6" + ], + [ + "deepseek-ai/DeepSeek-OCR-2", + "allenai/olmOCR-2-7B-1025-FP8" + ], + [ + "deepseek-ai/DeepSeek-OCR-2", + "zai-org/GLM-OCR" + ], + [ + "deepseek-ai/DeepSeek-OCR-2", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "deepseek-ai/DeepSeek-OCR-2", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "deepseek-ai/DeepSeek-OCR-2", + "tesseract-5" + ], + [ + "baidu/Qianfan-OCR", + "allenai/olmOCR-2-7B-1025-FP8" + ], + [ + "baidu/Qianfan-OCR", + "zai-org/GLM-OCR" + ], + [ + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "baidu/Qianfan-OCR", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "baidu/Qianfan-OCR", + "tesseract-5" + ], + [ + "numind/NuExtract3", + "PaddlePaddle/PaddleOCR-VL-1.6" + ], + [ + "numind/NuExtract3", + "allenai/olmOCR-2-7B-1025-FP8" + ], + [ + "numind/NuExtract3", + "zai-org/GLM-OCR" + ], + [ + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "numind/NuExtract3", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "numind/NuExtract3", + "tesseract-5" + ], + [ + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6" + ], + [ + "baidu/Unlimited-OCR", + "zai-org/GLM-OCR" + ], + [ + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "baidu/Unlimited-OCR", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "baidu/Unlimited-OCR", + "tesseract-5" + ], + [ + "FireRedTeam/FireRed-OCR", + "zai-org/GLM-OCR" + ], + [ + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "FireRedTeam/FireRed-OCR", + "tesseract-5" + ], + [ + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8" + ], + [ + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR" + ], + [ + "rednote-hilab/dots.mocr", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "rednote-hilab/dots.mocr", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "rednote-hilab/dots.mocr", + "tesseract-5" + ], + [ + "rednote-hilab/dots.ocr", + "deepseek-ai/DeepSeek-OCR" + ], + [ + "rednote-hilab/dots.ocr", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "rednote-hilab/dots.ocr", + "tesseract-5" + ], + [ + "PaddlePaddle/PaddleOCR-VL-1.6", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "PaddlePaddle/PaddleOCR-VL-1.6", + "tesseract-5" + ], + [ + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "allenai/olmOCR-2-7B-1025-FP8", + "tesseract-5" + ], + [ + "zai-org/GLM-OCR", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "zai-org/GLM-OCR", + "tesseract-5" + ], + [ + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium" + ], + [ + "deepseek-ai/DeepSeek-OCR", + "tesseract-5" + ], + [ + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ] + ], + "role": "independent 14-model collection", + "sample_count": 50, + "seed_results": [ + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8461538461538461, + "robust_pair_agreement_rate": 0.9788235294117646, + "topk_membership_rate": 0.325 + }, + "comparisons_consumed": 1126, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "max_absolute_elo_delta": 84.60404452855505, + "median_absolute_elo_delta": 16.11079065770923, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9208791208791209, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8241758241758242, + "robust_pair_agreement_rate": 0.9766666666666667, + "topk_membership_rate": 0.32 + }, + "comparisons_consumed": 1126, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "max_absolute_elo_delta": 51.17006770521766, + "median_absolute_elo_delta": 9.522717885973748, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "baidu/Qianfan-OCR", + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9428571428571428, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8241758241758242, + "robust_pair_agreement_rate": 0.9802941176470588, + "topk_membership_rate": 0.345 + }, + "comparisons_consumed": 1126, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "max_absolute_elo_delta": 43.531941958760854, + "median_absolute_elo_delta": 10.401269327578575, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9340659340659341, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8241758241758242, + "robust_pair_agreement_rate": 0.9755882352941176, + "topk_membership_rate": 0.34 + }, + "comparisons_consumed": 1126, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7582417582417583, + "max_absolute_elo_delta": 40.61281047495663, + "median_absolute_elo_delta": 14.208611462407589, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.8989010989010989, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.8241758241758242, + "robust_pair_agreement_rate": 0.9754901960784313, + "topk_membership_rate": 0.37 + }, + "comparisons_consumed": 1126, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "max_absolute_elo_delta": 40.87976704023163, + "median_absolute_elo_delta": 20.248387256132673, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "rednote-hilab/dots.mocr", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 0.9803921568627451, + "spearman_rho": 0.9296703296703297, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8901098901098902, + "robust_pair_agreement_rate": 0.9844117647058823, + "topk_membership_rate": 0.455 + }, + "comparisons_consumed": 1802, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "max_absolute_elo_delta": 37.23125338169325, + "median_absolute_elo_delta": 9.653540594321612, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9252747252747253, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8681318681318682, + "robust_pair_agreement_rate": 0.9844117647058823, + "topk_membership_rate": 0.435 + }, + "comparisons_consumed": 1802, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9120879120879122, + "max_absolute_elo_delta": 30.757182086236526, + "median_absolute_elo_delta": 6.137030487099651, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9780219780219781, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8791208791208791, + "robust_pair_agreement_rate": 0.9861764705882353, + "topk_membership_rate": 0.535 + }, + "comparisons_consumed": 1802, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "max_absolute_elo_delta": 37.53302303767623, + "median_absolute_elo_delta": 9.52553453103792, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9208791208791209, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8681318681318682, + "robust_pair_agreement_rate": 0.9837254901960785, + "topk_membership_rate": 0.48 + }, + "comparisons_consumed": 1802, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "max_absolute_elo_delta": 26.134090227832303, + "median_absolute_elo_delta": 14.395902540397856, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9252747252747253, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8681318681318682, + "robust_pair_agreement_rate": 0.9834313725490196, + "topk_membership_rate": 0.465 + }, + "comparisons_consumed": 1802, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "max_absolute_elo_delta": 28.4152162110247, + "median_absolute_elo_delta": 12.123770821153698, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9824175824175824, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9890196078431372, + "topk_membership_rate": 0.585 + }, + "comparisons_consumed": 2703, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "max_absolute_elo_delta": 22.029425586951447, + "median_absolute_elo_delta": 3.1641902267133446, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "rednote-hilab/dots.mocr", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.956043956043956, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.989313725490196, + "topk_membership_rate": 0.63 + }, + "comparisons_consumed": 2703, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9560439560439562, + "max_absolute_elo_delta": 18.323697797567775, + "median_absolute_elo_delta": 3.9949570822469695, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9912087912087912, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9887254901960785, + "topk_membership_rate": 0.645 + }, + "comparisons_consumed": 2703, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9120879120879122, + "max_absolute_elo_delta": 18.46202995345584, + "median_absolute_elo_delta": 5.615808407188638, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9736263736263736, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9877450980392157, + "topk_membership_rate": 0.58 + }, + "comparisons_consumed": 2703, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "max_absolute_elo_delta": 20.111822912160733, + "median_absolute_elo_delta": 6.668207319465068, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "rednote-hilab/dots.mocr", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9384615384615385, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9878431372549019, + "topk_membership_rate": 0.635 + }, + "comparisons_consumed": 2703, + "design": "pair-balanced", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9340659340659341, + "max_absolute_elo_delta": 25.11475250155536, + "median_absolute_elo_delta": 7.530233911158916, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9824175824175824, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 42, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.7802197802197802, + "robust_pair_agreement_rate": 0.9639215686274509, + "topk_membership_rate": 0.23 + }, + "comparisons_consumed": 1126, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "max_absolute_elo_delta": 75.07628080257223, + "median_absolute_elo_delta": 23.51300905580956, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "allenai/olmOCR-2-7B-1025-FP8", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9340659340659341, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.7802197802197802, + "robust_pair_agreement_rate": 0.9670588235294117, + "topk_membership_rate": 0.25 + }, + "comparisons_consumed": 1126, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "max_absolute_elo_delta": 82.4409623687352, + "median_absolute_elo_delta": 24.032203861665153, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9340659340659341, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 44, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.7802197802197802, + "robust_pair_agreement_rate": 0.965392156862745, + "topk_membership_rate": 0.245 + }, + "comparisons_consumed": 1126, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "max_absolute_elo_delta": 57.579922663682055, + "median_absolute_elo_delta": 18.421721836137408, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9296703296703297, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.7802197802197802, + "robust_pair_agreement_rate": 0.9664705882352941, + "topk_membership_rate": 0.235 + }, + "comparisons_consumed": 1126, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8021978021978022, + "max_absolute_elo_delta": 103.77096208907892, + "median_absolute_elo_delta": 22.498107619688767, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9252747252747253, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.25, + "clustered": { + "kendall_tau_median": 0.7802197802197802, + "robust_pair_agreement_rate": 0.9664705882352941, + "topk_membership_rate": 0.18 + }, + "comparisons_consumed": 1126, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 13, + "median_direct_comparisons": 12, + "min_direct_comparisons": 12, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.7802197802197802, + "max_absolute_elo_delta": 60.51432562169339, + "median_absolute_elo_delta": 22.647873460968867, + "percentage_saved": 75.00554938956715, + "rank_order": [ + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.ocr", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9208791208791209, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8461538461538461, + "robust_pair_agreement_rate": 0.9821568627450981, + "topk_membership_rate": 0.365 + }, + "comparisons_consumed": 1802, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "max_absolute_elo_delta": 51.03398034125166, + "median_absolute_elo_delta": 16.530667177120904, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9384615384615385, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8461538461538461, + "robust_pair_agreement_rate": 0.9836274509803922, + "topk_membership_rate": 0.375 + }, + "comparisons_consumed": 1802, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "max_absolute_elo_delta": 45.98383836109997, + "median_absolute_elo_delta": 18.500508277681547, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "numind/NuExtract3", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9428571428571428, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8681318681318682, + "robust_pair_agreement_rate": 0.9831372549019608, + "topk_membership_rate": 0.405 + }, + "comparisons_consumed": 1802, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8901098901098902, + "max_absolute_elo_delta": 26.785873493551208, + "median_absolute_elo_delta": 10.784672518576144, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9692307692307693, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8461538461538461, + "robust_pair_agreement_rate": 0.9828431372549019, + "topk_membership_rate": 0.38 + }, + "comparisons_consumed": 1802, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "max_absolute_elo_delta": 43.586791994669966, + "median_absolute_elo_delta": 12.40763082495755, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9472527472527472, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.4, + "clustered": { + "kendall_tau_median": 0.8461538461538461, + "robust_pair_agreement_rate": 0.9808823529411764, + "topk_membership_rate": 0.335 + }, + "comparisons_consumed": 1802, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 20, + "median_direct_comparisons": 20, + "min_direct_comparisons": 19, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8461538461538461, + "max_absolute_elo_delta": 43.66100877748113, + "median_absolute_elo_delta": 22.397146617674707, + "percentage_saved": 60.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "baidu/Qianfan-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9472527472527472, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 42, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9875490196078431, + "topk_membership_rate": 0.555 + }, + "comparisons_consumed": 2703, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.9120879120879122, + "max_absolute_elo_delta": 35.15264403241281, + "median_absolute_elo_delta": 9.537645802636007, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "FireRedTeam/FireRed-OCR", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9692307692307693, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 43, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9886274509803922, + "topk_membership_rate": 0.6 + }, + "comparisons_consumed": 2703, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8681318681318682, + "max_absolute_elo_delta": 30.356880157668, + "median_absolute_elo_delta": 10.745471364785772, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Qianfan-OCR", + "FireRedTeam/FireRed-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "rednote-hilab/dots.ocr", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.956043956043956, + "topk_membership_matches": true, + "topk_order_matches": true + }, + { + "allocation_seed": 44, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9877450980392157, + "topk_membership_rate": 0.54 + }, + "comparisons_consumed": 2703, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8901098901098902, + "max_absolute_elo_delta": 21.06096407512746, + "median_absolute_elo_delta": 9.032332913477717, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Unlimited-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9692307692307693, + "topk_membership_matches": true, + "topk_order_matches": false + }, + { + "allocation_seed": 45, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9890196078431372, + "topk_membership_rate": 0.565 + }, + "comparisons_consumed": 2703, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8901098901098902, + "max_absolute_elo_delta": 27.884895010562786, + "median_absolute_elo_delta": 6.2723616390209145, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "numind/NuExtract3", + "deepseek-ai/DeepSeek-OCR-2", + "FireRedTeam/FireRed-OCR", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "allenai/olmOCR-2-7B-1025-FP8", + "PaddlePaddle/PaddleOCR-VL-1.6", + "zai-org/GLM-OCR", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9648351648351648, + "topk_membership_matches": false, + "topk_order_matches": false + }, + { + "allocation_seed": 46, + "budget_fraction": 0.6, + "clustered": { + "kendall_tau_median": 0.9120879120879122, + "robust_pair_agreement_rate": 0.9866666666666667, + "topk_membership_rate": 0.545 + }, + "comparisons_consumed": 2703, + "design": "mixed-random", + "graph": { + "connected": true, + "connected_components": [ + [ + "FireRedTeam/FireRed-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "baidu/Qianfan-OCR", + "baidu/Unlimited-OCR", + "deepseek-ai/DeepSeek-OCR", + "deepseek-ai/DeepSeek-OCR-2", + "lightonai/LightOnOCR-2-1B", + "numind/NuExtract3", + "rednote-hilab/dots.mocr", + "rednote-hilab/dots.ocr", + "tesseract-5", + "zai-org/GLM-OCR" + ] + ], + "max_direct_comparisons": 30, + "median_direct_comparisons": 30, + "min_direct_comparisons": 29, + "observed_pairs": 91, + "pair_coverage_pct": 100.0, + "possible_pairs": 91 + }, + "kendall_tau": 0.8241758241758242, + "max_absolute_elo_delta": 29.58597113580481, + "median_absolute_elo_delta": 11.511059775810281, + "percentage_saved": 40.0, + "rank_order": [ + "lightonai/LightOnOCR-2-1B", + "deepseek-ai/DeepSeek-OCR-2", + "baidu/Unlimited-OCR", + "FireRedTeam/FireRed-OCR", + "numind/NuExtract3", + "baidu/Qianfan-OCR", + "rednote-hilab/dots.ocr", + "rednote-hilab/dots.mocr", + "zai-org/GLM-OCR", + "PaddlePaddle/PaddleOCR-VL-1.6", + "allenai/olmOCR-2-7B-1025-FP8", + "deepseek-ai/DeepSeek-OCR", + "PaddlePaddle/PP-OCRv6_medium", + "tesseract-5" + ], + "robust_pair_agreement": 1, + "spearman_rho": 0.9428571428571428, + "topk_membership_matches": false, + "topk_order_matches": false + } + ], + "sentinel_comparisons_detectable": 0, + "stored_comparisons": 4505 + } + ], + "collection_design_pass_counts": { + "bpl:mixed-random@25%": { + "passed": 0, + "total": 1 + }, + "bpl:mixed-random@40%": { + "passed": 0, + "total": 1 + }, + "bpl:mixed-random@60%": { + "passed": 0, + "total": 1 + }, + "bpl:pair-balanced@25%": { + "passed": 0, + "total": 1 + }, + "bpl:pair-balanced@40%": { + "passed": 0, + "total": 1 + }, + "bpl:pair-balanced@60%": { + "passed": 0, + "total": 1 + }, + "britannica:mixed-random@25%": { + "passed": 1, + "total": 1 + }, + "britannica:mixed-random@40%": { + "passed": 1, + "total": 1 + }, + "britannica:mixed-random@60%": { + "passed": 1, + "total": 1 + }, + "britannica:pair-balanced@25%": { + "passed": 1, + "total": 1 + }, + "britannica:pair-balanced@40%": { + "passed": 1, + "total": 1 + }, + "britannica:pair-balanced@60%": { + "passed": 1, + "total": 1 + }, + "moh:mixed-random@25%": { + "passed": 0, + "total": 1 + }, + "moh:mixed-random@40%": { + "passed": 0, + "total": 1 + }, + "moh:mixed-random@60%": { + "passed": 0, + "total": 1 + }, + "moh:pair-balanced@25%": { + "passed": 0, + "total": 1 + }, + "moh:pair-balanced@40%": { + "passed": 0, + "total": 1 + }, + "moh:pair-balanced@60%": { + "passed": 0, "total": 1 }, "rubenstein:mixed-random@25%": { @@ -11085,7 +13719,7 @@ 45, 46 ], - "boards": 6, + "boards": 7, "budget_fractions": [ 0.25, 0.4, @@ -11100,6 +13734,7 @@ "independent_collections": [ "bpl", "britannica", + "moh", "rubenstein", "ufo" ], diff --git a/experiments/adaptive-stopping/multi-board-summary.csv b/experiments/adaptive-stopping/multi-board-summary.csv index c01f18d..83eb7f5 100644 --- a/experiments/adaptive-stopping/multi-board-summary.csv +++ b/experiments/adaptive-stopping/multi-board-summary.csv @@ -35,3 +35,9 @@ britannica-qwen35,britannica,pair-balanced,0.6,432,40.0,13,15,1.0,5,1.0,0.993,0. britannica-qwen35,britannica,mixed-random,0.25,180,75.0,13,15,0.942857142857143,4,0.942857142857143,0.956,0.9699230769230769,91.67612246683814,2348.1679073592472,12,True britannica-qwen35,britannica,mixed-random,0.4,288,60.0,13,15,0.942857142857143,5,1.0,0.989,0.9826923076923078,60.51017695201898,197.6989721651328,19,True britannica-qwen35,britannica,mixed-random,0.6,432,40.0,13,15,0.942857142857143,5,1.0,0.999,0.9906153846153847,39.35275643631411,105.36616237876325,28,True +moh-table-fidelity,moh,pair-balanced,0.25,1126,75.00554938956715,51,91,0.9296703296703297,1,0.9384615384615385,0.34,0.9773725490196078,48.939915803189024,84.31332593481861,12,False +moh-table-fidelity,moh,pair-balanced,0.4,1802,60.0,51,91,0.9252747252747253,2,0.9604395604395607,0.474,0.9844313725490196,34.285550887453155,55.81184566831936,19,False +moh-table-fidelity,moh,pair-balanced,0.6,2703,40.0,51,91,0.9736263736263736,3,0.9780219780219781,0.615,0.9885294117647059,22.887958838301074,36.55624739703296,29,False +moh-table-fidelity,moh,mixed-random,0.25,1126,75.00554938956715,51,91,0.9296703296703297,0,0.9032967032967033,0.228,0.9658627450980392,67.13236783675984,113.83754428615104,12,False +moh-table-fidelity,moh,mixed-random,0.4,1802,60.0,51,91,0.9472527472527472,2,0.9494505494505494,0.372,0.9825294117647059,40.96385137541381,68.24805202727987,19,False +moh-table-fidelity,moh,mixed-random,0.6,2703,40.0,51,91,0.9648351648351648,2,0.9736263736263736,0.561,0.987921568627451,24.994304320712672,40.05965087285978,29,False diff --git a/experiments/adaptive-stopping/multi_board.py b/experiments/adaptive-stopping/multi_board.py index b89762f..5da98bd 100644 --- a/experiments/adaptive-stopping/multi_board.py +++ b/experiments/adaptive-stopping/multi_board.py @@ -83,6 +83,13 @@ class BoardSpec: "8b6602826e2b71c615ed229d388221a7aa7c69b9", "same-corpus judge/model-grid sensitivity", ), + BoardSpec( + "moh-table-fidelity", + "moh", + "davanstrien/ocr-bench-moh-results", + "05d24695f131432c38984dd2cb55f5a18a3c2580", + "independent 14-model collection", + ), ) BUDGET_FRACTIONS = (0.25, 0.40, 0.60)