remove pandas in code

Author: xzrderek

eval_protocol/pytest/utils.py

@@ -29,7 +29,8 @@
 
 import logging
 import json
-import pandas as pd
+import random
+import statistics
 
 
 AggregationMethod = Literal["mean", "max", "min", "bootstrap"]
@@ -122,30 +123,25 @@ async def execute_run_with_progress(run_idx: int, config: RolloutProcessorConfig
             raise
 
 
-def calculate_bootstrap_scores(all_scores: list[float]) -> float:
+def calculate_bootstrap_scores(all_scores: list[float], n_boot: int = 100, seed: int | None = None) -> float:
     """
-    Calculate bootstrap confidence intervals for individual scores.
+    Calculate the mean of bootstrap sample means for a list of scores.
 
     Args:
-        all_scores: List of individual scores from all rows
+        all_scores: List of individual scores from all rows.
+        n_boot: Number of bootstrap resamples to draw (default 100).
+        seed: Optional RNG seed for reproducibility.
 
     Returns:
-        Mean bootstrap score
+        Mean bootstrap score (float). Returns 0.0 if all_scores is empty.
     """
     if not all_scores:
         return 0.0
 
-    # Create DataFrame (single column of scores)
-    battles = pd.DataFrame({"score": all_scores})
-
-    # Bootstrap sampling for calculating relative performance
-    bootstrap_means = [battles.sample(frac=1.0, replace=True)["score"].mean() for _ in range(100)]
-
-    # Calculate final scores
-    bootstraps = pd.Series(bootstrap_means)
-    mean_score = bootstraps.mean()
-
-    return float(mean_score)
+    rng = random.Random(seed) if seed is not None else random
+    k = len(all_scores)
+    bootstrap_means = [statistics.fmean(rng.choices(all_scores, k=k)) for _ in range(n_boot)]
+    return float(statistics.fmean(bootstrap_means))
 
 
 def aggregate(scores: list[float], method: AggregationMethod) -> float:

eval_protocol/quickstart/utils.py

@@ -3,15 +3,10 @@
 """
 
 import os
-from datetime import datetime
 import re
-from typing import List, Dict, Any, Optional
-from openai import AsyncOpenAI
-import pandas as pd
+from typing import Dict, Any, Optional
 
-from eval_protocol.models import EvaluationRow, Message, EvaluateResult, MetricResult
-import asyncio
-from openai import OpenAI
+from eval_protocol.models import EvaluationRow, Message
 
 OG_ARENA_HARD_PROMPT = """Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.
 

tests/test_evaluation_postprocess.py

@@ -208,6 +208,33 @@ def test_all_invalid_scores(self):
         assert mock_logger.log.call_count == 2
 
 
+class TestBootstrapEquivalence:
+    def test_bootstrap_equivalence_pandas_vs_pure_python(self):
+        import random
+        import pandas as pd
+        from eval_protocol.pytest.utils import calculate_bootstrap_scores as py_bootstrap
+
+        # Deterministic synthetic scores
+        rng = random.Random(123)
+        scores = [rng.random() for _ in range(100)]
+
+        n_boot = 1000
+        seed = 42
+
+        # Old (pandas) style bootstrap: resample full column with replacement
+        df = pd.DataFrame({"score": scores})
+        pandas_means = [
+            df.sample(frac=1.0, replace=True, random_state=seed + i)["score"].mean() for i in range(n_boot)
+        ]
+        pandas_boot_mean = sum(pandas_means) / len(pandas_means)
+
+        # New pure-python implementation
+        py_boot_mean = py_bootstrap(scores, n_boot=n_boot, seed=seed)
+
+        # They estimate the same quantity; allow small Monte Carlo tolerance
+        assert abs(pandas_boot_mean - py_boot_mean) < 0.02
+
+
 class TestComputeFixedSetMuCi:
     """Tests for compute_fixed_set_mu_ci function."""