test_pytest_stable_row_ids

Author: Dylan Huang

eval_protocol/human_id/__init__.py

@@ -78,7 +78,7 @@ def generate_id(
     return separator.join(parts)
 
 
-def num_combinations(word_count: int) -> int:
+def num_combinations(word_count: int = 5) -> int:
     """
     Return the total number of unique IDs possible for the given word_count.
 

eval_protocol/models.py

@@ -209,7 +209,7 @@ class InputMetadata(BaseModel):
 
     model_config = ConfigDict(extra="allow")
 
-    row_id: Optional[str] = Field(default_factory=generate_id, description="Unique string to ID the row")
+    row_id: Optional[str] = Field(None, description="Unique string to ID the row")
     completion_params: CompletionParams = Field(
         default_factory=dict, description="Completion endpoint parameters used"
     )
@@ -422,9 +422,21 @@ def get_termination_reason(self) -> str:
         return "unknown"
 
     def __hash__(self) -> int:
-        json_str = self.model_dump_json(exclude_none=True)
+        # Use a stable hash by sorting keys and ensuring compact output
+        json_str = self.stable_json(self)
         return hash(json_str)
 
+    @classmethod
+    def stable_json(cls, row: "EvaluationRow") -> int:
+        json_str = row.model_dump_json(
+            exclude_none=True,
+            exclude_defaults=True,
+            by_alias=True,
+            indent=None,
+            exclude=["created_at", "execution_metadata"],
+        )
+        return json_str
+
 
 # Original dataclass-based models for backwards compatibility
 # These are deprecated and will be removed in a future version

eval_protocol/pytest/evaluation_test.py

@@ -15,7 +15,7 @@
 
 from eval_protocol.dataset_logger import default_logger
 from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
-from eval_protocol.human_id import generate_id
+from eval_protocol.human_id import generate_id, num_combinations
 from eval_protocol.models import (
     CompletionParams,
     EvalMetadata,
@@ -294,6 +294,16 @@ def _log_eval_error(
                     else:
                         raise ValueError("No input dataset or input messages provided")
 
+                    for row in data:
+                        # generate a stable row_id for each row
+                        if row.input_metadata.row_id is None:
+                            # Generate a stable, deterministic row_id using the row's hash and num_combinations
+                            index = hash(row)
+                            max_index = num_combinations() - 1
+                            # Ensure index is a non-negative integer within [0, max_index]
+                            index = abs(index) % (max_index + 1)
+                            row.input_metadata.row_id = generate_id(seed=0, index=index)
+
                     if "completion_params" not in kwargs or not kwargs["completion_params"]:
                         raise ValueError(
                             "No completion parameters provided. Please provide a completion parameters object."

tests/pytest/test_pytest_stable_row_id.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from eval_protocol.models import EvaluationRow
+from eval_protocol.models import EvaluationRow, Message
 from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
 from tests.pytest.test_markdown_highlighting import markdown_dataset_to_evaluation_row
 
@@ -37,5 +37,61 @@ def eval_fn(row: EvaluationRow) -> EvaluationRow:
         for params in completion_params_list:
             await eval_fn(dataset_path=[ds_path], completion_params=params)
 
+    # Second invocation to ensure that IDs are stable across multiple invocations
+    for ds_path in input_dataset:
+        for params in completion_params_list:
+            await eval_fn(dataset_path=[ds_path], completion_params=params)
+
     # Assertions on IDs generated by the decorator logic
     assert len(row_ids) == 19  # from the markdown dataset
+
+
+async def test_evaluation_test_generated_row_ids_without_dataset_keys():
+    from eval_protocol.pytest.evaluation_test import evaluation_test
+
+    # Adapter that does NOT set row_id; lets evaluation_test generate IDs
+    def markdown_dataset_no_row_id_adapter(data: List[dict]) -> List[EvaluationRow]:
+        return [
+            EvaluationRow(
+                messages=[Message(role="user", content=row["prompt"])],
+                ground_truth=str(row["num_highlights"]),
+            )
+            for row in data
+        ]
+
+    row_ids = set()
+
+    input_dataset = ["tests/pytest/data/markdown_dataset.jsonl", "tests/pytest/data/markdown_dataset.jsonl"]
+    completion_params = [
+        {"temperature": 0.0, "model": "dummy/local-model"},
+        {"temperature": 1.0, "model": "dummy/local-model"},
+    ]
+
+    @evaluation_test(
+        input_dataset=input_dataset,
+        completion_params=completion_params,
+        dataset_adapter=markdown_dataset_no_row_id_adapter,
+        rollout_processor=NoOpRolloutProcessor(),
+        mode="pointwise",
+        combine_datasets=False,
+        num_runs=5,
+    )
+    def eval_fn(row: EvaluationRow) -> EvaluationRow:
+        # row_id should be auto-generated by evaluation_test/InputMetadata
+        assert row.input_metadata is not None
+        assert row.input_metadata.row_id is not None and isinstance(row.input_metadata.row_id, str)
+        row_ids.add(row.input_metadata.row_id)
+        return row
+
+    # Single invocation (one dataset, one param set) with multiple runs
+    for ds_path in input_dataset:
+        for params in completion_params:
+            await eval_fn(dataset_path=[ds_path], completion_params=params)
+
+    # Second invocation to ensure that IDs are stable across multiple invocations
+    for ds_path in input_dataset:
+        for params in completion_params:
+            await eval_fn(dataset_path=[ds_path], completion_params=params)
+
+    # Even with multiple runs, generated row_ids should be stable within the invocation
+    assert len(row_ids) == 19  # equals dataset size when IDs are generated once and preserved across runs

tests/test_models.py

@@ -289,6 +289,23 @@ def test_evaluation_row_creation():
     assert not row.is_trajectory_evaluation()
 
 
+def test_stable_hash():
+    """Test the stable hash method."""
+    row = EvaluationRow(
+        messages=[Message(role="user", content="What is 2+2?"), Message(role="assistant", content="2+2 equals 4.")],
+        ground_truth="4",
+    )
+    row2 = EvaluationRow(
+        messages=[Message(role="user", content="What is 2+2?"), Message(role="assistant", content="2+2 equals 4.")],
+        ground_truth="4",
+    )
+    stable_json = EvaluationRow.stable_json(row)
+    stable_json2 = EvaluationRow.stable_json(row2)
+    assert stable_json == stable_json2
+    assert "created_at" not in stable_json
+    assert "execution_metadata" not in stable_json
+
+
 def test_evaluation_row_trajectory_evaluation():
     """Test EvaluationRow with trajectory evaluation."""
     messages = [