implement logger and use in agent rollout processor

Author: Dylan Huang

.eval_protocol/datasets/2025-08-05.jsonl

@@ -0,0 +1 @@
+{"messages":[{"role":"system","content":"You are a helpful assistant that can answer questions about Fireworks.\nALWAYS provide code or commands to execute to answer the question."},{"role":"user","content":"Can you teach me about how to manage deployments on Fireworks"},{"role":"assistant","content":"I'll help you learn about managing deployments on Fireworks. Let me search for comprehensive information about deployment management.","tool_calls":[{"id":"functions.search:0","type":"function","function":{"name":"search","arguments":"{\"query\": \"deployment management Fireworks AI\"}"}}]},{"role":"tool","content":"Title: Fireworks AI Developer Platform\nLink: https://docs.fireworks.ai/getting-started/introduction\nContent: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\n","tool_call_id":"functions.search:0"},{"role":"assistant","content":"Let me search for more specific information about deployment management features and processes.","tool_calls":[{"id":"functions.search:1","type":"function","function":{"name":"search","arguments":"{\"query\": \"deploy models Fireworks AI deployment management\"}"}}]},{"role":"tool","content":"Title: Fireworks AI Developer Platform\nLink: https://docs.fireworks.ai/getting-started/introduction\nContent: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\n","tool_call_id":"functions.search:1"},{"role":"assistant","content":"Let me search for more specific deployment-related terms:","tool_calls":[{"id":"functions.search:0","type":"function","function":{"name":"search","arguments":"{\"query\": \"inference endpoints deployment Fireworks AI\"}"}}]},{"role":"tool","content":"Title: Fireworks AI Developer Platform\nLink: https://docs.fireworks.ai/getting-started/introduction\nContent: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\n","tool_call_id":"functions.search:0"},{"role":"assistant","content":"Let me try searching for more specific deployment management topics:","tool_calls":[{"id":"functions.search:0","type":"function","function":{"name":"search","arguments":"{\"query\": \"model deployment configuration scaling Fireworks\"}"}}]},{"role":"tool","content":"Title: What should I expect for deployment and scaling performance?\nLink: https://docs.fireworks.ai/faq-new/deployment-infrastructure/what-should-i-expect-for-deployment-and-scaling-performance\nContent: Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long\n\nInitial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long\n\nWhat should I expect for deployment and scaling performance?\nInitial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long\n\n","tool_call_id":"functions.search:0"},{"role":"assistant","content":null,"tool_calls":[{"id":"functions.search:1","type":"function","function":{"name":"search","arguments":"{\"query\": \"deployment status monitoring Fireworks AI\"}"}}]},{"role":"tool","content":"Title: Fireworks AI Developer Platform\nLink: https://docs.fireworks.ai/getting-started/introduction\nContent: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\n","tool_call_id":"functions.search:1"},{"role":"assistant","content":null,"tool_calls":[{"id":"functions.search:0","type":"function","function":{"name":"search","arguments":"{\"query\": \"custom model deployment Fireworks AI\"}"}}]},{"role":"tool","content":"Title: Does Fireworks support custom base models?\nLink: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models\nContent: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models.\n\nYes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models.\n\nDoes Fireworks support custom base models?\nYes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models.\n\n","tool_call_id":"functions.search:0"},{"role":"assistant","content":null,"tool_calls":[{"id":"functions.search:0","type":"function","function":{"name":"search","arguments":"{\"query\": \"firectl deployment tool Fireworks AI\"}"}}]},{"role":"tool","content":"Title: Fireworks AI Developer Platform\nLink: https://docs.fireworks.ai/getting-started/introduction\nContent: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\n","tool_call_id":"functions.search:0"},{"role":"assistant","content":"Let me search for more specific deployment management commands and tools:","tool_calls":[{"id":"functions.search:0","type":"function","function":{"name":"search","arguments":"{\"query\": \"deployment commands CLI Fireworks AI\"}"}}]},{"role":"tool","content":"Title: Fireworks AI Developer Platform\nLink: https://docs.fireworks.ai/getting-started/introduction\nContent: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\nFireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code!\n\n","tool_call_id":"functions.search:0"},{"role":"assistant","content":"Let me try searching for deployment configuration and management:","tool_calls":[{"id":"functions.search:0","type":"function","function":{"name":"search","arguments":"{\"query\": \"deployment configuration YAML JSON Fireworks\"}"}}]},{"role":"tool","content":"Title: Create a fine-tuning job\nLink: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-finetune-job\nContent: Creates a fine-tuning job on Fireworks AI platform with the provided configuration yaml.\n\nCreates a fine-tuning job on Fireworks AI platform with the provided configuration yaml.\n\nCreates a fine-tuning job on Fireworks AI platform with the provided configuration yaml.\n\n","tool_call_id":"functions.search:0"},{"role":"assistant","content":"Based on my search results, here's what I can teach you about managing deployments on Fireworks AI:\n\n## Deployment Management on Fireworks AI\n\n### 1. **Deployment Performance Expectations**\n- **Initial deployment**: Should complete within minutes\n- **Scaling from zero**: May experience brief availability delays while the system scales up\n- **Troubleshooting**: If deployment takes over 1 hour, this typically indicates a crash and should be investigated\n- **Best practice**: Monitor deployment status and contact support if deployment times are unusually long\n\n### 2. **Custom Model Deployment**\nFireworks AI supports custom base models through `firectl` - their deployment tool. You can deploy custom models using this command-line interface.\n\n### 3. **Deployment Tools**\n- **firectl**: The primary deployment tool for managing models on Fireworks AI\n- Used for both standard and custom model deployments\n- Supports configuration via YAML files\n\n### 4. **Monitoring and Management**\n- Monitor deployment status regularly\n- Contact support for unusually long deployment times\n- Watch for scaling delays when starting from zero instances\n\n### 5. **Configuration Management**\nDeployments can be configured using YAML configuration files, similar to how fine-tuning jobs are configured.\n\n## Next Steps for Managing Deployments\n\nTo get started with deployment management, you should:\n\n1. **Install firectl**: The Fireworks AI command-line tool\n2. **Prepare your model**: Ensure your model is properly formatted and ready for deployment\n3. **Create deployment configuration**: Use YAML files to define deployment parameters\n4. **Monitor deployment**: Use built-in monitoring tools to track deployment status\n5. **Scale as needed**: Configure auto-scaling based on your traffic patterns\n\nWould you like me to search for more specific information about any of these deployment management aspects, such as specific firectl commands or configuration examples?","tool_calls":[]}],"input_metadata":{"row_id":"feel-might-gaseous-picture"},"created_at":"2025-08-05T10:43:36.343980"}

eval_protocol/dataset_logger/__init__.py

@@ -0,0 +1,3 @@
+from eval_protocol.dataset_logger.local_fs_dataset_logger_adapter import LocalFSDatasetLoggerAdapter
+
+default_logger = LocalFSDatasetLoggerAdapter()

eval_protocol/dataset_logger/dataset_logger.py

@@ -0,0 +1,35 @@
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, List, Optional
+
+if TYPE_CHECKING:
+    from eval_protocol.models import EvaluationRow
+
+
+class DatasetLogger(ABC):
+    """
+    Abstract base class for logging EvaluationRow objects.
+    Implementations should provide methods for storing and retrieving logs.
+    """
+
+    @abstractmethod
+    def log(self, row: "EvaluationRow") -> None:
+        """
+        Store a single EvaluationRow log.
+
+        Args:
+            row (EvaluationRow): The evaluation row to log.
+        """
+        pass
+
+    @abstractmethod
+    def read(self, row_id: Optional[str] = None) -> List["EvaluationRow"]:
+        """
+        Retrieve EvaluationRow logs.
+
+        Args:
+            row_id (Optional[str]): If provided, filter logs by this row_id.
+
+        Returns:
+            List[EvaluationRow]: List of retrieved evaluation rows.
+        """
+        pass

eval_protocol/dataset_logger/local_fs_dataset_logger_adapter.py

@@ -0,0 +1,102 @@
+from datetime import datetime, timezone
+import json
+import os
+import tempfile
+import shutil
+from typing import TYPE_CHECKING, List, Optional
+from eval_protocol.common_utils import load_jsonl
+from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
+
+if TYPE_CHECKING:
+    from eval_protocol.models import EvaluationRow
+
+
+class LocalFSDatasetLoggerAdapter(DatasetLogger):
+    """
+    Logger that stores logs in the local filesystem.
+    """
+
+    EVAL_PROTOCOL_DIR = ".eval_protocol"
+    PYTHON_FILES = ["pyproject.toml", "requirements.txt"]
+    DATASETS_DIR = "datasets"
+
+    def __init__(self):
+        # recursively look up for a .eval_protocol directory
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        while current_dir != "/":
+            if os.path.exists(os.path.join(current_dir, self.EVAL_PROTOCOL_DIR)):
+                self.log_dir = os.path.join(current_dir, self.EVAL_PROTOCOL_DIR)
+                break
+            current_dir = os.path.dirname(current_dir)
+
+        # if not found, recursively look up until a pyproject.toml or requirements.txt is found
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        while current_dir != "/":
+            if any(os.path.exists(os.path.join(current_dir, f)) for f in self.PYTHON_FILES):
+                self.log_dir = os.path.join(current_dir, self.EVAL_PROTOCOL_DIR)
+                break
+            current_dir = os.path.dirname(current_dir)
+
+        # get the PWD that this python process is running in
+        self.log_dir = os.path.join(os.getcwd(), self.EVAL_PROTOCOL_DIR)
+
+        # create the .eval_protocol directory if it doesn't exist
+        os.makedirs(self.log_dir, exist_ok=True)
+
+        # create the datasets subdirectory
+        self.datasets_dir = os.path.join(self.log_dir, self.DATASETS_DIR)
+        os.makedirs(self.datasets_dir, exist_ok=True)
+
+        # ensure that log file exists
+        if not os.path.exists(self.jsonl_path):
+            with open(self.jsonl_path, "w") as f:
+                f.write("")
+
+    @property
+    def current_date(self) -> str:
+        # Use UTC timezone to be consistent across local device/locations/CI
+        return datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+    @property
+    def jsonl_path(self) -> str:
+        return os.path.join(self.datasets_dir, f"{self.current_date}.jsonl")
+
+    def log(self, row: "EvaluationRow") -> None:
+        """Log a row, updating existing row with same ID or appending new row."""
+        row_id = row.input_metadata.row_id
+
+        # Check if row with this ID already exists
+        if os.path.exists(self.jsonl_path):
+            with open(self.jsonl_path, "r") as f:
+                lines = f.readlines()
+
+            # Find the line with matching ID
+            for i, line in enumerate(lines):
+                try:
+                    line_data = json.loads(line.strip())
+                    if line_data["input_metadata"]["row_id"] == row_id:
+                        # Update existing row
+                        lines[i] = row.model_dump_json(exclude_none=True) + os.linesep
+                        with open(self.jsonl_path, "w") as f:
+                            f.writelines(lines)
+                        return
+                except json.JSONDecodeError:
+                    continue
+
+        # If no existing row found, append new row
+        with open(self.jsonl_path, "a") as f:
+            f.write(row.model_dump_json(exclude_none=True) + os.linesep)
+
+    def read(self, row_id: Optional[str] = None) -> List["EvaluationRow"]:
+        """Read rows from the JSONL file."""
+        if not os.path.exists(self.jsonl_path):
+            return []
+
+        data = load_jsonl(self.jsonl_path)
+        rows = [EvaluationRow(**r) for r in data]
+
+        if row_id:
+            # Filter by row_id if specified
+            return [row for row in rows if row.id == row_id]
+        else:
+            return rows

eval_protocol/human_id/__init__.py

@@ -0,0 +1,34 @@
+import random
+import itertools
+from typing import Hashable
+from . import dictionary
+
+__all__ = ["generate_id"]
+
+system_random = random.SystemRandom()
+
+
+def generate_id(separator="-", seed: int | float | str | bytes | bytearray | None = None, word_count=4) -> str:
+    """
+    Generate a human readable ID
+
+    :param separator: The string to use to separate words
+    :param seed: The seed to use. The same seed will produce the same ID
+    :param word_count: The number of words to use. Minimum of 3.
+    :return: A human readable ID
+    """
+    if word_count < 3:
+        raise ValueError("word_count cannot be lower than 3")
+
+    random_obj = system_random
+    if seed:
+        random_obj = random.Random(seed)
+
+    parts = {dictionary.verbs: 1, dictionary.adjectives: 1, dictionary.nouns: 1}
+
+    for _ in range(3, word_count):
+        parts[random_obj.choice(list(parts.keys()))] += 1
+
+    parts = itertools.chain.from_iterable(random_obj.sample(part, count) for part, count in parts.items())
+
+    return separator.join(parts)