Merge branch 'eval-protocol:main' into main

Author: LLiuZheng

eval_protocol/adapters/fireworks_tracing.py

@@ -273,7 +273,12 @@ def search_logs(self, tags: List[str], limit: int = 100, hours_back: int = 24) -
         if not tags:
             raise ValueError("At least one tag is required to fetch logs")
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
         params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
 
         # Try /logs first, fall back to /v1/logs if not found
@@ -398,7 +403,12 @@ def get_evaluation_rows(
         else:
             url = f"{self.base_url}/v1/traces/pointwise"
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
 
         result = None
         try:

eval_protocol/auth.py

@@ -242,9 +242,16 @@ def verify_api_key_and_get_account_id(
         if not resolved_key:
             return None
         resolved_base = api_base or get_fireworks_api_base()
+
+        from .common_utils import get_user_agent
+
         url = f"{resolved_base.rstrip('/')}/verifyApiKey"
-        headers = {"Authorization": f"Bearer {resolved_key}"}
+        headers = {
+            "Authorization": f"Bearer {resolved_key}",
+            "User-Agent": get_user_agent(),
+        }
         resp = requests.get(url, headers=headers, timeout=10)
+
         if resp.status_code != 200:
             logger.debug("verifyApiKey returned status %s", resp.status_code)
             return None

eval_protocol/cli.py

@@ -355,6 +355,10 @@ def parse_args(args=None):
         action="store_true",
         help="Non-interactive: upload all discovered evaluation tests",
     )
+    upload_parser.add_argument(
+        "--env-file",
+        help="Path to .env file containing secrets to upload (default: .env in current directory)",
+    )
 
     # Create command group
     create_parser = subparsers.add_parser(
@@ -421,6 +425,7 @@ def parse_args(args=None):
     rft_parser.add_argument("--rft-job-id", help="Specify an explicit RFT job id")
     rft_parser.add_argument("--yes", "-y", action="store_true", help="Non-interactive mode")
     rft_parser.add_argument("--dry-run", action="store_true", help="Print planned REST calls without sending")
+    rft_parser.add_argument("--force", action="store_true", help="Overwrite existing evaluator with the same ID")
 
     # Run command (for Hydra-based evaluations)
     # This subparser intentionally defines no arguments itself.

eval_protocol/cli_commands/create_rft.py

@@ -5,12 +5,15 @@
 import argparse
 from typing import Any, Dict, Optional
 
+import requests
+
 from ..auth import (
     get_fireworks_account_id,
     get_fireworks_api_base,
     get_fireworks_api_key,
     verify_api_key_and_get_account_id,
 )
+from ..common_utils import get_user_agent
 from ..fireworks_rft import (
     _map_api_host_to_app_host,
     build_default_output_model,
@@ -263,10 +266,72 @@ def _auto_select_evaluator_id(cwd: str) -> Optional[str]:
     return None
 
 
+def _poll_evaluator_status(
+    evaluator_resource_name: str, api_key: str, api_base: str, timeout_minutes: int = 10
+) -> bool:
+    """
+    Poll evaluator status until it becomes ACTIVE or times out.
+
+    Args:
+        evaluator_resource_name: Full evaluator resource name (e.g., accounts/xxx/evaluators/yyy)
+        api_key: Fireworks API key
+        api_base: Fireworks API base URL
+        timeout_minutes: Maximum time to wait in minutes
+
+    Returns:
+        True if evaluator becomes ACTIVE, False if timeout or BUILD_FAILED
+    """
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "User-Agent": get_user_agent(),
+    }
+
+    check_url = f"{api_base}/v1/{evaluator_resource_name}"
+    timeout_seconds = timeout_minutes * 60
+    poll_interval = 10  # seconds
+    start_time = time.time()
+
+    print(f"Polling evaluator status (timeout: {timeout_minutes}m, interval: {poll_interval}s)...")
+
+    while time.time() - start_time < timeout_seconds:
+        try:
+            response = requests.get(check_url, headers=headers, timeout=30)
+            response.raise_for_status()
+
+            evaluator_data = response.json()
+            state = evaluator_data.get("state", "STATE_UNSPECIFIED")
+            status = evaluator_data.get("status", "")
+
+            if state == "ACTIVE":
+                print("✅ Evaluator is ACTIVE and ready!")
+                return True
+            elif state == "BUILD_FAILED":
+                print(f"❌ Evaluator build failed. Status: {status}")
+                return False
+            elif state == "BUILDING":
+                elapsed_minutes = (time.time() - start_time) / 60
+                print(f"⏳ Evaluator is still building... ({elapsed_minutes:.1f}m elapsed)")
+            else:
+                print(f"⏳ Evaluator state: {state}, status: {status}")
+
+        except requests.exceptions.RequestException as e:
+            print(f"Warning: Failed to check evaluator status: {e}")
+
+        # Wait before next poll
+        time.sleep(poll_interval)
+
+    # Timeout reached
+    elapsed_minutes = (time.time() - start_time) / 60
+    print(f"⏰ Timeout after {elapsed_minutes:.1f}m - evaluator is not yet ACTIVE")
+    return False
+
+
 def create_rft_command(args) -> int:
     evaluator_id: Optional[str] = getattr(args, "evaluator_id", None)
     non_interactive: bool = bool(getattr(args, "yes", False))
     dry_run: bool = bool(getattr(args, "dry_run", False))
+    force: bool = bool(getattr(args, "force", False))
 
     api_key = get_fireworks_api_key()
     if not api_key:
@@ -326,12 +391,34 @@ def create_rft_command(args) -> int:
             id=evaluator_id,
             display_name=None,
             description=None,
-            force=False,
+            force=force,  # Pass through the --force flag
             yes=True,
+            env_file=None,  # Add the new env_file parameter
         )
+
+        if force:
+            print(f"🔄 Force flag enabled - will overwrite existing evaluator '{evaluator_id}'")
+
         rc = upload_command(upload_args)
         if rc == 0:
             print(f"✓ Uploaded/ensured evaluator: {evaluator_id}")
+
+            # Poll for evaluator status
+            print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
+            is_active = _poll_evaluator_status(
+                evaluator_resource_name=evaluator_resource_name, api_key=api_key, api_base=api_base, timeout_minutes=10
+            )
+
+            if not is_active:
+                # Print helpful message with dashboard link
+                app_base = _map_api_host_to_app_host(api_base)
+                evaluator_slug = _extract_terminal_segment(evaluator_id)
+                dashboard_url = f"{app_base}/dashboard/evaluators/{evaluator_slug}"
+
+                print("\n❌ Evaluator is not ready within the timeout period.")
+                print(f"📊 Please check the evaluator status at: {dashboard_url}")
+                print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
+                return 1
         else:
             print("Warning: Evaluator upload did not complete successfully; proceeding to RFT creation.")
     except Exception as e:

eval_protocol/cli_commands/upload.py

@@ -9,7 +9,7 @@
 import sys
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Callable, Iterable, Optional
+from typing import Any, Dict, Iterable
 
 import pytest
 from eval_protocol.auth import (
@@ -551,6 +551,35 @@ def _prompt_select(tests: list[DiscoveredTest], non_interactive: bool) -> list[D
     return _prompt_select_interactive(tests)
 
 
+def _load_secrets_from_env_file(env_file_path: str) -> Dict[str, str]:
+    """
+    Load secrets from a .env file that should be uploaded to Fireworks.
+
+    Returns a dictionary of secret key-value pairs that contain 'API_KEY' in the name.
+    """
+    if not os.path.exists(env_file_path):
+        return {}
+
+    # Load the .env file into a temporary environment
+    env_vars = {}
+    with open(env_file_path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith("#") and "=" in line:
+                key, value = line.split("=", 1)
+                key = key.strip()
+                value = value.strip().strip('"').strip("'")  # Remove quotes
+                env_vars[key] = value
+
+    # Filter for secrets that look like API keys
+    secrets = {}
+    for key, value in env_vars.items():
+        if "API_KEY" in key.upper() and value:
+            secrets[key] = value
+
+    return secrets
+
+
 def upload_command(args: argparse.Namespace) -> int:
     root = os.path.abspath(getattr(args, "path", "."))
     entries_arg = getattr(args, "entry", None)
@@ -585,11 +614,27 @@ def upload_command(args: argparse.Namespace) -> int:
     display_name = getattr(args, "display_name", None)
     description = getattr(args, "description", None)
     force = bool(getattr(args, "force", False))
+    env_file = getattr(args, "env_file", None)
 
-    # Ensure FIREWORKS_API_KEY is available to the remote by storing it as a Fireworks secret
+    # Load secrets from .env file and ensure they're available on Fireworks
     try:
         fw_account_id = get_fireworks_account_id()
+
+        # Determine .env file path
+        if env_file:
+            env_file_path = env_file
+        else:
+            env_file_path = os.path.join(root, ".env")
+
+        # Load secrets from .env file
+        secrets_from_file = _load_secrets_from_env_file(env_file_path)
+        secrets_from_env_file = secrets_from_file.copy()  # Track what came from .env file
+
+        # Also ensure FIREWORKS_API_KEY from environment is included
         fw_api_key_value = get_fireworks_api_key()
+        if fw_api_key_value:
+            secrets_from_file["FIREWORKS_API_KEY"] = fw_api_key_value
+
         if not fw_account_id and fw_api_key_value:
             # Attempt to verify and resolve account id from server headers
             resolved = verify_api_key_and_get_account_id(api_key=fw_api_key_value, api_base=get_fireworks_api_base())
@@ -598,21 +643,27 @@ def upload_command(args: argparse.Namespace) -> int:
                 # Propagate to environment so downstream calls use it if needed
                 os.environ["FIREWORKS_ACCOUNT_ID"] = fw_account_id
                 print(f"Resolved FIREWORKS_ACCOUNT_ID via API verification: {fw_account_id}")
-        if fw_account_id and fw_api_key_value:
-            print("Ensuring FIREWORKS_API_KEY is registered as a secret on Fireworks for rollout...")
-            if create_or_update_fireworks_secret(
-                account_id=fw_account_id,
-                key_name="FIREWORKS_API_KEY",
-                secret_value=fw_api_key_value,
-            ):
-                print("✓ FIREWORKS_API_KEY secret created/updated on Fireworks.")
-            else:
-                print("Warning: Failed to create/update FIREWORKS_API_KEY secret on Fireworks.")
+
+        if fw_account_id and secrets_from_file:
+            print(f"Found {len(secrets_from_file)} API keys to upload as Fireworks secrets...")
+            if secrets_from_env_file and os.path.exists(env_file_path):
+                print(f"Loading secrets from: {env_file_path}")
+
+            for secret_name, secret_value in secrets_from_file.items():
+                print(f"Ensuring {secret_name} is registered as a secret on Fireworks for rollout...")
+                if create_or_update_fireworks_secret(
+                    account_id=fw_account_id,
+                    key_name=secret_name,
+                    secret_value=secret_value,
+                ):
+                    print(f"✓ {secret_name} secret created/updated on Fireworks.")
+                else:
+                    print(f"Warning: Failed to create/update {secret_name} secret on Fireworks.")
         else:
             if not fw_account_id:
-                print("Warning: FIREWORKS_ACCOUNT_ID not found; cannot register FIREWORKS_API_KEY secret.")
-            if not fw_api_key_value:
-                print("Warning: FIREWORKS_API_KEY not found locally; cannot register secret.")
+                print("Warning: FIREWORKS_ACCOUNT_ID not found; cannot register secrets.")
+            if not secrets_from_file:
+                print("Warning: No API keys found in environment or .env file; no secrets to register.")
     except Exception as e:
         print(f"Warning: Skipped Fireworks secret registration due to error: {e}")
 

eval_protocol/common_utils.py

@@ -5,6 +5,23 @@
 import requests
 
 
+def get_user_agent() -> str:
+    """
+    Returns the user-agent string for eval-protocol CLI requests.
+
+    Format: eval-protocol-cli/{version}
+
+    Returns:
+        User-agent string identifying the eval-protocol CLI and version.
+    """
+    try:
+        from . import __version__
+
+        return f"eval-protocol/{__version__}"
+    except Exception:
+        return "eval-protocol/unknown"
+
+
 def load_jsonl(file_path: str) -> List[Dict[str, Any]]:
     """
     Reads a JSONL file where each line is a valid JSON object and returns a list of these objects.

eval_protocol/evaluation.py

@@ -20,6 +20,7 @@
     get_fireworks_api_key,
     verify_api_key_and_get_account_id,
 )
+from eval_protocol.common_utils import get_user_agent
 from eval_protocol.typed_interface import EvaluationMode
 
 from eval_protocol.get_pep440_version import get_pep440_version
@@ -405,6 +406,7 @@ def preview(self, sample_file, max_samples=5):
         headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
+            "User-Agent": get_user_agent(),
         }
         logger.info(f"Previewing evaluator using API endpoint: {url} with account: {account_id}")
         logger.debug(f"Preview API Request URL: {url}")
@@ -748,6 +750,7 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
         headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
+            "User-Agent": get_user_agent(),
         }
 
         self._ensure_requirements_present(os.getcwd())