|
| 1 | +import argparse |
| 2 | +from fireworks._client import Fireworks |
| 3 | +from fireworks.types.evaluation_job_create_response import EvaluationJobCreateResponse |
| 4 | +import json |
| 5 | +import os |
| 6 | +import inspect |
| 7 | +from typing import Any, Dict, Optional |
| 8 | + |
| 9 | +from ..auth import get_fireworks_api_base, get_fireworks_api_key |
| 10 | +from ..fireworks_client import create_fireworks_client |
| 11 | +from .utils import ( |
| 12 | + _build_trimmed_dataset_id, |
| 13 | + _build_evaluator_dashboard_url, |
| 14 | + _ensure_account_id, |
| 15 | + _extract_terminal_segment, |
| 16 | + resolve_evaluator, |
| 17 | + validate_evaluator_locally, |
| 18 | +) |
| 19 | +from .create_rft import ( |
| 20 | + resolve_dataset, |
| 21 | + upload_dataset, |
| 22 | +) |
| 23 | + |
| 24 | + |
| 25 | +def _resolve_output_dataset( |
| 26 | + account_id: str, |
| 27 | + output_dataset_arg: Optional[str], |
| 28 | + evaluator_id: str, |
| 29 | +) -> tuple[str, str]: |
| 30 | + """Resolve output dataset id and resource name. |
| 31 | +
|
| 32 | + If not provided, auto-generates an output dataset ID based on the evaluator ID. |
| 33 | + """ |
| 34 | + if output_dataset_arg: |
| 35 | + if output_dataset_arg.startswith("accounts/"): |
| 36 | + output_dataset_resource = output_dataset_arg |
| 37 | + output_dataset_id = _extract_terminal_segment(output_dataset_arg) |
| 38 | + else: |
| 39 | + output_dataset_id = output_dataset_arg |
| 40 | + output_dataset_resource = f"accounts/{account_id}/datasets/{output_dataset_id}" |
| 41 | + else: |
| 42 | + # Auto-generate output dataset ID |
| 43 | + output_dataset_id = _build_trimmed_dataset_id(f"{evaluator_id}-results") |
| 44 | + output_dataset_resource = f"accounts/{account_id}/datasets/{output_dataset_id}" |
| 45 | + print(f"Auto-generated output dataset ID: {output_dataset_id}") |
| 46 | + |
| 47 | + return output_dataset_id, output_dataset_resource |
| 48 | + |
| 49 | + |
| 50 | +def _print_evj_links( |
| 51 | + evaluator_id: str, input_dataset_id: str, output_dataset_id: str, job_name: Optional[str] |
| 52 | +) -> None: |
| 53 | + """Print helpful links to the Fireworks dashboard.""" |
| 54 | + print("\n📊 Links:") |
| 55 | + print(f" Evaluator: {_build_evaluator_dashboard_url(evaluator_id)}") |
| 56 | + if job_name: |
| 57 | + # Extract job id from resource name if present |
| 58 | + job_id = _extract_terminal_segment(job_name) if "/" in job_name else job_name |
| 59 | + print(f" Evaluation Job: https://fireworks.ai/dashboard/evaluation-jobs/{job_id}") |
| 60 | + print(f" Input Dataset: https://fireworks.ai/dashboard/datasets/{input_dataset_id}") |
| 61 | + print(f" Output Dataset: https://fireworks.ai/dashboard/datasets/{output_dataset_id}") |
| 62 | + |
| 63 | + |
| 64 | +def _create_evj_job( |
| 65 | + account_id: str, |
| 66 | + api_key: str, |
| 67 | + api_base: str, |
| 68 | + evaluator_id: str, |
| 69 | + evaluator_resource_name: str, |
| 70 | + input_dataset_id: str, |
| 71 | + input_dataset_resource: str, |
| 72 | + output_dataset_id: str, |
| 73 | + output_dataset_resource: str, |
| 74 | + args: argparse.Namespace, |
| 75 | + dry_run: bool, |
| 76 | +) -> int: |
| 77 | + """Build and submit the Evaluation Job request (via Fireworks SDK).""" |
| 78 | + |
| 79 | + signature = inspect.signature(create_fireworks_client().evaluation_jobs.create) |
| 80 | + |
| 81 | + # Build top-level SDK kwargs |
| 82 | + sdk_kwargs: Dict[str, Any] = {} |
| 83 | + |
| 84 | + # Build the evaluation_job nested object |
| 85 | + evaluation_job: Dict[str, Any] = { |
| 86 | + "evaluator": evaluator_resource_name, |
| 87 | + "input_dataset": input_dataset_resource, |
| 88 | + "output_dataset": output_dataset_resource, |
| 89 | + } |
| 90 | + |
| 91 | + args_dict = vars(args) |
| 92 | + |
| 93 | + # Handle evaluation_job nested fields |
| 94 | + for k, v in args_dict.items(): |
| 95 | + if v is None: |
| 96 | + continue |
| 97 | + if k.startswith("evaluation_job_") and k != "evaluation_job_id": |
| 98 | + field_name = k[len("evaluation_job_") :] |
| 99 | + # Don't overwrite the normalized resources |
| 100 | + if field_name in ("evaluator", "input_dataset", "output_dataset"): |
| 101 | + continue |
| 102 | + evaluation_job[field_name] = v |
| 103 | + |
| 104 | + sdk_kwargs["evaluation_job"] = evaluation_job |
| 105 | + |
| 106 | + # Handle top-level fields |
| 107 | + for name in signature.parameters: |
| 108 | + if name in ("account_id", "evaluation_job", "extra_headers", "extra_query", "extra_body", "timeout"): |
| 109 | + continue |
| 110 | + |
| 111 | + value = args_dict.get(name) |
| 112 | + if value is not None: |
| 113 | + sdk_kwargs[name] = value |
| 114 | + |
| 115 | + print(f"Prepared Evaluation Job for evaluator '{evaluator_id}' using dataset '{input_dataset_id}'") |
| 116 | + |
| 117 | + if dry_run: |
| 118 | + print("--dry-run: would call Fireworks().evaluation_jobs.create with kwargs:") |
| 119 | + print(json.dumps(sdk_kwargs, indent=2)) |
| 120 | + _print_evj_links(evaluator_id, input_dataset_id, output_dataset_id, None) |
| 121 | + return 0 |
| 122 | + |
| 123 | + try: |
| 124 | + fw: Fireworks = create_fireworks_client(api_key=api_key, base_url=api_base) |
| 125 | + job: EvaluationJobCreateResponse = fw.evaluation_jobs.create(account_id=account_id, **sdk_kwargs) |
| 126 | + job_name = job.name |
| 127 | + print(f"\n✅ Created Evaluation Job: {job_name}") |
| 128 | + _print_evj_links(evaluator_id, input_dataset_id, output_dataset_id, job_name) |
| 129 | + return 0 |
| 130 | + except Exception as e: |
| 131 | + print(f"Error creating Evaluation Job: {e}") |
| 132 | + return 1 |
| 133 | + |
| 134 | + |
| 135 | +def create_evj_command(args) -> int: |
| 136 | + """Main entry point for the 'create evj' CLI command.""" |
| 137 | + # Pre-flight: resolve auth and environment |
| 138 | + api_key = get_fireworks_api_key() |
| 139 | + if not api_key: |
| 140 | + print("Error: FIREWORKS_API_KEY not set.") |
| 141 | + return 1 |
| 142 | + |
| 143 | + account_id = _ensure_account_id() |
| 144 | + if not account_id: |
| 145 | + print("Error: Could not resolve Fireworks account id from FIREWORKS_API_KEY.") |
| 146 | + return 1 |
| 147 | + |
| 148 | + api_base = get_fireworks_api_base() |
| 149 | + project_root = os.getcwd() |
| 150 | + evaluator_arg: Optional[str] = getattr(args, "evaluation_job_evaluator", None) |
| 151 | + input_dataset_arg: Optional[str] = getattr(args, "evaluation_job_input_dataset", None) |
| 152 | + output_dataset_arg: Optional[str] = getattr(args, "evaluation_job_output_dataset", None) |
| 153 | + non_interactive: bool = bool(getattr(args, "yes", False)) |
| 154 | + dry_run: bool = bool(getattr(args, "dry_run", False)) |
| 155 | + skip_validation: bool = bool(getattr(args, "skip_validation", False)) |
| 156 | + ignore_docker: bool = bool(getattr(args, "ignore_docker", False)) |
| 157 | + docker_build_extra: str = getattr(args, "docker_build_extra", "") or "" |
| 158 | + docker_run_extra: str = getattr(args, "docker_run_extra", "") or "" |
| 159 | + |
| 160 | + # 1) Resolve evaluator and associated local test |
| 161 | + ( |
| 162 | + evaluator_id, |
| 163 | + evaluator_resource_name, |
| 164 | + selected_test_file_path, |
| 165 | + selected_test_func_name, |
| 166 | + ) = resolve_evaluator(project_root, evaluator_arg, non_interactive, account_id, command_name="create evj") |
| 167 | + if not evaluator_id or not evaluator_resource_name: |
| 168 | + return 1 |
| 169 | + |
| 170 | + # 2) Resolve input dataset source (id or JSONL path) |
| 171 | + input_dataset_id, input_dataset_resource, dataset_jsonl = resolve_dataset( |
| 172 | + project_root=project_root, |
| 173 | + account_id=account_id, |
| 174 | + dataset_id_arg=input_dataset_arg, |
| 175 | + dataset_jsonl_arg=None, # EVJ doesn't support --dataset-jsonl flag yet |
| 176 | + selected_test_file_path=selected_test_file_path, |
| 177 | + selected_test_func_name=selected_test_func_name, |
| 178 | + ) |
| 179 | + # Require either an existing dataset id or a JSONL source to materialize from |
| 180 | + if dataset_jsonl is None and not input_dataset_id: |
| 181 | + return 1 |
| 182 | + |
| 183 | + # 3) Resolve output dataset |
| 184 | + output_dataset_id, output_dataset_resource = _resolve_output_dataset(account_id, output_dataset_arg, evaluator_id) |
| 185 | + |
| 186 | + # 4) Optional local validation |
| 187 | + if not skip_validation: |
| 188 | + if not validate_evaluator_locally( |
| 189 | + project_root=project_root, |
| 190 | + selected_test_file=selected_test_file_path, |
| 191 | + selected_test_func=selected_test_func_name, |
| 192 | + ignore_docker=ignore_docker, |
| 193 | + docker_build_extra=docker_build_extra, |
| 194 | + docker_run_extra=docker_run_extra, |
| 195 | + ): |
| 196 | + return 1 |
| 197 | + |
| 198 | + # 5) Upload dataset when using JSONL sources (no-op for existing datasets) |
| 199 | + input_dataset_id, input_dataset_resource = upload_dataset( |
| 200 | + project_root=project_root, |
| 201 | + account_id=account_id, |
| 202 | + api_key=api_key, |
| 203 | + api_base=api_base, |
| 204 | + evaluator_id=evaluator_id, |
| 205 | + dataset_id=input_dataset_id, |
| 206 | + dataset_resource=input_dataset_resource, |
| 207 | + dataset_jsonl=dataset_jsonl, |
| 208 | + dataset_display_name=None, # EVJ auto-generates display name |
| 209 | + dry_run=dry_run, |
| 210 | + ) |
| 211 | + if not input_dataset_id or not input_dataset_resource: |
| 212 | + return 1 |
| 213 | + |
| 214 | + # 6) Create the Evaluation Job |
| 215 | + return _create_evj_job( |
| 216 | + account_id=account_id, |
| 217 | + api_key=api_key, |
| 218 | + api_base=api_base, |
| 219 | + evaluator_id=evaluator_id, |
| 220 | + evaluator_resource_name=evaluator_resource_name, |
| 221 | + input_dataset_id=input_dataset_id, |
| 222 | + input_dataset_resource=input_dataset_resource, |
| 223 | + output_dataset_id=output_dataset_id, |
| 224 | + output_dataset_resource=output_dataset_resource, |
| 225 | + args=args, |
| 226 | + dry_run=dry_run, |
| 227 | + ) |
0 commit comments