Skip to content

Commit 0a00bcd

Browse files
author
Dylan Huang
committed
Refactor Fireworks client integration
- Introduced a new `fireworks_client.py` module to centralize Fireworks SDK client creation. - Updated CLI and evaluation modules to use the new `create_fireworks_client` function instead of direct instantiation of the Fireworks class. - Enhanced handling of API key, account ID, base URL, and extra headers through environment variables. - Added tests for the new Fireworks client factory to ensure proper functionality and configuration.
1 parent 77842b5 commit 0a00bcd

6 files changed

Lines changed: 310 additions & 17 deletions

File tree

eval_protocol/cli.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,9 @@
88
import sys
99
from pathlib import Path
1010

11-
from fireworks import Fireworks
12-
1311
from .cli_commands.common import setup_logging
1412
from .cli_commands.utils import add_args_from_callable_signature
13+
from .fireworks_client import create_fireworks_client
1514

1615
logger = logging.getLogger(__name__)
1716

@@ -88,7 +87,7 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
8887
)
8988

9089
# Auto-generate flags from SDK Fireworks().evaluators.create() signature
91-
create_evaluator_fn = Fireworks().evaluators.create
90+
create_evaluator_fn = create_fireworks_client().evaluators.create
9291

9392
upload_skip_fields = {
9493
"__top_level__": {
@@ -198,7 +197,7 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
198197
"loss_config.method": "RL loss method for underlying trainers. One of {grpo,dapo}.",
199198
}
200199

201-
create_rft_job_fn = Fireworks().reinforcement_fine_tuning_jobs.create
200+
create_rft_job_fn = create_fireworks_client().reinforcement_fine_tuning_jobs.create
202201

203202
add_args_from_callable_signature(
204203
rft_parser,

eval_protocol/cli_commands/create_rft.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from pydantic import ValidationError
1313

1414
from ..auth import get_fireworks_api_base, get_fireworks_api_key
15+
from ..fireworks_client import create_fireworks_client
1516
from ..common_utils import get_user_agent, load_jsonl
1617
from ..fireworks_rft import (
1718
create_dataset_from_jsonl,
@@ -35,8 +36,6 @@
3536
)
3637
from .local_test import run_evaluator_test
3738

38-
from fireworks import Fireworks
39-
4039

4140
def _extract_dataset_adapter(
4241
test_file_path: str, test_func_name: str
@@ -672,7 +671,7 @@ def _create_rft_job(
672671
) -> int:
673672
"""Build and submit the RFT job request (via Fireworks SDK)."""
674673

675-
signature = inspect.signature(Fireworks().reinforcement_fine_tuning_jobs.create)
674+
signature = inspect.signature(create_fireworks_client().reinforcement_fine_tuning_jobs.create)
676675

677676
# Build top-level SDK kwargs
678677
sdk_kwargs: Dict[str, Any] = {
@@ -711,7 +710,7 @@ def _create_rft_job(
711710
return 0
712711

713712
try:
714-
fw: Fireworks = Fireworks(api_key=api_key, base_url=api_base)
713+
fw: Fireworks = create_fireworks_client(api_key=api_key, base_url=api_base)
715714
job: ReinforcementFineTuningJob = fw.reinforcement_fine_tuning_jobs.create(account_id=account_id, **sdk_kwargs)
716715
job_name = job.name
717716
print(f"\n✅ Created Reinforcement Fine-tuning Job: {job_name}")

eval_protocol/evaluation.py

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,13 @@
55

66
import fireworks
77
import requests
8-
from fireworks import Fireworks
98

109
from eval_protocol.auth import (
1110
get_fireworks_account_id,
1211
get_fireworks_api_key,
1312
verify_api_key_and_get_account_id,
1413
)
14+
from eval_protocol.fireworks_client import create_fireworks_client
1515
from eval_protocol.get_pep440_version import get_pep440_version
1616

1717
logger = logging.getLogger(__name__)
@@ -164,7 +164,11 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
164164
logger.error("Authentication error: API credentials appear to be invalid or incomplete.")
165165
raise ValueError("Invalid or missing API credentials.")
166166

167-
client = Fireworks(api_key=auth_token, base_url=self.api_base, account_id=account_id)
167+
client = create_fireworks_client(
168+
api_key=auth_token,
169+
base_url=self.api_base,
170+
account_id=account_id,
171+
)
168172

169173
self.display_name = display_name or evaluator_id
170174
self.description = description or f"Evaluator created from {evaluator_id}"
@@ -239,9 +243,12 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
239243

240244
tar_size = self._create_tar_gz_with_ignores(tar_path, cwd)
241245

246+
version_id = "test"
247+
242248
# Call GetEvaluatorUploadEndpoint using SDK
243249
logger.info(f"Requesting upload endpoint for {tar_filename}")
244-
upload_response = client.evaluators.get_upload_endpoint(
250+
upload_response = client.evaluator_versions.get_upload_endpoint(
251+
version_id=version_id,
245252
evaluator_id=evaluator_id,
246253
filename_to_size={tar_filename: str(tar_size)},
247254
)
@@ -322,9 +329,9 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
322329
raise
323330

324331
# Step 3: Validate upload using SDK
325-
client.evaluators.validate_upload(
332+
client.evaluator_versions.validate_upload(
333+
version_id=version_id,
326334
evaluator_id=evaluator_id,
327-
body={},
328335
)
329336
logger.info("Upload validated successfully")
330337

eval_protocol/fireworks_client.py

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
"""
2+
Consolidated Fireworks client factory.
3+
4+
This module provides a single point of instantiation for the Fireworks SDK client,
5+
ensuring consistent handling of environment variables and configuration across the
6+
eval_protocol codebase.
7+
8+
Environment variables:
9+
FIREWORKS_API_KEY: API key for authentication (required)
10+
FIREWORKS_ACCOUNT_ID: Account ID (optional, can be derived from API key)
11+
FIREWORKS_API_BASE: Base URL for the API (default: https://api.fireworks.ai)
12+
FIREWORKS_EXTRA_HEADERS: JSON-encoded extra headers to include in requests
13+
Example: '{"X-Custom-Header": "value", "X-Another": "another-value"}'
14+
"""
15+
16+
import json
17+
import logging
18+
import os
19+
from typing import Mapping, Optional
20+
21+
from fireworks import Fireworks
22+
23+
from eval_protocol.auth import (
24+
get_fireworks_account_id,
25+
get_fireworks_api_base,
26+
get_fireworks_api_key,
27+
)
28+
29+
logger = logging.getLogger(__name__)
30+
31+
32+
def get_fireworks_extra_headers() -> Optional[Mapping[str, str]]:
33+
"""
34+
Retrieves extra headers from the FIREWORKS_EXTRA_HEADERS environment variable.
35+
36+
The value should be a JSON-encoded object mapping header names to values.
37+
Example: '{"X-Custom-Header": "value"}'
38+
39+
Returns:
40+
A mapping of header names to values if set and valid, otherwise None.
41+
"""
42+
extra_headers_str = os.environ.get("FIREWORKS_EXTRA_HEADERS")
43+
if not extra_headers_str or not extra_headers_str.strip():
44+
return None
45+
46+
try:
47+
headers = json.loads(extra_headers_str)
48+
if not isinstance(headers, dict):
49+
logger.warning(
50+
"FIREWORKS_EXTRA_HEADERS must be a JSON object, got %s. Ignoring.",
51+
type(headers).__name__,
52+
)
53+
return None
54+
# Validate all keys and values are strings
55+
for k, v in headers.items():
56+
if not isinstance(k, str) or not isinstance(v, str):
57+
logger.warning(
58+
"FIREWORKS_EXTRA_HEADERS contains non-string key or value: %s=%s. Ignoring all extra headers.",
59+
k,
60+
v,
61+
)
62+
return None
63+
logger.debug("Using FIREWORKS_EXTRA_HEADERS: %s", list(headers.keys()))
64+
return headers
65+
except json.JSONDecodeError as e:
66+
logger.warning("Failed to parse FIREWORKS_EXTRA_HEADERS as JSON: %s. Ignoring.", e)
67+
return None
68+
69+
70+
def create_fireworks_client(
71+
*,
72+
api_key: Optional[str] = None,
73+
account_id: Optional[str] = None,
74+
base_url: Optional[str] = None,
75+
extra_headers: Optional[Mapping[str, str]] = None,
76+
) -> Fireworks:
77+
"""
78+
Create a Fireworks client with consistent configuration.
79+
80+
This factory function centralizes the logic for creating Fireworks clients,
81+
ensuring that environment variables are handled consistently across the codebase.
82+
83+
Resolution order for each parameter:
84+
1. Explicit argument passed to this function
85+
2. Environment variable (via auth module helpers)
86+
3. SDK defaults (for base_url only)
87+
88+
Args:
89+
api_key: Fireworks API key. If not provided, resolves from FIREWORKS_API_KEY.
90+
account_id: Fireworks account ID. If not provided, resolves from FIREWORKS_ACCOUNT_ID
91+
or derives from the API key via the verifyApiKey endpoint.
92+
base_url: Base URL for the Fireworks API. If not provided, resolves from
93+
FIREWORKS_API_BASE or defaults to https://api.fireworks.ai.
94+
extra_headers: Additional headers to include in all requests. If not provided,
95+
resolves from FIREWORKS_EXTRA_HEADERS environment variable (JSON-encoded).
96+
97+
Returns:
98+
A configured Fireworks client instance.
99+
100+
Raises:
101+
fireworks.FireworksError: If api_key is not provided and FIREWORKS_API_KEY
102+
environment variable is not set.
103+
"""
104+
# Resolve parameters from environment if not explicitly provided
105+
resolved_api_key = api_key or get_fireworks_api_key()
106+
resolved_account_id = account_id or get_fireworks_account_id()
107+
resolved_base_url = base_url or get_fireworks_api_base()
108+
109+
# Merge extra headers: env var headers first, then explicit headers override
110+
env_extra_headers = get_fireworks_extra_headers()
111+
merged_headers: Optional[Mapping[str, str]] = None
112+
if env_extra_headers or extra_headers:
113+
merged = {}
114+
if env_extra_headers:
115+
merged.update(env_extra_headers)
116+
if extra_headers:
117+
merged.update(extra_headers)
118+
merged_headers = merged if merged else None
119+
120+
logger.debug(
121+
"Creating Fireworks client: base_url=%s, account_id=%s, extra_headers=%s",
122+
resolved_base_url,
123+
resolved_account_id,
124+
list(merged_headers.keys()) if merged_headers else None,
125+
)
126+
127+
return Fireworks(
128+
api_key=resolved_api_key,
129+
account_id=resolved_account_id,
130+
base_url=resolved_base_url,
131+
default_headers=merged_headers,
132+
)

eval_protocol/platform_api.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@
1010
get_fireworks_api_base,
1111
get_fireworks_api_key,
1212
)
13+
from eval_protocol.fireworks_client import create_fireworks_client
1314
from fireworks.types import Secret
14-
from fireworks import Fireworks, FireworksError, NotFoundError, InternalServerError
15+
from fireworks import FireworksError, NotFoundError, InternalServerError
1516

1617
logger = logging.getLogger(__name__)
1718

@@ -88,7 +89,11 @@ def create_or_update_fireworks_secret(
8889
resolved_api_key = api_key or get_fireworks_api_key()
8990
resolved_api_base = api_base or get_fireworks_api_base()
9091
resolved_account_id = account_id # Must be provided
91-
client = Fireworks(api_key=resolved_api_key, account_id=resolved_account_id, base_url=resolved_api_base)
92+
client = create_fireworks_client(
93+
api_key=resolved_api_key,
94+
account_id=resolved_account_id,
95+
base_url=resolved_api_base,
96+
)
9297

9398
if not all([resolved_api_key, resolved_api_base, resolved_account_id]):
9499
logger.error("Missing Fireworks API key, base URL, or account ID for creating/updating secret.")
@@ -173,7 +178,11 @@ def get_fireworks_secret(
173178
logger.error("Missing Fireworks API key, base URL, or account ID for getting secret.")
174179
return None
175180

176-
client = Fireworks(api_key=resolved_api_key, account_id=resolved_account_id, base_url=resolved_api_base)
181+
client = create_fireworks_client(
182+
api_key=resolved_api_key,
183+
account_id=resolved_account_id,
184+
base_url=resolved_api_base,
185+
)
177186
resource_id = _normalize_secret_resource_id(key_name)
178187

179188
try:
@@ -215,7 +224,11 @@ def delete_fireworks_secret(
215224
logger.error("Missing Fireworks API key, base URL, or account ID for deleting secret.")
216225
return False
217226

218-
client = Fireworks(api_key=resolved_api_key, account_id=resolved_account_id, base_url=resolved_api_base)
227+
client = create_fireworks_client(
228+
api_key=resolved_api_key,
229+
account_id=resolved_account_id,
230+
base_url=resolved_api_base,
231+
)
219232
resource_id = _normalize_secret_resource_id(key_name)
220233

221234
try:

0 commit comments

Comments
 (0)