diff --git a/README.md b/README.md index b279ef14..d617f1ed 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,24 @@ under `examples/code_review_agent/artifacts/hermes/`. Its complete base config and clone-based variants live in `examples/code_review_agent/config.py`. +## Supported Agent Harnesses + +Choose a bundled agent harness based on your model ecosystem and application +needs. Every harness supports persistent multi-turn local runtimes through the +same Fabric lifecycle and returns normalized results, artifacts, and telemetry +references. + +| Agent harness | Choose it for | Model ecosystem | Key capabilities | Observability | +| --- | --- | --- | --- | --- | +| Claude | Claude-native coding and tool-use workflows | Anthropic and NVIDIA-hosted Anthropic Messages-compatible models | Tool guardrails, MCP, skills, and persistent Claude sessions | NeMo Relay | +| Codex | Codex-native coding workflows | OpenAI and NVIDIA-hosted Responses-compatible models | MCP, skills, and persistent Codex threads | NeMo Relay and native OpenTelemetry | +| LangChain Deep Agents | Composable LangChain and LangGraph agents | LangChain model providers | Built-in and MCP tools, guardrails, skills, and local subagents | NeMo Relay and native OpenTelemetry/OpenInference | +| Hermes Agent | Hermes workflows with custom model endpoints | Configurable provider, model, and base URL | Toolsets, guardrails, MCP, skills, and persistent conversation history | NeMo Relay | + +For package names, exact compatibility and limitations, runtime ownership, and +individual harness guides, refer to the +[adapter compatibility reference](adapters/README.md). + ## Claude Adapter Build the local wheels and install Fabric with the independent Claude adapter: @@ -154,8 +172,8 @@ python -m pip install --find-links dist "nemo-fabric[claude]" ``` Refer to the [Claude adapter guide](adapters/claude/README.md) for -typed configuration, normalized tools, MCP and skills, multi-turn resume, -authentication, and execution details. +typed configuration, normalized tools, MCP and skills, persistent multi-turn +runtimes, authentication, and execution details. ## Core Concepts @@ -179,12 +197,10 @@ authentication, and execution details. `disallowed_tools`, Deep Agents enforces them with middleware, and adapters without a native deny mechanism route the policy as unsupported. - **Adapters:** harness-specific integrations selected by `harness.adapter_id`. - The Hermes adapter lives under `adapters/hermes/`; the Codex SDK - adapter lives under `adapters/codex/`; the - [Claude adapter](adapters/claude/README.md) - lives under `adapters/claude/`; the LangChain Deep Agents adapter lives under - `adapters/deepagents/`. Harness-specific extensions belong under - `harness.settings` so the normalized contract can remain stable. + Harness-specific extensions belong under `harness.settings` so the normalized + contract can remain stable. Refer to the + [adapter compatibility reference](adapters/README.md) for bundled package and + implementation details. - **Artifacts:** normalized output, logs, patches, and telemetry references returned through an `ArtifactManifest`. @@ -211,9 +227,8 @@ the [Python SDK guide](docs/sdk/python.mdx). Exact signatures are in the - [Harbor examples](examples/harbor/README.md): validate the integration with a deterministic, credential-free calculator smoke, optionally run the same task with Hermes or Claude, and evaluate real coding tasks with SWE-Bench. -- Adapter guides: [Hermes](adapters/hermes/README.md), - [Codex SDK](adapters/codex/README.md), and - [Deep Agents](adapters/deepagents/README.md). +- [Adapter compatibility and guides](adapters/README.md): compare bundled + harness support, runtime ownership, telemetry integration, and package guides. ## Tests diff --git a/adapters/README.md b/adapters/README.md new file mode 100644 index 00000000..d771fc02 --- /dev/null +++ b/adapters/README.md @@ -0,0 +1,63 @@ + + +# NVIDIA NeMo Fabric Agent Harness Adapters + +NeMo Fabric adapters translate the normalized Fabric contract into +harness-native models, tools, sessions, and telemetry. Use this reference to +compare the bundled adapters and then open the linked package guide for +installation, authentication, and configuration details. + +The adapter descriptor selected in `RunPlan` is authoritative for normalized +configuration and telemetry support. + +## Bundled Adapter Packages + +| Agent Harness | Adapter ID | Python Package | Supported Python | +| --- | --- | --- | --- | +| [Claude](claude/README.md) | `nvidia.fabric.claude` | `nemo-fabric-adapters-claude` | 3.11+ | +| [Codex](codex/README.md) | `nvidia.fabric.codex` | `nemo-fabric-adapters-codex` | 3.11+ | +| [LangChain Deep Agents](deepagents/README.md) | `nvidia.fabric.langchain.deepagents` | `nemo-fabric-adapters-deepagents` | 3.11+ | +| [Hermes Agent](hermes/README.md) | `nvidia.fabric.hermes` | `nemo-fabric-adapters-hermes` | 3.11-3.13 | + +## Configuration Compatibility + +| Agent Harness | Models | Tools / Blocked Tools | MCP | Skills | Subagents | +| --- | --- | --- | --- | --- | --- | +| [Claude](claude/README.md) | Anthropic and NVIDIA-hosted Anthropic Messages-compatible models | `allowed_tools` adapter setting / normalized `tools.blocked` | Normalized: stdio, HTTP, streamable HTTP, and SSE | Normalized `skills.paths` | Not exposed | +| [Codex](codex/README.md) | OpenAI; NVIDIA Responses-compatible models without Relay | Codex-native tools / configuring `tools.blocked` is unsupported and raises `UnsupportedToolsPolicy` | Normalized: stdio, HTTP, and streamable HTTP | Normalized `SKILL.md` directories | Not exposed | +| [LangChain Deep Agents](deepagents/README.md) | LangChain model providers | Built-ins and MCP / normalized middleware block list | Normalized through `langchain-mcp-adapters` | Normalized | Constrained declarative local delegation | +| [Hermes Agent](hermes/README.md) | Normalized provider, model, and base URL | Toolsets / normalized disabled toolsets | Normalized | Normalized | Not exposed | + +"Normalized" means that the adapter accepts the corresponding `FabricConfig` +field. "Not exposed" does not mean that the underlying harness lacks the +feature; it means that Fabric does not provide a portable configuration surface +for it. Fabric normalizes a blocked-tool list, not a portable tool-definition +catalog. Deep Agents subagents are limited to declarative local subagents that +inherit the parent agent's capabilities. + +## Runtime and Observability Compatibility + +All bundled adapters use one persistent Python adapter host with an ordered +`start` → `invoke*` → `stop` protocol. + +NeMo Relay records raw events in Agent Trajectory Observability Format (ATOF) +and produces normalized trajectories in Agent Trajectory Interchange Format +(ATIF). + +| Agent Harness | State Retained Across Turns | Relay Integration | Per-Turn Behavior | Stop Behavior | Remote Service | +| --- | --- | --- | --- | --- | --- | +| [Claude](claude/README.md) | `ClaudeSDKClient` and Claude session ID | Runtime-owned Relay CLI gateway and generated Claude hooks | Calls `client.query()`, validates the session ID, and collects ATOF and ATIF | Disconnects the client, stops the gateway, and removes the generated plugin | Not implemented | +| [Codex](codex/README.md) | `AsyncCodex` app-server client and SDK thread | Runtime-owned Relay CLI gateway and Codex SDK hooks | Reuses the SDK thread and persists its thread ID | Closes the SDK client and app server, then stops the gateway | Not implemented | +| [LangChain Deep Agents](deepagents/README.md) | Compiled LangGraph agent, checkpointer, and thread ID | NeMo Relay Python SDK integration added when the agent is compiled | Creates a fresh Relay request scope and callback for each invocation | Closes the checkpointer; no gateway process | Not implemented | +| [Hermes Agent](hermes/README.md) | `AIAgent`, `SessionDB`, and conversation history | Hermes NeMo Relay plugin context | Finalizes and flushes Relay after each invocation | Closes the agent and database, then exits the plugin context | Not implemented | + +Telemetry output names use the descriptor contract values. Claude, Codex, and +Hermes can emit NeMo Relay ATIF, OpenTelemetry, and OpenInference output. Deep +Agents supports the same Relay outputs plus native OpenTelemetry and +OpenInference; Codex also supports native OpenTelemetry. + +Shared lifecycle, Relay gateway, hook, and payload helpers are documented in +the [adapter utilities guide](common/README.md). diff --git a/adapters/claude/README.md b/adapters/claude/README.md index 77652142..1f3d37c0 100644 --- a/adapters/claude/README.md +++ b/adapters/claude/README.md @@ -48,7 +48,7 @@ that Claude Code and the Claude Agent SDK consume. This includes and `ANTHROPIC_IDENTITY_TOKEN` or `ANTHROPIC_IDENTITY_TOKEN_FILE`. Fabric reads selected environment values and forwards them to the Claude runtime, but it does not persist or log them in configuration or artifacts. Authentication is -validated when Claude starts the invocation. +validated when the Claude runtime starts. Unset unused `ANTHROPIC_API_KEY` and `ANTHROPIC_AUTH_TOKEN` variables before using WIF. Anthropic credential resolution treats an empty variable as selected, @@ -71,11 +71,18 @@ for other supported installation methods. ## Execution Model -Each `invoke` starts a fresh adapter process. The adapter persists the terminal -Claude session ID under the Fabric artifact root, keyed by `runtime_id`, and -passes it as `ClaudeAgentOptions.resume` on the next invocation. One Fabric -runtime therefore maps to one Claude session even though no adapter process -stays resident. +The Claude adapter implements Fabric's persistent local-host wire protocol. +`Fabric.start_runtime(...)` launches one adapter host, creates one +`ClaudeSDKClient`, and connects it once. Every `Runtime.invoke(...)` reuses that +client and its event loop; `Runtime.stop()` disconnects the client and exits the +host. `Fabric.run(...)` uses the same lifecycle around one invocation. + +One Fabric runtime maps to one live Claude session. The adapter records the +terminal Claude session ID under the Fabric artifact root for correlation, but +does not silently recreate a crashed host or replay an invocation. Start a new +Fabric runtime when the host or SDK connection becomes unusable. Runtime +hosting is adapter-declared; consumers do not configure a runtime strategy in +`FabricConfig` or `harness.settings`. ## Configuration @@ -90,7 +97,7 @@ Configure portable capabilities through the normalized `FabricConfig` fields: - `mcp` configures stdio, HTTP, streamable HTTP, or SSE servers. For stdio, Fabric parses `url` as a command plus arguments. - `skills.paths` names skill directories that contain `SKILL.md`. The adapter - stages these directories as a local Claude plugin for the invocation. + stages these directories as a local Claude plugin for the runtime. Only Claude-specific controls belong in `harness.settings`: @@ -121,12 +128,13 @@ config.enable_relay( ) ``` -For each Relay-enabled invocation, Fabric starts one `nemo-relay` gateway, -waits for its health endpoint, and stops it after Claude succeeds, fails, times -out, or is canceled. Fabric passes the gateway URL to Claude Code through -`ANTHROPIC_BASE_URL` and `NEMO_RELAY_GATEWAY_URL`. It also stages an -invocation-scoped Claude plugin that forwards lifecycle hooks with -`nemo-relay hook-forward claude`. +For each Relay-enabled Claude runtime, Fabric starts one `nemo-relay` gateway, +waits for its health endpoint, and stops it with the runtime. Fabric passes the +gateway URL to the connected Claude Code process through `ANTHROPIC_BASE_URL` +and `NEMO_RELAY_GATEWAY_URL`. It also stages a runtime-scoped Claude plugin that +forwards lifecycle hooks with `nemo-relay hook-forward claude`. +`Fabric.run(...)` starts the same runtime, invokes it once, and stops it, so the +gateway has the same lifecycle as that single invocation. The Fabric result includes `relay_runtime.gateway_config_path`, `relay_runtime.gateway_log_path`, and the collected `relay_artifacts`. Relay @@ -194,7 +202,7 @@ config = FabricConfig( fabric = Fabric() ``` -## One-Shot Run +## Single Invocation ```python result = await fabric.run( @@ -218,6 +226,6 @@ assert first.runtime_id == second.runtime_id assert first.output["session_id"] == second.output["session_id"] ``` -Resume requires the same workspace and Claude state directory on the same host. -The Fabric-to-Claude correlation record alone is insufficient if Claude's -underlying transcript store is removed. +The runtime must remain on the same local host for its lifetime. A persisted +Fabric-to-Claude correlation record is not an attach token and cannot recover a +stopped or crashed local host. diff --git a/adapters/claude/fabric-adapter.json b/adapters/claude/fabric-adapter.json index ed36b146..9f8996f9 100644 --- a/adapters/claude/fabric-adapter.json +++ b/adapters/claude/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "claude", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.claude.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.claude.adapter" }, "config": { "accepts": ["models", "tools", "tools.blocked", "mcp", "skills", "telemetry"] diff --git a/adapters/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/claude/src/nemo_fabric_adapters/claude/adapter.py index b76c0672..2a57fd10 100644 --- a/adapters/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Run Claude Agent SDK through the Fabric adapter process contract.""" +"""Run Claude Agent SDK through the Fabric adapter contract.""" from __future__ import annotations @@ -21,6 +21,7 @@ from typing import Any from claude_agent_sdk import ClaudeAgentOptions +from claude_agent_sdk import ClaudeSDKClient from claude_agent_sdk import ClaudeSDKError from claude_agent_sdk import CLIConnectionError from claude_agent_sdk import CLIJSONDecodeError @@ -28,8 +29,8 @@ from claude_agent_sdk import Message from claude_agent_sdk import ProcessError from claude_agent_sdk import ResultMessage -from claude_agent_sdk import query from claude_agent_sdk._errors import MessageParseError +from nemo_fabric_adapters.common import lifecycle from nemo_fabric_adapters.common import relay_gateway from nemo_fabric_adapters.common import relay_hooks from nemo_fabric_adapters.common import utils as common_utils @@ -97,7 +98,7 @@ @dataclass(frozen=True) class ClaudeRelaySettings: - """Invocation-scoped Relay gateway and Claude plugin settings.""" + """Relay gateway and Claude plugin settings owned by one adapter run.""" gateway: relay_gateway.RelayGatewayLaunch plugin_config: dict[str, Any] @@ -128,10 +129,6 @@ class AdapterConfigError(ClaudeAdapterError): """Invalid Claude adapter configuration.""" -class AdapterStateError(ClaudeAdapterError): - """Invalid persisted runtime state.""" - - class AdapterRelayError(ClaudeAdapterError): """NeMo Relay setup or lifecycle failure.""" @@ -415,7 +412,7 @@ def _stage_relay_plugin(plugin_path: Path, executable: Path) -> None: def prepare_claude_relay(payload: dict[str, Any]) -> ClaudeRelaySettings | None: - """Generate invocation-scoped Relay and Claude hook configuration.""" + """Generate Relay gateway and Claude hook configuration.""" if not common_utils.relay_enabled(payload): return None @@ -493,7 +490,6 @@ def discard_stderr(_: str) -> None: def build_options( payload: dict[str, Any], *, - resume: str | None, relay: ClaudeRelaySettings | None = None, ) -> ClaudeAgentOptions: settings = _settings(payload) @@ -536,7 +532,6 @@ def build_options( plugins.append({"type": "local", "path": str(relay.plugin_path)}) return ClaudeAgentOptions( - resume=resume, cwd=resolve_cwd(payload), model=selected_model(payload), system_prompt=system_prompt, @@ -573,58 +568,6 @@ def _artifact_root(payload: dict[str, Any]) -> Path: return Path(common_utils.base_dir(payload)) / "artifacts" / "claude" -def runtime_state_path(payload: dict[str, Any], fabric_runtime_id: str) -> Path: - digest = sha256(fabric_runtime_id.encode("utf-8")).hexdigest() - return ( - _artifact_root(payload) / ".fabric" / "claude" / "runtimes" / f"{digest}.json" - ) - - -def load_claude_session_id( - payload: dict[str, Any], fabric_runtime_id: str -) -> str | None: - path = runtime_state_path(payload, fabric_runtime_id) - if not path.exists(): - return None - try: - state = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(state, dict): - raise ValueError("state must be an object") - if state.get("runtime_id") != fabric_runtime_id: - raise ValueError("runtime mismatch") - session_id = state.get("claude_session_id") - if not isinstance(session_id, str) or not session_id: - raise ValueError("missing Claude session") - return session_id - except (OSError, ValueError, json.JSONDecodeError) as error: - raise AdapterStateError( - "claude_invalid_runtime_state", "Claude runtime state is invalid" - ) from error - - -def save_claude_session_id( - payload: dict[str, Any], fabric_runtime_id: str, claude_session_id: str -) -> None: - if not claude_session_id: - raise AdapterStateError( - "claude_invalid_runtime_state", "Claude session ID is missing" - ) - path = runtime_state_path(payload, fabric_runtime_id) - path.parent.mkdir(parents=True, exist_ok=True) - invocation_id = ( - common_utils.runtime_context(payload).get("invocation_id") or "invocation" - ) - temporary = path.with_suffix(f".{invocation_id}.tmp") - temporary.write_text( - json.dumps( - {"runtime_id": fabric_runtime_id, "claude_session_id": claude_session_id}, - sort_keys=True, - ), - encoding="utf-8", - ) - os.replace(temporary, path) - - def _json_safe(value: Any) -> Any: if is_dataclass(value) and not isinstance(value, type): return _json_safe(asdict(value)) @@ -819,126 +762,239 @@ def _cleanup_relay( return cleanup_error -def _merge_relay_output( - output: dict[str, Any], - relay: ClaudeRelaySettings | None, - cleanup_error: AdapterRelayError | None, -) -> dict[str, Any]: - if relay is None: - return output - output = _relay_output(output, relay) - if cleanup_error is None: - return output - cleanup: dict[str, Any] = { - "code": cleanup_error.code, - "message": cleanup_error.message, - "retryable": False, - } - if cleanup_error.metadata: - cleanup["metadata"] = cleanup_error.metadata - output["relay_runtime"]["cleanup_error"] = cleanup - if not output["failed"]: - output["completed"] = False - output["failed"] = True - output["error"] = cleanup - return output - - -def _persist_result_session( - payload: dict[str, Any], - fabric_runtime_id: str, - prior_session_id: str | None, - result: ResultMessage, +def _validate_result_session( + current_session_id: str | None, result: ResultMessage ) -> dict[str, Any] | None: - if prior_session_id is not None and result.session_id != prior_session_id: + if current_session_id is not None and result.session_id != current_session_id: return _failure( "claude_session_mismatch", - "Claude session identity changed during resume", + "Claude session identity changed during the runtime", ) - save_claude_session_id(payload, fabric_runtime_id, result.session_id) return None -async def run_claude(payload: dict[str, Any]) -> dict[str, Any]: - fabric_runtime_id = runtime_id(payload) - prior_session_id = load_claude_session_id(payload, fabric_runtime_id) - relay = prepare_claude_relay(payload) - gateway_process = None - cleanup_error: AdapterRelayError | None = None - messages: list[Message] = [] - result: ResultMessage | None = None - try: - gateway_process = _start_relay_gateway(payload, relay) - options = build_options(payload, resume=prior_session_id, relay=relay) +def _as_lifecycle_error(error: ClaudeAdapterError) -> lifecycle.LifecycleError: + return lifecycle.LifecycleError( + error.code, + error.message, + metadata=error.metadata, + ) + + +def _sdk_lifecycle_error(error: BaseException) -> lifecycle.LifecycleError: + output = sdk_failure(error) + reported = output["error"] + return lifecycle.LifecycleError( + reported["code"], + reported["message"], + retryable=reported["retryable"], + metadata=reported.get("metadata"), + ) + + +class ClaudeRuntime: + """One connected Claude SDK client owned by a Fabric runtime.""" + + def __init__(self) -> None: + self._start_payload: dict[str, Any] | None = None + self._fabric_runtime_id: str | None = None + self._claude_session_id: str | None = None + self._client: ClaudeSDKClient | None = None + self._relay: ClaudeRelaySettings | None = None + self._gateway_process: subprocess.Popen[Any] | None = None + self._unusable = False + + async def start(self, payload: dict[str, Any]) -> None: + if self._client is not None: + raise lifecycle.LifecycleError( + "claude_runtime_already_started", + "Claude runtime is already started", + ) + try: + fabric_runtime_id = runtime_id(payload) + relay = prepare_claude_relay(payload) + self._relay = relay + self._gateway_process = _start_relay_gateway(payload, relay) + options = build_options(payload, relay=relay) + client = ClaudeSDKClient(options) + await client.connect() + except ClaudeAdapterError as error: + self._cleanup_failed_start() + raise _as_lifecycle_error(error) from error + except ClaudeSDKError as error: + self._cleanup_failed_start() + raise _sdk_lifecycle_error(error) from error + except BaseException: + self._cleanup_failed_start() + raise + + self._start_payload = payload + self._fabric_runtime_id = fabric_runtime_id + self._client = client + + async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]: + client = self._client + start_payload = self._start_payload + fabric_runtime_id = self._fabric_runtime_id + if client is None or start_payload is None or fabric_runtime_id is None: + raise lifecycle.LifecycleError( + "claude_runtime_not_started", + "Claude runtime is not started", + ) + if runtime_id(invocation) != fabric_runtime_id: + raise lifecycle.LifecycleError( + "claude_runtime_mismatch", + "Claude invocation does not match the connected runtime", + ) + payload = { + **start_payload, + "runtime_context": invocation.get("runtime_context"), + "request": invocation.get("request"), + } + if self._unusable: + return _failure( + "claude_runtime_unavailable", + "Claude runtime cannot accept another invocation after an SDK failure", + ) + try: - async with asyncio.timeout(timeout_seconds(payload)): - async for message in query( - prompt=request_prompt(payload), options=options - ): + prompt = request_prompt(payload) + invocation_timeout = timeout_seconds(payload) + except ClaudeAdapterError as error: + output = adapter_failure(error) + else: + output = await self._run_query( + payload, + client, + prompt, + invocation_timeout, + ) + + if self._relay is not None: + output = _relay_output(output, self._relay) + return output + + async def _run_query( + self, + payload: dict[str, Any], + client: ClaudeSDKClient, + prompt: str, + invocation_timeout: float, + ) -> dict[str, Any]: + """Run one SDK query and normalize its terminal result.""" + + messages: list[Message] = [] + result: ResultMessage | None = None + try: + async with asyncio.timeout(invocation_timeout): + await client.query(prompt) + async for message in client.receive_response(): if isinstance(message, ResultMessage): result = message else: messages.append(message) except (TimeoutError, ClaudeSDKError) as error: + self._unusable = True + await self._interrupt_failed_invocation() output = sdk_failure(error) except Exception: # Claude Agent SDK 0.2.120 can yield an error ResultMessage and then - # raise a plain Exception while closing the query stream. Preserve - # the typed terminal result, but do not hide unrelated exceptions. + # raise a plain Exception while closing the response stream. + self._unusable = True if result is None or not _result_failed(result): raise LOGGER.exception("Claude SDK stream raised after a failed terminal result") - output = normalize_result(payload, messages, result) + output = self._normalize_invocation(payload, messages, result) else: if result is None: + self._unusable = True output = _failure( "claude_missing_result", "Claude returned no terminal result" ) else: - output = normalize_result(payload, messages, result) - if not output["failed"]: - output = ( - _persist_result_session( - payload, - fabric_runtime_id, - prior_session_id, - result, - ) - or output - ) - finally: - cleanup_error = _cleanup_relay(relay, gateway_process) - - return _merge_relay_output(output, relay, cleanup_error) - - -def run(payload: dict[str, Any]) -> dict[str, Any]: - """Run one Fabric invocation.""" + output = self._normalize_invocation(payload, messages, result) + return output - try: - return asyncio.run(run_claude(payload)) - except ClaudeAdapterError as error: - return adapter_failure(error) - except Exception: # Adapter boundary must always return normalized JSON. - return _failure( - "claude_adapter_internal_error", "Claude adapter failed unexpectedly" - ) + def _normalize_invocation( + self, + payload: dict[str, Any], + messages: list[Message], + result: ResultMessage, + ) -> dict[str, Any]: + try: + output = normalize_result(payload, messages, result) + if not output["failed"]: + invalid_session = _validate_result_session( + self._claude_session_id, result + ) + if invalid_session is not None: + self._unusable = True + return invalid_session + self._claude_session_id = result.session_id + return output + except ClaudeAdapterError as error: + self._unusable = True + return adapter_failure(error) + + async def stop(self) -> None: + client = self._client + self._client = None + self._start_payload = None + self._fabric_runtime_id = None + self._claude_session_id = None + self._unusable = True + + disconnect_error: BaseException | None = None + try: + if client is not None: + await client.disconnect() + except BaseException as error: + if not isinstance(error, asyncio.CancelledError): + LOGGER.exception("Claude SDK client failed to disconnect") + disconnect_error = error + finally: + cleanup_error = _cleanup_relay(self._relay, self._gateway_process) + self._relay = None + self._gateway_process = None + if isinstance(disconnect_error, asyncio.CancelledError): + raise disconnect_error + if disconnect_error is not None: + if cleanup_error is not None: + LOGGER.error( + "Claude Relay cleanup also failed during disconnect: %s", + cleanup_error.code, + ) + raise lifecycle.LifecycleError( + "claude_disconnect_failed", + "Claude SDK client failed to disconnect", + ) from disconnect_error + if cleanup_error is not None: + raise _as_lifecycle_error(cleanup_error) + + def _cleanup_failed_start(self) -> None: + cleanup_error = _cleanup_relay(self._relay, self._gateway_process) + self._relay = None + self._gateway_process = None + if cleanup_error is not None: + LOGGER.error( + "Claude runtime cleanup after start failure also failed: %s", + cleanup_error.code, + ) + + async def _interrupt_failed_invocation(self) -> None: + if self._client is None: + return + try: + async with asyncio.timeout(5): + await self._client.interrupt() + except Exception: + LOGGER.exception("Claude SDK invocation could not be interrupted") def main() -> None: - try: - payload = common_utils.load_payload() - except ( - Exception - ): # Malformed invocation input must still satisfy the process contract. - output = _failure( - "claude_adapter_internal_error", "Claude adapter failed unexpectedly" - ) - else: - output = run(payload) - print(json.dumps(output, sort_keys=True)) - if output.get("failed"): - raise SystemExit(2) + """Serve the persistent local-host lifecycle protocol.""" + + lifecycle.serve(ClaudeRuntime) if __name__ == "__main__": diff --git a/adapters/codex/README.md b/adapters/codex/README.md index e96c06b1..c26f36ea 100644 --- a/adapters/codex/README.md +++ b/adapters/codex/README.md @@ -49,11 +49,13 @@ the SDK runtime. The current real-agent acceptance path validates an existing Codex login; it does not yet claim a raw environment variable as a complete login flow. -When `models.default.provider` is `nvidia`, the adapter defines a request-scoped -Codex model provider for the configured NVIDIA Responses endpoint. It reads the -credential from `api_key_env` (default: `NVIDIA_API_KEY`) and isolates Codex -state under the Fabric artifact root, so the invocation does not depend on or -modify a user's Codex login. Set the endpoint in +When `models.default.provider` is `nvidia`, the adapter defines a Codex model +provider for the configured NVIDIA Responses endpoint. `Fabric.run(...)` owns +that provider for one invocation, while `Fabric.start_runtime(...)` fixes it for +the lifetime of the persistent runtime. The adapter reads the credential from +`api_key_env` (default: `NVIDIA_API_KEY`) and isolates Codex state under the +Fabric artifact root, so execution does not depend on or modify a user's Codex +login. Set the endpoint in `models.default.settings.base_url` or `NVIDIA_FRONTIER_BASE_URL`; the adapter does not assume a default frontier endpoint. @@ -69,11 +71,13 @@ to the explicit `base_dir`. Fabric passes the resolved path through ## Execution Model -Each Fabric invocation starts a fresh SDK client and closes its app-server -transport before returning. The first invocation creates a Codex thread and -persists its ID under the Fabric artifact root. Later invocations for the same -Fabric runtime resume that exact thread. Codex owns the transcript; Fabric owns -runtime-to-thread correlation, timeout, cancellation, and cleanup. +Each Fabric runtime currently starts one local adapter host and retains one +`AsyncCodex` client and one Codex thread. The Codex SDK starts and controls its +pinned local `codex app-server` subprocess over JSON-RPC. Ordered +`Runtime.invoke(...)` calls reuse that client and thread directly; the adapter +closes the SDK client and app-server transport during `Runtime.stop()`. Codex +owns the transcript; Fabric owns runtime-to-thread correlation, timeout, +cancellation, and cleanup. The result includes the SDK's typed terminal response, turn status, token usage, timing, and completed thread items. It does not expose CLI commands, @@ -87,7 +91,7 @@ Use normalized `FabricConfig` fields for portable configuration: provider and NVIDIA-hosted Responses-compatible models through the `nvidia` provider. - `environment.workspace` sets the working directory. -- `mcp` maps stdio, HTTP, and streamable HTTP servers into request-scoped Codex +- `mcp` maps stdio, HTTP, and streamable HTTP servers into the Codex thread's `mcp_servers` configuration. For stdio, Fabric parses `url` as a command plus arguments. - `skills.paths` names skill directories that contain `SKILL.md`. The adapter @@ -109,8 +113,8 @@ Codex-specific controls belong in `harness.settings`: - `personality`, `reasoning_effort`, `service_name`, and `service_tier` - `output_schema` for SDK-native structured output - `codex_bin` for an explicit Codex app-server runtime override -- `config_overrides` as dotted request-scoped Codex configuration keys, such as - Codex-only MCP timeout or required-server options +- `config_overrides` as dotted Codex configuration keys applied when the SDK + runtime starts, such as Codex-only MCP timeout or required-server options - `timeout_seconds`, defaulting to 1800 - `env` for variables explicitly forwarded to the Codex runtime - `nemo_relay_command` for the optional external Relay gateway executable @@ -118,6 +122,12 @@ Codex-specific controls belong in `harness.settings`: Set model selection through `models` and the working directory through `environment.workspace`. +For `Fabric.start_runtime(...)`, the model provider, MCP configuration, skill +roots, and `config_overrides` are fixed when the runtime starts and cannot vary +between `Runtime.invoke(...)` calls. Start a new runtime to change them. +`Fabric.run(...)` starts the same runtime, invokes it once, and stops it, so the +same settings are scoped to that single invocation. + The adapter filters the inherited environment. It retains portable OS and Codex state variables, the selected model's `api_key_env`, and explicit `settings.env` values while clearing unrelated parent-process secrets. @@ -125,15 +135,16 @@ Codex state variables, the selected model's `api_key_env`, and explicit ## Relay Observability Enable Relay through Fabric's normalized telemetry configuration. For each -Relay-enabled invocation, Fabric: +Relay-enabled Fabric runtime, the adapter: 1. Resolves one external `nemo-relay` executable. -2. Generates invocation-scoped gateway and plugin configuration. +2. Generates runtime-scoped gateway and plugin configuration. 3. Starts and health-checks `nemo-relay --config ... --bind ...`. -4. Redirects the built-in OpenAI provider with request-scoped +4. Redirects the built-in OpenAI provider with runtime-scoped `openai_base_url` and passes Relay hooks through the Codex SDK's `config` argument. -5. Interrupts timed-out turns, closes the SDK runtime, and stops the gateway. +5. Reuses the SDK client and gateway across turns, interrupting a timed-out turn. +6. Closes the SDK runtime and stops the gateway during runtime shutdown. The SDK remains the Codex execution driver. Relay is a supervised sidecar and hook forwarder; the adapter never invokes a `nemo-relay codex` wrapper. The diff --git a/adapters/codex/fabric-adapter.json b/adapters/codex/fabric-adapter.json index 96f2d7dd..251ab043 100644 --- a/adapters/codex/fabric-adapter.json +++ b/adapters/codex/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "codex", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.codex.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.codex.adapter" }, "config": { "accepts": ["models", "mcp", "skills", "telemetry"] diff --git a/adapters/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/codex/src/nemo_fabric_adapters/codex/adapter.py index 6f367192..0d08f927 100644 --- a/adapters/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -8,12 +8,13 @@ import asyncio import json +import logging import math import os import shlex +import subprocess from dataclasses import asdict, dataclass, is_dataclass from enum import Enum -from hashlib import sha256 from pathlib import Path from typing import Any @@ -32,6 +33,7 @@ import nemo_fabric_adapters.common.relay_gateway as relay_gateway import nemo_fabric_adapters.common.relay_hooks as relay_hooks import nemo_fabric_adapters.common.utils as common_utils +from nemo_fabric_adapters.common import lifecycle DEFAULT_TIMEOUT_SECONDS = 1800.0 @@ -89,11 +91,12 @@ "model_name": "FabricConfig.models", "skills": "FabricConfig.skills", } +LOGGER = logging.getLogger(__name__) @dataclass(frozen=True) class CodexRelaySettings: - """Invocation-scoped Relay state consumed by the Codex SDK adapter.""" + """Runtime-scoped Relay state consumed by the Codex SDK adapter.""" gateway: relay_gateway.RelayGatewayLaunch plugin_config: dict[str, Any] @@ -123,10 +126,6 @@ class AdapterConfigError(CodexAdapterError): """Invalid Codex adapter configuration.""" -class AdapterStateError(CodexAdapterError): - """Invalid persisted runtime state.""" - - class AdapterRelayError(CodexAdapterError): """NeMo Relay setup or lifecycle failure.""" @@ -355,9 +354,8 @@ def nvidia_model_provider_config(payload: dict[str, Any]) -> dict[str, Any]: model_settings = _mapping( model_config.get("settings"), name="selected model settings" ) - base_url = ( - model_settings.get("base_url") - or os.environ.get("NVIDIA_FRONTIER_BASE_URL") + base_url = model_settings.get("base_url") or os.environ.get( + "NVIDIA_FRONTIER_BASE_URL" ) if not isinstance(base_url, str) or not base_url: raise AdapterConfigError( @@ -490,54 +488,6 @@ def state_dir(payload: dict[str, Any]) -> Path: return _artifact_root(payload) / ".fabric" / "codex" -def runtime_state_path(payload: dict[str, Any], fabric_runtime_id: str) -> Path: - digest = sha256(fabric_runtime_id.encode("utf-8")).hexdigest() - return state_dir(payload) / "runtimes" / f"{digest}.json" - - -def load_thread_id(payload: dict[str, Any], fabric_runtime_id: str) -> str | None: - path = runtime_state_path(payload, fabric_runtime_id) - if not path.exists(): - return None - try: - state = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(state, dict): - raise ValueError("state must be an object") - if state.get("runtime_id") != fabric_runtime_id: - raise ValueError("runtime mismatch") - thread_id = state.get("codex_thread_id") - if not isinstance(thread_id, str) or not thread_id: - raise ValueError("missing Codex thread") - return thread_id - except (OSError, ValueError, json.JSONDecodeError) as error: - raise AdapterStateError( - "codex_invalid_runtime_state", "Codex runtime state is invalid" - ) from error - - -def save_thread_id( - payload: dict[str, Any], fabric_runtime_id: str, codex_thread_id: str -) -> None: - if not codex_thread_id: - raise AdapterStateError( - "codex_invalid_runtime_state", "Codex thread ID is missing" - ) - path = runtime_state_path(payload, fabric_runtime_id) - path.parent.mkdir(parents=True, exist_ok=True) - invocation_id = ( - common_utils.runtime_context(payload).get("invocation_id") or "invocation" - ) - temporary = path.with_suffix(f".{invocation_id}.tmp") - temporary.write_text( - json.dumps( - {"runtime_id": fabric_runtime_id, "codex_thread_id": codex_thread_id}, - sort_keys=True, - ), - encoding="utf-8", - ) - os.replace(temporary, path) - - def _merge_config(target: dict[str, Any], layer: dict[str, Any]) -> None: for key, value in layer.items(): existing = target.get(key) @@ -557,9 +507,7 @@ def _json_value(value: Any, *, name: str) -> Any: return value -def _apply_config_overrides( - config: dict[str, Any], overrides: dict[str, Any] -) -> None: +def _apply_config_overrides(config: dict[str, Any], overrides: dict[str, Any]) -> None: for dotted_key, value in sorted(overrides.items()): if not isinstance(dotted_key, str): raise AdapterConfigError( @@ -581,9 +529,7 @@ def _apply_config_overrides( f"Codex config override {dotted_key!r} conflicts with {part!r}", ) target = existing - target[parts[-1]] = _json_value( - value, name=f"config_overrides.{dotted_key}" - ) + target[parts[-1]] = _json_value(value, name=f"config_overrides.{dotted_key}") def native_codex_telemetry_config(payload: dict[str, Any]) -> dict[str, Any]: @@ -778,12 +724,11 @@ def _output_schema(payload: dict[str, Any]) -> dict[str, Any] | None: return _mapping(_json_value(value, name="output_schema"), name="output_schema") -def validate_payload(payload: dict[str, Any]) -> str: - """Validate pure invocation inputs before starting SDK or Relay processes.""" +def validate_runtime_payload(payload: dict[str, Any]) -> str: + """Validate runtime-owned configuration before starting SDK or Relay processes.""" settings = _settings(payload) _validate_settings_boundary(settings) - request_prompt(payload) _native_skill_paths(payload) fabric_runtime_id = runtime_id(payload) resolve_cwd(payload) @@ -892,7 +837,9 @@ def normalize_result( payload: dict[str, Any], *, thread_id: str, result: Any ) -> dict[str, Any]: status = _json_safe(result.status) - completed = result.status == TurnStatus.completed and result.final_response is not None + completed = ( + result.status == TurnStatus.completed and result.final_response is not None + ) error = None if not completed: message = ( @@ -940,88 +887,63 @@ async def _interrupt_turn(handle: Any) -> None: pass -async def invoke_codex_sdk( +def _thread_options( + payload: dict[str, Any], relay: CodexRelaySettings | None +) -> dict[str, Any]: + settings = _settings(payload) + return { + "approval_mode": approval_mode(payload), + "base_instructions": _optional_string(settings, "base_instructions"), + "config": thread_config(payload, relay) or None, + "cwd": str(resolve_cwd(payload)), + "developer_instructions": _optional_string(settings, "developer_instructions"), + "model": selected_model(payload), + "model_provider": selected_model_provider(payload), + "personality": _personality(payload), + "sandbox": sandbox(payload), + "service_tier": _optional_string(settings, "service_tier"), + } + + +async def _open_thread( + codex: AsyncCodex, payload: dict[str, Any], *, - prior_thread_id: str | None, relay: CodexRelaySettings | None, -) -> tuple[dict[str, Any], str | None]: - """Execute one SDK turn and always close the app-server transport.""" - +) -> Any: settings = _settings(payload) - config = thread_config(payload, relay) - skill_paths = _native_skill_paths(payload) - prompt = request_prompt(payload) - client_config = sdk_config(payload, relay) - codex: AsyncCodex | None = None + options = _thread_options(payload, relay) + return await codex.thread_start( + **options, + service_name=_optional_string(settings, "service_name"), + ) + + +async def _invoke_thread( + payload: dict[str, Any], thread: Any +) -> tuple[dict[str, Any], bool]: + """Run one turn and report whether the connected SDK transport remains usable.""" + handle = None - output: dict[str, Any] - thread_id: str | None = None try: - if selected_model_provider(payload) == "nvidia": - await asyncio.to_thread( - Path(client_config.env["CODEX_HOME"]).mkdir, - parents=True, - exist_ok=True, - ) - codex = AsyncCodex(config=client_config) async with asyncio.timeout(timeout_seconds(payload)): - await _register_skill_roots(codex, skill_paths) - common = { - "approval_mode": approval_mode(payload), - "base_instructions": _optional_string(settings, "base_instructions"), - "config": config or None, - "cwd": str(resolve_cwd(payload)), - "developer_instructions": _optional_string( - settings, "developer_instructions" - ), - "model": selected_model(payload), - "model_provider": selected_model_provider(payload), - "personality": _personality(payload), - "sandbox": sandbox(payload), - "service_tier": _optional_string(settings, "service_tier"), - } - if prior_thread_id is None: - thread = await codex.thread_start( - **common, - service_name=_optional_string(settings, "service_name"), - ) - else: - thread = await codex.thread_resume(prior_thread_id, **common) - if thread.id != prior_thread_id: - raise AdapterStateError( - "codex_thread_mismatch", - "Codex thread identity changed during resume", - ) - thread_id = thread.id handle = await thread.turn( - prompt, + request_prompt(payload), effort=_reasoning_effort(payload), output_schema=_output_schema(payload), ) result = await handle.run() - output = normalize_result(payload, thread_id=thread.id, result=result) + return normalize_result(payload, thread_id=thread.id, result=result), True except TimeoutError as error: await _interrupt_turn(handle) - output = sdk_failure(error) + return sdk_failure(error), False except CodexAdapterError: raise except (CodexError, RuntimeError, OSError) as error: - output = sdk_failure(error) - finally: - if codex is not None: - try: - await codex.close() - except Exception: - output = _failure( - "codex_sdk_stop_failed", "Codex SDK runtime failed to stop" - ) - return output, thread_id + return sdk_failure(error), False -def _relay_output( - output: dict[str, Any], relay: CodexRelaySettings -) -> dict[str, Any]: +def _relay_output(output: dict[str, Any], relay: CodexRelaySettings) -> dict[str, Any]: output["relay_runtime"] = { "enabled": True, "emitter": "codex-sdk/nemo-relay", @@ -1036,89 +958,212 @@ def _relay_output( return output -async def run_codex(payload: dict[str, Any]) -> dict[str, Any]: - """Run one Fabric invocation with SDK-owned Codex execution.""" +def _start_relay_gateway( + payload: dict[str, Any], relay: CodexRelaySettings | None +) -> subprocess.Popen[Any] | None: + if relay is None: + return None + try: + return relay_gateway.start_relay_gateway( + launch=relay.gateway, cwd=resolve_cwd(payload) + ) + except relay_gateway.RelayGatewayError as error: + raise AdapterRelayError( + "codex_relay_start_failed", + "NeMo Relay gateway failed to start", + metadata={"gateway_log_path": str(relay.gateway.log_path)}, + ) from error + - fabric_runtime_id = validate_payload(payload) - prior_thread_id = load_thread_id(payload, fabric_runtime_id) - relay = prepare_codex_relay(payload) - gateway_process = None - cleanup_error: AdapterRelayError | None = None +def _cleanup_relay( + relay: CodexRelaySettings | None, + process: subprocess.Popen[Any] | None, +) -> AdapterRelayError | None: + if process is None: + return None try: - if relay is not None: - try: - gateway_process = relay_gateway.start_relay_gateway( - launch=relay.gateway, cwd=resolve_cwd(payload) - ) - except relay_gateway.RelayGatewayError as error: - raise AdapterRelayError( - "codex_relay_start_failed", - "NeMo Relay gateway failed to start", - metadata={"gateway_log_path": str(relay.gateway.log_path)}, - ) from error - output, thread_id = await invoke_codex_sdk( - payload, prior_thread_id=prior_thread_id, relay=relay + relay_gateway.stop_relay_gateway(process) + except relay_gateway.RelayGatewayError: + return AdapterRelayError( + "codex_relay_stop_failed", + "NeMo Relay gateway failed to stop", + metadata={ + "gateway_log_path": str(relay.gateway.log_path) + if relay is not None + else "" + }, ) - if not output["failed"] and thread_id is not None: - save_thread_id(payload, fabric_runtime_id, thread_id) - finally: - if gateway_process is not None: - try: - relay_gateway.stop_relay_gateway(gateway_process) - except relay_gateway.RelayGatewayError: - cleanup_error = AdapterRelayError( - "codex_relay_stop_failed", - "NeMo Relay gateway failed to stop", - metadata={ - "gateway_log_path": str(relay.gateway.log_path) - if relay is not None - else "" - }, - ) + return None - if relay is not None: - output = _relay_output(output, relay) - if cleanup_error is not None: - cleanup: dict[str, Any] = { - "code": cleanup_error.code, - "message": cleanup_error.message, - "retryable": False, - } - if cleanup_error.metadata: - cleanup["metadata"] = cleanup_error.metadata - output["relay_runtime"]["cleanup_error"] = cleanup - if not output["failed"]: - output["completed"] = False - output["failed"] = True - output["error"] = cleanup - return output +def _as_lifecycle_error(error: CodexAdapterError) -> lifecycle.LifecycleError: + return lifecycle.LifecycleError( + error.code, + error.message, + metadata=error.metadata, + ) -def run(payload: dict[str, Any]) -> dict[str, Any]: - """Run one Fabric invocation from the synchronous adapter boundary.""" - try: - return asyncio.run(run_codex(payload)) - except CodexAdapterError as error: - return adapter_failure(error) - except Exception: - return _failure( - "codex_adapter_internal_error", "Codex adapter failed unexpectedly" - ) +class CodexRuntime: + """One Codex app-server client and thread owned by a Fabric runtime.""" + + def __init__(self) -> None: + self._start_payload: dict[str, Any] | None = None + self._fabric_runtime_id: str | None = None + self._client: AsyncCodex | None = None + self._thread: Any = None + self._relay: CodexRelaySettings | None = None + self._gateway_process: subprocess.Popen[Any] | None = None + self._unusable = False + + async def start(self, payload: dict[str, Any]) -> None: + if self._client is not None: + raise lifecycle.LifecycleError( + "codex_runtime_already_started", + "Codex runtime is already started", + ) + + try: + fabric_runtime_id = validate_runtime_payload(payload) + relay = prepare_codex_relay(payload) + self._relay = relay + self._gateway_process = _start_relay_gateway(payload, relay) + client_config = sdk_config(payload, relay) + if selected_model_provider(payload) == "nvidia": + await asyncio.to_thread( + Path(client_config.env["CODEX_HOME"]).mkdir, + parents=True, + exist_ok=True, + ) + client = AsyncCodex(config=client_config) + self._client = client + await _register_skill_roots(client, _native_skill_paths(payload)) + thread = await _open_thread( + client, + payload, + relay=relay, + ) + except CodexAdapterError as error: + await self._cleanup_failed_start() + raise _as_lifecycle_error(error) from error + except (CodexError, RuntimeError, OSError) as error: + await self._cleanup_failed_start() + reported = sdk_failure(error)["error"] + raise lifecycle.LifecycleError( + reported["code"], + reported["message"], + retryable=reported["retryable"], + metadata=reported.get("metadata"), + ) from error + except BaseException: + await self._cleanup_failed_start() + raise + + self._start_payload = payload + self._fabric_runtime_id = fabric_runtime_id + self._thread = thread + + async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]: + if ( + self._start_payload is None + or self._client is None + or self._thread is None + or self._fabric_runtime_id is None + ): + raise lifecycle.LifecycleError( + "codex_runtime_not_started", + "Codex runtime is not started", + ) + if runtime_id(invocation) != self._fabric_runtime_id: + raise lifecycle.LifecycleError( + "codex_runtime_mismatch", + "Codex invocation does not match the connected runtime", + ) + payload = { + **self._start_payload, + "runtime_context": invocation.get("runtime_context"), + "request": invocation.get("request"), + } + if self._unusable: + output = _failure( + "codex_runtime_unavailable", + "Codex runtime cannot accept another invocation after an SDK failure", + ) + return _relay_output(output, self._relay) if self._relay else output + + try: + request_prompt(payload) + timeout_seconds(payload) + _reasoning_effort(payload) + _output_schema(payload) + output, usable = await _invoke_thread(payload, self._thread) + except CodexAdapterError as error: + output = adapter_failure(error) + usable = True + + self._unusable = not usable + if self._relay is not None: + output = _relay_output(output, self._relay) + return output + + async def stop(self) -> None: + client = self._client + self._client = None + self._start_payload = None + self._thread = None + self._fabric_runtime_id = None + self._unusable = True + + close_error: BaseException | None = None + try: + if client is not None: + await client.close() + except BaseException as error: + if not isinstance(error, asyncio.CancelledError): + LOGGER.exception("Codex SDK client failed to close") + close_error = error + finally: + cleanup_error = _cleanup_relay(self._relay, self._gateway_process) + self._relay = None + self._gateway_process = None + + if isinstance(close_error, asyncio.CancelledError): + raise close_error + if close_error is not None: + if cleanup_error is not None: + LOGGER.error( + "Codex Relay cleanup also failed during close: %s", + cleanup_error.code, + ) + raise lifecycle.LifecycleError( + "codex_sdk_stop_failed", + "Codex SDK runtime failed to stop", + ) from close_error + if cleanup_error is not None: + raise _as_lifecycle_error(cleanup_error) + + async def _cleanup_failed_start(self) -> None: + client = self._client + self._client = None + if client is not None: + try: + await client.close() + except Exception: + LOGGER.exception("Codex SDK cleanup after start failure also failed") + cleanup_error = _cleanup_relay(self._relay, self._gateway_process) + self._relay = None + self._gateway_process = None + if cleanup_error is not None: + LOGGER.error( + "Codex Relay cleanup after start failure also failed: %s", + cleanup_error.code, + ) def main() -> None: - try: - payload = common_utils.load_payload() - except Exception: - output = _failure( - "codex_adapter_internal_error", "Codex adapter failed unexpectedly" - ) - else: - output = run(payload) - print(json.dumps(output, sort_keys=True)) - if output.get("failed"): - raise SystemExit(2) + """Serve the persistent local-host lifecycle protocol.""" + + lifecycle.serve(CodexRuntime) if __name__ == "__main__": diff --git a/adapters/codex/testing.md b/adapters/codex/testing.md index da71fe45..d5ccd987 100644 --- a/adapters/codex/testing.md +++ b/adapters/codex/testing.md @@ -18,9 +18,11 @@ RUN_FABRIC_CODEX_RELAY_INTEGRATION=1 \ Set `FABRIC_TEST_CODEX_BIN=/path/to/codex` on either opt-in command to validate an explicit app-server override instead of the SDK-pinned runtime. -The SDK test uses the current Codex authentication state and exercises both a -one-shot invocation and multi-turn thread resume. The Relay test additionally -requires an external gateway binary and verifies one-shot and resumed model -responses, stable thread identity, ATOF, and ATIF; gateway startup alone is not -a passing result. The semantic regression also requires decoded LLM request -content, a model, token usage, and the expected agent response in ATIF. \ No newline at end of file +The SDK test uses the current Codex authentication state and exercises both the +single-invocation convenience API and multiple turns against one started +runtime. The Relay test additionally requires an external gateway binary and +verifies model responses, stable thread identity across turns, Agent Trajectory +Observability Format (ATOF), and Agent Trajectory Interchange Format (ATIF); +gateway startup alone is not a passing result. The semantic regression requires +the LLM request content to be decoded. It also requires ATIF to contain the +model, token usage, and expected agent response. diff --git a/adapters/common/README.md b/adapters/common/README.md index 5f95d95a..0f91d19e 100644 --- a/adapters/common/README.md +++ b/adapters/common/README.md @@ -21,6 +21,42 @@ Alternately through the NeMo Fabric metapackage: pip install "nemo-fabric[adapters-common]" ``` +## Persistent Local Hosts + +Every local Process or Python adapter implements the ordered persistent-host +wire protocol. Python adapters can use +`nemo_fabric_adapters.common.lifecycle`. Supply a factory that creates one +adapter-owned runtime with asynchronous `start`, `invoke`, and `stop` methods: + +```python +from nemo_fabric_adapters.common import lifecycle + + +class AdapterRuntime: + async def start(self, payload): + self.client = await connect_client(payload) + + async def invoke(self, payload): + return await self.client.run(payload["request"]["input"]) + + async def stop(self): + await self.client.close() + + +lifecycle.serve(AdapterRuntime) +``` + +Fabric calls the factory once per local host to create one runtime instance and +serializes invocations through that instance. The host keeps one event loop +alive for the complete lifecycle so SDK clients, compiled graphs, +checkpointers, and harness databases can remain live safely. Fabric sends the +resolved configuration and capability plan during `start`. Each subsequent +`invoke` wire payload contains only `runtime_context` and `request`, and the +helper passes that payload to `AdapterRuntime.invoke` unchanged. An adapter that +needs configuration during invocation retains it as runtime-owned state during +`start`. Adapter stdout is reserved for the protocol; diagnostics are redirected +to stderr. A host crash or protocol timeout terminates that runtime. + Refer to the [NeMo Fabric documentation](https://nvidia-nemo-fabric.docs.buildwithfern.com/nemo/fabric) for adapter and configuration guidance. Source code is available in the [NVIDIA NeMo Fabric repository](https://github.com/NVIDIA/nemo-fabric/). diff --git a/adapters/common/src/nemo_fabric_adapters/common/lifecycle.py b/adapters/common/src/nemo_fabric_adapters/common/lifecycle.py new file mode 100644 index 00000000..0fcb7767 --- /dev/null +++ b/adapters/common/src/nemo_fabric_adapters/common/lifecycle.py @@ -0,0 +1,391 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Wire protocol for persistent local adapter hosts.""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +import traceback +from collections.abc import Awaitable +from collections.abc import Callable +from collections.abc import Iterator +from collections.abc import Mapping +from contextlib import contextmanager +from contextlib import redirect_stdout +from dataclasses import dataclass +from typing import Any +from typing import Protocol +from typing import TextIO + + +class AdapterRuntime(Protocol): + """One adapter-owned runtime living for the complete host lifetime.""" + + async def start(self, payload: dict[str, Any]) -> None: + """Initialize runtime-owned SDK clients and resources.""" + + async def invoke(self, payload: dict[str, Any]) -> Any: + """Execute one invocation against the initialized runtime.""" + + async def stop(self) -> None: + """Release all resources owned by the runtime.""" + + +RuntimeFactory = Callable[[], AdapterRuntime] + + +class LifecycleError(Exception): + """Adapter-supplied lifecycle failure safe to return across the protocol.""" + + def __init__( + self, + code: str, + message: str, + *, + retryable: bool = False, + metadata: Mapping[str, Any] | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retryable = retryable + self.metadata = dict(metadata or {}) + + +class _AdapterCallError(LifecycleError): + """Failure raised while executing an adapter runtime method.""" + + +@dataclass +class _HostState: + runtime: AdapterRuntime | None = None + runtime_id: str | None = None + failed: bool = False + + def clear(self) -> None: + self.runtime = None + self.runtime_id = None + self.failed = False + + +def _error( + stage: str, + code: str, + message: str, + *, + retryable: bool = False, + metadata: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + error: dict[str, Any] = { + "stage": stage, + "code": code, + "message": message, + "retryable": retryable, + } + if metadata: + error["metadata"] = dict(metadata) + return error + + +def _response( + operation: str, + *, + output: Any = None, + error: dict[str, Any] | None = None, +) -> dict[str, Any]: + outcome = ( + {"status": "succeeded", "output": output} + if error is None + else {"status": "failed", "error": error} + ) + return { + "operation": operation, + "outcome": outcome, + } + + +def _runtime_id(operation: str, payload: dict[str, Any]) -> str | None: + if operation in {"start", "invoke"}: + value = (payload.get("runtime_context") or {}).get("runtime_id") + else: + value = payload.get("runtime_id") + return value if isinstance(value, str) and value else None + + +@contextmanager +def _invocation_environment(payload: dict[str, Any]) -> Iterator[None]: + telemetry = (payload.get("runtime_context") or {}).get("telemetry") or {} + overlay = telemetry.get("env") if isinstance(telemetry, dict) else None + if not isinstance(overlay, dict) or any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in overlay.items() + ): + overlay = {} + previous = {key: os.environ.get(key) for key in overlay} + os.environ.update(overlay) + try: + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +async def _adapter_call(operation: str, call: Callable[[], Awaitable[Any]]) -> Any: + try: + # Protocol stdout is reserved for exactly one JSON response per line. + # Keep incidental adapter and library output as host diagnostics. + with redirect_stdout(sys.stderr): + return await call() + except LifecycleError as error: + raise _AdapterCallError( + error.code, + error.message, + retryable=error.retryable, + metadata=error.metadata, + ) from error + except Exception as error: + traceback.print_exc(file=sys.stderr) + raise _AdapterCallError( + f"lifecycle_adapter_{operation}_failed", + f"Adapter failed during lifecycle {operation}", + ) from error + + +def _failure_response(operation: str, error: LifecycleError) -> dict[str, Any]: + return _response( + operation, + error=_error( + operation, + error.code, + error.message, + retryable=error.retryable, + metadata=error.metadata, + ), + ) + + +async def _stop_after_eof(runtime: AdapterRuntime) -> None: + try: + await _adapter_call("stop", runtime.stop) + except LifecycleError: + traceback.print_exc(file=sys.stderr) + + +def _validated_request( + message: dict[str, Any], operation: str +) -> tuple[dict[str, Any], str]: + if operation not in {"start", "invoke", "stop"}: + raise LifecycleError( + "lifecycle_invalid_operation", + "Unknown lifecycle operation", + ) + payload = message.get("payload") + if not isinstance(payload, dict): + raise LifecycleError( + "lifecycle_invalid_payload", + "Lifecycle payload must be a mapping", + ) + message_runtime_id = _runtime_id(operation, payload) + if message_runtime_id is None: + raise LifecycleError( + "lifecycle_invalid_runtime", + "Lifecycle payload is missing a runtime ID", + ) + return payload, message_runtime_id + + +def _active_runtime(state: _HostState, message_runtime_id: str) -> AdapterRuntime: + if state.runtime is None or state.runtime_id is None: + raise LifecycleError( + "lifecycle_not_started", + "Lifecycle host has not started a runtime", + ) + if message_runtime_id != state.runtime_id: + raise LifecycleError( + "lifecycle_runtime_mismatch", + "Lifecycle payload does not match the active runtime", + ) + return state.runtime + + +async def _handle_start( + state: _HostState, + runtime_factory: RuntimeFactory, + payload: dict[str, Any], + message_runtime_id: str, +) -> dict[str, Any]: + if state.runtime is not None: + raise LifecycleError( + "lifecycle_already_started", + "Lifecycle host already owns a runtime", + ) + candidate = runtime_factory() + try: + await _adapter_call("start", lambda: candidate.start(payload)) + except LifecycleError: + await _stop_after_eof(candidate) + raise + state.runtime = candidate + state.runtime_id = message_runtime_id + state.failed = False + return _response("start") + + +async def _handle_invoke( + state: _HostState, + runtime: AdapterRuntime, + payload: dict[str, Any], +) -> dict[str, Any]: + if state.failed: + raise LifecycleError( + "lifecycle_runtime_failed", + "Lifecycle runtime cannot accept another invocation", + ) + with _invocation_environment(payload): + output = await _adapter_call("invoke", lambda: runtime.invoke(payload)) + return _response("invoke", output=output) + + +async def _handle_stop( + state: _HostState, + runtime: AdapterRuntime, +) -> dict[str, Any]: + try: + await _adapter_call("stop", runtime.stop) + finally: + state.clear() + return _response("stop") + + +async def _dispatch( + state: _HostState, + runtime_factory: RuntimeFactory, + operation: str, + payload: dict[str, Any], + message_runtime_id: str, +) -> dict[str, Any]: + if operation == "start": + return await _handle_start( + state, + runtime_factory, + payload, + message_runtime_id, + ) + runtime = _active_runtime(state, message_runtime_id) + if operation == "invoke": + return await _handle_invoke(state, runtime, payload) + return await _handle_stop(state, runtime) + + +def _encode_response( + state: _HostState, + operation: str, + response: dict[str, Any], +) -> str: + try: + return json.dumps(response, sort_keys=True) + except (TypeError, ValueError): + traceback.print_exc(file=sys.stderr) + if operation == "invoke" and state.runtime is not None: + state.failed = True + return json.dumps( + _response( + operation, + error=_error( + operation, + "lifecycle_invalid_response", + "Adapter returned a non-JSON lifecycle response", + ), + ), + sort_keys=True, + ) + + +async def _serve( + runtime_factory: RuntimeFactory, + *, + input_stream: TextIO, + output_stream: TextIO, +) -> None: + state = _HostState() + try: + while True: + # Keep this event loop alive while idle. Persistent SDK clients such + # as ClaudeSDKClient own background tasks tied to this exact loop. + line = await asyncio.to_thread(input_stream.readline) + if not line: + break + + operation = "start" + should_stop = False + try: + message = json.loads(line) + if not isinstance(message, dict): + raise TypeError("lifecycle request must be a mapping") + raw_operation = message.get("operation") + operation = raw_operation if isinstance(raw_operation, str) else "start" + payload, message_runtime_id = _validated_request(message, operation) + response = await _dispatch( + state, + runtime_factory, + operation, + payload, + message_runtime_id, + ) + should_stop = operation == "stop" + except LifecycleError as error: + if ( + operation == "invoke" + and state.runtime is not None + and isinstance(error, _AdapterCallError) + ): + state.failed = True + response = _failure_response(operation, error) + should_stop = should_stop or operation in {"start", "stop"} + except Exception as error: + traceback.print_exc(file=sys.stderr) + if operation == "invoke" and state.runtime is not None: + state.failed = True + response = _response( + operation, + error=_error( + operation, + "lifecycle_invalid_request", + "Invalid lifecycle request", + ), + ) + + encoded = _encode_response(state, operation, response) + print(encoded, file=output_stream, flush=True) + if should_stop: + break + finally: + if state.runtime is not None: + await _stop_after_eof(state.runtime) + + +def serve( + runtime_factory: RuntimeFactory, + *, + input_stream: TextIO = sys.stdin, + output_stream: TextIO = sys.stdout, +) -> None: + """Serve ordered lifecycle requests for exactly one Fabric runtime.""" + + # Reserve process stdout for the protocol for the entire host lifetime, + # including SDK background tasks running while the host is idle. + with redirect_stdout(sys.stderr): + asyncio.run( + _serve( + runtime_factory, + input_stream=input_stream, + output_stream=output_stream, + ) + ) diff --git a/adapters/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/common/src/nemo_fabric_adapters/common/relay_gateway.py index 3d7d1f71..a292fafb 100644 --- a/adapters/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -30,7 +30,7 @@ class RelayGatewayError(RuntimeError): @dataclass(frozen=True) class RelayGatewayLaunch: - """Complete invocation-scoped inputs for launching a Relay gateway.""" + """Complete inputs for launching one Relay gateway.""" executable: Path config_path: Path @@ -70,7 +70,7 @@ def find_available_tcp_port(host: str = "127.0.0.1") -> int: def relay_cli_contract(executable: Path) -> RelayCliContract: - """Resolve and validate the external Relay CLI contract once per invocation.""" + """Resolve and validate the external Relay CLI contract for one launch.""" try: completed = subprocess.run( @@ -150,7 +150,7 @@ def start_relay_gateway( launch: RelayGatewayLaunch, cwd: Path, ) -> subprocess.Popen[Any]: - """Launch and health-check one invocation-scoped Relay gateway.""" + """Launch and health-check one Relay gateway.""" if not launch.config_path.is_file(): raise RelayGatewayError("NeMo Relay gateway configuration was not generated") diff --git a/adapters/deepagents/README.md b/adapters/deepagents/README.md index 2077546e..1294290d 100644 --- a/adapters/deepagents/README.md +++ b/adapters/deepagents/README.md @@ -6,8 +6,8 @@ SPDX-License-Identifier: Apache-2.0 # NVIDIA NeMo Fabric LangChain Deep Agents Adapter Runs a [LangChain Deep Agents](https://github.com/langchain-ai/deepagents) agent -through Fabric's inline Python adapter lifecycle. The same adapter supports -one-shot, multi-turn, and resumed execution. +inside Fabric's persistent Python adapter host. One started runtime retains the +compiled graph, checkpointer, and LangGraph thread across ordered invocations. To install just the Deep Agents adapter by itself: @@ -40,8 +40,9 @@ never sent to the wrong endpoint. Because `models.default.api_key_env` is provider-specific, the adapter declares no static env requirement; a runtime **preflight** verifies that the `deepagents` -package is importable and the configured credential is set, and returns a -normalized failure otherwise. `fabric doctor` validates adapter resolution. +package is importable and the configured credential is set. A failed preflight +fails runtime start with a stable lifecycle error. `fabric doctor` validates +adapter resolution. Fabric maps the following into the harness: @@ -88,26 +89,23 @@ per-step events, LangGraph thread id, token usage (and cost when the provider reports it), and errors. Usage aggregates the current turn across the main agent and any delegated subagents (streamed with `subgraphs=True`). Configuration and preflight failures (a missing credential, an absent `deepagents` package, an -invalid MCP server, or a passthrough option) are returned as a -normalized failure result rather than a raw traceback. - -## Runtime Modes - -A one-shot `run` streams the agent with `astream` (buffering `updates` events and -`values` snapshots) and returns the final agent message, buffered messages and -per-step events, usage, and the LangGraph thread ID in the normalized Fabric -result. Each one-shot run gets a fresh Fabric `runtime_id`, so `resumed` is -`false`. - -Multi-turn and resume are keyed by the Fabric `runtime_id`, which is stable -across `invoke` calls in a started runtime (`start_runtime`) and fresh for each -one-shot run. On the first turn the adapter generates a LangGraph thread ID and -records it against the runtime; later turns of the same runtime reuse that thread -ID and a persistent LangGraph SQLite checkpointer to resume (`resumed` is `true`). -The checkpointer lives under `harness.settings.state_dir` (default the runtime -artifacts directory). Fabric owns the runtime-to-thread correlation record; +invalid MCP server, or a passthrough option) fail runtime start before an +invocation is accepted. + +## Runtime Lifecycle + +Fabric starts one local adapter host for every runtime. During runtime start, +the host compiles one Deep Agents graph, opens its async LangGraph checkpointer, +and creates one thread ID. Every invocation reuses those native objects; later +turns report `resumed` as `true`. The checkpointer lives under +`harness.settings.state_dir` (default the runtime artifacts directory) and is +closed during runtime stop. The live host owns the thread identity, and LangGraph owns the transcript. +`Fabric.run(...)` is a convenience over that same lifecycle: it starts the +runtime, invokes it once, and stops it. It does not use a separate adapter +entrypoint or execution path. + The `deepagents_config()` builder in `examples/code_review_agent` is the SDK example. Run it from the CLI with `python -m examples.code_review_agent --variant deepagents --input "..."`, or @@ -120,13 +118,13 @@ from nemo_fabric import Fabric config = deepagents_config() client = Fabric() -# One-shot: each run gets a fresh runtime, so `resumed` is False. +# Single invocation through the standard runtime lifecycle. result = await client.run( config, base_dir=BASE_DIR, input="Review the workspace changes." ) print(result["output"]["response"]) -# Multi-turn + resume: one started runtime keeps the LangGraph thread across turns. +# Multi-turn: one started runtime keeps the LangGraph thread across turns. async with await client.start_runtime(config, base_dir=BASE_DIR) as runtime: await runtime.invoke(input="Remember the value 42.") reply = await runtime.invoke(input="What value did I ask you to remember?") @@ -148,7 +146,7 @@ pip install "nemo-fabric-adapters-deepagents[relay]" # -> nemo-relay[deepagent - **Relay** (`telemetry.providers.relay`): the SDK-native integration attaches three complementary pieces around `create_deep_agent`, applied uniformly to - one-shot, multi-turn, resumed, and subagent-enabled runs: + single-invocation, multi-turn, and subagent-enabled runs: - `nemo_relay.integrations.deepagents.add_nemo_relay_integration(...)` injects Deep Agents-aware **middleware** that routes model and tool calls through Relay and emits skill/subagent configuration marks. diff --git a/adapters/deepagents/fabric-adapter.json b/adapters/deepagents/fabric-adapter.json index 78a09fe3..c32aa03b 100644 --- a/adapters/deepagents/fabric-adapter.json +++ b/adapters/deepagents/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "deepagents", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.deepagents.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.deepagents.adapter" }, "requirements": {}, "config": { diff --git a/adapters/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 0adc94c2..0a3523e7 100644 --- a/adapters/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -5,9 +5,8 @@ """LangChain Deep Agents adapter for Fabric. Maps Fabric's normalized invocation onto the ``deepagents`` SDK and returns a -normalized Fabric result. Supports one-shot, multi-turn, and resumed execution -via a persistent LangGraph checkpointer keyed by the Fabric session id, so -conversation state survives across separate adapter invocations. +normalized Fabric result. A started runtime retains one compiled graph and +LangGraph checkpointer across ordered invocations. """ from __future__ import annotations @@ -25,6 +24,7 @@ from langchain.agents.middleware import AgentMiddleware from langchain_core.messages import ToolMessage +from nemo_fabric_adapters.common import lifecycle import nemo_fabric_adapters.common.utils as common_utils HARNESS = "deepagents" @@ -46,7 +46,15 @@ # bypass the normalized model config, MCP tool resolution, workspace confinement, # or tool gating). FABRIC_OWNED_AGENT_KEYS = frozenset( - {"model", "tools", "backend", "skills", "system_prompt", "middleware", "checkpointer"} + { + "model", + "tools", + "backend", + "skills", + "system_prompt", + "middleware", + "checkpointer", + } ) # Documented, JSON-serializable create_deep_agent options callers may forward # through harness.settings.deepagents. Executable objects (AgentMiddleware, BaseTool, @@ -102,25 +110,16 @@ def resolve_api_key_env(settings: dict[str, Any], model_config: dict[str, Any]) provider = (settings.get("provider") or model_config.get("provider") or "").lower() default = PROVIDER_DEFAULT_API_KEY_ENV.get(provider) if default is None: - raise AdapterConfigError(f"models.default.api_key_env is required for provider '{provider}'.") + raise AdapterConfigError( + f"models.default.api_key_env is required for provider '{provider}'." + ) return default def main() -> None: - """Subprocess entrypoint used by the ``python -m`` process path.""" - - output = run(common_utils.load_payload()) - print(json.dumps(output, sort_keys=True)) - if output.get("failed"): - raise SystemExit(2) - + """Serve the persistent local-host lifecycle protocol.""" -def run(payload: dict[str, Any]) -> dict[str, Any]: - """Fabric adapter entrypoint used by the ``python -m`` and native SDK paths.""" - - import asyncio - - return asyncio.run(run_deepagents(payload)) + lifecycle.serve(DeepAgentsRuntime) def preflight_check(payload: dict[str, Any]) -> None: @@ -157,9 +156,13 @@ def selected_model_config(payload: dict[str, Any]) -> dict[str, Any]: return models.get(settings.get("model", "default"), {}) or {} -def resolve_base_url(settings: dict[str, Any], model_config: dict[str, Any]) -> str | None: +def resolve_base_url( + settings: dict[str, Any], model_config: dict[str, Any] +) -> str | None: base_url = ( - settings.get("base_url") or (model_config.get("settings") or {}).get("base_url") or model_config.get("base_url") + settings.get("base_url") + or (model_config.get("settings") or {}).get("base_url") + or model_config.get("base_url") ) if base_url: return base_url @@ -184,14 +187,18 @@ def build_chat_model(payload: dict[str, Any]) -> tuple[Any, str, str | None]: model_config = selected_model_config(payload) model_name = settings.get("model_name") or model_config.get("model") if not model_name: - raise RuntimeError("models.default.model is required for the Deep Agents adapter") + raise RuntimeError( + "models.default.model is required for the Deep Agents adapter" + ) api_key_env = resolve_api_key_env(settings, model_config) api_key = os.environ.get(api_key_env) if not api_key: raise RuntimeError(f"{api_key_env} is required for the Deep Agents adapter") - provider = (settings.get("provider") or model_config.get("provider") or "nvidia").lower() + provider = ( + settings.get("provider") or model_config.get("provider") or "nvidia" + ).lower() base_url = resolve_base_url(settings, model_config) temperature = settings.get("temperature", model_config.get("temperature")) @@ -204,7 +211,11 @@ def build_chat_model(payload: dict[str, Any]) -> tuple[Any, str, str | None]: kwargs["temperature"] = temperature if base_url: kwargs["base_url"] = base_url - return init_chat_model(**_supported_kwargs(init_chat_model, kwargs)), model_name, base_url + return ( + init_chat_model(**_supported_kwargs(init_chat_model, kwargs)), + model_name, + base_url, + ) from langchain_openai import ChatOpenAI @@ -220,7 +231,9 @@ def resolve_backend(payload: dict[str, Any]) -> Any: """Root the Deep Agents filesystem backend at the Fabric workspace, if set.""" environment = common_utils.environment_payload(payload) - workspace = environment.get("workspace") or common_utils.settings_payload(payload).get("workspace") + workspace = environment.get("workspace") or common_utils.settings_payload( + payload + ).get("workspace") if not workspace: return None root = Path(str(workspace)) @@ -287,7 +300,9 @@ def _mcp_connection(name: str, spec: dict[str, Any]) -> dict[str, Any]: # McpServerPlan carries the URL or command in ``url``; there is no ``command``. target = os.path.expandvars(str(spec.get("url") or "")).strip() if not target: - raise AdapterConfigError(f"MCP server '{name}' requires a url (or command in url).") + raise AdapterConfigError( + f"MCP server '{name}' requires a url (or command in url)." + ) if transport in ("stdio", "command", "process"): parts = shlex.split(target) if not parts: @@ -296,16 +311,13 @@ def _mcp_connection(name: str, spec: dict[str, Any]) -> dict[str, Any]: if transport in ("", "http", "streamable_http", "streamablehttp"): transport = "streamable_http" if transport not in VALID_MCP_TRANSPORTS: - raise AdapterConfigError(f"MCP server '{name}' has unsupported transport '{transport}'.") + raise AdapterConfigError( + f"MCP server '{name}' has unsupported transport '{transport}'." + ) return {"transport": transport, "url": target} -# --- runtime / resume state ------------------------------------------------ -# -# Resume is keyed by the Fabric ``runtime_id`` (stable across ``invoke`` calls in -# a started runtime, fresh for each one-shot ``run``), mirroring the Codex -# adapter. LangGraph owns the transcript via a persistent SQLite checkpointer; -# Fabric owns the runtime-to-LangGraph-thread correlation record. +# --- runtime state --------------------------------------------------------- def state_dir(payload: dict[str, Any]) -> Path: @@ -322,33 +334,14 @@ def state_dir(payload: dict[str, Any]) -> Path: return base_dir / "artifacts" / "deepagents" / ".fabric" -def runtime_state_paths(payload: dict[str, Any], runtime_id: str) -> tuple[Path, Path]: +def checkpointer_path(payload: dict[str, Any], runtime_id: str) -> Path: key = hashlib.sha256(runtime_id.encode("utf-8")).hexdigest() base = state_dir(payload) / "runtimes" - return base / f"{key}.json", base / f"{key}.sqlite" - - -def load_thread_id(payload: dict[str, Any], runtime_id: str) -> str | None: - json_path, _ = runtime_state_paths(payload, runtime_id) - if not json_path.is_file(): - return None - value = json.loads(json_path.read_text(encoding="utf-8")) - if not isinstance(value, dict) or value.get("runtime_id") != runtime_id or not value.get("thread_id"): - raise RuntimeError(f"invalid Deep Agents runtime state in {json_path}") - return str(value["thread_id"]) - - -def save_thread_id(payload: dict[str, Any], runtime_id: str, thread_id: str) -> None: - json_path, _ = runtime_state_paths(payload, runtime_id) - json_path.parent.mkdir(parents=True, exist_ok=True) - invocation_id = common_utils.runtime_context(payload).get("invocation_id") or "pending" - tmp = json_path.with_suffix(f".{invocation_id}.tmp") - tmp.write_text(json.dumps({"runtime_id": runtime_id, "thread_id": thread_id}, indent=2), encoding="utf-8") - os.replace(tmp, json_path) + return base / f"{key}.sqlite" async def open_checkpointer(state_sqlite: Path) -> Any: - """Open a persistent async LangGraph checkpointer so resume works across processes. + """Open the async LangGraph checkpointer owned by a started runtime. The agent is driven with ``astream``, so the checkpointer must be async: the synchronous ``SqliteSaver`` raises ``NotImplementedError`` from its async methods. @@ -359,7 +352,7 @@ async def open_checkpointer(state_sqlite: Path) -> Any: state_sqlite.parent.mkdir(parents=True, exist_ok=True) saver_cm = AsyncSqliteSaver.from_conn_string(str(state_sqlite)) saver = await saver_cm.__aenter__() - # Keep the context manager alive for the duration of the invocation. + # Keep the context manager alive for the duration of the runtime. saver._fabric_cm = saver_cm # type: ignore[attr-defined] return saver @@ -373,7 +366,9 @@ async def close_checkpointer(checkpointer: Any) -> None: # --- invocation ------------------------------------------------------------ -async def build_agent_kwargs(payload: dict[str, Any], model: Any, settings: dict[str, Any]) -> dict[str, Any]: +async def build_agent_kwargs( + payload: dict[str, Any], model: Any, settings: dict[str, Any] +) -> dict[str, Any]: kwargs: dict[str, Any] = { "model": model, "tools": await resolve_tools(payload), @@ -426,7 +421,10 @@ def _validated_passthrough(extra: Any) -> dict[str, Any]: def _block_subagent(subagent: dict[str, Any], blocked: set[str]) -> dict[str, Any]: gated = dict(subagent) - gated["middleware"] = [*(gated.get("middleware") or []), blocked_tools_middleware(blocked)] + gated["middleware"] = [ + *(gated.get("middleware") or []), + blocked_tools_middleware(blocked), + ] return gated @@ -443,12 +441,18 @@ def _gated_subagents(subagents: Any, blocked: set[str]) -> list[dict[str, Any]]: gated: list[dict[str, Any]] = [] for subagent in configured: if not isinstance(subagent, dict): - raise AdapterConfigError("Deep Agents subagents must be mappings when tools.blocked is configured.") + raise AdapterConfigError( + "Deep Agents subagents must be mappings when tools.blocked is configured." + ) name = str(subagent.get("name") or "") if "graph_id" in subagent: - raise AdapterConfigError(f"tools.blocked cannot be enforced for remote Deep Agents subagent '{name}'.") + raise AdapterConfigError( + f"tools.blocked cannot be enforced for remote Deep Agents subagent '{name}'." + ) if "runnable" in subagent: - raise AdapterConfigError(f"tools.blocked cannot be enforced for precompiled Deep Agents subagent '{name}'.") + raise AdapterConfigError( + f"tools.blocked cannot be enforced for precompiled Deep Agents subagent '{name}'." + ) gated.append(_block_subagent(subagent, blocked)) if not any(subagent.get("name") == "general-purpose" for subagent in gated): @@ -458,118 +462,217 @@ def _gated_subagents(subagents: Any, blocked: set[str]) -> list[dict[str, Any]]: return gated -async def run_deepagents(payload: dict[str, Any]) -> dict[str, Any]: - settings = common_utils.settings_payload(payload) - request = payload.get("request") or {} - telemetry_providers = common_utils.telemetry_providers(payload) - relay_enabled = common_utils.relay_enabled(payload) - telemetry_provider = "relay" if relay_enabled else ("native" if "native" in telemetry_providers else "") - - user_message = request.get("input") or "" - if not isinstance(user_message, str): - user_message = json.dumps(user_message, sort_keys=True) - - runtime_id = common_utils.runtime_context(payload).get("runtime_id") - model_name: str | None = None - base_url: str | None = None - prior_thread_id: str | None = None - thread_id: str | None = None - observability: Observability | None = None - result_state: Any = None - events: list[dict[str, Any]] = [] - turn_messages: list[dict[str, Any]] = [] - error: str | None = None - checkpointer = None - try: - # Preflight, model construction, and resume-state load run inside the guarded - # scope so a misconfiguration (e.g. a missing credential) returns a normalized - # failure result rather than raising a raw traceback. - preflight_check(payload) - model, model_name, base_url = build_chat_model(payload) - prior_thread_id = load_thread_id(payload, runtime_id) if runtime_id else None - thread_id = (prior_thread_id or uuid.uuid4().hex) if runtime_id else None - observability = resolve_observability(payload, telemetry_provider, relay_enabled) - - agent_kwargs = await build_agent_kwargs(payload, model, settings) - # Acquire the async checkpointer inside the guarded scope so a setup failure - # (tools, backend, observability) can never leak the SQLite connection. - if runtime_id: - _, state_sqlite = runtime_state_paths(payload, runtime_id) - checkpointer = await open_checkpointer(state_sqlite) - agent_kwargs["checkpointer"] = checkpointer - - if observability is not None: - # Both the relay and native telemetry providers run through nemo_relay's - # plugin, so nemo-relay is required for either. It is imported lazily only - # here (import-time dependency neutrality); fail with an actionable message - # when the optional Relay extra is not installed rather than a raw - # ModuleNotFoundError from the imports below. - import importlib.util - - if importlib.util.find_spec("nemo_relay") is None: - raise _relay_dependency_error() - try: - api_config = common_utils.relay_api_plugin_config(observability.plugin_config) - from nemo_relay import ScopeType, plugin, scope - from nemo_relay.integrations.deepagents import ( - NemoRelayDeepAgentsCallbackHandler, - add_nemo_relay_integration, +class DeepAgentsRuntime: + """One compiled Deep Agents graph and checkpointer owned by a Fabric runtime.""" + + def __init__(self) -> None: + self._started = False + self._start_payload: dict[str, Any] | None = None + self._runtime_id: str | None = None + self._model_name: str | None = None + self._base_url: str | None = None + self._thread_id: str | None = None + self._completed_invocations = 0 + self._checkpointer: Any = None + self._agent: Any = None + self._observability: Observability | None = None + self._telemetry_provider = "" + self._relay_plugin: Any = None + self._relay_scope: Any = None + self._relay_scope_type: Any = None + self._relay_api_config: Any = None + self._callback_handler_type: Any = None + + async def start(self, payload: dict[str, Any]) -> None: + if self._started: + raise lifecycle.LifecycleError( + "deepagents_runtime_already_started", + "Deep Agents runtime is already started", + ) + + try: + preflight_check(payload) + settings = common_utils.settings_payload(payload) + runtime_id = common_utils.runtime_context(payload).get("runtime_id") + model, self._model_name, self._base_url = build_chat_model(payload) + self._runtime_id = runtime_id + self._thread_id = uuid.uuid4().hex if runtime_id else None + + telemetry_providers = common_utils.telemetry_providers(payload) + relay_enabled = common_utils.relay_enabled(payload) + self._telemetry_provider = ( + "relay" + if relay_enabled + else "native" + if "native" in telemetry_providers + else "" + ) + self._observability = resolve_observability( + payload, + self._telemetry_provider, + relay_enabled, + ) + + agent_kwargs = await build_agent_kwargs(payload, model, settings) + if runtime_id: + self._checkpointer = await open_checkpointer( + checkpointer_path(payload, runtime_id) ) - except (ImportError, AttributeError) as exc: - raise _relay_dependency_error() from exc - - # add_nemo_relay_integration injects the Deep Agents middleware (model/tool - # calls, skill/subagent config marks) into create_deep_agent's kwargs and the - # same middleware into in-process dictionary-style subagents. The callback - # handler captures LangGraph scopes and human-in-the-loop interrupt/resume - # marks. The "deepagents-request" Agent scope contains the top-level Fabric - # invocation. All three apply uniformly to one-shot, multi-turn, resumed, and - # subagent-enabled runs because they share this single invocation path. - wrapped = add_nemo_relay_integration(agent_kwargs) - callback_handler = NemoRelayDeepAgentsCallbackHandler() - async with plugin.plugin(api_config): - with scope.scope("deepagents-request", ScopeType.Agent): - result_state, events, turn_messages = await invoke_agent( - wrapped, user_message, thread_id, callbacks=[callback_handler] - ) - else: - result_state, events, turn_messages = await invoke_agent(agent_kwargs, user_message, thread_id) - except Exception as exc: # normalized adapter failure - error = f"{type(exc).__name__}: {exc}" - finally: - if checkpointer is not None: - await close_checkpointer(checkpointer) - - if error is None and runtime_id and thread_id: - save_thread_id(payload, runtime_id, thread_id) + agent_kwargs["checkpointer"] = self._checkpointer + + if self._observability is not None: + agent_kwargs = self._configure_observability(agent_kwargs) + + from deepagents import create_deep_agent + + self._agent = create_deep_agent( + **_supported_kwargs(create_deep_agent, agent_kwargs) + ) + self._start_payload = payload + self._started = True + except BaseException: + await self.stop() + raise + + def _configure_observability(self, agent_kwargs: dict[str, Any]) -> dict[str, Any]: + import importlib.util + + if importlib.util.find_spec("nemo_relay") is None: + raise _relay_dependency_error() + try: + from nemo_relay import ScopeType, plugin, scope + from nemo_relay.integrations.deepagents import ( + NemoRelayDeepAgentsCallbackHandler, + add_nemo_relay_integration, + ) + except (ImportError, AttributeError) as exc: + raise _relay_dependency_error() from exc + + assert self._observability is not None + self._relay_api_config = common_utils.relay_api_plugin_config( + self._observability.plugin_config + ) + self._relay_plugin = plugin + self._relay_scope = scope + self._relay_scope_type = ScopeType + self._callback_handler_type = NemoRelayDeepAgentsCallbackHandler + return add_nemo_relay_integration(agent_kwargs) + + async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]: + start_payload = self._start_payload + if not self._started or self._agent is None or start_payload is None: + raise lifecycle.LifecycleError( + "deepagents_runtime_not_started", + "Deep Agents runtime is not started", + ) + runtime_id = common_utils.runtime_context(invocation).get("runtime_id") + if runtime_id != self._runtime_id: + raise lifecycle.LifecycleError( + "deepagents_runtime_mismatch", + "Deep Agents invocation does not match the active runtime", + ) + + payload = { + **start_payload, + "runtime_context": invocation.get("runtime_context"), + "request": invocation.get("request"), + } + request = payload.get("request") or {} + user_message = request.get("input") or "" + if not isinstance(user_message, str): + user_message = json.dumps(user_message, sort_keys=True) + + result_state: Any = None + events: list[dict[str, Any]] = [] + turn_messages: list[dict[str, Any]] = [] + error: str | None = None + resumed = self._completed_invocations > 0 + try: + if self._observability is not None: + callback_handler = self._callback_handler_type() + async with self._relay_plugin.plugin(self._relay_api_config): + with self._relay_scope.scope( + "deepagents-request", self._relay_scope_type.Agent + ): + ( + result_state, + events, + turn_messages, + ) = await invoke_compiled_agent( + self._agent, + user_message, + self._thread_id, + callbacks=[callback_handler], + ) + else: + result_state, events, turn_messages = await invoke_compiled_agent( + self._agent, + user_message, + self._thread_id, + ) + except Exception as exc: # normalized adapter failure + error = f"{type(exc).__name__}: {exc}" + + if error is None: + self._completed_invocations += 1 + + telemetry_runtime, relay_artifacts = self._telemetry_output() + return normalize_output( + model_name=self._model_name, + base_url=self._base_url, + runtime_id=self._runtime_id, + thread_id=self._thread_id, + resumed=resumed, + result_state=result_state, + events=events, + turn_messages=turn_messages, + error=error, + telemetry_runtime=telemetry_runtime, + relay_artifacts=relay_artifacts, + ) - telemetry_runtime: dict[str, Any] | None = None - relay_artifacts: list[dict[str, str]] | None = None - if observability is not None: + def _telemetry_output( + self, + ) -> tuple[dict[str, Any] | None, list[dict[str, str]] | None]: + if self._observability is None: + return None, None telemetry_runtime = { "enabled": True, - "provider": telemetry_provider, - "emitter": observability.emitter, + "provider": self._telemetry_provider, + "emitter": self._observability.emitter, } - if observability.collect_artifacts: - relay_artifacts = common_utils.collect_relay_artifacts(observability.plugin_config) - - return normalize_output( - model_name=model_name, - base_url=base_url, - runtime_id=runtime_id, - thread_id=thread_id, - resumed=bool(prior_thread_id), - result_state=result_state, - events=events, - turn_messages=turn_messages, - error=error, - telemetry_runtime=telemetry_runtime, - relay_artifacts=relay_artifacts, - ) + relay_artifacts = ( + common_utils.collect_relay_artifacts(self._observability.plugin_config) + if self._observability.collect_artifacts + else None + ) + return telemetry_runtime, relay_artifacts + + async def stop(self) -> None: + checkpointer = self._checkpointer + self._start_payload = None + self._runtime_id = None + self._model_name = None + self._base_url = None + self._thread_id = None + self._completed_invocations = 0 + self._checkpointer = None + self._agent = None + self._observability = None + self._telemetry_provider = "" + self._relay_plugin = None + self._relay_scope = None + self._relay_scope_type = None + self._relay_api_config = None + self._callback_handler_type = None + self._started = False + if checkpointer is not None: + await close_checkpointer(checkpointer) -def _apply_callbacks(config: dict[str, Any], callbacks: list[Any] | None) -> dict[str, Any]: +def _apply_callbacks( + config: dict[str, Any], callbacks: list[Any] | None +) -> dict[str, Any]: """Append ``callbacks`` to a LangGraph run config, preserving any already set. The Relay callback handler is added after any consumer-provided callbacks so @@ -582,26 +685,24 @@ def _apply_callbacks(config: dict[str, Any], callbacks: list[Any] | None) -> dic return config -async def invoke_agent( - agent_kwargs: dict[str, Any], +async def invoke_compiled_agent( + agent: Any, user_message: str, thread_id: str | None, callbacks: list[Any] | None = None, ) -> tuple[Any, list[dict[str, Any]], list[dict[str, Any]]]: - """Run the agent; return the final state, per-step events, and this turn's messages. + """Run a compiled agent and return state, events, and current-turn messages. - On a resumed run the final ``values`` state also replays prior-turn messages, - so usage/cost must be aggregated from the messages emitted *this* turn — the - ``updates`` deltas — rather than the full final state. + On a later runtime invocation the final ``values`` state also replays + prior-turn messages, so usage/cost must be aggregated from the messages + emitted *this* turn — the ``updates`` deltas — rather than the full final + state. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than dropped. """ - from deepagents import create_deep_agent - - agent = create_deep_agent(**_supported_kwargs(create_deep_agent, agent_kwargs)) inputs = {"messages": [{"role": "user", "content": user_message}]} config: dict[str, Any] = {} if thread_id: @@ -673,7 +774,9 @@ def resolve_observability( if telemetry_provider == "native": native_config = common_utils.native_telemetry_config(payload) if native_config.get("components"): - return Observability(native_config, "deepagents.observability/native", False) + return Observability( + native_config, "deepagents.observability/native", False + ) return None @@ -696,7 +799,7 @@ def normalize_output( ) -> dict[str, Any]: messages = _extract_messages(result_state) response = _final_response(messages) - # Aggregate usage/cost from this turn's messages only; the resumed final state + # Aggregate usage/cost from this turn's messages only; the replayed final state # replays prior-turn messages that must not be re-counted. usage = _aggregate_usage(turn_messages) @@ -759,7 +862,9 @@ def _dedup_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: def _message_to_dict(message: Any) -> dict[str, Any]: - role = getattr(message, "type", None) or getattr(message, "role", None) or "assistant" + role = ( + getattr(message, "type", None) or getattr(message, "role", None) or "assistant" + ) content = getattr(message, "content", "") entry: dict[str, Any] = {"role": role, "content": content} message_id = getattr(message, "id", None) @@ -801,7 +906,9 @@ def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: totals[target] = int(totals.get(target, 0)) + value # Cost is not part of LangChain's UsageMetadata; surface it only when a # model/provider reports it on the usage or response metadata. - candidate = usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) + candidate = ( + usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) + ) if isinstance(candidate, (int, float)) and not isinstance(candidate, bool): cost += float(candidate) has_cost = True diff --git a/adapters/hermes/README.md b/adapters/hermes/README.md index 4d560ba2..de6d14ce 100644 --- a/adapters/hermes/README.md +++ b/adapters/hermes/README.md @@ -37,6 +37,19 @@ The adapter receives a normalized payload from Fabric and materializes Hermes-na `runtimes/` so invocations in one Fabric runtime share Hermes state without sharing config or the session database with another runtime. +## Execution Model + +Each Fabric runtime starts one local adapter host, constructs one Hermes +`AIAgent`, and opens one `SessionDB`. Ordered `Runtime.invoke(...)` calls reuse +those native objects and pass the prior turn's returned transcript back to +`run_conversation(...)`. Runtime stop calls the agent's idempotent `close()` +method, closes the session database, and releases the Relay plugin context when +enabled. + +Hermes Relay telemetry is finalized after each Fabric invocation so its ATOF +and ATIF artifacts are complete when that invocation returns. This telemetry +boundary does not recreate the `AIAgent` or `SessionDB`. + ## Maintaining The Adapter Keep `fabric-adapter.json` aligned with the Python implementation: @@ -44,8 +57,8 @@ Keep `fabric-adapter.json` aligned with the Python implementation: - `contract_version` must match the adapter contract supported by Fabric core. - `adapter_id` is the stable id selected by `harness.adapter_id`. - `adapter_kind` is `python` because Fabric can invoke it through Python. -- `runner.module` names the module that Fabric invokes with `python -m`. - `runner.callable` names the equivalent reusable Python function. +- `runner.module` names the persistent host module that Fabric invokes with + `python -m`. - `requirements` powers `fabric doctor`; keep required env vars, binaries, or packages current. - `config.accepts` must match the Fabric sections this adapter maps into Hermes. diff --git a/adapters/hermes/fabric-adapter.json b/adapters/hermes/fabric-adapter.json index f2857470..ef60fb92 100644 --- a/adapters/hermes/fabric-adapter.json +++ b/adapters/hermes/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "hermes", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.hermes.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.hermes.adapter" }, "requirements": { "env": [ diff --git a/adapters/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 1faf92ea..07fd53aa 100755 --- a/adapters/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -13,13 +13,14 @@ import asyncio import inspect import json +import logging import os -import sys from contextlib import redirect_stdout from io import StringIO from pathlib import Path from typing import Any +from nemo_fabric_adapters.common import lifecycle import nemo_fabric_adapters.common.utils as common_utils # Default agent loop budget when harness.settings.max_iterations is unset. @@ -27,6 +28,7 @@ # as 1 silently starves multi-step tasks (they run out of budget before # answering while the trial still reports success). See FABRIC-85. DEFAULT_MAX_ITERATIONS: int = 90 +LOGGER = logging.getLogger(__name__) def validate_hermes_telemetry_provider(payload: dict[str, Any]) -> None: @@ -43,7 +45,9 @@ def disabled_toolsets(payload: dict[str, Any]) -> list[str]: ) -def build_hermes_config(payload: dict[str, Any], *, relay_enabled: bool = False) -> dict[str, Any]: +def build_hermes_config( + payload: dict[str, Any], *, relay_enabled: bool = False +) -> dict[str, Any]: settings = common_utils.settings_payload(payload) model_config = common_utils.selected_model_config(payload) native = common_utils.capability_plan(payload).get("native") or {} @@ -71,7 +75,9 @@ def build_hermes_config(payload: dict[str, Any], *, relay_enabled: bool = False) "terminal": common_utils.without_none( { "backend": settings.get("terminal_backend", "local"), - "cwd": str(environment.get("workspace") or settings.get("workspace") or "."), + "cwd": str( + environment.get("workspace") or settings.get("workspace") or "." + ), "timeout": settings.get("terminal_timeout", 60), } ), @@ -90,7 +96,9 @@ def build_hermes_config(payload: dict[str, Any], *, relay_enabled: bool = False) if "enabled_toolsets" in settings: config["platform_toolsets"] = { - settings.get("toolset_platform", "cli"): common_utils.normalize_list(settings.get("enabled_toolsets")) + settings.get("toolset_platform", "cli"): common_utils.normalize_list( + settings.get("enabled_toolsets") + ) } plugins = common_utils.normalize_list(settings.get("plugins_enabled")) @@ -148,20 +156,14 @@ def summarize_hermes_config(config: dict[str, Any]) -> dict[str, Any]: def main() -> None: - payload = json.load(sys.stdin) - output = run(payload) - print(json.dumps(output, sort_keys=True)) - if output.get("failed"): - raise SystemExit(2) - - -def run(payload: dict[str, Any]) -> dict[str, Any]: - """Fabric adapter entrypoint used by script and native SDK runtime calls.""" + """Serve the persistent local-host lifecycle protocol.""" - return asyncio.run(run_hermes(payload)) + lifecycle.serve(HermesRuntime) -def resolve_hermes_toolsets(settings: dict[str, Any], config: dict[str, Any]) -> list[str] | None: +def resolve_hermes_toolsets( + settings: dict[str, Any], config: dict[str, Any] +) -> list[str] | None: if "enabled_toolsets" in settings: return common_utils.normalize_list(settings.get("enabled_toolsets")) @@ -171,190 +173,335 @@ def resolve_hermes_toolsets(settings: dict[str, Any], config: dict[str, Any]) -> return sorted(_get_platform_tools(config, platform)) -def load_runtime_history(session_db: Any, session_id: str | None) -> list[dict[str, Any]] | None: - if not session_id: - return None - - resolved_id = session_id - resolve_session = getattr(session_db, "resolve_resume_session_id", None) - if resolve_session is not None: - resolved_id = resolve_session(session_id) or session_id - if session_db.get_session(resolved_id) is None: - return None - - messages = session_db.get_messages_as_conversation(resolved_id) - messages = [message for message in messages if message.get("role") != "session_meta"] - return messages or None +class HermesRuntime: + """One Hermes agent and session database owned by a Fabric runtime.""" + + def __init__(self) -> None: + self._started = False + self._start_payload: dict[str, Any] | None = None + self._runtime_id: str | None = None + self._settings: dict[str, Any] = {} + self._model_config: dict[str, Any] = {} + self._base_url: str | None = None + self._hermes_home: Path | None = None + self._hermes_config_path: Path | None = None + self._hermes_config: dict[str, Any] = {} + self._enabled_toolsets: list[str] | None = None + self._conversation_history: list[dict[str, Any]] | None = None + self._session_db: Any = None + self._agent: Any = None + self._invoke_hook: Any = None + self._relay_plugin_config: dict[str, Any] | None = None + self._relay_context: Any = None + self._relay_context_entered = False + self._relay_session_pending = False + self._relay_finalize_hook_invoked = False + self._relay_model_name = "unknown" + + async def start(self, payload: dict[str, Any]) -> None: + if self._started: + raise lifecycle.LifecycleError( + "hermes_runtime_already_started", + "Hermes runtime is already started", + ) + try: + self._relay_session_pending = False + self._relay_finalize_hook_invoked = False + validate_hermes_telemetry_provider(payload) + self._settings = common_utils.settings_payload(payload) + self._model_config = common_utils.selected_model_config(payload) + self._runtime_id = common_utils.runtime_id(payload) + hermes_home_base = Path(common_utils.base_dir(payload)).joinpath( + self._settings.get("hermes_home", "./artifacts/hermes-home") + ) + self._hermes_home = common_utils.runtime_state_directory( + hermes_home_base, payload + ) + self._hermes_home.mkdir(parents=True, exist_ok=True) + os.environ["HOME"] = str(self._hermes_home) + os.environ["HERMES_HOME"] = str(self._hermes_home) + os.environ.setdefault("HERMES_YOLO_MODE", "1") + os.environ.setdefault("HERMES_ACCEPT_HOOKS", "1") + os.environ["HERMES_SESSION_SOURCE"] = "fabric" + os.environ.setdefault( + "TERMINAL_ENV", + self._settings.get("terminal_backend", "local"), + ) + os.environ.setdefault( + "TERMINAL_TIMEOUT", + str(self._settings.get("terminal_timeout", 60)), + ) -async def run_hermes(payload: dict[str, Any]) -> dict[str, Any]: - validate_hermes_telemetry_provider(payload) - settings = common_utils.settings_payload(payload) - request = common_utils.request_payload(payload) - model_config = common_utils.selected_model_config(payload) - hermes_home_base = Path(common_utils.base_dir(payload)).joinpath( - settings.get("hermes_home", "./artifacts/hermes-home") - ) - hermes_home = common_utils.runtime_state_directory(hermes_home_base, payload) - hermes_home.mkdir(parents=True, exist_ok=True) - os.environ["HOME"] = str(hermes_home) - os.environ["HERMES_HOME"] = str(hermes_home) - os.environ.setdefault("HERMES_YOLO_MODE", "1") - os.environ.setdefault("HERMES_ACCEPT_HOOKS", "1") - os.environ["HERMES_SESSION_SOURCE"] = "fabric" - os.environ.setdefault("TERMINAL_ENV", settings.get("terminal_backend", "local")) - os.environ.setdefault("TERMINAL_TIMEOUT", str(settings.get("terminal_timeout", 60))) - relay_enabled = common_utils.relay_enabled(payload) - - relay_plugin_config = None - if relay_enabled: - relay_plugin_config = common_utils.load_relay_plugin_config(payload) - - hermes_config_path, hermes_config = write_hermes_config( - payload, - hermes_home, - relay_enabled=relay_enabled, - ) + relay_enabled = common_utils.relay_enabled(payload) + if relay_enabled: + self._relay_plugin_config = common_utils.load_relay_plugin_config( + payload + ) + relay_api_config = common_utils.relay_api_plugin_config( + self._relay_plugin_config + ) + from nemo_relay import plugin - api_key_env = settings.get("api_key_env") or model_config.get("api_key_env") or "NVIDIA_API_KEY" - api_key = os.environ.get(api_key_env) - if not api_key: - raise RuntimeError(f"{api_key_env} is required for Hermes mode") + self._relay_context = plugin.plugin(relay_api_config) + await self._relay_context.__aenter__() + self._relay_context_entered = True - base_url = common_utils.get_base_url(settings, model_config) - user_message = request.get("input") or "" - if not isinstance(user_message, str): - user_message = json.dumps(user_message, sort_keys=True) - - hermes_kwargs = { - "payload": payload, - "settings": settings, - "model_config": model_config, - "base_url": base_url, - "api_key": api_key, - "user_message": user_message, - "relay_plugin_config": relay_plugin_config, - } + self._hermes_config_path, self._hermes_config = write_hermes_config( + payload, + self._hermes_home, + relay_enabled=relay_enabled, + ) + api_key_env = ( + self._settings.get("api_key_env") + or self._model_config.get("api_key_env") + or "NVIDIA_API_KEY" + ) + api_key = os.environ.get(api_key_env) + if not api_key: + raise RuntimeError(f"{api_key_env} is required for Hermes mode") + self._base_url = common_utils.get_base_url( + self._settings, self._model_config + ) + self._relay_model_name = common_utils.relay_model_name(payload) + + from hermes_cli.config import load_config + from hermes_cli.plugins import discover_plugins + from hermes_cli.plugins import invoke_hook + from hermes_state import SessionDB + from run_agent import AIAgent + + with redirect_stdout(StringIO()): + discover_plugins(force=True) + loaded_hermes_config = load_config() + self._enabled_toolsets = resolve_hermes_toolsets( + self._settings, loaded_hermes_config + ) + self._session_db = SessionDB() + self._conversation_history = None + max_iterations = self._settings.get("max_iterations") + if max_iterations is None: + max_iterations = DEFAULT_MAX_ITERATIONS + self._agent = AIAgent( + **filter_supported_kwargs( + AIAgent, + base_url=self._base_url, + api_key=api_key, + provider=self._settings.get("provider") + or self._model_config.get("provider"), + model=self._settings.get("model_name") + or self._model_config.get("model", ""), + max_iterations=int(max_iterations), + enabled_toolsets=self._enabled_toolsets, + disabled_toolsets=disabled_toolsets(payload) or None, + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + save_trajectories=bool( + self._settings.get("save_trajectories", False) + ), + max_tokens=self._settings.get("max_tokens", 512), + temperature=self._settings.get( + "temperature", + self._model_config.get("temperature", 0.0), + ), + reasoning_config=self._settings.get( + "reasoning_config", {"effort": "none"} + ), + insert_reasoning=bool( + self._settings.get("insert_reasoning", False) + ), + platform="fabric", + session_id=self._runtime_id, + session_db=self._session_db, + ) + ) + self._invoke_hook = invoke_hook + self._start_payload = payload + self._started = True + except BaseException: + await self.stop() + raise + + async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]: + start_payload = self._start_payload + if not self._started or self._agent is None or start_payload is None: + raise lifecycle.LifecycleError( + "hermes_runtime_not_started", + "Hermes runtime is not started", + ) + if common_utils.runtime_id(invocation) != self._runtime_id: + raise lifecycle.LifecycleError( + "hermes_runtime_mismatch", + "Hermes invocation does not match the active runtime", + ) - if relay_enabled: - relay_api_config = common_utils.relay_api_plugin_config(relay_plugin_config or {}) - from nemo_relay import plugin - - async with plugin.plugin(relay_api_config): - (result, enabled_toolsets, relay_artifacts, adapter_stdout) = _invoke_hermes(**hermes_kwargs) - else: - (result, enabled_toolsets, relay_artifacts, adapter_stdout) = _invoke_hermes(**hermes_kwargs) - - if relay_plugin_config is not None: - relay_artifacts = common_utils.collect_relay_artifacts(relay_plugin_config) - - response = result.get("response") or result.get("final_response") - messages = result.get("messages") or [] - output = { - "harness": "hermes", - "adapter": "python", - "mode": "hermes", - "model": model_config.get("model"), - "base_url": base_url, - "response": response, - "completed": bool(result.get("completed")), - "failed": bool(result.get("failed")), - "api_calls": result.get("api_calls"), - "messages": messages, - "message_count": len(messages), - "error": result.get("error"), - "adapter_stdout": adapter_stdout, - "hermes_home": str(hermes_home), - "hermes_config_path": str(hermes_config_path), - "hermes_native_config": summarize_hermes_config(hermes_config), - "enabled_toolsets": enabled_toolsets, - } - if relay_plugin_config is not None: - output["relay_runtime"] = { - "enabled": True, - "config_path": os.environ.get("FABRIC_RELAY_CONFIG_PATH"), - "emitter": "hermes.observability/nemo_relay", + payload = { + **start_payload, + "runtime_context": invocation.get("runtime_context"), + "request": invocation.get("request"), + } + request = common_utils.request_payload(payload) + user_message = request.get("input") or "" + if not isinstance(user_message, str): + user_message = json.dumps(user_message, sort_keys=True) + try: + self._relay_session_pending = self._relay_plugin_config is not None + self._relay_finalize_hook_invoked = False + result, adapter_stdout = _invoke_hermes_turn( + agent=self._agent, + settings=self._settings, + user_message=user_message, + conversation_history=self._conversation_history, + ) + finally: + # Hermes' Relay plugin materializes ATIF when its session-finalize + # hook runs. Finalize the telemetry session for each Fabric + # invocation while retaining the native AIAgent and SessionDB. + self._finalize_relay_session() + messages = result.get("messages") or [] + if isinstance(messages, list): + self._conversation_history = messages + + output = { + "harness": "hermes", + "adapter": "python", + "mode": "hermes", + "model": self._model_config.get("model"), + "base_url": self._base_url, + "response": result.get("response") or result.get("final_response"), + "completed": bool(result.get("completed")), + "failed": bool(result.get("failed")), + "api_calls": result.get("api_calls"), + "messages": messages, + "message_count": len(messages), + "error": result.get("error"), + "adapter_stdout": adapter_stdout, + "hermes_home": str(self._hermes_home), + "hermes_config_path": str(self._hermes_config_path), + "hermes_native_config": summarize_hermes_config(self._hermes_config), + "enabled_toolsets": self._enabled_toolsets, } - output["relay_artifacts"] = relay_artifacts - return output + if self._relay_plugin_config is not None: + output["relay_runtime"] = { + "enabled": True, + "config_path": os.environ.get("FABRIC_RELAY_CONFIG_PATH"), + "emitter": "hermes.observability/nemo_relay", + } + output["relay_artifacts"] = common_utils.collect_relay_artifacts( + self._relay_plugin_config + ) + return output + + def _finalize_relay_session(self) -> None: + if ( + self._relay_plugin_config is None + or self._agent is None + or self._invoke_hook is None + or not self._relay_session_pending + ): + return + if not self._relay_finalize_hook_invoked: + self._invoke_hook( + "on_session_finalize", + session_id=getattr(self._agent, "session_id", ""), + model=getattr(self._agent, "model", None) or self._relay_model_name, + platform=getattr(self._agent, "platform", None) or "fabric", + ) + self._relay_finalize_hook_invoked = True + # Relay subscriber callbacks are queued. The long-lived plugin context + # does not flush them until runtime shutdown, but invocation results + # must include artifacts produced by this turn. + from nemo_relay import subscribers + + subscribers.flush() + self._relay_session_pending = False + self._relay_finalize_hook_invoked = False + + async def stop(self) -> None: + agent = self._agent + session_db = self._session_db + relay_context = self._relay_context + relay_context_entered = self._relay_context_entered + relay_plugin_config = self._relay_plugin_config + errors: list[BaseException] = [] + if relay_plugin_config is not None and agent is not None: + try: + self._finalize_relay_session() + except BaseException as error: + errors.append(error) + self._agent = None + self._session_db = None + self._start_payload = None + self._runtime_id = None + self._settings = {} + self._model_config = {} + self._base_url = None + self._hermes_home = None + self._hermes_config_path = None + self._hermes_config = {} + self._enabled_toolsets = None + self._conversation_history = None + self._relay_context = None + self._relay_context_entered = False + self._relay_session_pending = False + self._relay_finalize_hook_invoked = False + self._invoke_hook = None + self._relay_plugin_config = None + self._relay_model_name = "unknown" + self._started = False + + if agent is not None: + try: + agent.close() + except BaseException as error: + errors.append(error) + if session_db is not None: + try: + session_db.close() + except BaseException as error: + errors.append(error) + if relay_context is not None and relay_context_entered: + try: + await relay_context.__aexit__(None, None, None) + except BaseException as error: + errors.append(error) + + if errors: + for error in errors: + if isinstance(error, asyncio.CancelledError): + raise error + LOGGER.error( + "Hermes runtime cleanup failed", + exc_info=(type(error), error, error.__traceback__), + ) + raise lifecycle.LifecycleError( + "hermes_runtime_stop_failed", + "Hermes runtime failed to stop cleanly", + ) from errors[0] -def _invoke_hermes( +def _invoke_hermes_turn( *, - payload: dict[str, Any], + agent: Any, settings: dict[str, Any], - model_config: dict[str, Any], - base_url: str | None, - api_key: str, user_message: str, - relay_plugin_config: dict[str, Any] | None, -) -> tuple[dict[str, Any], list[str] | None, list[dict[str, str]], str]: - from hermes_cli.config import load_config - from hermes_cli.plugins import discover_plugins - from hermes_cli.plugins import invoke_hook - from hermes_state import SessionDB - from run_agent import AIAgent - - relay_artifacts: list[dict[str, str]] = [] + conversation_history: list[dict[str, Any]] | None, +) -> tuple[dict[str, Any], str]: hermes_stdout = StringIO() with redirect_stdout(hermes_stdout): - discover_plugins(force=True) - loaded_hermes_config = load_config() - enabled_toolsets = resolve_hermes_toolsets(settings, loaded_hermes_config) - blocked_toolsets = disabled_toolsets(payload) - session_id = common_utils.runtime_id(payload) - session_db = SessionDB() - conversation_history = load_runtime_history(session_db, session_id) - # Treat an explicit null max_iterations like an unset one (avoid int(None)). - max_iterations = settings.get("max_iterations") - if max_iterations is None: - max_iterations = DEFAULT_MAX_ITERATIONS - agent = None - agent = AIAgent( - **filter_supported_kwargs( - AIAgent, - base_url=base_url, - api_key=api_key, - provider=settings.get("provider") or model_config.get("provider"), - model=settings.get("model_name") or model_config.get("model", ""), - max_iterations=int(max_iterations), - enabled_toolsets=enabled_toolsets, - disabled_toolsets=blocked_toolsets or None, - quiet_mode=True, - skip_context_files=True, - skip_memory=True, - save_trajectories=bool(settings.get("save_trajectories", False)), - max_tokens=settings.get("max_tokens", 512), - temperature=settings.get("temperature", model_config.get("temperature", 0.0)), - reasoning_config=settings.get("reasoning_config", {"effort": "none"}), - insert_reasoning=bool(settings.get("insert_reasoning", False)), - platform="fabric", - session_id=session_id, - session_db=session_db, - ) + conversation_kwargs = filter_supported_call_kwargs( + agent.run_conversation, + system_message=settings.get("system_prompt"), + conversation_history=conversation_history, + sync_honcho=False, + dont_review=True, ) - try: - conversation_kwargs = filter_supported_call_kwargs( - agent.run_conversation, - system_message=settings.get("system_prompt"), - conversation_history=conversation_history, - sync_honcho=False, - dont_review=True, - ) - result = agent.run_conversation( - user_message, - **conversation_kwargs, - ) - finally: - if relay_plugin_config is not None and agent is not None: - invoke_hook( - "on_session_finalize", - session_id=getattr(agent, "session_id", ""), - model=getattr(agent, "model", None) or common_utils.relay_model_name(payload), - platform=getattr(agent, "platform", None) or "fabric", - ) - - return result, enabled_toolsets, relay_artifacts, hermes_stdout.getvalue() + result = agent.run_conversation( + user_message, + **conversation_kwargs, + ) + return result, hermes_stdout.getvalue() def filter_supported_kwargs(callable_obj: Any, **kwargs: Any) -> dict[str, Any]: diff --git a/crates/fabric-cli/assets/adapters/claude/fabric-adapter.json b/crates/fabric-cli/assets/adapters/claude/fabric-adapter.json index ed36b146..9f8996f9 100644 --- a/crates/fabric-cli/assets/adapters/claude/fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/claude/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "claude", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.claude.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.claude.adapter" }, "config": { "accepts": ["models", "tools", "tools.blocked", "mcp", "skills", "telemetry"] diff --git a/crates/fabric-cli/assets/adapters/codex/fabric-adapter.json b/crates/fabric-cli/assets/adapters/codex/fabric-adapter.json index 96f2d7dd..251ab043 100644 --- a/crates/fabric-cli/assets/adapters/codex/fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/codex/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "codex", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.codex.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.codex.adapter" }, "config": { "accepts": ["models", "mcp", "skills", "telemetry"] diff --git a/crates/fabric-cli/assets/adapters/deepagents/fabric-adapter.json b/crates/fabric-cli/assets/adapters/deepagents/fabric-adapter.json index 78a09fe3..c32aa03b 100644 --- a/crates/fabric-cli/assets/adapters/deepagents/fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/deepagents/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "deepagents", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.deepagents.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.deepagents.adapter" }, "requirements": {}, "config": { diff --git a/crates/fabric-cli/assets/adapters/hermes/fabric-adapter.json b/crates/fabric-cli/assets/adapters/hermes/fabric-adapter.json index f2857470..ef60fb92 100644 --- a/crates/fabric-cli/assets/adapters/hermes/fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/hermes/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "hermes", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.hermes.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.hermes.adapter" }, "requirements": { "env": [ diff --git a/crates/fabric-cli/assets/adapters/scripted/fabric-adapter.json b/crates/fabric-cli/assets/adapters/scripted/fabric-adapter.json index 298b1e4e..3a74fbb6 100644 --- a/crates/fabric-cli/assets/adapters/scripted/fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/scripted/fabric-adapter.json @@ -5,7 +5,6 @@ "adapter_kind": "process", "runner": { "command": "python3", - "script": "run.py", - "stdin_payload": "fabric_request" + "script": "run.py" } } diff --git a/crates/fabric-cli/assets/adapters/scripted/run.py b/crates/fabric-cli/assets/adapters/scripted/run.py index 1f0f3cce..eb8782d1 100644 --- a/crates/fabric-cli/assets/adapters/scripted/run.py +++ b/crates/fabric-cli/assets/adapters/scripted/run.py @@ -6,15 +6,47 @@ import json import sys +from typing import Any -payload = json.load(sys.stdin) -request = payload["request"] -print( - json.dumps( - { - "response": request.get("input"), - "request_id": request["request_id"], - } +def response(operation: str, *, output: Any = None) -> None: + print( + json.dumps( + { + "operation": operation, + "outcome": {"status": "succeeded", "output": output}, + } + ), + flush=True, ) -) + + +def main() -> None: + runtime_id = None + for line in sys.stdin: + message = json.loads(line) + operation = message["operation"] + payload = message["payload"] + if operation == "start": + runtime_id = payload["runtime_context"]["runtime_id"] + response("start") + elif operation == "invoke": + if payload["runtime_context"]["runtime_id"] != runtime_id: + raise RuntimeError("invoke does not match the active runtime") + request = payload["request"] + response( + "invoke", + output={ + "response": request.get("input"), + "request_id": request["request_id"], + }, + ) + elif operation == "stop": + if payload["runtime_id"] != runtime_id: + raise RuntimeError("stop does not match the active runtime") + response("stop") + break + + +if __name__ == "__main__": + main() diff --git a/crates/fabric-cli/src/app.rs b/crates/fabric-cli/src/app.rs index 6093f356..93ab91eb 100644 --- a/crates/fabric-cli/src/app.rs +++ b/crates/fabric-cli/src/app.rs @@ -250,7 +250,22 @@ fn run(cli: Cli) -> Result<(), Box> { selected.config().clone(), ResolveContext::new(selected.base_dir()), )?; - let result = run_plan(&plan, RunRequest::text(input))?; + let result = match run_plan(&plan, RunRequest::text(input)) { + Ok(result) => result, + Err(error) => { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "status": "failed", + "error": { + "code": "runtime_error", + "message": "run failed before a normalized result was available", + }, + }))? + ); + return Err(error.into()); + } + }; println!("{}", serde_json::to_string_pretty(&result)?); require_successful_run(result.status)?; } diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 49b5592d..5316c433 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -377,11 +377,11 @@ impl ResolveContext { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AdapterKind { - /// Launch and supervise a CLI process. + /// Launch and supervise a persistent adapter process. Process, /// Connect to a service or HTTP-backed harness. Http, - /// Call a Python SDK/plugin adapter. + /// Launch and supervise a persistent Python adapter host. Python, /// Delegate to a harness-native plugin package. NativePlugin, diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index 41140959..ce2de11b 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -82,6 +82,27 @@ pub enum FabricError { /// Adapter kind. adapter_kind: AdapterKind, }, + /// A persistent local-host lifecycle operation failed. + #[error( + "adapter lifecycle {operation} failed for runtime `{runtime_id}` ({code}): {message}{diagnostics_suffix}", + diagnostics_suffix = if diagnostics.is_empty() { + String::new() + } else { + format!("; diagnostics: {diagnostics}") + } + )] + AdapterLifecycleOperation { + /// Lifecycle operation that failed. + operation: &'static str, + /// Runtime whose host failed. + runtime_id: String, + /// Stable failure code. + code: String, + /// Human-readable failure message. + message: String, + /// Bounded adapter-host diagnostics. + diagnostics: String, + }, /// The selected harness cannot enforce the configured blocked-tools policy. #[error("harness `{harness}` cannot enforce configured blocked tools: {reason}")] UnsupportedToolsPolicy { diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 8c07fd40..4c5b9b80 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5,13 +5,15 @@ use std::collections::BTreeMap; use std::ffi::OsString; -use std::io::{ErrorKind, Write}; +use std::fs::File; +use std::io::{BufRead, BufReader, ErrorKind, Write}; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -#[cfg(test)] -use std::sync::Mutex; +use std::process::{Child, ChildStdin, Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::sync::{Arc, LazyLock, Mutex}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -26,6 +28,13 @@ use crate::error::{FabricError, Result}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; +const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); +// Protocol liveness backstop. Adapters remain responsible for normal request +// timeouts and should return a normalized response before this bound. +const LOCAL_HOST_INVOKE_TIMEOUT: Duration = Duration::from_secs(60 * 60); +const LOCAL_HOST_STOP_TIMEOUT: Duration = Duration::from_secs(10); +const LOCAL_HOST_EXIT_GRACE: Duration = Duration::from_secs(2); +const LOCAL_HOST_DIAGNOSTIC_LIMIT: usize = 16 * 1024; #[cfg(not(windows))] const VENV_BIN_DIR: &str = "bin"; @@ -45,6 +54,8 @@ const DEFAULT_PYTHON: &str = "python3"; const DEFAULT_PYTHON: &str = "python.exe"; #[cfg(test)] static TEST_STOPPED_AGENTS: Mutex> = Mutex::new(Vec::new()); +static LOCAL_HOSTS: LazyLock>>>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); /// A request passed to a Fabric-managed harness runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, Default)] @@ -272,7 +283,7 @@ pub struct InvocationHandle { pub runtime_id: String, } -/// Per-run/per-invocation context passed to harness adapters. +/// Context generated for one invocation of a started runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RuntimeContext { /// Runtime handle id. @@ -306,25 +317,107 @@ pub struct RuntimeTelemetryContext { pub metadata: BTreeMap, } -/// Adapter-facing invocation payload. +/// One invocation against an initialized adapter runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct AdapterInvocation { - /// Stable agent name. - pub agent_name: String, - /// Absolute base directory used to resolve relative Fabric paths. - pub base_dir: PathBuf, - /// Complete typed Fabric config. - pub config: FabricConfig, - /// Per-runtime/per-invocation execution context. + /// Invocation context generated by Fabric. pub runtime_context: RuntimeContext, - /// Per-invocation request. + /// Typed caller request for this invocation. pub request: RunRequest, - /// Derived capability routing plan for the selected adapter. - #[serde(default)] - pub capability_plan: CapabilityPlan, - /// Derived telemetry plan for the selected adapter. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub telemetry_plan: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum AdapterLifecycleOperation { + Start, + Invoke, + Stop, +} + +impl AdapterLifecycleOperation { + fn as_str(self) -> &'static str { + match self { + Self::Start => "start", + Self::Invoke => "invoke", + Self::Stop => "stop", + } + } + + fn error_stage(self) -> ErrorStage { + match self { + Self::Start => ErrorStage::Start, + Self::Invoke => ErrorStage::Invoke, + Self::Stop => ErrorStage::Stop, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +struct AdapterLifecycleStart { + agent_name: String, + base_dir: PathBuf, + config: FabricConfig, + runtime_context: RuntimeContext, + capability_plan: CapabilityPlan, + #[serde(skip_serializing_if = "Option::is_none")] + telemetry_plan: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +struct AdapterLifecycleStop { + runtime_id: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(tag = "operation", content = "payload", rename_all = "snake_case")] +enum AdapterLifecycleRequestKind { + Start(Box), + Invoke(Box), + Stop(AdapterLifecycleStop), +} + +impl AdapterLifecycleRequestKind { + fn operation(&self) -> AdapterLifecycleOperation { + match self { + Self::Start(_) => AdapterLifecycleOperation::Start, + Self::Invoke(_) => AdapterLifecycleOperation::Invoke, + Self::Stop(_) => AdapterLifecycleOperation::Stop, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +struct AdapterLifecycleRequest { + #[serde(flatten)] + request: AdapterLifecycleRequestKind, +} + +impl AdapterLifecycleRequest { + fn new(request: AdapterLifecycleRequestKind) -> Self { + Self { request } + } + + fn operation(&self) -> AdapterLifecycleOperation { + self.request.operation() + } +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +enum AdapterLifecycleOutcome { + Succeeded { + #[serde(default)] + output: Value, + }, + Failed { + error: ErrorInfo, + }, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +struct AdapterLifecycleResponse { + operation: AdapterLifecycleOperation, + outcome: AdapterLifecycleOutcome, } trait RuntimeAdapter { @@ -338,8 +431,7 @@ trait RuntimeAdapter { fn stop(&self, runtime: &RuntimeHandle) -> Result>; } -struct ProcessAdapter; -struct PythonAdapter; +struct LocalHostAdapter; #[derive(Debug, Clone)] struct RelayRuntimeConfig { @@ -347,6 +439,18 @@ struct RelayRuntimeConfig { env: BTreeMap, } +struct LocalAdapterHost { + child: Child, + stdin: ChildStdin, + responses: Receiver>, + command: String, + runtime_dir: PathBuf, + stderr_path: PathBuf, + stderr_offset: usize, + artifacts: ArtifactManifest, + relay_config: Option, +} + /// Invoke a Fabric run plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { let runtime = start_runtime(plan)?; @@ -430,14 +534,13 @@ pub fn prepare_environment(plan: &RunPlan) -> Result { pub fn start_runtime(plan: &RunPlan) -> Result { validate_blocked_tools_support(plan)?; let environment = prepare_environment(plan)?; - match adapter_kind(plan) { - AdapterKind::Process => ProcessAdapter.start(plan, environment), - AdapterKind::Python => PythonAdapter.start(plan, environment), - adapter_kind => Err(FabricError::UnsupportedRuntimeAdapter { - harness: harness(plan), - adapter_kind, - }), + if uses_local_host(plan) { + return LocalHostAdapter.start(plan, environment); } + Err(FabricError::UnsupportedRuntimeAdapter { + harness: harness(plan), + adapter_kind: adapter_kind(plan), + }) } /// Invoke a started harness runtime. @@ -448,14 +551,13 @@ pub fn invoke_runtime( ) -> Result { validate_blocked_tools_support(plan)?; validate_runtime_handle(plan, runtime)?; - match adapter_kind(plan) { - AdapterKind::Process => ProcessAdapter.invoke(plan, runtime, request), - AdapterKind::Python => PythonAdapter.invoke(plan, runtime, request), - adapter_kind => Err(FabricError::UnsupportedRuntimeAdapter { - harness: harness(plan), - adapter_kind, - }), + if uses_local_host(plan) { + return LocalHostAdapter.invoke(plan, runtime, request); } + Err(FabricError::UnsupportedRuntimeAdapter { + harness: harness(plan), + adapter_kind: adapter_kind(plan), + }) } fn validate_blocked_tools_support(plan: &RunPlan) -> Result<()> { @@ -473,14 +575,20 @@ fn validate_blocked_tools_support(plan: &RunPlan) -> Result<()> { /// Stop or detach from a harness runtime. pub fn stop_runtime(plan: &RunPlan, runtime: &RuntimeHandle) -> Result> { validate_runtime_handle(plan, runtime)?; - match runtime.adapter_kind { - AdapterKind::Process => ProcessAdapter.stop(runtime), - AdapterKind::Python => PythonAdapter.stop(runtime), - adapter_kind => Err(FabricError::UnsupportedRuntimeAdapter { - harness: runtime.harness.clone(), - adapter_kind, - }), + if uses_local_host(plan) { + return LocalHostAdapter.stop(runtime); } + Err(FabricError::UnsupportedRuntimeAdapter { + harness: runtime.harness.clone(), + adapter_kind: runtime.adapter_kind, + }) +} + +fn uses_local_host(plan: &RunPlan) -> bool { + matches!( + adapter_kind(plan), + AdapterKind::Process | AdapterKind::Python + ) } fn validate_runtime_handle(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<()> { @@ -593,16 +701,6 @@ struct ProcessAdapterSettings { cwd: Option, #[serde(default)] env: BTreeMap, - #[serde(default)] - stdin_payload: ProcessStdinPayload, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -enum ProcessStdinPayload { - #[default] - Input, - FabricRequest, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -661,65 +759,30 @@ impl PythonSource { } } -impl RuntimeAdapter for ProcessAdapter { +impl RuntimeAdapter for LocalHostAdapter { fn start(&self, plan: &RunPlan, environment: EnvironmentHandle) -> Result { if environment.provider != "local" { return Err(FabricError::UnsupportedEnvironmentProvider { provider: environment.provider, - adapter_kind: AdapterKind::Process, + adapter_kind: adapter_kind(plan), }); } - let runtime_id = new_id("runtime"); - let runtime_binding = runtime_binding(&runtime_id, plan, &environment)?; - Ok(RuntimeHandle { - runtime_id, - runtime_binding, - agent_name: plan.agent_name.clone(), - harness: harness(plan), - adapter_kind: adapter_kind(plan), - adapter_id: adapter_id(plan), - environment, - }) - } - - fn invoke( - &self, - plan: &RunPlan, - runtime: &RuntimeHandle, - request: RunRequest, - ) -> Result { - run_process_adapter(plan, runtime, request) - } - - fn stop(&self, runtime: &RuntimeHandle) -> Result> { - #[cfg(test)] - TEST_STOPPED_AGENTS - .lock() - .expect("stop tracker") - .push(runtime.agent_name.clone()); - Ok(vec![event_with_metadata( - "runtime_stop", - format!("stopped runtime {}", runtime.runtime_id), - BTreeMap::from([( - "runtime_id".to_string(), - Value::String(runtime.runtime_id.clone()), - )]), - )]) - } -} - -impl RuntimeAdapter for PythonAdapter { - fn start(&self, plan: &RunPlan, environment: EnvironmentHandle) -> Result { - if environment.provider != "local" { - return Err(FabricError::UnsupportedEnvironmentProvider { - provider: environment.provider, - adapter_kind: AdapterKind::Python, + if !matches!( + adapter_kind(plan), + AdapterKind::Process | AdapterKind::Python + ) { + return Err(FabricError::UnsupportedRuntimeAdapter { + harness: harness(plan), + adapter_kind: adapter_kind(plan), }); } - preflight_python_adapter(plan)?; + if adapter_kind(plan) == AdapterKind::Python { + preflight_python_adapter(plan)?; + } + let runtime_id = new_id("runtime"); let runtime_binding = runtime_binding(&runtime_id, plan, &environment)?; - Ok(RuntimeHandle { + let runtime = RuntimeHandle { runtime_id, runtime_binding, agent_name: plan.agent_name.clone(), @@ -727,7 +790,40 @@ impl RuntimeAdapter for PythonAdapter { adapter_kind: adapter_kind(plan), adapter_id: adapter_id(plan), environment, - }) + }; + + let start_invocation = InvocationHandle { + invocation_id: new_id("runtime-start"), + request_id: new_id("runtime-start-request"), + runtime_id: runtime.runtime_id.clone(), + }; + let mut artifacts = artifact_manifest(plan)?; + let fabric_home = prepare_fabric_home(&artifacts, &runtime, &start_invocation)?; + let relay_config = + prepare_relay_runtime_config(plan, &runtime, &fabric_home, &mut artifacts)?; + let start = adapter_lifecycle_start( + plan, + &runtime, + &start_invocation, + &artifacts, + relay_config.as_ref(), + )?; + let request = + AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Start(Box::new(start))); + let mut host = spawn_local_host(plan, &runtime, artifacts, relay_config)?; + if let Err(error) = exchange_lifecycle_message( + &mut host, + &runtime.runtime_id, + &request, + LOCAL_HOST_START_TIMEOUT, + ) { + let _ = terminate_local_host(&mut host); + let _ = remove_local_host_files(&host); + return Err(error); + } + + local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); + Ok(runtime) } fn invoke( @@ -736,30 +832,76 @@ impl RuntimeAdapter for PythonAdapter { runtime: &RuntimeHandle, request: RunRequest, ) -> Result { - run_python_adapter(plan, runtime, request) + run_local_host_adapter(plan, runtime, request) } fn stop(&self, runtime: &RuntimeHandle) -> Result> { + let Some(host) = local_hosts().remove(&runtime.runtime_id) else { + return Ok(vec![local_host_stop_event(runtime, true, false)]); + }; + let mut host = host.lock().unwrap_or_else(|error| error.into_inner()); + let request = + AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { + runtime_id: runtime.runtime_id.clone(), + })); + let result = exchange_lifecycle_message( + &mut host, + &runtime.runtime_id, + &request, + LOCAL_HOST_STOP_TIMEOUT, + ); + let termination = terminate_local_host(&mut host); + let diagnostics = local_host_diagnostics(&host); + let removal = remove_local_host_files(&host); + let host_crashed = matches!( + &result, + Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "host_crashed" + ); + if !host_crashed { + result?; + } + termination.map_err(|source| { + lifecycle_error( + AdapterLifecycleOperation::Stop, + &runtime.runtime_id, + "host_termination_failed", + format!("persistent local adapter host could not be terminated: {source}"), + diagnostics, + ) + })?; + removal.map_err(|source| { + lifecycle_error( + AdapterLifecycleOperation::Stop, + &runtime.runtime_id, + "host_cleanup_failed", + format!("persistent local adapter host files could not be removed: {source}"), + "", + ) + })?; + #[cfg(test)] TEST_STOPPED_AGENTS .lock() .expect("stop tracker") .push(runtime.agent_name.clone()); - Ok(vec![event_with_metadata( - "runtime_stop", - format!("stopped runtime {}", runtime.runtime_id), - BTreeMap::from([( - "runtime_id".to_string(), - Value::String(runtime.runtime_id.clone()), - )]), - )]) + Ok(vec![local_host_stop_event(runtime, false, host_crashed)]) } } -fn run_process_adapter( +fn run_local_host_adapter( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, +) -> Result { + run_local_host_adapter_with_timeout(plan, runtime, request, LOCAL_HOST_INVOKE_TIMEOUT) +} + +fn run_local_host_adapter_with_timeout( plan: &RunPlan, runtime: &RuntimeHandle, mut request: RunRequest, + invoke_timeout: Duration, ) -> Result { if request.request_id.is_empty() { request.request_id = new_id("request"); @@ -769,53 +911,78 @@ fn run_process_adapter( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; - let settings = parse_process_settings(plan)?; - let command_path = resolve_command_path( - adapter_setting_root(plan, "command"), - Path::new(&settings.command), - ); - let command_display = command_path.to_string_lossy().into_owned(); - let command_args = process_command_args(plan, &settings); - let cwd = settings - .cwd - .as_ref() - .map(|path| resolve_path(&plan.base_dir, path)) - .or_else(|| runtime.environment.workspace.clone()) - .unwrap_or_else(|| plan.base_dir.clone()); - let mut artifacts = artifact_manifest(plan)?; - let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; - let relay_config = prepare_relay_runtime_config( - plan, - runtime, - &invocation, - &request, - &fabric_home, - &mut artifacts, - )?; - let adapter_payload = fabric_adapter_payload( - plan, - runtime, - &invocation, - &request, - &artifacts, - relay_config.as_ref(), - )?; - let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; + let host = local_hosts() + .get(&runtime.runtime_id) + .cloned() + .ok_or_else(|| { + lifecycle_error( + AdapterLifecycleOperation::Invoke, + &runtime.runtime_id, + "host_unavailable", + "persistent local adapter host is not active", + "", + ) + })?; - let mut command = Command::new(&command_path); - command - .args(&command_args) - .current_dir(&cwd) - .envs(&settings.env) - .envs(relay_env(&relay_config)) - .env("FABRIC_HOME", &fabric_home) - .env("FABRIC_INVOCATION", &fabric_invocation) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - if let Some(root) = artifacts.root.as_ref() { - command.env("FABRIC_ARTIFACTS", root); - } + let exchange_result = { + let mut host_guard = host.lock().unwrap_or_else(|error| error.into_inner()); + let artifacts = host_guard.artifacts.clone(); + let relay_config = host_guard.relay_config.clone(); + let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; + let adapter_invocation = adapter_invocation( + plan, + runtime, + &invocation, + &request, + &artifacts, + relay_config.as_ref(), + )?; + let adapter_payload = serde_json::to_string_pretty(&adapter_invocation) + .map_err(FabricError::SerializeJson)?; + let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; + let lifecycle_request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Invoke( + Box::new(adapter_invocation), + )); + match exchange_lifecycle_message( + &mut host_guard, + &runtime.runtime_id, + &lifecycle_request, + invoke_timeout, + ) { + Ok(output) => { + let stderr = take_local_host_stderr(&mut host_guard); + Ok(( + output, + stderr, + host_guard.command.clone(), + host_guard.child.id(), + artifacts, + relay_config, + fabric_home, + fabric_invocation, + )) + } + Err(error) => { + if matches!( + &error, + FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" + ) { + invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); + } + Err(error) + } + } + }; + let ( + output, + stderr, + host_command, + host_pid, + mut artifacts, + relay_config, + fabric_home, + fabric_invocation, + ) = exchange_result?; let mut events = vec![event_with_metadata( "runtime_start", @@ -833,11 +1000,12 @@ fn run_process_adapter( "environment_provider".to_string(), Value::String(runtime.environment.provider.clone()), ), + ("host_pid".to_string(), Value::from(host_pid)), ]), )]; events.push(event_with_metadata( "invocation_start", - format!("starting process adapter for {}", harness(plan)), + format!("invoking persistent local host for {}", harness(plan)), BTreeMap::from([ ( "runtime_id".to_string(), @@ -849,39 +1017,10 @@ fn run_process_adapter( ), ]), )); - let mut child = command - .spawn() - .map_err(|source| FabricError::ProcessRunner { - command: command_display.clone(), - source, - })?; - - let stdin_payload = match settings.stdin_payload { - ProcessStdinPayload::Input => value_to_stdin(&request.input)?, - ProcessStdinPayload::FabricRequest => adapter_payload, - }; - if !stdin_payload.is_empty() { - if let Some(mut stdin) = child.stdin.take() { - write_child_stdin(&mut stdin, &stdin_payload, &command_display)?; - } - } - - let output = child - .wait_with_output() - .map_err(|source| FabricError::ProcessRunner { - command: command_display.clone(), - source, - })?; - - let exit_code = output.status.code(); - let status = if output.status.success() { - RunStatus::Succeeded - } else { - RunStatus::Failed - }; + let (status, error) = adapter_output_status(&output); events.push(event_with_metadata( "invocation_end", - format!("process adapter completed with status {:?}", status), + format!("persistent local host completed with status {status:?}"), BTreeMap::from([ ( "runtime_id".to_string(), @@ -893,20 +1032,17 @@ fn run_process_adapter( ), ]), )); - - let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); - let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); - if !stdout.is_empty() { - write_artifact( - &mut artifacts, - &fabric_home, - "stdout", - "log", - "stdout.txt", - &stdout, - "text/plain", - )?; - } + let mut stdout = serde_json::to_string(&output).map_err(FabricError::SerializeJson)?; + stdout.push('\n'); + write_artifact( + &mut artifacts, + &fabric_home, + "stdout", + "log", + "stdout.txt", + &stdout, + "text/plain", + )?; if !stderr.is_empty() { write_artifact( &mut artifacts, @@ -919,51 +1055,28 @@ fn run_process_adapter( )?; } collect_workspace_artifacts(&mut artifacts, &fabric_home, runtime, &mut events)?; + promote_relay_artifacts_to_manifest(&output, &mut artifacts); - let mut metadata = BTreeMap::new(); - metadata.insert( - "adapter_runner".to_string(), - Value::String("process".to_string()), - ); - metadata.insert("command".to_string(), Value::String(command_display)); - metadata.insert( - "args".to_string(), - Value::Array(command_args.into_iter().map(Value::String).collect()), - ); - metadata.insert( - "fabric_home".to_string(), - Value::String(fabric_home.to_string_lossy().into_owned()), - ); - metadata.insert( - "fabric_invocation".to_string(), - Value::String(fabric_invocation.to_string_lossy().into_owned()), - ); - if let Some(exit_code) = exit_code { - metadata.insert("exit_code".to_string(), Value::from(exit_code)); - } - metadata.insert( - "cwd".to_string(), - Value::String(cwd.to_string_lossy().into_owned()), - ); - metadata.insert( - "environment_provider".to_string(), - Value::String(runtime.environment.provider.clone()), - ); - - let error = if status == RunStatus::Failed { - Some(adapter_exit_error( - "process_exit_nonzero", - "process exited with non-zero status", - &stderr, - &metadata, - )) - } else { - None - }; - - let parsed_output = parse_stdout_output(&stdout); - promote_relay_artifacts_to_manifest(&parsed_output, &mut artifacts); - + let metadata = BTreeMap::from([ + ( + "adapter_runner".to_string(), + Value::String("persistent_local_host".to_string()), + ), + ("host_command".to_string(), Value::String(host_command)), + ("host_pid".to_string(), Value::from(host_pid)), + ( + "fabric_home".to_string(), + Value::String(fabric_home.to_string_lossy().into_owned()), + ), + ( + "fabric_invocation".to_string(), + Value::String(fabric_invocation.to_string_lossy().into_owned()), + ), + ( + "environment_provider".to_string(), + Value::String(runtime.environment.provider.clone()), + ), + ]); Ok(RunResult { agent_name: plan.agent_name.clone(), harness: harness(plan), @@ -973,7 +1086,7 @@ fn run_process_adapter( invocation_id: invocation.invocation_id, request_id: request.request_id, status, - output: parsed_output, + output, error, artifacts, telemetry: telemetry_ref(plan, relay_config.as_ref()), @@ -982,214 +1095,485 @@ fn run_process_adapter( }) } -fn run_python_adapter( +fn invalidate_timed_out_local_host( + runtime_id: &str, + expected_host: &Arc>, + host: &mut LocalAdapterHost, +) { + { + let mut hosts = local_hosts(); + if hosts + .get(runtime_id) + .is_some_and(|host| Arc::ptr_eq(host, expected_host)) + { + hosts.remove(runtime_id); + } + } + + let _ = terminate_local_host(host); + let _ = remove_local_host_files(host); +} + +fn adapter_output_status(output: &Value) -> (RunStatus, Option) { + let failed = output + .as_object() + .and_then(|output| output.get("failed")) + .and_then(Value::as_bool) + .unwrap_or(false); + if !failed { + return (RunStatus::Succeeded, None); + } + + let reported = output + .as_object() + .and_then(|output| output.get("error")) + .and_then(Value::as_object); + let code = reported + .and_then(|error| error.get("code")) + .and_then(Value::as_str) + .unwrap_or("adapter_reported_failure") + .to_string(); + let message = reported + .and_then(|error| error.get("message")) + .and_then(Value::as_str) + .unwrap_or("adapter reported an invocation failure") + .to_string(); + let retryable = reported + .and_then(|error| error.get("retryable")) + .and_then(Value::as_bool) + .unwrap_or(false); + let metadata = reported + .and_then(|error| error.get("metadata")) + .and_then(Value::as_object) + .map(|metadata| { + metadata + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect() + }) + .unwrap_or_default(); + ( + RunStatus::Failed, + Some(ErrorInfo { + stage: ErrorStage::Invoke, + code, + message, + retryable, + metadata, + }), + ) +} + +fn local_host_stop_event( + runtime: &RuntimeHandle, + already_stopped: bool, + host_crashed: bool, +) -> FabricEvent { + event_with_metadata( + "runtime_stop", + format!("stopped runtime {}", runtime.runtime_id), + BTreeMap::from([ + ( + "runtime_id".to_string(), + Value::String(runtime.runtime_id.clone()), + ), + ("already_stopped".to_string(), Value::Bool(already_stopped)), + ("host_crashed".to_string(), Value::Bool(host_crashed)), + ]), + ) +} + +fn local_hosts() -> std::sync::MutexGuard<'static, BTreeMap>>> { + LOCAL_HOSTS + .lock() + .unwrap_or_else(|error| error.into_inner()) +} + +fn spawn_local_host( plan: &RunPlan, runtime: &RuntimeHandle, - mut request: RunRequest, -) -> Result { - if request.request_id.is_empty() { - request.request_id = new_id("request"); + artifacts: ArtifactManifest, + relay_config: Option, +) -> Result { + let runtime_dir = std::env::temp_dir() + .join("nemo-fabric") + .join("hosts") + .join(&runtime.runtime_id); + std::fs::create_dir_all(&runtime_dir).map_err(|source| FabricError::Write { + path: runtime_dir.clone(), + source, + })?; + let stderr_path = runtime_dir.join("host.stderr.log"); + let stderr = File::create(&stderr_path).map_err(|source| FabricError::Write { + path: stderr_path.clone(), + source, + })?; + let (mut command, command_display) = match local_host_command(plan, runtime) { + Ok(command) => command, + Err(error) => { + let _ = std::fs::remove_dir_all(&runtime_dir); + return Err(error); + } + }; + command + .env("FABRIC_RUNTIME_ID", &runtime.runtime_id) + .env("FABRIC_HOME", &runtime_dir) + .envs(relay_env(&relay_config)) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::from(stderr)); + if let Some(root) = artifacts.root.as_ref() { + command.env("FABRIC_ARTIFACTS", root); } - let invocation = InvocationHandle { - invocation_id: new_id("invocation"), - request_id: request.request_id.clone(), - runtime_id: runtime.runtime_id.clone(), + let mut child = match command.spawn() { + Ok(child) => child, + Err(source) => { + let _ = std::fs::remove_dir_all(&runtime_dir); + return Err(FabricError::ProcessRunner { + command: command_display, + source, + }); + } }; - let settings = parse_python_settings(plan)?; + let Some(stdin) = child.stdin.take() else { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_dir_all(&runtime_dir); + return Err(lifecycle_error( + AdapterLifecycleOperation::Start, + &runtime.runtime_id, + "host_io", + "persistent local adapter host stdin was not available", + "", + )); + }; + let Some(stdout) = child.stdout.take() else { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_dir_all(&runtime_dir); + return Err(lifecycle_error( + AdapterLifecycleOperation::Start, + &runtime.runtime_id, + "host_io", + "persistent local adapter host stdout was not available", + "", + )); + }; + let (sender, responses) = mpsc::channel(); + if let Err(source) = thread::Builder::new() + .name(format!("fabric-host-{}", runtime.runtime_id)) + .spawn(move || { + let mut stdout = BufReader::new(stdout); + loop { + let mut line = String::new(); + match stdout.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + while line.ends_with(['\n', '\r']) { + line.pop(); + } + if sender.send(Ok(line)).is_err() { + break; + } + } + Err(error) => { + let _ = sender.send(Err(error.to_string())); + break; + } + } + } + }) + { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_dir_all(&runtime_dir); + return Err(FabricError::ProcessRunner { + command: command_display, + source, + }); + } + Ok(LocalAdapterHost { + child, + stdin, + responses, + command: command_display, + runtime_dir, + stderr_path, + stderr_offset: 0, + artifacts, + relay_config, + }) +} + +fn local_host_command(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<(Command, String)> { + match adapter_kind(plan) { + AdapterKind::Process => process_local_host_command(plan, runtime), + AdapterKind::Python => python_local_host_command(plan, runtime), + adapter_kind => Err(FabricError::UnsupportedRuntimeAdapter { + harness: harness(plan), + adapter_kind, + }), + } +} + +fn process_local_host_command( + plan: &RunPlan, + runtime: &RuntimeHandle, +) -> Result<(Command, String)> { + let settings = parse_process_settings(plan)?; + let command_path = resolve_command_path( + adapter_setting_root(plan, "command"), + Path::new(&settings.command), + ); + let command_args = process_command_args(plan, &settings); let cwd = settings .cwd .as_ref() .map(|path| resolve_path(&plan.base_dir, path)) .or_else(|| runtime.environment.workspace.clone()) .unwrap_or_else(|| plan.base_dir.clone()); + let mut command = Command::new(&command_path); + command + .args(&command_args) + .current_dir(cwd) + .envs(&settings.env); + let display = std::iter::once(command_path.to_string_lossy().into_owned()) + .chain(command_args) + .collect::>() + .join(" "); + Ok((command, display)) +} +fn python_local_host_command(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<(Command, String)> { + let settings = parse_python_settings(plan)?; let python = resolve_python_command(&plan.base_dir, &settings).path; - let mut artifacts = artifact_manifest(plan)?; - let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; - let relay_config = prepare_relay_runtime_config( - plan, - runtime, - &invocation, - &request, - &fabric_home, - &mut artifacts, - )?; - + let cwd = settings + .cwd + .as_ref() + .map(|path| resolve_path(&plan.base_dir, path)) + .or_else(|| runtime.environment.workspace.clone()) + .unwrap_or_else(|| plan.base_dir.clone()); let mut command = Command::new(&python); command .arg("-m") .arg(&settings.module) .args(&settings.args) - .current_dir(&cwd) - .envs(&settings.env) - .envs(relay_env(&relay_config)) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + .current_dir(cwd) + .envs(&settings.env); + Ok(( + command, + format!("{} -m {}", python.to_string_lossy(), settings.module), + )) +} - let mut events = vec![event_with_metadata( - "runtime_start", - format!("started runtime {}", runtime.runtime_id), - BTreeMap::from([ - ( - "runtime_id".to_string(), - Value::String(runtime.runtime_id.clone()), - ), - ( - "environment_id".to_string(), - Value::String(runtime.environment.environment_id.clone()), - ), - ( - "environment_provider".to_string(), - Value::String(runtime.environment.provider.clone()), +fn exchange_lifecycle_message( + host: &mut LocalAdapterHost, + runtime_id: &str, + request: &AdapterLifecycleRequest, + timeout: Duration, +) -> Result { + let operation = request.operation(); + if let Some(status) = host.child.try_wait().map_err(|source| { + lifecycle_error( + operation, + runtime_id, + "host_io", + format!("failed to inspect persistent local adapter host: {source}"), + local_host_diagnostics(host), + ) + })? { + return Err(lifecycle_error( + operation, + runtime_id, + "host_crashed", + format!( + "persistent local adapter host exited before {} ({status})", + operation.as_str() ), - ]), - )]; - events.push(event_with_metadata( - "invocation_start", - format!("starting python adapter for {}", harness(plan)), - BTreeMap::from([ - ( - "runtime_id".to_string(), - Value::String(runtime.runtime_id.clone()), + local_host_diagnostics(host), + )); + } + let mut message = serde_json::to_string(request).map_err(FabricError::SerializeJson)?; + message.push('\n'); + if let Err(source) = host + .stdin + .write_all(message.as_bytes()) + .and_then(|()| host.stdin.flush()) + { + let code = match (source.kind(), host.child.try_wait()) { + // A closed lifecycle pipe is terminal even when the child exit + // status is not observable yet. + (ErrorKind::BrokenPipe, _) | (_, Ok(Some(_))) => "host_crashed", + _ => "host_io", + }; + return Err(lifecycle_error( + operation, + runtime_id, + code, + format!( + "failed to send {} to persistent local adapter host: {source}", + operation.as_str() ), - ( - "invocation_id".to_string(), - Value::String(invocation.invocation_id.clone()), + local_host_diagnostics(host), + )); + } + + let line = match host.responses.recv_timeout(timeout) { + Ok(line) => line, + Err(RecvTimeoutError::Timeout) => { + return Err(lifecycle_error( + operation, + runtime_id, + "host_timeout", + format!( + "persistent local adapter host did not complete {} within {} ms", + operation.as_str(), + timeout.as_millis() + ), + local_host_diagnostics(host), + )); + } + Err(RecvTimeoutError::Disconnected) => { + return Err(lifecycle_error( + operation, + runtime_id, + "host_crashed", + format!( + "persistent local adapter host exited while processing {}", + operation.as_str() + ), + local_host_diagnostics(host), + )); + } + } + .map_err(|message| { + lifecycle_error( + operation, + runtime_id, + "host_io", + message, + local_host_diagnostics(host), + ) + })?; + let response: AdapterLifecycleResponse = serde_json::from_str(&line).map_err(|source| { + lifecycle_error( + operation, + runtime_id, + "protocol_error", + format!("invalid lifecycle response: {source}"), + local_host_diagnostics(host), + ) + })?; + if response.operation != operation { + return Err(lifecycle_error( + operation, + runtime_id, + "protocol_error", + format!( + "expected `{}` response but host returned `{}`", + operation.as_str(), + response.operation.as_str() ), - ("module".to_string(), Value::String(settings.module.clone())), - ]), - )); - - let mut child = command - .spawn() - .map_err(|source| FabricError::ProcessRunner { - command: python.to_string_lossy().into_owned(), - source, - })?; + local_host_diagnostics(host), + )); + } + match response.outcome { + AdapterLifecycleOutcome::Succeeded { output } => Ok(output), + AdapterLifecycleOutcome::Failed { error } => { + if error.stage != operation.error_stage() { + return Err(lifecycle_error( + operation, + runtime_id, + "protocol_error", + format!( + "{} failure reported the wrong lifecycle stage `{:?}`", + operation.as_str(), + error.stage + ), + local_host_diagnostics(host), + )); + } + let mut diagnostics = local_host_diagnostics(host); + if !error.metadata.is_empty() + && let Ok(metadata) = serde_json::to_string(&error.metadata) + { + if !diagnostics.is_empty() { + diagnostics.push('\n'); + } + diagnostics.push_str("adapter metadata: "); + diagnostics.push_str(&metadata); + } + Err(lifecycle_error( + operation, + runtime_id, + error.code, + error.message, + diagnostics, + )) + } + } +} - let stdin_payload = fabric_adapter_payload( - plan, - runtime, - &invocation, - &request, - &artifacts, - relay_config.as_ref(), - )?; - if let Some(mut stdin) = child.stdin.take() { - write_child_stdin(&mut stdin, &stdin_payload, &python.to_string_lossy())?; +fn lifecycle_error( + operation: AdapterLifecycleOperation, + runtime_id: &str, + code: impl Into, + message: impl Into, + diagnostics: impl Into, +) -> FabricError { + FabricError::AdapterLifecycleOperation { + operation: operation.as_str(), + runtime_id: runtime_id.to_string(), + code: code.into(), + message: message.into(), + diagnostics: diagnostics.into(), } +} - let output = child - .wait_with_output() - .map_err(|source| FabricError::ProcessRunner { - command: python.to_string_lossy().into_owned(), - source, - })?; +fn local_host_diagnostics(host: &LocalAdapterHost) -> String { + let Ok(bytes) = std::fs::read(&host.stderr_path) else { + return String::new(); + }; + let start = bytes.len().saturating_sub(LOCAL_HOST_DIAGNOSTIC_LIMIT); + String::from_utf8_lossy(&bytes[start..]).trim().to_string() +} - let exit_code = output.status.code(); - let status = if output.status.success() { - RunStatus::Succeeded - } else { - RunStatus::Failed +fn take_local_host_stderr(host: &mut LocalAdapterHost) -> String { + let Ok(bytes) = std::fs::read(&host.stderr_path) else { + return String::new(); }; - events.push(event_with_metadata( - "invocation_end", - format!("python adapter completed with status {:?}", status), - BTreeMap::from([ - ( - "runtime_id".to_string(), - Value::String(runtime.runtime_id.clone()), - ), - ( - "invocation_id".to_string(), - Value::String(invocation.invocation_id.clone()), - ), - ]), - )); + let start = host.stderr_offset.min(bytes.len()); + host.stderr_offset = bytes.len(); + String::from_utf8_lossy(&bytes[start..]).into_owned() +} - let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); - let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); - if !stdout.is_empty() { - write_artifact( - &mut artifacts, - &fabric_home, - "stdout", - "log", - "stdout.txt", - &stdout, - "text/plain", - )?; - } - if !stderr.is_empty() { - write_artifact( - &mut artifacts, - &fabric_home, - "stderr", - "log", - "stderr.txt", - &stderr, - "text/plain", - )?; +fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { + let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; + loop { + if host.child.try_wait()?.is_some() { + return Ok(()); + } + if Instant::now() >= deadline { + break; + } + thread::sleep(Duration::from_millis(10)); } - collect_workspace_artifacts(&mut artifacts, &fabric_home, runtime, &mut events)?; - let mut metadata = BTreeMap::new(); - metadata.insert( - "adapter_runner".to_string(), - Value::String("python".to_string()), - ); - metadata.insert( - "python".to_string(), - Value::String(python.to_string_lossy().into_owned()), - ); - metadata.insert("module".to_string(), Value::String(settings.module)); - metadata.insert( - "args".to_string(), - Value::Array(settings.args.into_iter().map(Value::String).collect()), - ); - if let Some(exit_code) = exit_code { - metadata.insert("exit_code".to_string(), Value::from(exit_code)); + if let Err(source) = host.child.kill() + && host.child.try_wait()?.is_none() + { + return Err(source); } - metadata.insert( - "cwd".to_string(), - Value::String(cwd.to_string_lossy().into_owned()), - ); - metadata.insert( - "environment_provider".to_string(), - Value::String(runtime.environment.provider.clone()), - ); - - let error = if status == RunStatus::Failed { - Some(adapter_exit_error( - "python_adapter_exit_nonzero", - "python adapter exited with non-zero status", - &stderr, - &metadata, - )) - } else { - None - }; - - let parsed_output = parse_stdout_output(&stdout); - promote_relay_artifacts_to_manifest(&parsed_output, &mut artifacts); + host.child.wait()?; + Ok(()) +} - Ok(RunResult { - agent_name: plan.agent_name.clone(), - harness: harness(plan), - adapter_kind: adapter_kind(plan), - adapter_id: adapter_id(plan), - runtime_id: invocation.runtime_id, - invocation_id: invocation.invocation_id, - request_id: request.request_id, - status, - output: parsed_output, - error, - artifacts, - telemetry: telemetry_ref(plan, relay_config.as_ref()), - events, - metadata, - }) +fn remove_local_host_files(host: &LocalAdapterHost) -> std::io::Result<()> { + match std::fs::remove_dir_all(&host.runtime_dir) { + Ok(()) => Ok(()), + Err(source) if source.kind() == ErrorKind::NotFound => Ok(()), + Err(source) => Err(source), + } } fn adapter_id(plan: &RunPlan) -> Option { @@ -1199,17 +1583,6 @@ fn adapter_id(plan: &RunPlan) -> Option { .or_else(|| Some(plan.config.harness.adapter_id.clone())) } -fn write_child_stdin(stdin: &mut impl Write, payload: &str, command: &str) -> Result<()> { - match stdin.write_all(payload.as_bytes()) { - Ok(()) => Ok(()), - Err(source) if source.kind() == ErrorKind::BrokenPipe => Ok(()), - Err(source) => Err(FabricError::ProcessRunner { - command: command.to_string(), - source, - }), - } -} - fn harness(plan: &RunPlan) -> String { plan.adapter_descriptor .as_ref() @@ -1238,25 +1611,6 @@ fn optional_runtime_value(value: Option<&str>) -> String { value.unwrap_or("").to_string() } -fn adapter_exit_error( - code: &str, - default_message: &str, - stderr: &str, - metadata: &BTreeMap, -) -> ErrorInfo { - ErrorInfo { - stage: ErrorStage::Invoke, - code: code.to_string(), - message: if stderr.is_empty() { - default_message.to_string() - } else { - stderr.to_string() - }, - retryable: false, - metadata: metadata.clone(), - } -} - fn parse_python_settings(plan: &RunPlan) -> Result { let value = Value::Object(merged_adapter_settings(plan)); serde_json::from_value(value).map_err(|source| FabricError::InvalidPythonSettings { @@ -1293,23 +1647,27 @@ fn adapter_setting_root<'a>(plan: &'a RunPlan, key: &str) -> &'a Path { .unwrap_or(&plan.base_dir) } -fn fabric_adapter_payload( +fn adapter_lifecycle_start( plan: &RunPlan, runtime: &RuntimeHandle, invocation: &InvocationHandle, - request: &RunRequest, artifacts: &ArtifactManifest, relay_config: Option<&RelayRuntimeConfig>, -) -> Result { - serde_json::to_string_pretty(&adapter_invocation( - plan, - runtime, - invocation, - request, - artifacts, - relay_config, - )?) - .map_err(FabricError::SerializeJson) +) -> Result { + Ok(AdapterLifecycleStart { + agent_name: plan.agent_name.clone(), + base_dir: absolute_path(plan.base_dir.clone())?, + config: plan.config.clone(), + runtime_context: adapter_runtime_context( + plan, + runtime, + invocation, + artifacts, + relay_config, + ), + capability_plan: plan.capability_plan.clone(), + telemetry_plan: plan.telemetry_plan.clone(), + }) } fn adapter_invocation( @@ -1321,23 +1679,34 @@ fn adapter_invocation( relay_config: Option<&RelayRuntimeConfig>, ) -> Result { Ok(AdapterInvocation { - agent_name: plan.agent_name.clone(), - base_dir: absolute_path(plan.base_dir.clone())?, - config: plan.config.clone(), - runtime_context: RuntimeContext { - runtime_id: runtime.runtime_id.clone(), - invocation_id: invocation.invocation_id.clone(), - request_id: request.request_id.clone(), - environment: runtime.environment.clone(), - artifacts: artifacts.clone(), - telemetry: runtime_telemetry_context(plan, relay_config), - }, + runtime_context: adapter_runtime_context( + plan, + runtime, + invocation, + artifacts, + relay_config, + ), request: request.clone(), - capability_plan: plan.capability_plan.clone(), - telemetry_plan: plan.telemetry_plan.clone(), }) } +fn adapter_runtime_context( + plan: &RunPlan, + runtime: &RuntimeHandle, + invocation: &InvocationHandle, + artifacts: &ArtifactManifest, + relay_config: Option<&RelayRuntimeConfig>, +) -> RuntimeContext { + RuntimeContext { + runtime_id: runtime.runtime_id.clone(), + invocation_id: invocation.invocation_id.clone(), + request_id: invocation.request_id.clone(), + environment: runtime.environment.clone(), + artifacts: artifacts.clone(), + telemetry: runtime_telemetry_context(plan, relay_config), + } +} + fn runtime_telemetry_context( plan: &RunPlan, relay_config: Option<&RelayRuntimeConfig>, @@ -1537,10 +1906,6 @@ fn absolute_path(path: PathBuf) -> Result { Ok(cwd.join(path)) } -fn parse_stdout_output(stdout: &str) -> Value { - serde_json::from_str(stdout).unwrap_or_else(|_| Value::String(stdout.to_string())) -} - #[derive(Debug, Default, Deserialize)] struct RelayArtifactOutput { #[serde(default)] @@ -1642,14 +2007,6 @@ fn process_command_args(plan: &RunPlan, settings: &ProcessAdapterSettings) -> Ve args } -fn value_to_stdin(value: &Value) -> Result { - match value { - Value::Null => Ok(String::new()), - Value::String(text) => Ok(text.clone()), - value => serde_json::to_string_pretty(value).map_err(FabricError::SerializeJson), - } -} - fn artifact_manifest(plan: &RunPlan) -> Result { let root = plan .config @@ -1859,8 +2216,6 @@ fn untracked_workspace_patch(workspace: &Path) -> Result { fn prepare_relay_runtime_config( plan: &RunPlan, runtime: &RuntimeHandle, - invocation: &InvocationHandle, - request: &RunRequest, artifact_directory: &Path, artifacts: &mut ArtifactManifest, ) -> Result> { @@ -1873,6 +2228,13 @@ fn prepare_relay_runtime_config( if artifacts.root.is_none() { return Ok(None); } + let fabric = serde_json::json!({ + "agent_name": plan.agent_name.clone(), + "harness": harness(plan), + "adapter_id": adapter_id(plan), + "runtime_id": runtime.runtime_id.clone(), + "adapter_outputs": telemetry.adapter_outputs.clone(), + }); let relay_config = serde_json::json!({ "schema_version": "fabric.relay/v1alpha1", "relay": { @@ -1887,15 +2249,7 @@ fn prepare_relay_runtime_config( .clone() .unwrap_or_else(|| Value::Object(Default::default())), }, - "fabric": { - "agent_name": plan.agent_name.clone(), - "harness": harness(plan), - "adapter_id": adapter_id(plan), - "runtime_id": runtime.runtime_id.clone(), - "invocation_id": invocation.invocation_id.clone(), - "request_id": request.request_id.clone(), - "adapter_outputs": telemetry.adapter_outputs.clone(), - } + "fabric": fabric, }); let contents = serde_json::to_string_pretty(&relay_config).map_err(FabricError::SerializeJson)?; @@ -1995,10 +2349,9 @@ fn event_with_metadata( } fn new_id(prefix: &str) -> String { - // The atomic counter only differentiates ids within a single process; a - // process-backed runner spawns a fresh `fabric` process per call, resetting it - // to 1. Include the process id (distinct across concurrently running - // processes) so ids stay unique when two runs land in the same millisecond. + // The atomic counter only differentiates ids within one Fabric process. + // Include the process id so independently running Fabric processes cannot + // collide when they generate ids in the same millisecond. let counter = NEXT_ID.fetch_add(1, Ordering::Relaxed); format!("{prefix}-{}-{}-{counter}", now_millis(), std::process::id()) } @@ -2009,3 +2362,544 @@ fn now_millis() -> u128 { .map(|duration| duration.as_millis()) .unwrap_or_default() } + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + use crate::config::{ResolveContext, resolve_run_plan_from_config}; + + fn local_host_plan(mode: &str) -> (PathBuf, RunPlan) { + local_host_plan_with_relay(mode, false) + } + + fn local_host_plan_with_relay(mode: &str, relay: bool) -> (PathBuf, RunPlan) { + let root = std::env::temp_dir().join(new_id("fabric-local-host-test")); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(root.join("adapters/local-host")).expect("create adapters dir"); + fs::write( + root.join("adapters/local-host/fabric-adapter.json"), + r#"{ + "contract_version": "fabric.adapter/v1alpha1", + "adapter_id": "acme.fabric.local-host", + "harness": "local-host-test", + "adapter_kind": "python", + "runner": {"module": "fake_host"}, + "telemetry": { + "providers": { + "relay": {"outputs": ["atif"]} + } + } +}"#, + ) + .expect("write adapter descriptor"); + fs::write( + root.join("fake_host.py"), + r#"import json +import os +import sys +import time + +MODE = os.environ.get("FABRIC_FAKE_HOST_MODE", "success") +invocations = 0 + +def response(operation, *, output=None, error=None): + outcome = ( + {"status": "succeeded", "output": output} + if error is None + else {"status": "failed", "error": error} + ) + print(json.dumps({ + "operation": operation, + "outcome": outcome, + }), flush=True) + +def failure(stage, code, message): + return { + "stage": stage, + "code": code, + "message": message, + "retryable": False, + } + +for line in sys.stdin: + message = json.loads(line) + operation = message["operation"] + if operation == "start": + if MODE == "start_failure": + print("start diagnostic", file=sys.stderr, flush=True) + response("start", error=failure("start", "fake_start", "start rejected")) + sys.exit(16) + response("start") + if MODE == "crash_after_start": + os.close(0) + print("host crashed intentionally", file=sys.stderr, flush=True) + time.sleep(1) + sys.exit(17) + elif operation == "invoke": + invocations += 1 + if MODE == "invoke_stderr": + print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) + if MODE == "invoke_timeout": + print("invoke accepted without response", file=sys.stderr, flush=True) + continue + if MODE == "invoke_failure": + response("invoke", error=failure("invoke", "fake_invoke", "invoke rejected")) + continue + invocation = message["payload"] + if set(invocation) != {"runtime_context", "request"}: + response("invoke", error=failure( + "invoke", "fake_invoke_shape", "invoke payload contains runtime config" + )) + continue + output = { + "host_pid": os.getpid(), + "invocation_count": invocations, + "input": invocation["request"]["input"], + "runtime_id": invocation["runtime_context"]["runtime_id"], + "invocation_id": invocation["runtime_context"]["invocation_id"], + "request_id": invocation["runtime_context"]["request_id"], + } + if MODE == "adapter_reported_failure": + output.update({ + "failed": True, + "error": { + "code": "fake_adapter_failure", + "message": "adapter rejected the invocation", + "retryable": True, + "metadata": {"source": "fake-host"}, + }, + }) + response("invoke", output=output) + elif operation == "stop": + if MODE == "stop_failure": + response("stop", error=failure("stop", "fake_stop", "stop rejected")) + sys.exit(18) + response("stop") + break +"#, + ) + .expect("write fake host"); + + let mut config_value = serde_json::json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "local-host-test-agent"}, + "harness": { + "adapter_id": "acme.fabric.local-host", + "resolution": "preinstalled", + "settings": { + "python": "python3", + "cwd": ".", + "env": {"FABRIC_FAKE_HOST_MODE": mode}, + }, + }, + "models": { + "default": { + "provider": "test", + "model": "test-model", + }, + }, + "runtime": { + "input_schema": "text", + "output_schema": "text", + "artifacts": "./artifacts", + }, + }); + if relay { + config_value["telemetry"] = serde_json::json!({ + "providers": {"relay": {}}, + }); + config_value["relay"] = serde_json::json!({ + "observability": {"atif": {"enabled": true}}, + }); + } + let config: FabricConfig = serde_json::from_value(config_value).expect("typed config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) + .expect("resolve local-host plan"); + (root, plan) + } + + fn stopped_agents() -> Vec { + TEST_STOPPED_AGENTS.lock().expect("stop tracker").clone() + } + + #[test] + fn local_host_reuses_one_process_and_stops_idempotently() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let first = + invoke_runtime(&plan, &runtime, RunRequest::text("first")).expect("first invocation"); + let second = + invoke_runtime(&plan, &runtime, RunRequest::text("second")).expect("second invocation"); + + assert_eq!(first.output["host_pid"], second.output["host_pid"]); + assert_eq!(first.output["invocation_count"], serde_json::json!(1)); + assert_eq!(second.output["invocation_count"], serde_json::json!(2)); + assert_eq!(first.output["input"], serde_json::json!("first")); + assert_eq!(second.output["input"], serde_json::json!("second")); + assert_eq!(first.metadata["host_pid"], second.metadata["host_pid"]); + assert_eq!( + first.metadata["adapter_runner"], + serde_json::json!("persistent_local_host") + ); + let stdout = first + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "stdout") + .expect("stdout artifact"); + let captured: Value = + serde_json::from_str(&fs::read_to_string(&stdout.path).expect("read stdout artifact")) + .expect("parse stdout artifact"); + assert_eq!(captured, first.output); + assert!( + first + .artifacts + .artifacts + .iter() + .all(|artifact| artifact.name != "stderr") + ); + + let first_stop = stop_runtime(&plan, &runtime).expect("first stop"); + let second_stop = stop_runtime(&plan, &runtime).expect("idempotent stop"); + assert_eq!(first_stop[0].metadata["already_stopped"], false); + assert_eq!(second_stop[0].metadata["already_stopped"], true); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn process_adapter_uses_the_same_persistent_local_host_protocol() { + let (root, mut plan) = local_host_plan("success"); + let descriptor = plan + .adapter_descriptor + .as_mut() + .expect("resolved adapter descriptor"); + descriptor.descriptor.adapter_kind = AdapterKind::Process; + descriptor.descriptor.runner = serde_json::from_value(serde_json::json!({ + "command": "python3", + "script": "../../fake_host.py", + })) + .expect("process runner settings"); + + let runtime = start_runtime(&plan).expect("start process host"); + let first = + invoke_runtime(&plan, &runtime, RunRequest::text("first")).expect("first invocation"); + let second = + invoke_runtime(&plan, &runtime, RunRequest::text("second")).expect("second invocation"); + stop_runtime(&plan, &runtime).expect("stop process host"); + + assert_eq!(first.output["host_pid"], second.output["host_pid"]); + assert_eq!(first.output["invocation_count"], 1); + assert_eq!(second.output["invocation_count"], 2); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_relay_config_is_runtime_scoped() { + let (root, plan) = local_host_plan_with_relay("success", true); + let runtime = start_runtime(&plan).expect("start local host"); + let host = local_hosts() + .get(&runtime.runtime_id) + .cloned() + .expect("active local host"); + let relay_config_path = host + .lock() + .expect("local host") + .relay_config + .as_ref() + .expect("Relay config") + .path + .clone(); + let relay_config: Value = serde_json::from_str( + &fs::read_to_string(relay_config_path).expect("read Relay config"), + ) + .expect("parse Relay config"); + + assert_eq!( + relay_config["fabric"]["runtime_id"], + serde_json::json!(runtime.runtime_id) + ); + assert!(relay_config["fabric"].get("invocation_id").is_none()); + assert!(relay_config["fabric"].get("request_id").is_none()); + + let result = + invoke_runtime(&plan, &runtime, RunRequest::text("first")).expect("invoke local host"); + assert_eq!(result.output["runtime_id"], result.runtime_id); + assert_eq!(result.output["invocation_id"], result.invocation_id); + assert_eq!(result.output["request_id"], result.request_id); + + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_start_failure_preserves_stage_and_diagnostics() { + let (root, plan) = local_host_plan("start_failure"); + + let error = start_runtime(&plan).expect_err("start must fail"); + let message = error.to_string(); + assert!(message.contains("lifecycle start"), "{message}"); + assert!(message.contains("fake_start"), "{message}"); + assert!(message.contains("start diagnostic"), "{message}"); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_invoke_timeout_evicts_and_terminates_host() { + let (root, plan) = local_host_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + let host = local_hosts() + .get(&runtime.runtime_id) + .cloned() + .expect("active local host"); + let runtime_dir = host.lock().expect("local host").runtime_dir.clone(); + + let error = run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("unresponsive host must time out"); + + assert!(matches!( + &error, + FabricError::AdapterLifecycleOperation { + code, + diagnostics, + .. + } if code == "host_timeout" && diagnostics.contains("invoke accepted without response") + )); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!( + host.lock() + .expect("local host") + .child + .try_wait() + .expect("inspect timed-out host") + .is_some(), + "timed-out host process must be terminated" + ); + assert!(!runtime_dir.exists()); + + let retry = invoke_runtime(&plan, &runtime, RunRequest::text("retry")) + .expect_err("timed-out runtime must remain unavailable"); + assert!(matches!( + retry, + FabricError::AdapterLifecycleOperation { code, .. } if code == "host_unavailable" + )); + let stopped = stop_runtime(&plan, &runtime).expect("stop after timeout is idempotent"); + assert_eq!(stopped[0].metadata["already_stopped"], true); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_timeout_prevents_waiting_invocation_from_reusing_host() { + let (root, plan) = local_host_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + let host = local_hosts() + .get(&runtime.runtime_id) + .cloned() + .expect("active local host"); + let stderr_path = host.lock().expect("local host").stderr_path.clone(); + + let first_plan = plan.clone(); + let first_runtime = runtime.clone(); + let first = thread::spawn(move || { + run_local_host_adapter_with_timeout( + &first_plan, + &first_runtime, + RunRequest::text("first"), + Duration::from_millis(500), + ) + }); + let accepted_deadline = Instant::now() + Duration::from_secs(2); + while !fs::read_to_string(&stderr_path) + .expect("read host stderr") + .contains("invoke accepted without response") + { + assert!( + Instant::now() < accepted_deadline, + "first invocation was not accepted" + ); + thread::sleep(Duration::from_millis(10)); + } + + let second_plan = plan.clone(); + let second_runtime = runtime.clone(); + let second = thread::spawn(move || { + run_local_host_adapter_with_timeout( + &second_plan, + &second_runtime, + RunRequest::text("second"), + Duration::from_millis(100), + ) + }); + // The registry, this test, and the first invocation own three host + // references. A fourth proves that the second invocation passed the + // registry lookup and is waiting for the host mutex. + let waiting_deadline = Instant::now() + Duration::from_secs(1); + while Arc::strong_count(&host) < 4 { + assert!( + Instant::now() < waiting_deadline, + "second invocation did not acquire the host reference" + ); + thread::sleep(Duration::from_millis(10)); + } + + let first_error = first + .join() + .expect("first invocation thread") + .expect_err("first invocation must time out"); + let second_error = second + .join() + .expect("second invocation thread") + .expect_err("waiting invocation must not reuse the timed-out host"); + + assert!(matches!( + first_error, + FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" + )); + assert!(matches!( + second_error, + FabricError::AdapterLifecycleOperation { code, .. } if code == "host_crashed" + )); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + let stopped = stop_runtime(&plan, &runtime).expect("stop after timeout is idempotent"); + assert_eq!(stopped[0].metadata["already_stopped"], true); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_captures_each_stderr_delta_once() { + let (root, plan) = local_host_plan("invoke_stderr"); + let runtime = start_runtime(&plan).expect("start local host"); + + let first = + invoke_runtime(&plan, &runtime, RunRequest::text("first")).expect("first invocation"); + let second = + invoke_runtime(&plan, &runtime, RunRequest::text("second")).expect("second invocation"); + let read_stderr = |result: &RunResult| { + let artifact = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "stderr") + .expect("stderr artifact"); + fs::read_to_string(&artifact.path).expect("read stderr artifact") + }; + + assert_eq!(read_stderr(&first), "diagnostic-1\n"); + assert_eq!(read_stderr(&second), "diagnostic-2\n"); + stop_runtime(&plan, &runtime).expect("stop local host"); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn run_plan_stops_local_host_after_invoke_failure() { + let (root, mut plan) = local_host_plan("invoke_failure"); + let agent_name = new_id("local-host-invoke-error-agent"); + plan.agent_name = agent_name.clone(); + plan.config.metadata.name = agent_name.clone(); + + let error = run_plan(&plan, RunRequest::text("fail")).expect_err("invoke must fail"); + assert!(error.to_string().contains("fake_invoke"), "{error}"); + assert!( + stopped_agents().contains(&agent_name), + "run_plan must stop the local host after invocation failure" + ); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_preserves_normalized_adapter_failure() { + let (root, plan) = local_host_plan("adapter_reported_failure"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("normalized result"); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.output["failed"], true); + assert_eq!( + result.error, + Some(ErrorInfo { + stage: ErrorStage::Invoke, + code: "fake_adapter_failure".to_string(), + message: "adapter rejected the invocation".to_string(), + retryable: true, + metadata: BTreeMap::from([("source".to_string(), serde_json::json!("fake-host"))]), + }) + ); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_crash_is_terminal_for_runtime() { + let (root, plan) = local_host_plan("crash_after_start"); + let runtime = start_runtime(&plan).expect("start local host"); + let host = local_hosts() + .get(&runtime.runtime_id) + .cloned() + .expect("active local host"); + let stderr_path = host.lock().expect("local host").stderr_path.clone(); + let closed_deadline = Instant::now() + Duration::from_secs(2); + while !fs::read_to_string(&stderr_path) + .expect("read host stderr") + .contains("host crashed intentionally") + { + assert!( + Instant::now() < closed_deadline, + "host did not close its lifecycle pipe" + ); + thread::sleep(Duration::from_millis(10)); + } + assert!( + host.lock() + .expect("local host") + .child + .try_wait() + .expect("inspect crashed host") + .is_none(), + "host must still be alive while its lifecycle pipe is closed" + ); + + let first = invoke_runtime(&plan, &runtime, RunRequest::text("first")) + .expect_err("crashed host must reject invocation"); + let second = invoke_runtime(&plan, &runtime, RunRequest::text("second")) + .expect_err("dead runtime must remain unusable"); + assert!(first.to_string().contains("host_crashed"), "{first}"); + assert!( + first.to_string().contains("failed to send invoke"), + "{first}" + ); + assert!(second.to_string().contains("host_crashed"), "{second}"); + + let stopped = stop_runtime(&plan, &runtime).expect("crashed host cleanup"); + assert_eq!(stopped[0].metadata["host_crashed"], true); + stop_runtime(&plan, &runtime).expect("cleanup remains idempotent"); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_stop_failure_still_releases_host() { + let (root, plan) = local_host_plan("stop_failure"); + let runtime = start_runtime(&plan).expect("start local host"); + + let error = stop_runtime(&plan, &runtime).expect_err("stop must fail"); + assert!(error.to_string().contains("fake_stop"), "{error}"); + let retry = stop_runtime(&plan, &runtime).expect("cleanup retry is idempotent"); + assert_eq!(retry[0].metadata["already_stopped"], true); + + let _ = fs::remove_dir_all(root); + } +} diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index f248466f..3ad59955 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -25,7 +25,7 @@ pub enum SchemaName { AdapterDescriptor, /// Resolved run plan schema. RunPlan, - /// Adapter-facing invocation payload schema. + /// Initialized-runtime invocation payload schema. AdapterInvocation, /// Runtime context schema. RuntimeContext, diff --git a/docs/about-nemo-fabric/overview.mdx b/docs/about-nemo-fabric/overview.mdx index 324dca0e..cb10aaa4 100644 --- a/docs/about-nemo-fabric/overview.mdx +++ b/docs/about-nemo-fabric/overview.mdx @@ -28,7 +28,7 @@ into the caller. SDK instead of embedding harness launch logic in every consumer. - Resolve configs, inspect capabilities, run one-shot jobs, and hold + Resolve configs, inspect capabilities, run single-invocation jobs, and hold multi-turn runtimes with typed requests, plans, handles, and results. diff --git a/docs/integrations/claude.mdx b/docs/integrations/claude.mdx index cabf0957..deb7bca1 100644 --- a/docs/integrations/claude.mdx +++ b/docs/integrations/claude.mdx @@ -85,11 +85,13 @@ variables take precedence over federation even when their value is empty. ## Use Authentication with Relay -NeMo Relay does not authenticate Claude. A Relay-enabled NeMo Fabric invocation -starts the gateway as a supervised sidecar and sets `ANTHROPIC_BASE_URL` for the -Claude runtime. Claude still resolves its credential through the selected mode, -and NeMo Fabric does not write authentication values to Relay configuration or -artifacts. +NeMo Relay does not authenticate Claude. A Relay-enabled NeMo Fabric runtime +starts one gateway as a supervised sidecar, sets `ANTHROPIC_BASE_URL` for the +Claude runtime, and reuses the gateway across ordered invocations. +`Fabric.run(...)` starts the same runtime, invokes it once, and stops it, so the +gateway is scoped to that single invocation. Claude still resolves its credential +through the selected mode, and NeMo Fabric does not write authentication values +to Relay configuration or artifacts. NeMo Fabric supports the external NeMo Relay CLI from `0.6.0` up to, but not including, `0.7.0`. The Python package named `nemo-relay` does not install this diff --git a/docs/integrations/codex.mdx b/docs/integrations/codex.mdx index 158e75b9..008de52b 100644 --- a/docs/integrations/codex.mdx +++ b/docs/integrations/codex.mdx @@ -31,10 +31,16 @@ config.add_mcp_server( ``` The adapter maps stdio, HTTP, and streamable HTTP servers into the Codex -thread's request-scoped `mcp_servers` configuration. It resolves each skill -directory, verifies that it contains `SKILL.md`, and registers it as a -process-scoped Codex skill root. Codex uses its normal skill discovery behavior. -Invalid skill directories and MCP transports fail before the SDK runtime starts. +thread's `mcp_servers` configuration. It resolves each skill directory, verifies +that it contains `SKILL.md`, and registers it as a process-scoped Codex skill +root. Codex uses its normal skill discovery behavior. Invalid skill directories +and MCP transports fail before the SDK runtime starts. + +For `Fabric.start_runtime(...)`, the model provider, MCP configuration, skill +roots, and `harness.settings.config_overrides` are fixed when the runtime starts +and cannot vary between `Runtime.invoke(...)` calls. Start a new runtime to +change them. `Fabric.run(...)` starts the same runtime, invokes it once, and +stops it, so the same settings are scoped to that single invocation. The Codex adapter does not declare `tools.blocked` support. Codex can filter individual MCP server tools, but the pinned runtime does not provide one deny @@ -103,24 +109,27 @@ storage. Do not commit or copy it into NeMo Fabric configuration or artifacts. ## Use Authentication with Relay -NeMo Relay does not replace OpenAI authentication. A Relay-enabled invocation -starts the Relay gateway as a supervised sidecar and directs the Codex SDK's -built-in OpenAI provider through that gateway. The SDK still obtains credentials -from the selected Codex authentication mode. +NeMo Relay does not replace OpenAI authentication. A Relay-enabled NeMo Fabric +runtime starts one gateway as a supervised sidecar and directs the Codex SDK's +built-in OpenAI provider through that gateway. The runtime reuses the gateway +and SDK client across ordered invocations. `Fabric.run(...)` starts the same +runtime, invokes it once, and stops it, so the gateway is scoped to that single +invocation. The SDK still obtains credentials from the selected Codex +authentication mode. Relay-enabled Codex runs require `models.default.provider` to be `openai`. The custom `nvidia` provider uses its configured NVIDIA Responses endpoint and does not support the built-in provider redirect that Relay requires. Use the `nvidia` provider without Relay. -NeMo Fabric supplies Relay configuration to the SDK for only the current request. It -does not copy the Codex credential store into Relay configuration or persist -credentials in Relay artifacts. +NeMo Fabric supplies runtime-scoped Relay configuration to the SDK. It does not +copy the Codex credential store into Relay configuration or persist credentials +in Relay artifacts. NeMo Fabric supports the external NeMo Relay CLI from `0.6.0` up to, but not including, `0.7.0`. The Python package named `nemo-relay` is a separate library dependency and does not install the CLI. NeMo Fabric owns sidecar supervision and -request-scoped SDK configuration. Relay owns gateway transport behavior, +runtime-scoped SDK configuration. Relay owns gateway transport behavior, including decoding `Content-Encoding` before it constructs managed LLM events. There is no NeMo Fabric compression setting. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.client.md b/docs/reference/api/python-library-reference/nemo_fabric.client.md index 59d91106..f8fec00e 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.client.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.client.md @@ -21,7 +21,7 @@ Every lifecycle method accepts a complete, typed ``FabricConfig`` plus an option ``Fabric`` uses the native Rust extension. SDK calls raise ``FabricNativeUnavailableError`` when the native extension is not installed. -See the Getting Started overview for runnable one-shot, typed-config, and multi-turn examples. +See the Getting Started overview for runnable single-invocation, typed-config, and multi-turn examples. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-adapterkind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-adapterkind.mdx index c786a243..bdb4d02b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-adapterkind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-adapterkind.mdx @@ -26,7 +26,7 @@ Adapter implementation kind.
-Launch and supervise a CLI process. +Launch and supervise a persistent adapter process. ### `Http` @@ -38,7 +38,7 @@ Connect to a service or HTTP-backed harness.
-Call a Python SDK/plugin adapter. +Launch and supervise a persistent Python adapter host. ### `NativePlugin` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 4ff429c4..e0854e2f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedToolsPolicy {\n        harness: String,\n        reason: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    UnsupportedToolsPolicy {\n        harness: String,\n        reason: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by Fabric config loading and validation. @@ -145,6 +145,34 @@ Harness type. Adapter kind. +### `AdapterLifecycleOperation` + +
+ +A persistent local-host lifecycle operation failed. + +#### Fields + +### `operation: &'static str` + +Lifecycle operation that failed. + +### `runtime_id: String` + +Runtime whose host failed. + +### `code: String` + +Stable failure code. + +### `message: String` + +Human-readable failure message. + +### `diagnostics: String` + +Bounded adapter-host diagnostics. + ### `UnsupportedToolsPolicy`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 05f895e7..aa347d62 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -13,7 +13,7 @@ Runtime invocation helpers. ## Structs -- [AdapterInvocation](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation): Adapter-facing invocation payload. +- [AdapterInvocation](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation): One invocation against an initialized adapter runtime. - [ArtifactManifest](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactmanifest): Manifest of run artifacts. - [ArtifactRef](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactref): Reference to one artifact. - [EnvironmentHandle](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle): Resolved execution environment context. @@ -22,7 +22,7 @@ Runtime invocation helpers. - [InvocationHandle](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle): One request sent to a runtime. - [RunRequest](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest): A request passed to a Fabric-managed harness runtime. - [RunResult](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult): Result from a Fabric-managed harness invocation. -- [RuntimeContext](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext): Per-run/per-invocation context passed to harness adapters. +- [RuntimeContext](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext): Context generated for one invocation of a started runtime. - [RuntimeHandle](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle): Active or resumable harness runtime. - [RuntimeTelemetryContext](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext): Runtime telemetry config passed to adapters. - [TelemetryRef](/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref): Reference to telemetry emitted by Relay or another configured telemetry path. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation.mdx index 24052b9d..866c95b9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation.mdx @@ -1,7 +1,7 @@ --- title: "Struct Adapter Invocation" sidebar-title: "AdapterInvocation" -description: "Adapter-facing invocation payload." +description: "One invocation against an initialized adapter runtime." position: 1 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -9,39 +9,19 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub runtime_context: RuntimeContext,\n    pub request: RunRequest,\n    pub capability_plan: CapabilityPlan,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
+
RuntimeContext,\n    pub request: RunRequest,\n}"}} />
-Adapter-facing invocation payload. +One invocation against an initialized adapter runtime. ## Fields -### `agent_name: String` - -Stable agent name. - -### `base_dir: PathBuf` - -Absolute base directory used to resolve relative Fabric paths. - -### `config: FabricConfig` - -Complete typed Fabric config. - ### `runtime_context: RuntimeContext` -Per-runtime/per-invocation execution context. +Invocation context generated by Fabric. ### `request: RunRequest` -Per-invocation request. - -### `capability_plan: CapabilityPlan` - -Derived capability routing plan for the selected adapter. - -### `telemetry_plan: Option` - -Derived telemetry plan for the selected adapter. +Typed caller request for this invocation. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 3b27d359..c5349ecd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -1,7 +1,7 @@ --- title: "Struct Runtime Context" sidebar-title: "RuntimeContext" -description: "Per-run/per-invocation context passed to harness adapters." +description: "Context generated for one invocation of a started runtime." position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -11,7 +11,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. 
String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub environment: EnvironmentHandle,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<RuntimeTelemetryContext>,\n}"}} />
-Per-run/per-invocation context passed to harness adapters. +Context generated for one invocation of a started runtime. ## Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index b7cc4cbf..5f8f36a7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -53,7 +53,7 @@ Resolved run plan schema.
-Adapter-facing invocation payload schema. +Initialized-runtime invocation payload schema. ### `RuntimeContext` diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 8c7bfaa0..724eff3b 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -101,10 +101,10 @@ Most application code works with four objects: | `RunResult` | The normalized outcome of one invocation. | Read its status, output, error, events, artifacts, and correlation IDs. | `Fabric` is a lightweight, reusable SDK facade. It resolves configuration, -creates plans and runtimes, and runs one-shot requests, but it does not -represent a started execution and does not require cleanup. A `Runtime` owns -stateful execution and shutdown, so it is the object used as an async context -manager. +creates plans and runtimes, and provides a single-invocation convenience API, +but it does not represent a started execution and does not require cleanup. A +`Runtime` owns stateful execution and shutdown, so it is the object used as an +async context manager. `RuntimeHandle` and `InvocationHandle` carry lifecycle identity across the native boundary. Most Python callers use their `runtime_id` and @@ -115,6 +115,42 @@ A runtime is a logical execution boundary, not necessarily an operating-system process. An adapter may use an in-process SDK, a process, or shared service infrastructure while preserving isolated state for each NeMo Fabric runtime. +Runtime hosting is selected by the adapter kind, not by a public `FabricConfig` +setting. Every local Process or Python adapter implements the persistent +local-host wire protocol. Fabric starts one host during +`start_runtime(...)`, reuses its adapter-owned native resources across ordered +invocations, and attempts to release them during `stop()`. The lifecycle start +operation carries the resolved configuration and capability plan. Each +subsequent `AdapterInvocation` carries only `runtime_context` and `request`. + +### Bundled Adapter Capability Matrix + +The following matrix summarizes the current bundled adapter descriptors. The +descriptor selected in `RunPlan` remains authoritative. Telemetry output names +use the descriptor contract values. + +| Adapter ID | Models | Tools / Blocked Tools | MCP | Skills | Subagents | Telemetry | Persistent Local Host | Remote Service | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `nvidia.fabric.claude` | Anthropic provider and Claude models | `allowed_tools` adapter setting / normalized `tools.blocked` | Normalized | Normalized | Not exposed | Relay: `atif`, `otel`, and `openinference` through hooks and gateway | Yes: connected `ClaudeSDKClient`, session, and optional Relay gateway | Not implemented | +| `nvidia.fabric.codex` | Built-in OpenAI and custom NVIDIA Responses providers with Codex-compatible models | Not normalized | Normalized: stdio, HTTP, and streamable HTTP | Normalized: `SKILL.md` directories | Not exposed | Relay: `atif`, `otel`, and `openinference` through hooks and gateway; native `otel` | Yes: `AsyncCodex` app-server client, thread, and optional Relay gateway | Not implemented | +| `nvidia.fabric.langchain.deepagents` | NVIDIA, OpenAI, OpenAI-compatible, and other LangChain providers | Built-ins and MCP / normalized middleware block list | Normalized | Normalized | Constrained: declarative local subagents inherit parent capabilities | Relay SDK: `atif`, `otel`, and `openinference`; native `otel` and `openinference` | Yes: compiled graph and async LangGraph checkpointer | Not implemented | +| `nvidia.fabric.hermes` | Normalized provider, model, and base URL | Toolsets / normalized disabled toolsets | Normalized | Normalized | Not exposed | Relay plugin: `atif`, `otel`, and `openinference` | Yes: `AIAgent`, `SessionDB`, and Relay plugin context | Not implemented | + +"Normalized" means the adapter accepts the corresponding `FabricConfig` +field and maps it to the harness. "Not normalized" does not mean that the +underlying harness lacks the feature; it means that Fabric does not expose a +portable configuration surface for it. Fabric currently normalizes +`tools.blocked`, not a portable tool definition catalog. Deep Agents supports +only JSON-shaped, in-process subagent definitions through +`harness.settings.deepagents.subagents`; independently configured or remote +subagent capabilities are not exposed. + +Third-party local adapters must implement the persistent local-host contract. +NeMo Fabric does not currently define a remote-service adapter contract. A +crashed persistent host is terminal for that runtime. The same applies when the +host exceeds the protocol response timeout. NeMo Fabric does not silently +respawn the host or replay a request. + Harness-native threads, sessions, and conversations remain adapter-owned state associated with the NeMo Fabric runtime. They are not additional NeMo Fabric lifecycle objects. @@ -211,7 +247,7 @@ multiple independent runtimes. | `Runtime.invoke(...)` | Yes | You need one turn on an existing runtime. | A runtime permits one active invocation at a time. | | `Runtime.stop()` | Yes | You need to stop or detach from the runtime. | Called automatically when using `async with`. | -## One-Shot Runs +## Single-Invocation Runs Use `run(...)` when the application has one input and does not need to preserve runtime state after the result is collected. @@ -266,9 +302,10 @@ async with await fabric.start_runtime( print(first.status, second.status) ``` -The adapter reuses its native state between calls. For example, the Codex -adapter uses one Codex thread for the runtime and maps each invocation to one -turn. That thread ID remains adapter-internal. +The adapter reuses its native state between calls. Codex maps the calls to turns +on one live thread, Deep Agents invokes one compiled graph and checkpointer, +Hermes reuses one agent and session database, and Claude keeps one connected SDK +client. Harness-native identifiers remain adapter-internal. ## Application-Owned Parallelism @@ -420,12 +457,12 @@ All public SDK errors inherit from `FabricError`. | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -Consumers own job-level retries and rollout-level failure policy. One-shot runs -attempt to stop the runtime before returning. A `Runtime` used with `async with` -also attempts cleanup after an invocation error; if cleanup fails, that failure -is attached to the original exception rather than replacing it. NeMo Fabric records -structured error metadata when possible and returns enough detail for the -consumer to decide what to do next. +Consumers own job-level retries and rollout-level failure policy. +Single-invocation runs attempt to stop the runtime before returning. A `Runtime` +used with `async with` also attempts cleanup after an invocation error; if cleanup +fails, that failure is attached to the original exception rather than replacing +it. NeMo Fabric records structured error metadata when possible and returns enough +detail for the consumer to decide what to do next. ## SDK Contract Boundaries diff --git a/examples/harbor/calculator/task/environment/fabric/adapters/scripted/fabric-adapter.json b/examples/harbor/calculator/task/environment/fabric/adapters/scripted/fabric-adapter.json index e456718d..f41862d3 100644 --- a/examples/harbor/calculator/task/environment/fabric/adapters/scripted/fabric-adapter.json +++ b/examples/harbor/calculator/task/environment/fabric/adapters/scripted/fabric-adapter.json @@ -5,7 +5,6 @@ "adapter_kind": "process", "runner": { "command": "python3", - "script": "run.py", - "stdin_payload": "fabric_request" + "script": "run.py" } } diff --git a/examples/harbor/calculator/task/environment/fabric/adapters/scripted/run.py b/examples/harbor/calculator/task/environment/fabric/adapters/scripted/run.py index 7c6bc74c..82102e78 100644 --- a/examples/harbor/calculator/task/environment/fabric/adapters/scripted/run.py +++ b/examples/harbor/calculator/task/environment/fabric/adapters/scripted/run.py @@ -8,22 +8,56 @@ import json import sys from pathlib import Path +from typing import Any -payload = json.load(sys.stdin) -calculator = Path("/app/calculator.py") -calculator.write_text( - "def add(a, b):\n" - " return a + b\n\n\n" - "def multiply(a, b):\n" - " return a * b\n", - encoding="utf-8", -) -print( - json.dumps( - { - "harness": "scripted", - "response": "Fixed multiply(a, b) in /app/calculator.py", - "request_id": payload["request"]["request_id"], - } + +def response(operation: str, *, output: Any = None) -> None: + print( + json.dumps( + { + "operation": operation, + "outcome": {"status": "succeeded", "output": output}, + } + ), + flush=True, + ) + + +def invoke(payload: dict[str, Any]) -> dict[str, Any]: + calculator = Path("/app/calculator.py") + calculator.write_text( + "def add(a, b):\n" + " return a + b\n\n\n" + "def multiply(a, b):\n" + " return a * b\n", + encoding="utf-8", ) -) + return { + "harness": "scripted", + "response": "Fixed multiply(a, b) in /app/calculator.py", + "request_id": payload["request"]["request_id"], + } + + +def main() -> None: + runtime_id = None + for line in sys.stdin: + message = json.loads(line) + operation = message["operation"] + payload = message["payload"] + if operation == "start": + runtime_id = payload["runtime_context"]["runtime_id"] + response("start") + elif operation == "invoke": + if payload["runtime_context"]["runtime_id"] != runtime_id: + raise RuntimeError("invoke does not match the active runtime") + response("invoke", output=invoke(payload)) + elif operation == "stop": + if payload["runtime_id"] != runtime_id: + raise RuntimeError("stop does not match the active runtime") + response("stop") + break + + +if __name__ == "__main__": + main() diff --git a/examples/harbor/swebench/adapters/claude/fabric-adapter.json b/examples/harbor/swebench/adapters/claude/fabric-adapter.json index ed36b146..9f8996f9 100644 --- a/examples/harbor/swebench/adapters/claude/fabric-adapter.json +++ b/examples/harbor/swebench/adapters/claude/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "claude", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.claude.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.claude.adapter" }, "config": { "accepts": ["models", "tools", "tools.blocked", "mcp", "skills", "telemetry"] diff --git a/examples/harbor/swebench/adapters/hermes/fabric-adapter.json b/examples/harbor/swebench/adapters/hermes/fabric-adapter.json index f2857470..ef60fb92 100644 --- a/examples/harbor/swebench/adapters/hermes/fabric-adapter.json +++ b/examples/harbor/swebench/adapters/hermes/fabric-adapter.json @@ -4,8 +4,7 @@ "harness": "hermes", "adapter_kind": "python", "runner": { - "module": "nemo_fabric_adapters.hermes.adapter", - "callable": "run" + "module": "nemo_fabric_adapters.hermes.adapter" }, "requirements": { "env": [ diff --git a/examples/notebooks/01_quickstart.ipynb b/examples/notebooks/01_quickstart.ipynb index ec207bf8..063037aa 100644 --- a/examples/notebooks/01_quickstart.ipynb +++ b/examples/notebooks/01_quickstart.ipynb @@ -253,7 +253,7 @@ "\n", "`run()` performs one full lifecycle in a single call: it starts a runtime, sends\n", "your input, collects the result, and stops the runtime. Reach for it for\n", - "one-shot work. It is async, and Jupyter lets you `await` it directly.\n", + "single-invocation work. It is async, and Jupyter lets you `await` it directly.\n", "\n", "It returns a **`RunResult`**: a normalized record of the invocation with the same\n", "shape regardless of harness. Always check `status` and `error` first, then read\n", diff --git a/python/src/nemo_fabric/client.py b/python/src/nemo_fabric/client.py index 64dc088a..f6d23c10 100644 --- a/python/src/nemo_fabric/client.py +++ b/python/src/nemo_fabric/client.py @@ -50,8 +50,8 @@ class Fabric: ``FabricNativeUnavailableError`` when the native extension is not installed. - See the Getting Started overview for runnable one-shot, typed-config, and - multi-turn examples. + See the Getting Started overview for runnable single-invocation, + typed-config, and multi-turn examples. """ def plan( diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index bd534e92..54ebe4b3 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -22,14 +22,16 @@ The core schema generator exports the current public typed contract. - `agent`: complete typed `FabricConfig`. - `adapter-descriptor`: minimal adapter descriptor consumed by Fabric. Each descriptor declares a `contract_version`; Fabric rejects descriptors for - unsupported adapter contracts during planning. + unsupported adapter contracts during planning. The `process` and `python` + adapter kinds use Fabric's persistent local-host wire protocol. - `run-plan`: executable plan containing the canonical typed config, absolute base directory, selected adapter, and derived execution metadata. ### Adapter Invocation -- `adapter-invocation`: adapter-facing payload sent to inline or process - adapters. +- `adapter-invocation`: per-turn payload sent to an initialized persistent + local adapter host. It contains only `runtime_context` and `request`; Fabric + sends configuration and capability planning data during lifecycle start. - `runtime-context`: per-run/per-invocation context included in adapter invocations. - `run-request`: per-invocation request/input. diff --git a/schemas/adapter-descriptor.schema.json b/schemas/adapter-descriptor.schema.json index 6c93bfe1..97d42cb9 100644 --- a/schemas/adapter-descriptor.schema.json +++ b/schemas/adapter-descriptor.schema.json @@ -26,7 +26,7 @@ "oneOf": [ { "const": "process", - "description": "Launch and supervise a CLI process.", + "description": "Launch and supervise a persistent adapter process.", "type": "string" }, { @@ -36,7 +36,7 @@ }, { "const": "python", - "description": "Call a Python SDK/plugin adapter.", + "description": "Launch and supervise a persistent Python adapter host.", "type": "string" }, { diff --git a/schemas/adapter-invocation.schema.json b/schemas/adapter-invocation.schema.json index 44162719..756caa97 100644 --- a/schemas/adapter-invocation.schema.json +++ b/schemas/adapter-invocation.schema.json @@ -50,1167 +50,84 @@ ], "type": "object" }, - "CapabilityKind": { - "description": "Capability kind.", - "oneOf": [ - { - "const": "tools", - "description": "Tool config.", - "type": "string" - }, - { - "const": "skills", - "description": "Skill paths.", - "type": "string" - }, - { - "const": "mcp", - "description": "MCP server.", - "type": "string" - } - ] - }, - "CapabilityPlan": { - "description": "Resolved capability configuration.", - "properties": { - "managed": { - "$ref": "#/$defs/CapabilityTargetPlan", - "default": { - "tools_configured": false - }, - "description": "Capabilities that Fabric must expose or manage outside the native harness config." - }, - "mcp_servers": { - "additionalProperties": { - "$ref": "#/$defs/McpServerPlan" - }, - "description": "MCP server exposure plan.", - "type": "object" - }, - "native": { - "$ref": "#/$defs/CapabilityTargetPlan", - "default": { - "tools_configured": false - }, - "description": "Capabilities mapped into the harness-native surface." - }, - "routes": { - "description": "Routing decisions made while planning the configured capabilities.", - "items": { - "$ref": "#/$defs/CapabilityRoute" - }, - "type": "array" - }, - "skill_paths": { - "description": "Resolved skill paths.", - "items": { - "type": "string" - }, - "type": "array" - }, - "tools": { - "$ref": "#/$defs/ToolsPlan", - "default": {}, - "description": "Normalized tool policy." - }, - "tools_configured": { - "default": false, - "description": "Whether tool configuration was provided.", - "type": "boolean" - }, - "unsupported": { - "$ref": "#/$defs/CapabilityTargetPlan", - "default": { - "tools_configured": false - }, - "description": "Capabilities that are configured but not executable by this Fabric build." - } - }, - "type": "object" - }, - "CapabilityRoute": { - "description": "One capability routing decision.", - "properties": { - "kind": { - "$ref": "#/$defs/CapabilityKind", - "description": "Capability kind." - }, - "name": { - "description": "Capability name.", - "type": "string" - }, - "reason": { - "description": "Human-readable reason for the selected route.", - "type": "string" - }, - "target": { - "$ref": "#/$defs/CapabilityTarget", - "description": "Routing target." - } - }, - "required": [ - "kind", - "name", - "target", - "reason" - ], - "type": "object" - }, - "CapabilityTarget": { - "description": "Capability routing target.", - "oneOf": [ - { - "const": "harness_native", - "description": "Adapter maps the capability into harness-native config.", - "type": "string" - }, - { - "const": "fabric_managed", - "description": "Fabric exposes or manages the capability around the harness.", - "type": "string" - }, - { - "const": "unsupported", - "description": "Capability is configured but no executable surface exists.", - "type": "string" - } - ] - }, - "CapabilityTargetPlan": { - "description": "Capabilities routed to one target.", - "properties": { - "mcp_servers": { - "additionalProperties": { - "$ref": "#/$defs/McpServerPlan" - }, - "description": "MCP servers for this target.", - "type": "object" - }, - "skill_paths": { - "description": "Resolved skill paths for this target.", - "items": { - "type": "string" - }, - "type": "array" - }, - "tools_configured": { - "default": false, - "description": "Whether tool configuration was provided for this target.", - "type": "boolean" - } - }, - "type": "object" - }, - "ControlLocation": { - "description": "Where Fabric control code runs relative to the environment.", - "oneOf": [ - { - "const": "external_control", - "description": "Fabric runs on the host/control plane and starts or connects to the harness in the environment.", - "type": "string" - }, - { - "const": "in_env_control", - "description": "Fabric runs inside the prepared environment with the harness.", - "type": "string" - } - ] - }, - "EnvironmentConfig": { - "additionalProperties": true, - "description": "Execution environment configuration.", - "properties": { - "artifacts": { - "description": "Artifact path inside or outside the provider.", - "type": [ - "string", - "null" - ] - }, - "connection": { - "additionalProperties": true, - "description": "Provider connection metadata, such as server URL, credential reference, or namespace.", - "type": "object" - }, - "control_location": { - "$ref": "#/$defs/ControlLocation", - "default": "in_env_control", - "description": "Where Fabric control code runs relative to the environment." - }, - "metadata": { - "additionalProperties": true, - "description": "Consumer-provided environment metadata.", - "type": "object" - }, - "ownership": { - "$ref": "#/$defs/EnvironmentOwnership", - "default": "caller_owned", - "description": "Whether Fabric owns the environment resource." - }, - "provider": { - "description": "Environment provider, for example `local`, `docker`, `opensandbox`, or `k8s`.", - "type": "string" - }, - "settings": { - "additionalProperties": true, - "description": "Provider-specific settings.", - "type": "object" - }, - "workspace": { - "description": "Workspace path inside or outside the provider.", - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "provider" - ], - "type": "object" - }, - "EnvironmentHandle": { - "description": "Resolved execution environment context.", - "properties": { - "artifacts": { - "description": "Artifact root visible to the harness runtime.", - "type": [ - "string", - "null" - ] - }, - "connection": { - "additionalProperties": true, - "description": "Provider connection metadata.", - "type": "object" - }, - "control_location": { - "$ref": "#/$defs/ControlLocation", - "description": "Where Fabric control code runs." - }, - "environment_id": { - "description": "Environment handle id.", - "type": "string" - }, - "metadata": { - "additionalProperties": true, - "description": "Provider-specific metadata.", - "type": "object" - }, - "ownership": { - "$ref": "#/$defs/EnvironmentOwnership", - "description": "Whether Fabric owns the environment resource." - }, - "provider": { - "description": "Environment provider.", - "type": "string" - }, - "workspace": { - "description": "Workspace visible to the harness runtime.", - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "environment_id", - "provider", - "control_location", - "ownership" - ], - "type": "object" - }, - "EnvironmentOwnership": { - "description": "Whether Fabric owns the underlying environment resource.", - "oneOf": [ - { - "const": "caller_owned", - "description": "The caller or a surrounding system owns the environment resource.", - "type": "string" - }, - { - "const": "fabric_owned", - "description": "Fabric created or leased the environment resource and may release it.", - "type": "string" - } - ] - }, - "FabricConfig": { - "additionalProperties": true, - "description": "Versioned Fabric agent config.", - "properties": { - "environment": { - "anyOf": [ - { - "$ref": "#/$defs/EnvironmentConfig" - }, - { - "type": "null" - } - ], - "description": "Environment where the harness or its tools execute." - }, - "harness": { - "$ref": "#/$defs/HarnessConfig", - "description": "Harness selection and harness-specific settings." - }, - "mcp": { - "anyOf": [ - { - "$ref": "#/$defs/McpConfig" - }, - { - "type": "null" - } - ], - "description": "MCP capability configuration." - }, - "metadata": { - "$ref": "#/$defs/MetadataConfig", - "description": "Human-readable metadata." - }, - "models": { - "additionalProperties": { - "$ref": "#/$defs/ModelConfig" - }, - "description": "Model aliases.", - "type": "object" - }, - "relay": { - "anyOf": [ - { - "$ref": "#/$defs/RelayConfig" - }, - { - "type": "null" - } - ], - "description": "First-class NeMo Relay integration configuration." - }, - "runtime": { - "$ref": "#/$defs/RuntimeConfig", - "description": "Runtime input/output contract." - }, - "schema_version": { - "description": "Config schema version.", - "type": "string" - }, - "skills": { - "anyOf": [ - { - "$ref": "#/$defs/SkillConfig" - }, - { - "type": "null" - } - ], - "description": "Skill capability configuration." - }, - "telemetry": { - "anyOf": [ - { - "$ref": "#/$defs/TelemetryConfig" - }, - { - "type": "null" - } - ], - "description": "Telemetry configuration." - }, - "tools": { - "anyOf": [ - { - "$ref": "#/$defs/ToolsConfig" - }, - { - "type": "null" - } - ], - "description": "Tool capability configuration." - } - }, - "required": [ - "schema_version", - "metadata", - "harness", - "runtime" - ], - "type": "object" - }, - "HarnessConfig": { - "additionalProperties": true, - "description": "Harness selection.", - "properties": { - "adapter_id": { - "description": "Adapter implementation id.", - "type": "string" - }, - "resolution": { - "anyOf": [ - { - "$ref": "#/$defs/ResolutionStrategy" - }, - { - "type": "null" - } - ], - "description": "Selected install or availability strategy." - }, - "settings": { - "additionalProperties": true, - "description": "Harness-specific settings.", - "type": "object" - } - }, - "required": [ - "adapter_id" - ], - "type": "object" - }, - "McpConfig": { - "additionalProperties": true, - "description": "MCP capability configuration.", - "properties": { - "servers": { - "additionalProperties": { - "$ref": "#/$defs/McpServerConfig" - }, - "description": "Named MCP servers.", - "type": "object" - } - }, - "type": "object" - }, - "McpExposure": { - "description": "MCP exposure strategy.", - "oneOf": [ - { - "const": "harness_native", - "description": "Map into harness-native MCP config through the selected adapter.", - "type": "string" - }, - { - "const": "fabric_managed", - "description": "Fabric manages MCP and exposes basic tools/actions.", - "type": "string" - } - ] - }, - "McpServerConfig": { - "additionalProperties": true, - "description": "MCP server configuration.", - "properties": { - "exposure": { - "$ref": "#/$defs/McpExposure", - "description": "How Fabric exposes the MCP capability to the harness." - }, - "transport": { - "description": "MCP transport.", - "type": "string" - }, - "url": { - "description": "MCP server URL or process command, depending on transport.", - "type": "string" - } - }, - "required": [ - "transport", - "url", - "exposure" - ], - "type": "object" - }, - "McpServerPlan": { - "description": "Resolved MCP server exposure.", - "properties": { - "exposure": { - "$ref": "#/$defs/McpExposure", - "description": "Exposure strategy." - }, - "transport": { - "description": "MCP transport.", - "type": "string" - }, - "url": { - "description": "MCP URL or command.", - "type": "string" - } - }, - "required": [ - "transport", - "url", - "exposure" - ], - "type": "object" - }, - "MetadataConfig": { - "additionalProperties": true, - "description": "Human-readable metadata.", - "properties": { - "description": { - "description": "Optional description.", - "type": [ - "string", - "null" - ] - }, - "name": { - "description": "Agent/config name.", - "type": "string" - } - }, - "required": [ - "name" - ], - "type": "object" - }, - "ModelConfig": { - "additionalProperties": true, - "description": "Model configuration.", - "properties": { - "api_key_env": { - "description": "Optional environment variable containing an API key.", - "type": [ - "string", - "null" - ] - }, - "model": { - "description": "Provider model identifier.", - "type": "string" - }, - "provider": { - "description": "Model provider name.", - "type": "string" - }, - "settings": { - "additionalProperties": true, - "description": "Provider-specific settings.", - "type": "object" - }, - "temperature": { - "description": "Optional temperature.", - "format": "double", - "type": [ - "number", - "null" - ] - } - }, - "required": [ - "provider", - "model" - ], - "type": "object" - }, - "RelayAtifConfig": { - "additionalProperties": true, - "description": "Relay ATIF export configuration.", - "properties": { - "agent_name": { - "default": "NeMo Relay", - "description": "Agent name written into ATIF.", - "type": "string" - }, - "agent_version": { - "description": "Agent version written into ATIF.", - "type": [ - "string", - "null" - ] - }, - "enabled": { - "default": false, - "description": "Whether ATIF export is enabled.", - "type": "boolean" - }, - "extra": { - "description": "Extra ATIF metadata." - }, - "filename_template": { - "default": "nemo-relay-atif-{session_id}.json", - "description": "ATIF file name template.", - "type": "string" - }, - "model_name": { - "default": "unknown", - "description": "Model name written into ATIF.", - "type": "string" - }, - "output_directory": { - "description": "Directory used for ATIF files.", - "type": [ - "string", - "null" - ] - }, - "storage": { - "description": "Optional ATIF remote storage.", - "items": { - "$ref": "#/$defs/RelayAtifStorageConfig" - }, - "type": [ - "array", - "null" - ] - }, - "tool_definitions": { - "description": "Tool definitions written into ATIF.", - "items": true, - "type": [ - "array", - "null" - ] - } - }, - "type": "object" - }, - "RelayAtifStorageConfig": { - "description": "Relay ATIF remote storage configuration.", - "oneOf": [ - { - "additionalProperties": true, - "description": "Upload ATIF artifacts to HTTP storage.", - "properties": { - "endpoint": { - "default": "", - "description": "HTTP storage endpoint.", - "type": "string" - }, - "header_env": { - "additionalProperties": { - "type": "string" - }, - "description": "Environment-variable-backed HTTP headers.", - "type": "object" - }, - "headers": { - "additionalProperties": { - "type": "string" - }, - "description": "Static HTTP headers.", - "type": "object" - }, - "timeout_millis": { - "default": 3000, - "description": "Request timeout in milliseconds.", - "format": "uint64", - "minimum": 0, - "type": "integer" - }, - "type": { - "const": "http", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "additionalProperties": true, - "description": "Upload ATIF artifacts to S3-compatible storage.", - "properties": { - "access_key_id": { - "description": "AWS access key id.", - "type": [ - "string", - "null" - ] - }, - "allow_http": { - "description": "Allow HTTP endpoints for S3-compatible storage.", - "type": [ - "boolean", - "null" - ] - }, - "bucket": { - "default": "", - "description": "S3 bucket name.", - "type": "string" - }, - "endpoint_url": { - "description": "S3-compatible endpoint URL.", - "type": [ - "string", - "null" - ] - }, - "key_prefix": { - "description": "Optional S3 object key prefix.", - "type": [ - "string", - "null" - ] - }, - "region": { - "description": "AWS region.", - "type": [ - "string", - "null" - ] - }, - "secret_access_key_var": { - "description": "Environment variable containing the AWS secret access key.", - "type": [ - "string", - "null" - ] - }, - "session_token_var": { - "description": "Environment variable containing the AWS session token.", - "type": [ - "string", - "null" - ] - }, - "type": { - "const": "s3", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - } - ] - }, - "RelayAtofConfig": { - "additionalProperties": true, - "description": "Relay ATOF export configuration.", - "properties": { - "enabled": { - "default": false, - "description": "Whether ATOF export is enabled.", - "type": "boolean" - }, - "endpoints": { - "description": "Optional remote ATOF endpoints.", - "items": { - "$ref": "#/$defs/RelayAtofEndpointConfig" - }, - "type": "array" - }, - "filename": { - "description": "ATOF file name.", - "type": [ - "string", - "null" - ] - }, - "mode": { - "$ref": "#/$defs/RelayAtofMode", - "default": "append", - "description": "File write mode." - }, - "output_directory": { - "description": "Directory used for ATOF files.", - "type": [ - "string", - "null" - ] - } - }, - "type": "object" - }, - "RelayAtofEndpointConfig": { - "additionalProperties": true, - "description": "Relay ATOF endpoint configuration.", - "properties": { - "field_name_policy": { - "$ref": "#/$defs/RelayAtofEndpointFieldNamePolicy", - "default": "preserve", - "description": "Field-name handling policy." - }, - "headers": { - "additionalProperties": { - "type": "string" - }, - "description": "Endpoint headers.", - "type": "object" - }, - "timeout_millis": { - "default": 3000, - "description": "Request timeout in milliseconds.", - "format": "uint64", - "minimum": 0, - "type": "integer" - }, - "transport": { - "$ref": "#/$defs/RelayAtofEndpointTransport", - "default": "http_post", - "description": "Endpoint transport." - }, - "url": { - "description": "Endpoint URL.", - "type": "string" - } - }, - "required": [ - "url" - ], - "type": "object" - }, - "RelayAtofEndpointFieldNamePolicy": { - "description": "Relay ATOF endpoint field-name policy.", - "oneOf": [ - { - "const": "preserve", - "description": "Preserve field names.", - "type": "string" - }, - { - "const": "replace_dots", - "description": "Replace dots in field names.", - "type": "string" - } - ] - }, - "RelayAtofEndpointTransport": { - "description": "Relay ATOF endpoint transport.", - "oneOf": [ - { - "const": "http_post", - "description": "HTTP POST transport.", - "type": "string" - }, - { - "const": "websocket", - "description": "WebSocket transport.", - "type": "string" - }, - { - "const": "ndjson", - "description": "NDJSON transport.", - "type": "string" - } - ] - }, - "RelayAtofMode": { - "description": "Relay ATOF file mode.", - "oneOf": [ - { - "const": "append", - "description": "Append to an existing ATOF file.", - "type": "string" - }, - { - "const": "overwrite", - "description": "Overwrite an existing ATOF file.", - "type": "string" - } - ] - }, - "RelayComponentConfig": { - "additionalProperties": true, - "description": "Generic NeMo Relay plugin component configuration.", - "properties": { - "config": { - "additionalProperties": true, - "description": "Component-local Relay plugin config.", - "type": "object" - }, - "enabled": { - "default": true, - "description": "Whether this Relay component should be activated.", - "type": "boolean" - }, - "kind": { - "description": "Registered Relay plugin kind.", - "type": "string" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "RelayConfig": { - "additionalProperties": true, - "description": "NeMo Relay integration configuration.", - "properties": { - "components": { - "description": "Additional Relay plugin components.", - "items": { - "$ref": "#/$defs/RelayComponentConfig" - }, - "type": "array" - }, - "observability": { - "anyOf": [ - { - "$ref": "#/$defs/RelayObservabilityConfig" - }, - { - "type": "null" - } - ], - "description": "Relay observability component configuration." - }, - "output_dir": { - "description": "Optional Relay output directory.", - "type": [ - "string", - "null" - ] - }, - "policy": { - "anyOf": [ - { - "$ref": "#/$defs/RelayConfigPolicy" - }, - { - "type": "null" - } - ], - "description": "Relay plugin validation policy." - }, - "project": { - "description": "Optional project name for Relay backends.", - "type": [ - "string", - "null" - ] - } - }, - "type": "object" - }, - "RelayConfigPolicy": { - "description": "Relay validation policy.", - "properties": { - "unknown_component": { - "$ref": "#/$defs/RelayUnsupportedBehavior", - "default": "warn", - "description": "Policy for unknown components." - }, - "unknown_field": { - "$ref": "#/$defs/RelayUnsupportedBehavior", - "default": "warn", - "description": "Policy for unknown fields." - }, - "unsupported_value": { - "$ref": "#/$defs/RelayUnsupportedBehavior", - "default": "error", - "description": "Policy for unsupported values." - } - }, - "type": "object" - }, - "RelayObservabilityConfig": { - "additionalProperties": true, - "description": "NeMo Relay observability component configuration.", - "properties": { - "atif": { - "anyOf": [ - { - "$ref": "#/$defs/RelayAtifConfig" - }, - { - "type": "null" - } - ], - "description": "ATIF export configuration." - }, - "atof": { - "anyOf": [ - { - "$ref": "#/$defs/RelayAtofConfig" - }, - { - "type": "null" - } - ], - "description": "ATOF export configuration." - }, - "openinference": { - "anyOf": [ - { - "$ref": "#/$defs/RelayOtlpConfig" - }, - { - "type": "null" - } - ], - "description": "OpenInference export configuration." - }, - "opentelemetry": { - "anyOf": [ - { - "$ref": "#/$defs/RelayOtlpConfig" - }, - { - "type": "null" - } - ], - "description": "OpenTelemetry export configuration." - }, - "policy": { - "anyOf": [ - { - "$ref": "#/$defs/RelayConfigPolicy" - }, - { - "type": "null" - } - ], - "description": "Relay config validation policy." + "ControlLocation": { + "description": "Where Fabric control code runs relative to the environment.", + "oneOf": [ + { + "const": "external_control", + "description": "Fabric runs on the host/control plane and starts or connects to the harness in the environment.", + "type": "string" }, - "version": { - "default": 1, - "description": "Relay observability config version.", - "format": "uint32", - "minimum": 0, - "type": "integer" + { + "const": "in_env_control", + "description": "Fabric runs inside the prepared environment with the harness.", + "type": "string" } - }, - "type": "object" + ] }, - "RelayOtlpConfig": { - "additionalProperties": true, - "description": "Relay OpenTelemetry/OpenInference export configuration.", + "EnvironmentHandle": { + "description": "Resolved execution environment context.", "properties": { - "enabled": { - "default": false, - "description": "Whether OTLP export is enabled.", - "type": "boolean" - }, - "endpoint": { - "description": "OTLP endpoint.", + "artifacts": { + "description": "Artifact root visible to the harness runtime.", "type": [ "string", "null" ] }, - "headers": { - "additionalProperties": { - "type": "string" - }, - "description": "OTLP headers.", + "connection": { + "additionalProperties": true, + "description": "Provider connection metadata.", "type": "object" }, - "instrumentation_scope": { - "description": "OTLP instrumentation scope.", - "type": [ - "string", - "null" - ] + "control_location": { + "$ref": "#/$defs/ControlLocation", + "description": "Where Fabric control code runs." }, - "resource_attributes": { - "additionalProperties": { - "type": "string" - }, - "description": "OTLP resource attributes.", + "environment_id": { + "description": "Environment handle id.", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "description": "Provider-specific metadata.", "type": "object" }, - "service_name": { - "default": "nemo-relay", - "description": "OTLP service name.", - "type": "string" + "ownership": { + "$ref": "#/$defs/EnvironmentOwnership", + "description": "Whether Fabric owns the environment resource." }, - "service_namespace": { - "description": "OTLP service namespace.", - "type": [ - "string", - "null" - ] + "provider": { + "description": "Environment provider.", + "type": "string" }, - "service_version": { - "description": "OTLP service version.", + "workspace": { + "description": "Workspace visible to the harness runtime.", "type": [ "string", "null" ] - }, - "timeout_millis": { - "default": 3000, - "description": "Request timeout in milliseconds.", - "format": "uint64", - "minimum": 0, - "type": "integer" - }, - "transport": { - "$ref": "#/$defs/RelayOtlpTransport", - "default": "http_binary", - "description": "OTLP transport." } }, + "required": [ + "environment_id", + "provider", + "control_location", + "ownership" + ], "type": "object" }, - "RelayOtlpTransport": { - "description": "Relay OTLP transport.", - "oneOf": [ - { - "const": "http_binary", - "description": "OTLP HTTP binary transport.", - "type": "string" - }, - { - "const": "grpc", - "description": "OTLP gRPC transport.", - "type": "string" - } - ] - }, - "RelayUnsupportedBehavior": { - "description": "Relay unsupported/unknown config handling.", - "oneOf": [ - { - "const": "ignore", - "description": "Ignore the unsupported or unknown value.", - "type": "string" - }, - { - "const": "warn", - "description": "Warn on the unsupported or unknown value.", - "type": "string" - }, - { - "const": "error", - "description": "Error on the unsupported or unknown value.", - "type": "string" - } - ] - }, - "ResolutionStrategy": { - "description": "Adapter install or availability strategy.", + "EnvironmentOwnership": { + "description": "Whether Fabric owns the underlying environment resource.", "oneOf": [ { - "const": "preinstalled", - "description": "Harness is already available in the prepared environment.", - "type": "string" - }, - { - "const": "image_provided", - "description": "Environment image already contains the harness and dependencies.", - "type": "string" - }, - { - "const": "pip_uv", - "description": "Adapter may install a Python package with pip or uv.", - "type": "string" - }, - { - "const": "npm", - "description": "Adapter may install a Node package.", - "type": "string" - }, - { - "const": "source", - "description": "Adapter may install from source.", - "type": "string" - }, - { - "const": "service", - "description": "Adapter connects to an already-running service.", + "const": "caller_owned", + "description": "The caller or a surrounding system owns the environment resource.", "type": "string" }, { - "const": "native_plugin", - "description": "Adapter is installed through a harness-native plugin manager.", + "const": "fabric_owned", + "description": "Fabric created or leased the environment resource and may release it.", "type": "string" } ] @@ -1240,32 +157,8 @@ ], "type": "object" }, - "RuntimeConfig": { - "additionalProperties": true, - "description": "Runtime input/output contract.", - "properties": { - "artifacts": { - "description": "Artifact directory.", - "type": [ - "string", - "null" - ] - }, - "input_schema": { - "default": "text", - "description": "Input schema label.", - "type": "string" - }, - "output_schema": { - "default": "text", - "description": "Output schema label.", - "type": "string" - } - }, - "type": "object" - }, "RuntimeContext": { - "description": "Per-run/per-invocation context passed to harness adapters.", + "description": "Context generated for one invocation of a started runtime.", "properties": { "artifacts": { "$ref": "#/$defs/ArtifactManifest", @@ -1339,192 +232,21 @@ "relay_enabled" ], "type": "object" - }, - "SkillConfig": { - "additionalProperties": true, - "description": "Skill capability configuration.", - "properties": { - "paths": { - "description": "Skill paths resolved relative to the config base directory.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "TelemetryConfig": { - "additionalProperties": true, - "description": "Telemetry configuration.", - "properties": { - "providers": { - "additionalProperties": { - "$ref": "#/$defs/TelemetryProviderConfig" - }, - "description": "Telemetry providers enabled for this run.", - "type": "object" - } - }, - "type": "object" - }, - "TelemetryPlan": { - "description": "Resolved telemetry plan.", - "properties": { - "adapter_outputs": { - "description": "Telemetry outputs declared by the selected adapter descriptor.", - "items": { - "type": "string" - }, - "type": "array" - }, - "native_config": { - "description": "Native telemetry pass-through config." - }, - "providers": { - "description": "Telemetry providers selected for this run.", - "items": { - "$ref": "#/$defs/TelemetryProvider" - }, - "type": "array" - }, - "relay_config": { - "description": "Relay pass-through config." - }, - "relay_enabled": { - "description": "Whether Relay is enabled.", - "type": "boolean" - }, - "relay_output_dir": { - "description": "Relay output directory, when configured.", - "type": [ - "string", - "null" - ] - }, - "relay_project": { - "description": "Relay project, when configured.", - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "providers", - "relay_enabled" - ], - "type": "object" - }, - "TelemetryProvider": { - "description": "Telemetry runtime provider.", - "oneOf": [ - { - "const": "relay", - "description": "Use NeMo Relay for telemetry integration.", - "type": "string" - }, - { - "const": "native", - "description": "Let the selected adapter handle telemetry natively.", - "type": "string" - } - ] - }, - "TelemetryProviderConfig": { - "additionalProperties": true, - "description": "Provider-specific telemetry configuration.", - "properties": { - "config": { - "description": "Provider-specific pass-through config." - } - }, - "type": "object" - }, - "ToolsConfig": { - "additionalProperties": true, - "description": "Harness-neutral tool capability configuration.", - "properties": { - "blocked": { - "description": "Adapter-native tool names or toolset names to block.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "ToolsPlan": { - "description": "Normalized tool policy for a run.", - "properties": { - "blocked": { - "description": "Adapter-native tool names or toolset names to block.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Adapter-facing invocation payload.", + "description": "One invocation against an initialized adapter runtime.", "properties": { - "agent_name": { - "description": "Stable agent name.", - "type": "string" - }, - "base_dir": { - "description": "Absolute base directory used to resolve relative Fabric paths.", - "type": "string" - }, - "capability_plan": { - "$ref": "#/$defs/CapabilityPlan", - "default": { - "managed": { - "tools_configured": false - }, - "native": { - "tools_configured": false - }, - "tools": {}, - "tools_configured": false, - "unsupported": { - "tools_configured": false - } - }, - "description": "Derived capability routing plan for the selected adapter." - }, - "config": { - "$ref": "#/$defs/FabricConfig", - "description": "Complete typed Fabric config." - }, "request": { "$ref": "#/$defs/RunRequest", - "description": "Per-invocation request." + "description": "Typed caller request for this invocation." }, "runtime_context": { "$ref": "#/$defs/RuntimeContext", - "description": "Per-runtime/per-invocation execution context." - }, - "telemetry_plan": { - "anyOf": [ - { - "$ref": "#/$defs/TelemetryPlan" - }, - { - "type": "null" - } - ], - "description": "Derived telemetry plan for the selected adapter." + "description": "Invocation context generated by Fabric." } }, "required": [ - "agent_name", - "base_dir", - "config", "runtime_context", "request" ], diff --git a/schemas/run-plan.schema.json b/schemas/run-plan.schema.json index 110290d0..db8bb4ea 100644 --- a/schemas/run-plan.schema.json +++ b/schemas/run-plan.schema.json @@ -103,7 +103,7 @@ "oneOf": [ { "const": "process", - "description": "Launch and supervise a CLI process.", + "description": "Launch and supervise a persistent adapter process.", "type": "string" }, { @@ -113,7 +113,7 @@ }, { "const": "python", - "description": "Call a Python SDK/plugin adapter.", + "description": "Launch and supervise a persistent Python adapter host.", "type": "string" }, { diff --git a/schemas/run-result.schema.json b/schemas/run-result.schema.json index 54272bb3..ed2da451 100644 --- a/schemas/run-result.schema.json +++ b/schemas/run-result.schema.json @@ -5,7 +5,7 @@ "oneOf": [ { "const": "process", - "description": "Launch and supervise a CLI process.", + "description": "Launch and supervise a persistent adapter process.", "type": "string" }, { @@ -15,7 +15,7 @@ }, { "const": "python", - "description": "Call a Python SDK/plugin adapter.", + "description": "Launch and supervise a persistent Python adapter host.", "type": "string" }, { diff --git a/schemas/runtime-context.schema.json b/schemas/runtime-context.schema.json index c6be1070..cda168be 100644 --- a/schemas/runtime-context.schema.json +++ b/schemas/runtime-context.schema.json @@ -166,7 +166,7 @@ } }, "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Per-run/per-invocation context passed to harness adapters.", + "description": "Context generated for one invocation of a started runtime.", "properties": { "artifacts": { "$ref": "#/$defs/ArtifactManifest", diff --git a/schemas/runtime-handle.schema.json b/schemas/runtime-handle.schema.json index 85ba2296..9cda47df 100644 --- a/schemas/runtime-handle.schema.json +++ b/schemas/runtime-handle.schema.json @@ -5,7 +5,7 @@ "oneOf": [ { "const": "process", - "description": "Launch and supervise a CLI process.", + "description": "Launch and supervise a persistent adapter process.", "type": "string" }, { @@ -15,7 +15,7 @@ }, { "const": "python", - "description": "Call a Python SDK/plugin adapter.", + "description": "Launch and supervise a persistent Python adapter host.", "type": "string" }, { diff --git a/skills/README.md b/skills/README.md index 0bbb3f74..8f00e7a9 100644 --- a/skills/README.md +++ b/skills/README.md @@ -49,7 +49,7 @@ symlink or `.agents/skills/` set); those serve Fabric's own contributors. | Skill | Use it when | |---|---| -| [`nemo-fabric-integrate`](nemo-fabric-integrate/SKILL.md) | You are adding NeMo Fabric to a consumer application, service, evaluation harness, or platform through the typed Python SDK — building an in-memory `FabricConfig`, choosing one-shot versus stateful-runtime execution, validating with `plan`/`doctor`, and consuming normalized results. | +| [`nemo-fabric-integrate`](nemo-fabric-integrate/SKILL.md) | You are adding NeMo Fabric to a consumer application, service, evaluation harness, or platform through the typed Python SDK — building an in-memory `FabricConfig`, choosing the single-invocation convenience API or an explicitly started runtime, validating with `plan`/`doctor`, and consuming normalized results. | ## Conventions diff --git a/skills/nemo-fabric-integrate/SKILL.md b/skills/nemo-fabric-integrate/SKILL.md index 110d5a21..5cc6bf1c 100644 --- a/skills/nemo-fabric-integrate/SKILL.md +++ b/skills/nemo-fabric-integrate/SKILL.md @@ -1,6 +1,6 @@ --- name: nemo-fabric-integrate -description: Use this skill when integrating NeMo Fabric into a consumer application, service, evaluation harness, or platform through the typed Python SDK — translating the consumer's own application, job, or deployment config into an in-memory FabricConfig, choosing one-shot run versus a stateful runtime, validating with plan and doctor, and consuming normalized results, artifacts, and telemetry. +description: Use this skill when integrating NeMo Fabric into a consumer application, service, evaluation harness, or platform through the typed Python SDK — translating the consumer's own application, job, or deployment config into an in-memory FabricConfig, choosing the single-invocation convenience API or an explicitly started runtime, validating with plan and doctor, and consuming normalized results, artifacts, and telemetry. license: Apache-2.0 metadata: author: NVIDIA Corporation and Affiliates @@ -115,17 +115,26 @@ construction. Pick the smallest lifecycle the consumer needs: -- **One-shot** — one input, no retained state. `await Fabric().run(config, input=...)` - runs the full start, invoke, and stop cycle and returns a `RunResult`. Pass +- **Single invocation** — one input, no retained state after the call. + `await Fabric().run(config, input=...)` runs the full start, invoke, and stop + cycle and returns a `RunResult`. Pass `request=RunRequest(...)` instead of `input=...` when the invocation needs a caller-owned request ID or context (the two are mutually exclusive). -- **Stateful runtime** — ordered turns over one live harness. Start it with +- **Stateful runtime** — ordered turns over one logical harness lifecycle. Start it with `start_runtime(...)` and use the returned `Runtime` as an async context manager so cleanup runs on exit — shutdown is attempted, not guaranteed (`stop()` can raise `FabricRuntimeError`; see Consume Results And Handle Errors). A runtime accepts one active invocation at a time; overlapping calls raise `FabricStateError`. +The selected adapter owns the execution topology. The bundled Claude, Codex, +Deep Agents, and Hermes adapters retain their native client, graph/checkpointer, +or agent/database inside one local host for the full runtime. Local `process` +and `python` adapters use this host lifecycle; consumers do not select another +local execution mechanism in `FabricConfig`. Do not replay an invocation after +a runtime failure. Stop the failed runtime and explicitly start a new one +according to the application's retry policy. + The lifecycle fragment below shows both forms. It assumes the caller has already set `config = to_fabric_config(job)` and chosen `base`, as described in the configuration example above: @@ -139,7 +148,7 @@ from nemo_fabric import Fabric async def main() -> None: fabric = Fabric() - # One-shot + # Single invocation result = await fabric.run(config, base_dir=base, input="Review the changes.") # Multi-turn @@ -236,7 +245,7 @@ result-field and error inventory, and - [ ] The consumer config object is translated directly into an in-memory `FabricConfig`. - [ ] Only public `nemo_fabric` symbols are imported; no `_native` or adapter internals. - [ ] The consumer config is built in memory and passed directly to Fabric. -- [ ] The right lifecycle is chosen: `run(...)` for one-shot, `start_runtime(...)` with `async with` for multi-turn. +- [ ] The right lifecycle is chosen: `run(...)` for a single invocation, `start_runtime(...)` with `async with` for multi-turn. - [ ] `plan(...)` and `doctor(...)` validate adapter selection, capabilities, and environment before execution. - [ ] Installation, adapter dependencies, and credentials are owned by the environment, not consumer code. - [ ] `RunResult` status, error, and events are inspected before output; artifacts and telemetry are captured. diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 99f6b1ad..7656ae33 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -66,5 +66,10 @@ FabricConfig -> plan() -> RunPlan -> start_runtime() -> Runtime -> invoke() -> R - A runtime is a logical execution boundary, not necessarily an operating-system process. Harness-native threads, sessions, and conversations remain adapter-owned state associated with the runtime. +- Runtime hosting is adapter-declared, not consumer-configured. The bundled + Claude, Codex, Deep Agents, and Hermes adapters retain their native runtime + resources in one local host. Local `process` and `python` adapters use the + same host lifecycle. A host crash or protocol timeout terminates the runtime; + the application may explicitly start a new runtime according to its policy. - The application owns scheduling, queues, retries, and how many runtimes to run. Fabric provides only the runtime contract. diff --git a/tests/adapters/test_adapters_common_lifecycle.py b/tests/adapters/test_adapters_common_lifecycle.py new file mode 100644 index 00000000..138e031c --- /dev/null +++ b/tests/adapters/test_adapters_common_lifecycle.py @@ -0,0 +1,342 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import io +import json +import os +from typing import Any + +import pytest +from nemo_fabric_adapters.common import lifecycle + + +def _request(operation: str, payload: dict[str, Any]) -> dict[str, Any]: + return { + "operation": operation, + "payload": payload, + } + + +def _streams(requests: list[dict[str, Any]]) -> tuple[io.StringIO, io.StringIO]: + input_stream = io.StringIO("".join(f"{json.dumps(item)}\n" for item in requests)) + return input_stream, io.StringIO() + + +def test_lifecycle_host_reuses_one_runtime_and_one_event_loop(): + runtime_id = "runtime-1" + input_stream, output_stream = _streams( + [ + _request( + "start", + {"runtime_context": {"runtime_id": runtime_id}}, + ), + _request( + "invoke", + { + "runtime_context": {"runtime_id": runtime_id}, + "request": {"input": "first"}, + }, + ), + _request( + "invoke", + { + "runtime_context": {"runtime_id": runtime_id}, + "request": {"input": "second"}, + }, + ), + _request("stop", {"runtime_id": runtime_id}), + ] + ) + instances = [] + + class Runtime: + def __init__(self): + self.loop_ids: list[int] = [] + self.invocations = 0 + instances.append(self) + + async def start(self, _payload): + self.loop_ids.append(id(asyncio.get_running_loop())) + + async def invoke(self, payload): + self.loop_ids.append(id(asyncio.get_running_loop())) + self.invocations += 1 + return { + "count": self.invocations, + "input": payload["request"]["input"], + } + + async def stop(self): + self.loop_ids.append(id(asyncio.get_running_loop())) + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + responses = [json.loads(line) for line in output_stream.getvalue().splitlines()] + assert [item["operation"] for item in responses] == [ + "start", + "invoke", + "invoke", + "stop", + ] + assert all(item["outcome"]["status"] == "succeeded" for item in responses) + assert all(set(item) == {"operation", "outcome"} for item in responses) + assert responses[1]["outcome"]["output"] == {"count": 1, "input": "first"} + assert responses[2]["outcome"]["output"] == {"count": 2, "input": "second"} + assert len(instances) == 1 + assert len(set(instances[0].loop_ids)) == 1 + + +def test_lifecycle_host_passes_minimal_invoke_payload_unchanged(): + runtime_id = "runtime-1" + start_payload = { + "agent_name": "agent", + "base_dir": "/workspace", + "config": {"harness": {"settings": {"mode": "retained"}}}, + "runtime_context": { + "runtime_id": runtime_id, + "invocation_id": "runtime-start", + }, + "capability_plan": {"native": ["tools"]}, + } + invoke_payload = { + "runtime_context": { + "runtime_id": runtime_id, + "invocation_id": "invocation-1", + }, + "request": {"input": "hello"}, + } + input_stream, output_stream = _streams( + [ + _request("start", start_payload), + _request("invoke", invoke_payload), + _request("stop", {"runtime_id": runtime_id}), + ] + ) + invocations = [] + + class Runtime: + async def start(self, _payload) -> None: + pass + + async def invoke(self, payload) -> dict[str, str]: + invocations.append(payload) + return {"input": payload["request"]["input"]} + + async def stop(self) -> None: + pass + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + assert invocations == [invoke_payload] + + +def test_lifecycle_host_rejects_runtime_mismatch_without_poisoning_runtime(): + input_stream, output_stream = _streams( + [ + _request("start", {"runtime_context": {"runtime_id": "runtime-1"}}), + _request( + "invoke", + { + "runtime_context": {"runtime_id": "runtime-2"}, + "request": {"input": "do not run"}, + }, + ), + _request( + "invoke", + { + "runtime_context": {"runtime_id": "runtime-1"}, + "request": {"input": "run"}, + }, + ), + _request("stop", {"runtime_id": "runtime-1"}), + ] + ) + invocations = [] + + class Runtime: + async def start(self, _payload): + pass + + async def invoke(self, payload): + invocations.append(payload) + return {"input": payload["request"]["input"]} + + async def stop(self): + pass + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + responses = [json.loads(line) for line in output_stream.getvalue().splitlines()] + assert responses[1]["outcome"]["status"] == "failed" + assert responses[1]["outcome"]["error"]["code"] == "lifecycle_runtime_mismatch" + assert responses[2]["outcome"] == { + "status": "succeeded", + "output": {"input": "run"}, + } + assert len(invocations) == 1 + + +def test_lifecycle_host_keeps_adapter_stdout_out_of_protocol(capsys): + runtime_id = "runtime-1" + input_stream, output_stream = _streams( + [ + _request("start", {"runtime_context": {"runtime_id": runtime_id}}), + _request( + "invoke", + { + "runtime_context": {"runtime_id": runtime_id}, + "request": {"input": "hello"}, + }, + ), + _request("stop", {"runtime_id": runtime_id}), + ] + ) + + class Runtime: + async def start(self, _payload): + pass + + async def invoke(self, _payload): + print("adapter diagnostic") + return {"failed": False} + + async def stop(self): + pass + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + assert "adapter diagnostic" not in output_stream.getvalue() + assert "adapter diagnostic" in capsys.readouterr().err + + +def test_lifecycle_host_scopes_invocation_telemetry_environment(): + runtime_id = "runtime-1" + variable = "FABRIC_TEST_LIFECYCLE_ENV" + os.environ[variable] = "host-value" + input_stream, output_stream = _streams( + [ + _request("start", {"runtime_context": {"runtime_id": runtime_id}}), + _request( + "invoke", + { + "runtime_context": { + "runtime_id": runtime_id, + "telemetry": {"env": {variable: "invocation-value"}}, + }, + "request": {"input": "hello"}, + }, + ), + _request("stop", {"runtime_id": runtime_id}), + ] + ) + + class Runtime: + async def start(self, _payload): + pass + + async def invoke(self, _payload): + return {"value": os.environ[variable]} + + async def stop(self): + pass + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + responses = [json.loads(line) for line in output_stream.getvalue().splitlines()] + assert responses[1]["outcome"]["output"] == {"value": "invocation-value"} + assert os.environ[variable] == "host-value" + + +def test_lifecycle_host_stops_runtime_when_fabric_closes_stdin(): + input_stream, output_stream = _streams( + [_request("start", {"runtime_context": {"runtime_id": "runtime-1"}})] + ) + stopped = [] + + class Runtime: + async def start(self, _payload): + pass + + async def invoke(self, _payload): + return None + + async def stop(self): + stopped.append(True) + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + assert stopped == [True] + + +@pytest.mark.parametrize( + ("failure", "expected_code"), + [ + (RuntimeError("adapter failed"), "lifecycle_adapter_invoke_failed"), + ( + lifecycle.LifecycleError("adapter_known_failure", "Adapter failed"), + "adapter_known_failure", + ), + ], +) +def test_lifecycle_host_rejects_invoke_after_adapter_failure(failure, expected_code): + runtime_id = "runtime-1" + invoke_payload = { + "runtime_context": {"runtime_id": runtime_id}, + "request": {"input": "fail"}, + } + input_stream, output_stream = _streams( + [ + _request("start", {"runtime_context": {"runtime_id": runtime_id}}), + _request("invoke", invoke_payload), + _request("invoke", invoke_payload), + _request("stop", {"runtime_id": runtime_id}), + ] + ) + invocations = [] + + class Runtime: + async def start(self, _payload): + pass + + async def invoke(self, payload): + invocations.append(payload) + raise failure + + async def stop(self) -> None: + pass + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + responses = [json.loads(line) for line in output_stream.getvalue().splitlines()] + assert responses[1]["outcome"]["error"]["code"] == expected_code + assert responses[2]["outcome"]["error"]["code"] == "lifecycle_runtime_failed" + assert len(invocations) == 1 + + +def test_lifecycle_host_cleans_up_and_exits_after_start_failure(): + runtime_id = "runtime-1" + start = _request("start", {"runtime_context": {"runtime_id": runtime_id}}) + input_stream, output_stream = _streams([start, start]) + stopped = [] + + class Runtime: + async def start(self, _payload): + raise RuntimeError("start failed") + + async def invoke(self, _payload): + raise AssertionError("failed runtime must not be invoked") + + async def stop(self): + stopped.append(True) + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + responses = [json.loads(line) for line in output_stream.getvalue().splitlines()] + assert len(responses) == 1 + assert responses[0]["outcome"]["error"]["code"] == ( + "lifecycle_adapter_start_failed" + ) + assert stopped == [True] diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index 22d388f6..f575fe0b 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -7,16 +7,21 @@ import json import os import tomllib +from collections.abc import AsyncIterator +from collections.abc import Callable from pathlib import Path from typing import Any +from unittest.mock import AsyncMock from unittest.mock import MagicMock import pytest from claude_agent_sdk import AssistantMessage +from claude_agent_sdk import ClaudeAgentOptions from claude_agent_sdk import ClaudeSDKError from claude_agent_sdk import CLIConnectionError from claude_agent_sdk import CLIJSONDecodeError from claude_agent_sdk import CLINotFoundError +from claude_agent_sdk import Message from claude_agent_sdk import ProcessError from claude_agent_sdk import ResultMessage from claude_agent_sdk import SystemMessage @@ -39,6 +44,36 @@ } +def lifecycle_invocation(payload: dict[str, Any]) -> dict[str, Any]: + return { + "runtime_context": payload["runtime_context"], + "request": payload["request"], + } + + +def install_fake_client( + monkeypatch: pytest.MonkeyPatch, + response_factory: Callable[[MagicMock], AsyncIterator[Message]], +) -> list[MagicMock]: + clients: list[MagicMock] = [] + client_type = adapter.ClaudeSDKClient + + def build_client(options: ClaudeAgentOptions) -> MagicMock: + client = MagicMock(spec=client_type) + client.options = options + client.prompts = [] + client.connect = AsyncMock() + client.query = AsyncMock(side_effect=client.prompts.append) + client.receive_response.side_effect = lambda: response_factory(client) + client.disconnect = AsyncMock() + client.interrupt = AsyncMock() + clients.append(client) + return client + + monkeypatch.setattr(adapter, "ClaudeSDKClient", build_client) + return clients + + def test_claude_descriptor_is_narrow_and_versioned(): descriptor_path = ROOT / "adapters" / "claude" / "fabric-adapter.json" descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) @@ -50,7 +85,6 @@ def test_claude_descriptor_is_narrow_and_versioned(): "adapter_kind": "python", "runner": { "module": "nemo_fabric_adapters.claude.adapter", - "callable": "run", }, "config": { "accepts": [ @@ -84,27 +118,27 @@ def claude_payload_fixture(tmp_path) -> dict[str, Any]: "agent_name": "claude-test", "base_dir": str(tmp_path), "config": { - "harness": { - "adapter_id": "nvidia.fabric.claude", - "settings": { - "system_prompt": "Review carefully.", - "allowed_tools": ["Read"], - "permission_mode": "dontAsk", - "max_turns": 4, - "max_budget_usd": 1.5, - "setting_sources": [], - "timeout_seconds": 30, - "env": {"ANTHROPIC_API_KEY": "configured-secret"}, - }, - }, - "models": { - "default": { - "provider": "anthropic", - "model": "anthropic/claude-test-model", - "api_key_env": "ANTHROPIC_API_KEY", - } + "harness": { + "adapter_id": "nvidia.fabric.claude", + "settings": { + "system_prompt": "Review carefully.", + "allowed_tools": ["Read"], + "permission_mode": "dontAsk", + "max_turns": 4, + "max_budget_usd": 1.5, + "setting_sources": [], + "timeout_seconds": 30, + "env": {"ANTHROPIC_API_KEY": "configured-secret"}, }, - "tools": {"blocked": ["Bash"]}, + }, + "models": { + "default": { + "provider": "anthropic", + "model": "anthropic/claude-test-model", + "api_key_env": "ANTHROPIC_API_KEY", + } + }, + "tools": {"blocked": ["Bash"]}, }, "runtime_context": { "runtime_id": "runtime-claude-1", @@ -135,9 +169,7 @@ def claude_payload_fixture(tmp_path) -> dict[str, Any]: def test_build_options_maps_normalized_capabilities_and_claude_settings(claude_payload): - options = adapter.build_options(claude_payload, resume="claude-session") - - assert options.resume == "claude-session" + options = adapter.build_options(claude_payload) assert options.cwd == Path( claude_payload["runtime_context"]["environment"]["workspace"] ) @@ -295,7 +327,7 @@ def test_build_options_adds_relay_plugin_and_gateway_environment( ) relay = adapter.prepare_claude_relay(relay_payload) - options = adapter.build_options(relay_payload, resume=None, relay=relay) + options = adapter.build_options(relay_payload, relay=relay) assert options.env["NEMO_RELAY_GATEWAY_URL"] == relay.gateway.url assert options.env["ANTHROPIC_BASE_URL"] == relay.gateway.url @@ -322,7 +354,7 @@ def test_build_options_does_not_enable_skills_for_relay_plugin_alone( plugin_path=tmp_path / "relay-plugin", ) - options = adapter.build_options(relay_payload, resume=None, relay=relay) + options = adapter.build_options(relay_payload, relay=relay) assert options.tools is None assert options.skills is None @@ -330,11 +362,9 @@ def test_build_options_does_not_enable_skills_for_relay_plugin_alone( def test_build_options_maps_blocked_tools_to_disallowed_tools(claude_payload): - claude_payload["config"]["tools"] = { - "blocked": ["Bash", "WebFetch"] - } + claude_payload["config"]["tools"] = {"blocked": ["Bash", "WebFetch"]} - options = adapter.build_options(claude_payload, resume=None) + options = adapter.build_options(claude_payload) assert options.tools is None assert options.disallowed_tools == ["Bash", "WebFetch"] @@ -359,7 +389,7 @@ def test_build_options_rejects_normalized_capabilities_in_harness_settings( with pytest.raises( adapter.AdapterConfigError, match=normalized_field.replace(".", r"\.") ): - adapter.build_options(claude_payload, resume=None) + adapter.build_options(claude_payload) def test_build_options_rejects_skill_path_without_skill_manifest(claude_payload): @@ -367,7 +397,7 @@ def test_build_options_rejects_skill_path_without_skill_manifest(claude_payload) (skill_path / "SKILL.md").unlink() with pytest.raises(adapter.AdapterConfigError, match="SKILL.md"): - adapter.build_options(claude_payload, resume=None) + adapter.build_options(claude_payload) def test_build_options_maps_nvidia_provider_to_claude_gateway_environment( @@ -384,7 +414,7 @@ def test_build_options_maps_nvidia_provider_to_claude_gateway_environment( ) os.environ["NVIDIA_API_KEY"] = "nvidia-secret" - options = adapter.build_options(claude_payload, resume=None) + options = adapter.build_options(claude_payload) assert options.model == "aws/anthropic/claude-opus-4-5" assert options.env["ANTHROPIC_BASE_URL"] == "https://nvidia.example" @@ -407,7 +437,7 @@ def test_build_options_uses_nvidia_provider_endpoint_and_default_credential( os.environ["NVIDIA_API_KEY"] = "nvidia-secret" os.environ["NVIDIA_FRONTIER_BASE_URL"] = "https://frontier.example/v1" - options = adapter.build_options(claude_payload, resume=None) + options = adapter.build_options(claude_payload) assert options.env["ANTHROPIC_BASE_URL"] == "https://frontier.example" assert options.env["ANTHROPIC_API_KEY"] == "nvidia-secret" @@ -427,7 +457,7 @@ def test_build_options_requires_nvidia_provider_endpoint(claude_payload): os.environ.pop("NVIDIA_FRONTIER_BASE_URL", None) with pytest.raises(adapter.AdapterConfigError, match="NVIDIA_FRONTIER_BASE_URL"): - adapter.build_options(claude_payload, resume=None) + adapter.build_options(claude_payload) def test_build_options_requires_nvidia_provider_credential(claude_payload): @@ -442,7 +472,7 @@ def test_build_options_requires_nvidia_provider_credential(claude_payload): os.environ.pop("NVIDIA_API_KEY", None) with pytest.raises(adapter.AdapterConfigError, match="NVIDIA_API_KEY is required"): - adapter.build_options(claude_payload, resume=None) + adapter.build_options(claude_payload) def test_selected_model_rejects_unsupported_provider(claude_payload): @@ -455,28 +485,6 @@ def test_selected_model_rejects_unsupported_provider(claude_payload): adapter.selected_model(claude_payload) -def test_state_round_trip_is_keyed_by_fabric_runtime(claude_payload): - runtime_id = adapter.runtime_id(claude_payload) - adapter.save_claude_session_id(claude_payload, runtime_id, "claude-session") - - assert ( - adapter.load_claude_session_id(claude_payload, runtime_id) == "claude-session" - ) - state_path = adapter.runtime_state_path(claude_payload, runtime_id) - assert state_path.parent.name == "runtimes" - assert runtime_id not in state_path.name - - -def test_state_loader_rejects_non_object_json(claude_payload): - runtime_id = adapter.runtime_id(claude_payload) - state_path = adapter.runtime_state_path(claude_payload, runtime_id) - state_path.parent.mkdir(parents=True) - state_path.write_text("[]", encoding="utf-8") - - with pytest.raises(adapter.AdapterStateError, match="runtime state is invalid"): - adapter.load_claude_session_id(claude_payload, runtime_id) - - def test_normalize_result_exposes_session_usage_cost_and_buffered_events( claude_payload, ): @@ -514,54 +522,148 @@ def test_normalize_result_exposes_session_usage_cost_and_buffered_events( ] -async def test_run_claude_resumes_and_persists_session(claude_payload, monkeypatch): - captured = [] +async def test_claude_runtime_reuses_one_connected_sdk_client( + claude_payload, monkeypatch +): + clients = [] - async def query_result(*, prompt, options): - captured.append((prompt, options.resume, dict(options.env), dict(os.environ))) - yield AssistantMessage( - content=[TextBlock(text="done")], model="claude-test-model" - ) - yield ResultMessage( - subtype="success", - duration_ms=100, - duration_api_ms=80, - is_error=False, - num_turns=1, - session_id="claude-session", - total_cost_usd=0.02, - usage={"input_tokens": 1, "output_tokens": 1}, - result="done", - ) + class FakeClient: + def __init__(self, options): + self.options = options + self.connect_count = 0 + self.disconnect_count = 0 + self.prompts = [] + clients.append(self) - mock_query = MagicMock(side_effect=query_result) - monkeypatch.setattr(adapter, "query", mock_query) - monkeypatch.setenv("FABRIC_UNRELATED_SECRET", "do-not-forward") + async def connect(self): + self.connect_count += 1 - first = await adapter.run_claude(claude_payload) - claude_payload["runtime_context"]["invocation_id"] = "invocation-2" - second = await adapter.run_claude(claude_payload) + async def query(self, prompt): + self.prompts.append(prompt) - assert first["failed"] is False - assert second["failed"] is False - assert [entry[0] for entry in captured] == [ - "Inspect the patch", - "Inspect the patch", - ] - assert [entry[1] for entry in captured] == [None, "claude-session"] - assert all(entry[2]["FABRIC_UNRELATED_SECRET"] == "" for entry in captured) - assert all( - entry[2]["ANTHROPIC_API_KEY"] == "configured-secret" for entry in captured - ) - assert all( - entry[3]["FABRIC_UNRELATED_SECRET"] == "do-not-forward" for entry in captured - ) - assert os.environ["FABRIC_UNRELATED_SECRET"] == "do-not-forward" + async def receive_response(self): + yield AssistantMessage( + content=[TextBlock(text="done")], model="claude-test-model" + ) + yield ResultMessage( + subtype="success", + duration_ms=100, + duration_api_ms=80, + is_error=False, + num_turns=1, + session_id="claude-session", + total_cost_usd=0.02, + usage={"input_tokens": 1, "output_tokens": 1}, + result=f"done-{len(self.prompts)}", + ) + async def disconnect(self): + self.disconnect_count += 1 -async def test_run_claude_supervises_relay_and_reports_artifacts( + async def interrupt(self): + raise AssertionError("successful invocation must not be interrupted") + + monkeypatch.setattr(adapter, "ClaudeSDKClient", FakeClient) + + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(start_payload) + first = await runtime.invoke(lifecycle_invocation(claude_payload)) + claude_payload["runtime_context"]["invocation_id"] = "invocation-2" + claude_payload["request"]["input"] = {"not": "text"} + invalid = await runtime.invoke(lifecycle_invocation(claude_payload)) + claude_payload["runtime_context"]["invocation_id"] = "invocation-3" + claude_payload["request"]["input"] = "Inspect the tests" + second = await runtime.invoke(lifecycle_invocation(claude_payload)) + await runtime.stop() + + assert len(clients) == 1 + assert clients[0].connect_count == 1 + assert clients[0].disconnect_count == 1 + assert clients[0].prompts == ["Inspect the patch", "Inspect the tests"] + assert first["response"] == "done-1" + assert second["response"] == "done-2" + assert invalid["error"]["code"] == "claude_invalid_request" + assert first["session_id"] == second["session_id"] == "claude-session" + + +async def test_claude_runtime_owns_one_relay_gateway_until_stop( relay_payload, monkeypatch, tmp_path ): + relay = adapter.ClaudeRelaySettings( + gateway=adapter.relay_gateway.RelayGatewayLaunch( + executable=tmp_path / "nemo-relay", + config_path=tmp_path / "relay-config" / "config.toml", + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "relay-config" / "gateway.log", + ), + plugin_config={"version": 1, "components": []}, + plugin_path=tmp_path / "relay-plugin", + ) + relay.plugin_path.mkdir() + process = MagicMock() + mock_start = MagicMock(return_value=process) + mock_stop = MagicMock() + + class FakeClient: + def __init__(self, options): + self.options = options + + async def connect(self): + pass + + async def query(self, _prompt): + pass + + async def receive_response(self): + yield ResultMessage( + subtype="success", + duration_ms=10, + duration_api_ms=8, + is_error=False, + num_turns=1, + session_id="claude-session", + total_cost_usd=0.01, + usage={"input_tokens": 1, "output_tokens": 1}, + result="done", + ) + + async def disconnect(self): + pass + + async def interrupt(self): + raise AssertionError("successful invocation must not be interrupted") + + monkeypatch.setattr(adapter, "ClaudeSDKClient", FakeClient) + monkeypatch.setattr(adapter, "prepare_claude_relay", MagicMock(return_value=relay)) + monkeypatch.setattr(adapter.relay_gateway, "start_relay_gateway", mock_start) + monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", mock_stop) + + start_payload = dict(relay_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(start_payload) + first = await runtime.invoke(lifecycle_invocation(relay_payload)) + relay_payload["runtime_context"]["invocation_id"] = "invocation-2" + second = await runtime.invoke(lifecycle_invocation(relay_payload)) + + mock_start.assert_called_once_with( + launch=relay.gateway, + cwd=Path(relay_payload["runtime_context"]["environment"]["workspace"]), + ) + mock_stop.assert_not_called() + assert first["relay_runtime"]["gateway_url"] == relay.gateway.url + assert second["relay_runtime"]["gateway_url"] == relay.gateway.url + + await runtime.stop() + + mock_stop.assert_called_once_with(process) + assert not relay.plugin_path.exists() + + +async def test_runtime_reports_relay_artifacts(relay_payload, monkeypatch, tmp_path): executable = tmp_path / "nemo-relay" executable.touch() relay = adapter.ClaudeRelaySettings( @@ -602,9 +704,9 @@ async def test_run_claude_supervises_relay_and_reports_artifacts( monkeypatch.setattr(adapter.relay_gateway, "start_relay_gateway", mock_start) monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", mock_stop) - async def query_result(*, prompt, options): - assert options.env["ANTHROPIC_BASE_URL"] == relay.gateway.url - assert Path(options.plugins[-1]["path"]) == relay.plugin_path + async def responses(client) -> AsyncIterator[ResultMessage]: + assert client.options.env["ANTHROPIC_BASE_URL"] == relay.gateway.url + assert Path(client.options.plugins[-1]["path"]) == relay.plugin_path yield ResultMessage( subtype="success", duration_ms=10, @@ -617,9 +719,14 @@ async def query_result(*, prompt, options): result="done", ) - monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_result)) - - output = await adapter.run_claude(relay_payload) + install_fake_client(monkeypatch, responses) + runtime = adapter.ClaudeRuntime() + start_payload = { + key: value for key, value in relay_payload.items() if key != "request" + } + await runtime.start(start_payload) + output = await runtime.invoke(lifecycle_invocation(relay_payload)) + await runtime.stop() assert output["relay_runtime"] == { "enabled": True, @@ -638,7 +745,7 @@ async def query_result(*, prompt, options): assert not relay.plugin_path.exists() -async def test_run_claude_preserves_result_when_relay_stop_fails( +async def test_runtime_stop_reports_relay_gateway_failure( relay_payload, monkeypatch, tmp_path ): relay = adapter.ClaudeRelaySettings( @@ -667,7 +774,7 @@ async def test_run_claude_preserves_result_when_relay_stop_fails( ), ) - async def query_result(*, prompt, options): + async def responses(_client) -> AsyncIterator[ResultMessage]: yield ResultMessage( subtype="success", duration_ms=10, @@ -680,25 +787,25 @@ async def query_result(*, prompt, options): result="done", ) - monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_result)) - - output = await adapter.run_claude(relay_payload) + install_fake_client(monkeypatch, responses) + runtime = adapter.ClaudeRuntime() + start_payload = { + key: value for key, value in relay_payload.items() if key != "request" + } + await runtime.start(start_payload) + output = await runtime.invoke(lifecycle_invocation(relay_payload)) + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.stop() assert output["response"] == "done" - assert output["completed"] is False - assert output["failed"] is True - assert output["error"] == { - "code": "claude_relay_stop_failed", - "message": "NeMo Relay gateway failed to stop", - "retryable": False, - "metadata": {"gateway_log_path": str(relay.gateway.log_path)}, - } - assert output["relay_runtime"]["cleanup_error"] == output["error"] - assert "raw stop failure" not in json.dumps(output) + assert output["completed"] is True + assert caught.value.code == "claude_relay_stop_failed" + assert caught.value.metadata == {"gateway_log_path": str(relay.gateway.log_path)} + assert "raw stop failure" not in str(caught.value) assert not relay.plugin_path.exists() -async def test_run_claude_preserves_result_when_relay_plugin_cleanup_fails( +async def test_runtime_stop_reports_relay_plugin_cleanup_failure( relay_payload, monkeypatch, tmp_path ): relay = adapter.ClaudeRelaySettings( @@ -725,7 +832,7 @@ async def test_run_claude_preserves_result_when_relay_plugin_cleanup_fails( monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", mock_stop) monkeypatch.setattr(adapter.shutil, "rmtree", mock_rmtree) - async def query_result(**_): + async def responses(_client) -> AsyncIterator[ResultMessage]: yield ResultMessage( subtype="success", duration_ms=10, @@ -738,20 +845,20 @@ async def query_result(**_): result="done", ) - monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_result)) - - output = await adapter.run_claude(relay_payload) + install_fake_client(monkeypatch, responses) + runtime = adapter.ClaudeRuntime() + start_payload = { + key: value for key, value in relay_payload.items() if key != "request" + } + await runtime.start(start_payload) + output = await runtime.invoke(lifecycle_invocation(relay_payload)) + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.stop() assert output["response"] == "done" - assert output["completed"] is False - assert output["failed"] is True - assert output["error"] == { - "code": "claude_relay_cleanup_failed", - "message": "Claude Relay hook configuration could not be removed", - "retryable": False, - } - assert output["relay_runtime"]["cleanup_error"] == output["error"] - assert "raw plugin cleanup failure" not in json.dumps(output) + assert output["completed"] is True + assert caught.value.code == "claude_relay_cleanup_failed" + assert "raw plugin cleanup failure" not in str(caught.value) mock_stop.assert_called_once_with(process) mock_rmtree.assert_called_once_with(relay.plugin_path) assert relay.plugin_path.exists() @@ -760,7 +867,7 @@ async def query_result(**_): @pytest.mark.parametrize( "failure", [ClaudeSDKError("sdk failed"), asyncio.CancelledError()] ) -async def test_run_claude_stops_relay_on_sdk_failure_or_cancellation( +async def test_runtime_stops_relay_after_sdk_failure_or_cancellation( relay_payload, monkeypatch, tmp_path, failure ): relay = adapter.ClaudeRelaySettings( @@ -785,19 +892,26 @@ async def test_run_claude_stops_relay_on_sdk_failure_or_cancellation( ) monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", mock_stop) - async def query_failure(*, prompt, options): + async def responses(_client) -> AsyncIterator[ResultMessage]: raise failure yield - monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_failure)) - - if isinstance(failure, asyncio.CancelledError): - with pytest.raises(asyncio.CancelledError): - await adapter.run_claude(relay_payload) - else: - output = await adapter.run_claude(relay_payload) - assert output["error"]["code"] == "claude_failed" - assert output["relay_runtime"]["enabled"] is True + install_fake_client(monkeypatch, responses) + runtime = adapter.ClaudeRuntime() + start_payload = { + key: value for key, value in relay_payload.items() if key != "request" + } + await runtime.start(start_payload) + try: + if isinstance(failure, asyncio.CancelledError): + with pytest.raises(asyncio.CancelledError): + await runtime.invoke(lifecycle_invocation(relay_payload)) + else: + output = await runtime.invoke(lifecycle_invocation(relay_payload)) + assert output["error"]["code"] == "claude_failed" + assert output["relay_runtime"]["enabled"] is True + finally: + await runtime.stop() mock_stop.assert_called_once_with(process) assert not relay.plugin_path.exists() @@ -807,14 +921,14 @@ async def query_failure(*, prompt, options): ("subtype", "is_error"), [("success", True), ("error_max_budget_usd", False)], ) -async def test_run_claude_preserves_failed_result_when_sdk_stream_raises( +async def test_runtime_preserves_failed_result_when_sdk_stream_raises( claude_payload, monkeypatch, caplog, subtype, is_error, ): - async def query_error_result(**_): + async def responses(_client) -> AsyncIterator[ResultMessage]: yield ResultMessage( subtype=subtype, duration_ms=10, @@ -826,9 +940,14 @@ async def query_error_result(**_): ) raise RuntimeError("raw SDK stream error") - monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_error_result)) - - output = await adapter.run_claude(claude_payload) + install_fake_client(monkeypatch, responses) + runtime = adapter.ClaudeRuntime() + start_payload = { + key: value for key, value in claude_payload.items() if key != "request" + } + await runtime.start(start_payload) + output = await runtime.invoke(lifecycle_invocation(claude_payload)) + await runtime.stop() assert output["response"] == "Not logged in" assert output["error"] == { @@ -841,7 +960,7 @@ async def query_error_result(**_): assert "raw SDK stream error" in caplog.text -def test_run_reports_relay_start_failure_without_raw_diagnostic( +async def test_runtime_start_reports_relay_failure_without_raw_diagnostic( relay_payload, monkeypatch, tmp_path ): executable = tmp_path / "nemo-relay" @@ -869,15 +988,17 @@ def test_run_reports_relay_start_failure_without_raw_diagnostic( ), ) - output = adapter.run(relay_payload) - - assert output["error"] == { - "code": "claude_relay_start_failed", - "message": "NeMo Relay gateway failed to start", - "retryable": False, - "metadata": {"gateway_log_path": str(relay.gateway.log_path)}, + runtime = adapter.ClaudeRuntime() + start_payload = { + key: value for key, value in relay_payload.items() if key != "request" } - assert "secret" not in json.dumps(output) + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.start(start_payload) + + assert caught.value.code == "claude_relay_start_failed" + assert caught.value.message == "NeMo Relay gateway failed to start" + assert caught.value.metadata == {"gateway_log_path": str(relay.gateway.log_path)} + assert "secret" not in str(caught.value) assert not relay.plugin_path.exists() @@ -926,7 +1047,7 @@ def test_build_options_forwards_anthropic_auth_environment( os.environ["FABRIC_UNRELATED_SECRET"] = "do-not-forward" os.environ.update(auth_environment) - options = adapter.build_options(claude_payload, resume=None) + options = adapter.build_options(claude_payload) forwarded_auth_environment = { name: options.env[name] @@ -942,7 +1063,7 @@ def test_build_options_preserves_unix_user_for_cached_login( ): os.environ["USER"] = "fabric-user" - options = adapter.build_options(claude_payload, resume=None) + options = adapter.build_options(claude_payload) assert options.env["USER"] == "fabric-user" @@ -1012,29 +1133,10 @@ def test_error_subtype_is_failure_when_sdk_flag_is_false(claude_payload): assert output["error"]["metadata"] == {"subtype": "error_max_budget_usd"} -def test_run_normalizes_unexpected_exception(claude_payload, monkeypatch): - monkeypatch.setattr( - adapter, - "run_claude", - MagicMock(side_effect=RuntimeError("secret")), - ) - - output = adapter.run(claude_payload) - - assert output["error"]["code"] == "claude_adapter_internal_error" - assert "secret" not in json.dumps(output) - - -def test_main_normalizes_payload_load_failure(monkeypatch, capsys): - monkeypatch.setattr( - adapter.common_utils, - "load_payload", - MagicMock(side_effect=ValueError("secret payload")), - ) +def test_main_serves_persistent_runtime(monkeypatch): + serve = MagicMock() + monkeypatch.setattr(adapter.lifecycle, "serve", serve) - with pytest.raises(SystemExit, match="2"): - adapter.main() + adapter.main() - output = json.loads(capsys.readouterr().out) - assert output["error"]["code"] == "claude_adapter_internal_error" - assert "secret" not in json.dumps(output) + serve.assert_called_once_with(adapter.ClaudeRuntime) diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 4b8913cc..ea977d23 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -6,6 +6,7 @@ import os from pathlib import Path from types import SimpleNamespace +from typing import Any from unittest.mock import AsyncMock from unittest.mock import MagicMock @@ -16,6 +17,40 @@ from openai_codex.types import TurnStatus +def lifecycle_start_payload(payload): + return {key: value for key, value in payload.items() if key != "request"} + + +def lifecycle_invocation(payload): + return { + "runtime_context": payload["runtime_context"], + "request": payload["request"], + } + + +async def invoke_once_async(payload): + runtime = adapter.CodexRuntime() + await runtime.start(lifecycle_start_payload(payload)) + try: + return await runtime.invoke(lifecycle_invocation(payload)) + finally: + await runtime.stop() + + +def invoke_once(payload): + return asyncio.run(invoke_once_async(payload)) + + +def runtime_start_error(payload): + async def scenario() -> adapter.lifecycle.LifecycleError: + runtime = adapter.CodexRuntime() + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.start(lifecycle_start_payload(payload)) + return caught.value + + return asyncio.run(scenario()) + + @pytest.fixture(name="codex_payload") def codex_payload_fixture(tmp_path): workspace = tmp_path / "workspace" @@ -24,23 +59,23 @@ def codex_payload_fixture(tmp_path): "agent_name": "codex-test", "base_dir": str(tmp_path), "config": { - "harness": { - "adapter_id": "nvidia.fabric.codex", - "settings": { - "sandbox": "workspace-write", - "config_overrides": { - "features.web_search": False, - "model_reasoning_effort": "high", - }, + "harness": { + "adapter_id": "nvidia.fabric.codex", + "settings": { + "sandbox": "workspace-write", + "config_overrides": { + "features.web_search": False, + "model_reasoning_effort": "high", }, }, - "models": { - "default": { - "provider": "openai", - "model": "openai/gpt-5.4", - } - }, - "runtime": {}, + }, + "models": { + "default": { + "provider": "openai", + "model": "openai/gpt-5.4", + } + }, + "runtime": {}, }, "runtime_context": { "runtime_id": "runtime-1", @@ -105,8 +140,8 @@ def mock_codex_fixture(monkeypatch): mock_codex.next_thread_id = "thread-123" mock_codex.next_result = None mock_codex.next_thread = None - mock_codex.resume_thread_id = None mock_codex.skill_request = AsyncMock() + mock_codex.close_error = None def build_client(*, config): mock_client = MagicMock(spec=AsyncCodex) @@ -116,6 +151,8 @@ def build_client(*, config): mock_client._client = SimpleNamespace(request=mock_codex.skill_request) async def close(): + if mock_codex.close_error is not None: + raise mock_codex.close_error mock_client.closed = True async def thread_start(**_kwargs): @@ -126,16 +163,8 @@ async def thread_start(**_kwargs): ) return mock_client.thread - async def thread_resume(thread_id, **_kwargs): - resumed_thread_id = mock_codex.resume_thread_id or thread_id - mock_client.thread = mock_thread( - resumed_thread_id, mock_codex.next_result - ) - return mock_client.thread - mock_client.close.side_effect = close mock_client.thread_start.side_effect = thread_start - mock_client.thread_resume.side_effect = thread_resume mock_codex.instances.append(mock_client) return mock_client @@ -144,7 +173,7 @@ async def thread_resume(thread_id, **_kwargs): return mock_codex -def test_sdk_oneshot_uses_native_thread_and_turn_contract( +def test_single_invocation_uses_native_thread_and_turn_contract( codex_payload, mock_codex, tmp_path ): os.environ["CODEX_HOME"] = str(tmp_path / "codex-home") @@ -154,7 +183,7 @@ def test_sdk_oneshot_uses_native_thread_and_turn_contract( "CODEX_EXPLICIT": "forward-me" } - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["completed"] is True assert output["adapter"] == "sdk" @@ -175,10 +204,7 @@ def test_sdk_oneshot_uses_native_thread_and_turn_contract( ) assert client.config.env["CODEX_HOME"] == str(tmp_path / "codex-home") assert client.config.env["CODEX_EXPLICIT"] == "forward-me" - assert ( - client.config.env["CODEX_INTERNAL_ORIGINATOR_OVERRIDE"] - == "codex_python_sdk" - ) + assert client.config.env["CODEX_INTERNAL_ORIGINATOR_OVERRIDE"] == "codex_python_sdk" assert client.config.env["FABRIC_UNRELATED_SECRET"] == "" start = client.thread_start.await_args.kwargs assert start["model"] == "gpt-5.4" @@ -195,6 +221,58 @@ def test_sdk_oneshot_uses_native_thread_and_turn_contract( client._client.request.assert_not_awaited() +def test_runtime_stop_reports_close_failure_after_completed_turn( + codex_payload, mock_codex, caplog +): + mock_codex.close_error = RuntimeError("close failed") + + async def scenario() -> tuple[dict[str, Any], adapter.lifecycle.LifecycleError]: + runtime = adapter.CodexRuntime() + await runtime.start(lifecycle_start_payload(codex_payload)) + output = await runtime.invoke(lifecycle_invocation(codex_payload)) + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.stop() + return output, caught.value + + output, error = asyncio.run(scenario()) + + assert output["completed"] is True + assert output["failed"] is False + assert output["thread_id"] == "thread-123" + assert output["response"] == "done" + assert output["error"] is None + assert error.code == "codex_sdk_stop_failed" + assert "Codex SDK client failed to close" in caplog.text + mock_codex.instances[0].close.assert_awaited_once_with() + + +def test_start_failure_is_not_masked_by_sdk_close_failure( + codex_payload, mock_codex, monkeypatch, caplog +): + mock_codex.close_error = RuntimeError("close failed") + register_skills = AsyncMock( + side_effect=adapter.AdapterConfigError( + "codex_skill_registration_failed", + "Codex skill registration failed", + ) + ) + monkeypatch.setattr(adapter, "_register_skill_roots", register_skills) + + async def scenario() -> adapter.lifecycle.LifecycleError: + runtime = adapter.CodexRuntime() + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.start(lifecycle_start_payload(codex_payload)) + return caught.value + + error = asyncio.run(scenario()) + + assert error.code == "codex_skill_registration_failed" + assert error.message == "Codex skill registration failed" + assert "Codex SDK cleanup after start failure also failed" in caplog.text + register_skills.assert_awaited_once() + mock_codex.instances[0].close.assert_awaited_once_with() + + def test_sdk_maps_native_mcp_servers_into_thread_config(codex_payload, mock_codex): os.environ["FABRIC_TEST_MCP_URL"] = "https://mcp.example.test/mcp" codex_payload["capability_plan"] = { @@ -211,11 +289,11 @@ def test_sdk_maps_native_mcp_servers_into_thread_config(codex_payload, mock_code } } } - codex_payload["config"]["harness"]["settings"][ - "config_overrides" - ]["mcp_servers.remote.required"] = True + codex_payload["config"]["harness"]["settings"]["config_overrides"][ + "mcp_servers.remote.required" + ] = True - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["completed"] is True config = mock_codex.instances[0].thread_start.await_args.kwargs["config"] @@ -244,7 +322,7 @@ def test_sdk_registers_native_skill_roots(codex_payload, mock_codex, tmp_path): "native": {"skill_paths": ["skills/review", "skills/test"]} } - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["completed"] is True mock_codex.instances[0].thread.turn.assert_awaited_once_with( @@ -269,14 +347,12 @@ def test_sdk_closes_when_skill_registration_is_unavailable( "---\nname: review\ndescription: Test skill.\n---\n", encoding="utf-8", ) - codex_payload["capability_plan"] = { - "native": {"skill_paths": ["skills/review"]} - } + codex_payload["capability_plan"] = {"native": {"skill_paths": ["skills/review"]}} mock_codex.skill_request = None - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" + assert error.code == "codex_invalid_configuration" client = mock_codex.instances[0] client.thread_start.assert_not_awaited() assert client.closed is True @@ -292,10 +368,10 @@ def test_sdk_rejects_unsupported_mcp_transport(codex_payload, mock_codex, transp } } - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert f"unsupported Codex MCP transport: {transport}" in output["error"]["message"] + assert error.code == "codex_invalid_configuration" + assert f"unsupported Codex MCP transport: {transport}" in error.message mock_codex.assert_not_called() @@ -303,25 +379,21 @@ def test_sdk_rejects_invalid_native_skill_path(codex_payload, mock_codex, tmp_pa missing = tmp_path / "skills" / "missing" codex_payload["capability_plan"] = {"native": {"skill_paths": [str(missing)]}} - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert "directory containing SKILL.md" in output["error"]["message"] + assert error.code == "codex_invalid_configuration" + assert "directory containing SKILL.md" in error.message mock_codex.assert_not_called() @pytest.mark.parametrize("skill_paths", [None, "", {}, False]) -def test_sdk_rejects_falsy_non_list_skill_paths( - codex_payload, mock_codex, skill_paths -): - codex_payload["capability_plan"] = { - "native": {"skill_paths": skill_paths} - } +def test_sdk_rejects_falsy_non_list_skill_paths(codex_payload, mock_codex, skill_paths): + codex_payload["capability_plan"] = {"native": {"skill_paths": skill_paths}} - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert output["error"]["message"] == "native skill_paths must be a list of paths" + assert error.code == "codex_invalid_configuration" + assert error.message == "native skill_paths must be a list of paths" mock_codex.assert_not_called() @@ -329,23 +401,17 @@ def test_sdk_can_use_an_explicit_codex_runtime(codex_payload, mock_codex, tmp_pa codex_bin = tmp_path / "bin" / "codex" codex_bin.parent.mkdir() codex_bin.touch() - codex_payload["config"]["harness"]["settings"][ - "codex_bin" - ] = str(codex_bin) + codex_payload["config"]["harness"]["settings"]["codex_bin"] = str(codex_bin) - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["completed"] is True assert mock_codex.instances[0].config.codex_bin == str(codex_bin) @pytest.mark.parametrize("codex_bin", ["bin/codex", "~/bin/codex"]) -def test_sdk_resolves_relative_codex_runtime_from_base_dir( - codex_payload, codex_bin -): - codex_payload["config"]["harness"]["settings"][ - "codex_bin" - ] = codex_bin +def test_sdk_resolves_relative_codex_runtime_from_base_dir(codex_payload, codex_bin): + codex_payload["config"]["harness"]["settings"]["codex_bin"] = codex_bin config = adapter.sdk_config(codex_payload, relay=None) @@ -355,46 +421,120 @@ def test_sdk_resolves_relative_codex_runtime_from_base_dir( def test_sdk_keeps_absolute_codex_runtime_path(codex_payload, tmp_path): codex_bin = tmp_path / "bin" / ".." / "codex" - codex_payload["config"]["harness"]["settings"][ - "codex_bin" - ] = str(codex_bin) + codex_payload["config"]["harness"]["settings"]["codex_bin"] = str(codex_bin) config = adapter.sdk_config(codex_payload, relay=None) assert config.codex_bin == str(codex_bin) -def test_runtime_resumes_sdk_thread_across_invocations( +async def test_persistent_runtime_reuses_one_client_and_thread( codex_payload, mock_codex ): - first = adapter.run(codex_payload) + start_payload = dict(codex_payload) + start_payload.pop("request") + runtime = adapter.CodexRuntime() + + await runtime.start(start_payload) + first = await runtime.invoke(lifecycle_invocation(codex_payload)) codex_payload["runtime_context"]["invocation_id"] = "invocation-2" codex_payload["request"]["input"] = "Continue." - second = adapter.run(codex_payload) + second = await runtime.invoke(lifecycle_invocation(codex_payload)) + await runtime.stop() assert first["thread_id"] == second["thread_id"] == "thread-123" - mock_codex.instances[0].thread_start.assert_awaited_once() - assert mock_codex.instances[1].thread_resume.await_args.args[0] == "thread-123" - assert mock_codex.instances[1].thread.turn.await_args.args[0] == "Continue." - state = json.loads( - adapter.runtime_state_path(codex_payload, "runtime-1").read_text( - encoding="utf-8" - ) + assert len(mock_codex.instances) == 1 + client = mock_codex.instances[0] + client.thread_start.assert_awaited_once() + assert client.thread.turn.await_count == 2 + assert client.thread.turn.await_args_list[1].args[0] == "Continue." + client.close.assert_awaited_once() + + +async def test_persistent_runtime_registers_skills_once_and_maps_mcp( + codex_payload, mock_codex, tmp_path +): + skill = tmp_path / "skills" / "review" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text( + "---\nname: review\ndescription: Test skill.\n---\n", + encoding="utf-8", + ) + codex_payload["capability_plan"] = { + "native": { + "skill_paths": ["skills/review"], + "mcp_servers": { + "review": { + "transport": "streamable-http", + "url": "https://mcp.example.test/review", + } + }, + } + } + start_payload = dict(codex_payload) + start_payload.pop("request") + runtime = adapter.CodexRuntime() + + await runtime.start(start_payload) + await runtime.invoke(lifecycle_invocation(codex_payload)) + codex_payload["runtime_context"]["invocation_id"] = "invocation-2" + await runtime.invoke(lifecycle_invocation(codex_payload)) + await runtime.stop() + + client = mock_codex.instances[0] + client.models.assert_awaited_once_with() + client._client.request.assert_awaited_once_with( + "skills/extraRoots/set", + {"extraRoots": [str(skill)]}, + response_model=adapter.SkillsExtraRootsSetResponse, ) - assert state == { - "runtime_id": "runtime-1", - "codex_thread_id": "thread-123", + assert client.thread_start.await_args.kwargs["config"]["mcp_servers"] == { + "review": {"url": "https://mcp.example.test/review"} } + assert client.thread.turn.await_count == 2 -def test_runtime_rejects_corrupt_thread_state(codex_payload): - state_path = adapter.runtime_state_path(codex_payload, "runtime-1") - state_path.parent.mkdir(parents=True) - state_path.write_text("{", encoding="utf-8") +async def test_persistent_runtime_owns_one_relay_gateway( + codex_payload, mock_codex, monkeypatch, tmp_path +): + codex_payload["telemetry_plan"] = { + "providers": ["relay"], + "relay_enabled": True, + } + gateway = adapter.relay_gateway.RelayGatewayLaunch( + executable=tmp_path / "nemo-relay", + config_path=tmp_path / "relay" / "config.toml", + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "relay" / "gateway.log", + ) + relay = adapter.CodexRelaySettings( + gateway=gateway, + plugin_config={"version": 1, "components": []}, + ) + process = MagicMock() + start_gateway = MagicMock(return_value=process) + stop_gateway = MagicMock() + monkeypatch.setattr(adapter, "prepare_codex_relay", MagicMock(return_value=relay)) + monkeypatch.setattr(adapter.relay_gateway, "start_relay_gateway", start_gateway) + monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", stop_gateway) + start_payload = dict(codex_payload) + start_payload.pop("request") + runtime = adapter.CodexRuntime() - output = adapter.run(codex_payload) + await runtime.start(start_payload) + await runtime.invoke(lifecycle_invocation(codex_payload)) + codex_payload["runtime_context"]["invocation_id"] = "invocation-2" + await runtime.invoke(lifecycle_invocation(codex_payload)) + stop_gateway.assert_not_called() + await runtime.stop() - assert output["error"]["code"] == "codex_invalid_runtime_state" + assert len(mock_codex.instances) == 1 + start_gateway.assert_called_once_with( + launch=gateway, + cwd=Path(codex_payload["runtime_context"]["environment"]["workspace"]), + ) + stop_gateway.assert_called_once_with(process) def test_failed_sdk_turn_is_normalized_and_transport_is_closed( @@ -402,7 +542,7 @@ def test_failed_sdk_turn_is_normalized_and_transport_is_closed( ): mock_codex.next_result = RuntimeError("model request failed") - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["error"] == { "code": "codex_turn_failed", @@ -410,30 +550,26 @@ def test_failed_sdk_turn_is_normalized_and_transport_is_closed( "retryable": False, } assert mock_codex.instances[0].closed is True - assert not adapter.runtime_state_path(codex_payload, "runtime-1").exists() -def test_incomplete_sdk_turn_is_failed_without_persisting_thread( - codex_payload, mock_codex -): +def test_incomplete_sdk_turn_is_failed(codex_payload, mock_codex): result = successful_result(response=None) mock_codex.next_result = result - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["error"]["code"] == "codex_turn_incomplete" assert output["turn_status"] == "completed" - assert not adapter.runtime_state_path(codex_payload, "runtime-1").exists() def test_selected_model_rejects_unsupported_provider(codex_payload, mock_codex): model = codex_payload["config"]["models"]["default"] model["provider"] = "anthropic" - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert "provider must be openai or nvidia" in output["error"]["message"] + assert error.code == "codex_invalid_configuration" + assert "provider must be openai or nvidia" in error.message mock_codex.assert_not_called() @@ -451,7 +587,7 @@ def test_nvidia_provider_uses_responses_api_and_nvidia_credential( ) os.environ["NVIDIA_API_KEY"] = "nvidia-secret" - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["completed"] is True client = mock_codex.instances[0] @@ -490,9 +626,9 @@ async def fail_to_create_home(*_args, **_kwargs): monkeypatch.setattr(adapter.asyncio, "to_thread", fail_to_create_home) - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_runtime_unavailable" + assert error.code == "codex_runtime_unavailable" mock_codex.assert_not_called() @@ -507,10 +643,10 @@ def test_nvidia_provider_requires_credential(codex_payload, mock_codex): ) os.environ.pop("NVIDIA_API_KEY", None) - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert "NVIDIA_API_KEY is required" in output["error"]["message"] + assert error.code == "codex_invalid_configuration" + assert "NVIDIA_API_KEY is required" in error.message assert not (adapter.state_dir(codex_payload) / "nvidia-home").exists() mock_codex.assert_not_called() @@ -528,26 +664,14 @@ def test_nvidia_provider_requires_endpoint(codex_payload, mock_codex): os.environ["NVIDIA_API_KEY"] = "nvidia-secret" os.environ.pop("NVIDIA_FRONTIER_BASE_URL", None) - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert "NVIDIA_FRONTIER_BASE_URL" in output["error"]["message"] + assert error.code == "codex_invalid_configuration" + assert "NVIDIA_FRONTIER_BASE_URL" in error.message assert not (adapter.state_dir(codex_payload) / "nvidia-home").exists() mock_codex.assert_not_called() -def test_resume_rejects_changed_sdk_thread_identity( - codex_payload, mock_codex -): - adapter.save_thread_id(codex_payload, "runtime-1", "thread-persisted") - mock_codex.resume_thread_id = "thread-replaced" - - output = adapter.run(codex_payload) - - assert output["error"]["code"] == "codex_thread_mismatch" - assert mock_codex.instances[0].closed is True - - def test_relay_uses_gateway_and_request_scoped_sdk_config( codex_payload, mock_codex, monkeypatch, tmp_path ): @@ -572,13 +696,11 @@ def test_relay_uses_gateway_and_request_scoped_sdk_config( start_gateway = MagicMock(return_value=process) stop_gateway = MagicMock() monkeypatch.setattr(adapter, "prepare_codex_relay", MagicMock(return_value=relay)) - monkeypatch.setattr( - adapter.relay_gateway, "start_relay_gateway", start_gateway - ) + monkeypatch.setattr(adapter.relay_gateway, "start_relay_gateway", start_gateway) monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", stop_gateway) os.environ["FABRIC_RELAY_CONFIG_PATH"] = str(tmp_path / "relay.json") - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) client = mock_codex.instances[0] start = client.thread_start.await_args.kwargs @@ -628,12 +750,10 @@ def test_relay_rejects_nvidia_provider(codex_payload, mock_codex): } os.environ["NVIDIA_API_KEY"] = "nvidia-secret" - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert output["error"]["message"] == ( - "NeMo Relay requires the built-in openai model provider" - ) + assert error.code == "codex_invalid_configuration" + assert error.message == ("NeMo Relay requires the built-in openai model provider") mock_codex.assert_not_called() @@ -655,9 +775,7 @@ def test_prepare_relay_reuses_one_resolved_executable( ) write = MagicMock(return_value=(config_path, plugin_path)) monkeypatch.setattr(adapter.relay_gateway, "resolve_relay_command", resolve) - monkeypatch.setattr( - adapter.relay_gateway, "relay_cli_contract", contract - ) + monkeypatch.setattr(adapter.relay_gateway, "relay_cli_contract", contract) monkeypatch.setattr(adapter.relay_gateway, "find_available_tcp_port", lambda: 43210) monkeypatch.setattr( adapter.common_utils, @@ -684,7 +802,7 @@ def test_prepare_relay_reuses_one_resolved_executable( @pytest.mark.usefixtures("mock_codex") -def test_relay_cleanup_failure_changes_success_to_failure( +def test_relay_stop_failure_is_reported_by_runtime_stop( codex_payload, monkeypatch, tmp_path ): gateway = adapter.relay_gateway.RelayGatewayLaunch( @@ -708,12 +826,18 @@ def test_relay_cleanup_failure_changes_success_to_failure( MagicMock(side_effect=adapter.relay_gateway.RelayGatewayError("stuck")), ) - output = adapter.run(codex_payload) + async def scenario() -> tuple[dict[str, Any], adapter.lifecycle.LifecycleError]: + runtime = adapter.CodexRuntime() + await runtime.start(lifecycle_start_payload(codex_payload)) + output = await runtime.invoke(lifecycle_invocation(codex_payload)) + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.stop() + return output, caught.value + + output, error = asyncio.run(scenario()) - assert output["failed"] is True - assert output["completed"] is False - assert output["error"]["code"] == "codex_relay_stop_failed" - assert output["relay_runtime"]["cleanup_error"] == output["error"] + assert output["completed"] is True + assert error.code == "codex_relay_stop_failed" def test_native_sdk_controls_and_telemetry_are_request_scoped( @@ -745,9 +869,7 @@ def test_native_sdk_controls_and_telemetry_are_request_scoped( "enabled": True, "endpoint": "http://localhost:4318/v1/traces", "transport": "http_binary", - "resource_attributes": { - "deployment.environment": "test" - }, + "resource_attributes": {"deployment.environment": "test"}, } }, } @@ -755,7 +877,7 @@ def test_native_sdk_controls_and_telemetry_are_request_scoped( }, } - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["failed"] is False client = mock_codex.instances[0] @@ -776,9 +898,7 @@ def test_native_sdk_controls_and_telemetry_are_request_scoped( assert turn["output_schema"]["required"] == ["summary"] -def test_timeout_interrupts_native_turn_and_closes_sdk( - codex_payload, mock_codex -): +def test_timeout_interrupts_native_turn_and_closes_sdk(codex_payload, mock_codex): mock_blocking_thread = mock_thread("thread-timeout") async def block(): @@ -786,11 +906,9 @@ async def block(): mock_blocking_thread.handle.run.side_effect = block mock_codex.next_thread = mock_blocking_thread - codex_payload["config"]["harness"]["settings"][ - "timeout_seconds" - ] = 0.01 + codex_payload["config"]["harness"]["settings"]["timeout_seconds"] = 0.01 - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) client = mock_codex.instances[0] assert output["error"]["code"] == "codex_timed_out" @@ -802,14 +920,12 @@ async def block(): "setting", ["codex_command", "codex_args", "codex_profile", "skip_git_repo_check"] ) def test_cli_only_settings_are_rejected(codex_payload, setting): - codex_payload["config"]["harness"]["settings"][setting] = ( - "legacy" - ) + codex_payload["config"]["harness"]["settings"][setting] = "legacy" - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert setting in output["error"]["message"] + assert error.code == "codex_invalid_configuration" + assert setting in error.message @pytest.mark.parametrize( @@ -821,10 +937,10 @@ def test_normalized_capabilities_reject_harness_settings( ): codex_payload["config"]["harness"]["settings"][setting] = {} - output = adapter.run(codex_payload) + error = runtime_start_error(codex_payload) - assert output["error"]["code"] == "codex_invalid_configuration" - assert normalized_field in output["error"]["message"] + assert error.code == "codex_invalid_configuration" + assert normalized_field in error.message mock_codex.assert_not_called() @@ -833,7 +949,7 @@ def test_adapter_rejects_structured_input(codex_payload): "messages": [{"role": "user", "content": "Inspect the change."}] } - output = adapter.run(codex_payload) + output = invoke_once(codex_payload) assert output["error"]["code"] == "codex_invalid_request" @@ -848,7 +964,6 @@ def test_descriptor_has_no_codex_binary_requirement(): assert descriptor["adapter_id"] == "nvidia.fabric.codex" assert descriptor["runner"] == { "module": "nemo_fabric_adapters.codex.adapter", - "callable": "run", } assert descriptor["config"]["accepts"] == [ "models", @@ -939,9 +1054,7 @@ def test_environment_rejects_non_string_runtime_telemetry_env( @pytest.mark.parametrize("telemetry", [[], "invalid"]) -def test_environment_rejects_non_mapping_runtime_telemetry( - codex_payload, telemetry -): +def test_environment_rejects_non_mapping_runtime_telemetry(codex_payload, telemetry): codex_payload["runtime_context"]["telemetry"] = telemetry with pytest.raises( @@ -949,3 +1062,12 @@ def test_environment_rejects_non_mapping_runtime_telemetry( match=r"runtime_context\.telemetry must be a mapping", ): adapter.child_environment(codex_payload) + + +def test_main_serves_persistent_runtime(monkeypatch): + serve = MagicMock() + monkeypatch.setattr(adapter.lifecycle, "serve", serve) + + adapter.main() + + serve.assert_called_once_with(adapter.CodexRuntime) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index bffa8c10..e0373fe3 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -5,13 +5,14 @@ These tests stub the ``deepagents``/``langchain``/``langgraph`` SDKs so they run without the real harness installed; they assert the normalized Fabric result and -the session/resume thread-id handling. The real SDK is exercised by the opt-in +the live runtime's thread continuity. The real SDK is exercised by the opt-in integration test in ``tests/e2e/test_deepagents.py``. """ from __future__ import annotations import importlib.machinery +import os import sys import types from collections.abc import Iterator @@ -24,6 +25,26 @@ from nemo_fabric_adapters.deepagents import adapter # noqa: E402 +def lifecycle_start_payload(payload: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in payload.items() if key != "request"} + + +def lifecycle_invocation(payload: dict[str, Any]) -> dict[str, Any]: + return { + "runtime_context": payload["runtime_context"], + "request": payload["request"], + } + + +async def invoke_once(payload: dict[str, Any]) -> dict[str, Any]: + runtime = adapter.DeepAgentsRuntime() + await runtime.start(lifecycle_start_payload(payload)) + try: + return await runtime.invoke(lifecycle_invocation(payload)) + finally: + await runtime.stop() + + @pytest.fixture(name="fake_sdks", autouse=True) def fake_sdks_fixture(monkeypatch): """Stub the deepagents/langchain/langgraph SDKs with mocks. @@ -55,7 +76,7 @@ async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): } # subgraphs=True yields 3-tuples ``(namespace, mode, chunk)``; the main # graph has an empty namespace. ``updates`` carries the message produced - # this turn; ``values`` is the full (on resume, replayed) state. + # this turn; ``values`` is the full replayed state. yield ((), "updates", {"agent": {"messages": [ai]}}) yield ((), "values", {"messages": [{"role": "user", "content": user}, ai]}) @@ -129,14 +150,14 @@ def make(tmp_path: Path, *, runtime_id: str = "run-1") -> dict[str, Any]: return { "base_dir": str(tmp_path), "config": { - "harness": {"settings": {"system_prompt": "be concise"}}, - "models": { - "default": { - "provider": "nvidia", - "model": "nvidia/nemotron-3-nano-30b-a3b", - "api_key_env": "NVIDIA_API_KEY", - } - }, + "harness": {"settings": {"system_prompt": "be concise"}}, + "models": { + "default": { + "provider": "nvidia", + "model": "nvidia/nemotron-3-nano-30b-a3b", + "api_key_env": "NVIDIA_API_KEY", + } + }, }, "runtime_context": { "runtime_id": runtime_id, @@ -162,12 +183,17 @@ def add_nemo_relay_integration(kwargs, **_): merged = dict(kwargs) merged["middleware"] = [*(merged.get("middleware") or []), "relay-mw"] calls["wrapped"] = True + calls["integration_adds"] = calls.get("integration_adds", 0) + 1 return merged @contextlib.asynccontextmanager async def plugin_ctx(_config): calls["plugin_open"] = True - yield + calls["plugin_enters"] = calls.get("plugin_enters", 0) + 1 + try: + yield + finally: + calls["plugin_exits"] = calls.get("plugin_exits", 0) + 1 class ScopeType: Agent = "agent" @@ -224,20 +250,26 @@ def use_real_langgraph_fixture(fake_sdks, monkeypatch): monkeypatch.delitem(sys.modules, name, raising=False) -async def test_oneshot_normalizes_response_usage_and_thread(tmp_path, make_payload, fake_sdks): - output = await adapter.run_deepagents(make_payload(tmp_path)) +async def test_single_invocation_normalizes_response_usage_and_thread( + tmp_path, make_payload, fake_sdks +): + output = await invoke_once(make_payload(tmp_path)) assert output["harness"] == "deepagents" assert output["mode"] == "deepagents" assert output["model"] == "nvidia/nemotron-3-nano-30b-a3b" assert output["response"] == "reply to hello" assert output["message_count"] == 2 - assert output["usage"] == {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12} + assert output["usage"] == { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12, + } # streamed events are buffered assert output["events"] == [{"nodes": ["agent"]}] assert output["event_count"] == 1 assert output["runtime_id"] == "run-1" - # a LangGraph thread id is assigned and reported; a fresh runtime is not a resume + # The first invocation receives a newly assigned LangGraph thread id. assert output["thread_id"] assert output["resumed"] is False assert output["completed"] is True @@ -248,22 +280,20 @@ async def test_oneshot_normalizes_response_usage_and_thread(tmp_path, make_paylo assert "instructions" not in fake_sdks["create_kwargs"] -async def test_missing_api_key_is_normalized(tmp_path, make_payload, monkeypatch): - # Missing model-provider auth is caught by the adapter preflight. Because the - # preflight runs inside the guarded scope, the failure is normalized into the - # Fabric result rather than raising a raw traceback. - monkeypatch.delenv("NVIDIA_API_KEY", raising=False) - output = await adapter.run_deepagents(make_payload(tmp_path)) +async def test_missing_api_key_fails_runtime_start(tmp_path, make_payload): + os.environ.pop("NVIDIA_API_KEY", None) - assert output["failed"] is True - assert output["completed"] is False - assert "NVIDIA_API_KEY" in output["error"] + with pytest.raises(RuntimeError, match="NVIDIA_API_KEY"): + await adapter.DeepAgentsRuntime().start( + lifecycle_start_payload(make_payload(tmp_path)) + ) -async def test_missing_deepagents_package_is_normalized(tmp_path, make_payload, monkeypatch): - # Preflight reports a clear, normalized error when the deepagents package is - # absent. Force find_spec("deepagents") -> None so the test holds whether or - # not the real package is installed in the environment. +async def test_missing_deepagents_package_fails_runtime_start( + tmp_path, make_payload, monkeypatch +): + # Force find_spec("deepagents") -> None so the test holds whether or not the + # real package is installed in the environment. import importlib.util as importlib_util real_find_spec = importlib_util.find_spec @@ -276,34 +306,42 @@ def fake_find_spec( return real_find_spec(name, *args, **kwargs) monkeypatch.setattr(importlib_util, "find_spec", fake_find_spec) - output = await adapter.run_deepagents(make_payload(tmp_path)) - - assert output["failed"] is True - assert "deepagents" in output["error"] + with pytest.raises(RuntimeError, match="deepagents"): + await adapter.DeepAgentsRuntime().start( + lifecycle_start_payload(make_payload(tmp_path)) + ) -async def test_invocation_error_is_normalized(tmp_path, make_payload, monkeypatch): - # Errors raised during the agent run are normalized into the Fabric result. +async def test_agent_creation_error_fails_runtime_start( + tmp_path, make_payload, monkeypatch +): import deepagents def boom(**_kwargs): raise RuntimeError("agent exploded") monkeypatch.setattr(deepagents, "create_deep_agent", boom) - output = await adapter.run_deepagents(make_payload(tmp_path)) - - assert output["failed"] is True - assert output["completed"] is False - assert "agent exploded" in output["error"] + with pytest.raises(RuntimeError, match="agent exploded"): + await adapter.DeepAgentsRuntime().start( + lifecycle_start_payload(make_payload(tmp_path)) + ) async def test_relay_telemetry_wraps_agent_and_reports_artifacts( tmp_path, make_payload, monkeypatch, fake_sdks, fake_relay ): artifacts = [{"kind": "atof", "path": str(tmp_path / "events.atof.jsonl")}] - monkeypatch.setattr(adapter.common_utils, "load_relay_plugin_config", lambda _p: {"version": 1, "components": []}) - monkeypatch.setattr(adapter.common_utils, "relay_api_plugin_config", lambda _c: object()) - monkeypatch.setattr(adapter.common_utils, "collect_relay_artifacts", lambda _c: artifacts) + monkeypatch.setattr( + adapter.common_utils, + "load_relay_plugin_config", + lambda _p: {"version": 1, "components": []}, + ) + monkeypatch.setattr( + adapter.common_utils, "relay_api_plugin_config", lambda _c: object() + ) + monkeypatch.setattr( + adapter.common_utils, "collect_relay_artifacts", lambda _c: artifacts + ) payload = make_payload(tmp_path) payload["telemetry_plan"] = { "providers": ["relay"], @@ -315,7 +353,7 @@ async def test_relay_telemetry_wraps_agent_and_reports_artifacts( "adapter_outputs": [], } - output = await adapter.run_deepagents(payload) + output = await invoke_once(payload) assert fake_relay["wrapped"] assert fake_relay["plugin_open"] @@ -332,11 +370,17 @@ async def test_relay_telemetry_wraps_agent_and_reports_artifacts( assert fake_relay["scopes"] == [("deepagents-request", "agent")] # the Deep Agents callback handler is added to the LangGraph run config so # LangGraph scopes and human-in-the-loop interrupt/resume marks are captured - assert fake_relay["callback_handler"] in (fake_sdks["config"] or {}).get("callbacks", []) + assert fake_relay["callback_handler"] in (fake_sdks["config"] or {}).get( + "callbacks", [] + ) -async def test_native_telemetry_exports_without_artifacts(tmp_path, make_payload, monkeypatch, fake_sdks, fake_relay): - monkeypatch.setattr(adapter.common_utils, "relay_api_plugin_config", lambda _c: object()) +async def test_native_telemetry_exports_without_artifacts( + tmp_path, make_payload, monkeypatch, fake_sdks, fake_relay +): + monkeypatch.setattr( + adapter.common_utils, "relay_api_plugin_config", lambda _c: object() + ) payload = make_payload(tmp_path) payload["telemetry_plan"] = { @@ -364,7 +408,7 @@ async def test_native_telemetry_exports_without_artifacts(tmp_path, make_payload "adapter_outputs": [], } - output = await adapter.run_deepagents(payload) + output = await invoke_once(payload) assert fake_relay["wrapped"] assert fake_relay["plugin_open"] @@ -378,13 +422,17 @@ async def test_native_telemetry_exports_without_artifacts(tmp_path, make_payload assert "relay-mw" in fake_sdks["create_kwargs"]["middleware"] # the scope + callback handler apply to any observability-enabled run, native included assert fake_relay["scopes"] == [("deepagents-request", "agent")] - assert fake_relay["callback_handler"] in (fake_sdks["config"] or {}).get("callbacks", []) + assert fake_relay["callback_handler"] in (fake_sdks["config"] or {}).get( + "callbacks", [] + ) -async def test_relay_disabled_adds_no_scope_or_callbacks(tmp_path, make_payload, fake_sdks): +async def test_relay_disabled_adds_no_scope_or_callbacks( + tmp_path, make_payload, fake_sdks +): # With telemetry disabled the invocation runs without a Relay scope, callback # handler, or middleware, preserving the Relay-neutral default behavior. - output = await adapter.run_deepagents(make_payload(tmp_path)) + output = await invoke_once(make_payload(tmp_path)) assert output["completed"] is True assert "telemetry" not in output @@ -392,7 +440,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks(tmp_path, make_payload, assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) -async def test_missing_nemo_relay_with_native_telemetry_is_normalized(tmp_path, make_payload, monkeypatch): +async def test_missing_nemo_relay_with_native_telemetry_fails_runtime_start( + tmp_path, make_payload, monkeypatch +): # Native telemetry also runs through the nemo_relay plugin, so a core-only # install configured with native telemetry must fail with the actionable # extra-install message rather than a raw ModuleNotFoundError -- even though @@ -417,20 +467,20 @@ def fake_find_spec( "relay_enabled": False, "native_config": { "version": 1, - "components": [{"kind": "observability", "enabled": True, "config": {"version": 1}}], + "components": [ + {"kind": "observability", "enabled": True, "config": {"version": 1}} + ], }, "adapter_outputs": [], } - output = await adapter.run_deepagents(payload) - - assert output["failed"] is True - assert "nemo-relay" in output["error"] - assert "[relay]" in output["error"] + with pytest.raises(RuntimeError, match="nemo-relay.*\\[relay\\]"): + await adapter.DeepAgentsRuntime().start(lifecycle_start_payload(payload)) -async def test_incomplete_nemo_relay_install_is_normalized( - tmp_path, make_payload, monkeypatch, fake_relay +@pytest.mark.usefixtures("fake_relay") +async def test_incomplete_nemo_relay_install_fails_runtime_start( + tmp_path, make_payload, monkeypatch ): monkeypatch.delitem(sys.modules, "nemo_relay.integrations.deepagents") payload = make_payload(tmp_path) @@ -446,11 +496,8 @@ async def test_incomplete_nemo_relay_install_is_normalized( "adapter_outputs": [], } - output = await adapter.run_deepagents(payload) - - assert output["failed"] is True - assert "compatible 'nemo-relay' package" in output["error"] - assert "[relay]" in output["error"] + with pytest.raises(RuntimeError, match="compatible 'nemo-relay'.*\\[relay\\]"): + await adapter.DeepAgentsRuntime().start(lifecycle_start_payload(payload)) def test_apply_callbacks_preserves_existing_ahead_of_new(): @@ -465,28 +512,36 @@ def test_apply_callbacks_preserves_existing_ahead_of_new(): def test_apply_callbacks_without_callbacks_leaves_config_untouched(): config = {"configurable": {"thread_id": "t"}} - assert adapter._apply_callbacks(config, None) == {"configurable": {"thread_id": "t"}} + assert adapter._apply_callbacks(config, None) == { + "configurable": {"thread_id": "t"} + } + +async def test_invoke_compiled_agent_wires_callbacks_into_run_config(fake_sdks): + from deepagents import create_deep_agent -async def test_invoke_agent_wires_callbacks_into_run_config(fake_sdks): - # invoke_agent threads the supplied callbacks into the LangGraph run config. - agent_kwargs = {"model": object()} - await adapter.invoke_agent(agent_kwargs, "hello", "thread-1", callbacks=["cb-a", "cb-b"]) + agent = create_deep_agent(model=object()) + await adapter.invoke_compiled_agent( + agent, "hello", "thread-1", callbacks=["cb-a", "cb-b"] + ) config = fake_sdks["config"] assert config["configurable"]["thread_id"] == "thread-1" assert config["callbacks"] == ["cb-a", "cb-b"] -async def test_invoke_agent_without_callbacks_sets_no_callbacks_key(fake_sdks): - await adapter.invoke_agent({"model": object()}, "hello", None, callbacks=None) +async def test_invoke_compiled_agent_without_callbacks_sets_no_callbacks_key(fake_sdks): + from deepagents import create_deep_agent + + agent = create_deep_agent(model=object()) + await adapter.invoke_compiled_agent(agent, "hello", None, callbacks=None) # No thread and no callbacks means the agent is streamed without a config. assert fake_sdks["config"] is None async def test_workspace_roots_filesystem_backend(tmp_path, make_payload, fake_sdks): - await adapter.run_deepagents(make_payload(tmp_path)) + await invoke_once(make_payload(tmp_path)) backend_kwargs = fake_sdks["fs_backend"].call_args.kwargs assert backend_kwargs["root_dir"] == str(tmp_path) # virtual_mode=True confines the agent to root_dir: absolute paths and ``..`` @@ -494,9 +549,11 @@ async def test_workspace_roots_filesystem_backend(tmp_path, make_payload, fake_s assert backend_kwargs["virtual_mode"] is True -async def test_checkpointer_closed_on_success_and_failure(tmp_path, make_payload, monkeypatch, fake_sdks): +async def test_checkpointer_closed_on_success_and_failure( + tmp_path, make_payload, monkeypatch, fake_sdks +): # The async checkpointer must be closed on both the success and error paths. - await adapter.run_deepagents(make_payload(tmp_path)) + await invoke_once(make_payload(tmp_path)) assert fake_sdks["saver_exits"] == 1 import deepagents @@ -505,12 +562,16 @@ def boom(**_kwargs): raise RuntimeError("boom") monkeypatch.setattr(deepagents, "create_deep_agent", boom) - output = await adapter.run_deepagents(make_payload(tmp_path)) - assert output["failed"] is True + with pytest.raises(RuntimeError, match="boom"): + await adapter.DeepAgentsRuntime().start( + lifecycle_start_payload(make_payload(tmp_path)) + ) assert fake_sdks["saver_exits"] == 2 -async def test_mcp_servers_become_adapter_tools(tmp_path, make_payload, monkeypatch, fake_sdks): +async def test_mcp_servers_become_adapter_tools( + tmp_path, make_payload, monkeypatch, fake_sdks +): tool_read = MagicMock() tool_read.name = "read_file" tool_write = MagicMock() @@ -521,7 +582,11 @@ async def test_mcp_servers_become_adapter_tools(tmp_path, make_payload, monkeypa client_mod = types.ModuleType("langchain_mcp_adapters.client") client_mod.MultiServerMCPClient = mock_client_cls - monkeypatch.setitem(sys.modules, "langchain_mcp_adapters", types.ModuleType("langchain_mcp_adapters")) + monkeypatch.setitem( + sys.modules, + "langchain_mcp_adapters", + types.ModuleType("langchain_mcp_adapters"), + ) monkeypatch.setitem(sys.modules, "langchain_mcp_adapters.client", client_mod) payload = make_payload(tmp_path) @@ -535,7 +600,7 @@ async def test_mcp_servers_become_adapter_tools(tmp_path, make_payload, monkeypa } } - output = await adapter.run_deepagents(payload) + output = await invoke_once(payload) assert output["failed"] is False # Fabric MCP transport is normalized; stdio command/args come from ``url`` @@ -558,7 +623,9 @@ async def handler(_request: types.SimpleNamespace) -> str: return "executed" def request(name: str) -> types.SimpleNamespace: - return types.SimpleNamespace(tool_call={"name": name, "id": "call-1", "args": {}}) + return types.SimpleNamespace( + tool_call={"name": name, "id": "call-1", "args": {}} + ) blocked = await middleware.awrap_tool_call(request("write_file"), handler) assert isinstance(blocked, ToolMessage) @@ -596,14 +663,16 @@ def build(**kwargs): monkeypatch.setattr(deepagents, "create_deep_agent", build) - output = await adapter.run_deepagents(make_payload(tmp_path)) + output = await invoke_once(make_payload(tmp_path)) assert output["failed"] is False, output["error"] assert output["response"] == "ok" assert output["thread_id"] -async def test_openai_provider_keeps_openai_endpoint(tmp_path, make_payload, monkeypatch, fake_sdks): +async def test_openai_provider_keeps_openai_endpoint( + tmp_path, make_payload, monkeypatch, fake_sdks +): monkeypatch.setenv("OPENAI_API_KEY", "sk-test") payload = make_payload(tmp_path) payload["config"]["models"]["default"] = { @@ -612,7 +681,7 @@ async def test_openai_provider_keeps_openai_endpoint(tmp_path, make_payload, mon "api_key_env": "OPENAI_API_KEY", } - output = await adapter.run_deepagents(payload) + output = await invoke_once(payload) # openai must NOT be redirected to NVIDIA's endpoint assert output["base_url"] is None @@ -623,12 +692,14 @@ async def test_skill_paths_map_to_skills(tmp_path, make_payload, fake_sdks): payload = make_payload(tmp_path) payload["capability_plan"] = {"native": {"skill_paths": ["/skills/a", "/skills/b"]}} - await adapter.run_deepagents(payload) + await invoke_once(payload) assert fake_sdks["create_kwargs"]["skills"] == ["/skills/a", "/skills/b"] -async def test_cost_is_extracted_from_response_metadata(tmp_path, make_payload, monkeypatch): +async def test_cost_is_extracted_from_response_metadata( + tmp_path, make_payload, monkeypatch +): import deepagents message = { @@ -645,13 +716,15 @@ async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): agent = MagicMock() agent.astream = astream monkeypatch.setattr(deepagents, "create_deep_agent", MagicMock(return_value=agent)) - output = await adapter.run_deepagents(make_payload(tmp_path)) + output = await invoke_once(make_payload(tmp_path)) assert output["usage"]["cost"] == 0.0025 -async def test_resumed_usage_counts_current_turn_only(tmp_path, make_payload, monkeypatch): - # On a resumed run the final state replays the prior turn's messages; usage and +async def test_replayed_state_usage_counts_current_turn_only( + tmp_path, make_payload, monkeypatch +): + # On a later turn the final state replays prior messages; usage and # cost must reflect only the message emitted this turn, not the replayed one. import deepagents @@ -671,14 +744,14 @@ async def test_resumed_usage_counts_current_turn_only(tmp_path, make_payload, mo async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): # Only the current turn's message is emitted as an update... yield ((), "updates", {"agent": {"messages": [current]}}) - # ...but the resumed final state also replays the prior turn. + # ...but the final state also replays the prior turn. yield ((), "values", {"messages": [prior, current]}) agent = MagicMock() agent.astream = astream monkeypatch.setattr(deepagents, "create_deep_agent", MagicMock(return_value=agent)) - output = await adapter.run_deepagents(make_payload(tmp_path)) + output = await invoke_once(make_payload(tmp_path)) assert output["usage"] == { "prompt_tokens": 2, @@ -690,27 +763,85 @@ async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): assert output["message_count"] == 2 -async def test_runtime_resume_reuses_thread_id(tmp_path, make_payload, fake_sdks): - # Two invocations of the same runtime_id (a started runtime) resume the same - # LangGraph thread; a one-shot run gets a fresh runtime_id and never resumes. - payload = make_payload(tmp_path, runtime_id="run-42") +async def test_persistent_runtime_reuses_compiled_agent_and_checkpointer( + tmp_path, make_payload, fake_sdks +): + import deepagents + + payload = make_payload(tmp_path, runtime_id="run-persistent") + runtime = adapter.DeepAgentsRuntime() - first = await adapter.run_deepagents(payload) - assert first["resumed"] is False - thread_id = first["thread_id"] - assert thread_id - # thread id was threaded into the LangGraph config on the invocation - assert fake_sdks["config"] == {"configurable": {"thread_id": thread_id}} + await runtime.start(lifecycle_start_payload(payload)) + first = await runtime.invoke(lifecycle_invocation(payload)) + payload["runtime_context"]["invocation_id"] = "inv-2" + payload["request"]["input"] = "continue" + second = await runtime.invoke(lifecycle_invocation(payload)) - second = await adapter.run_deepagents(payload) + assert first["resumed"] is False assert second["resumed"] is True - assert second["thread_id"] == thread_id + assert first["thread_id"] == second["thread_id"] + assert deepagents.create_deep_agent.call_count == 1 + assert fake_sdks["saver_exits"] == 0 + checkpointer = fake_sdks["create_kwargs"]["checkpointer"] + assert checkpointer is fake_sdks["checkpointer"] + + await runtime.stop() + + assert fake_sdks["saver_exits"] == 1 + + +async def test_persistent_runtime_scopes_relay_per_invocation( + tmp_path, make_payload, monkeypatch, fake_sdks, fake_relay +): + artifacts = [{"kind": "atif", "path": str(tmp_path / "trajectory.json")}] + monkeypatch.setattr( + adapter.common_utils, + "load_relay_plugin_config", + lambda _payload: {"version": 1, "components": []}, + ) + monkeypatch.setattr( + adapter.common_utils, "relay_api_plugin_config", lambda _config: object() + ) + monkeypatch.setattr( + adapter.common_utils, + "collect_relay_artifacts", + lambda _config: artifacts, + ) + payload = make_payload(tmp_path, runtime_id="run-relay-persistent") + payload["telemetry_plan"] = { + "providers": ["relay"], + "relay_enabled": True, + "relay_project": None, + "relay_output_dir": None, + "relay_config": {}, + "native_config": None, + "adapter_outputs": ["atif"], + } + runtime = adapter.DeepAgentsRuntime() + + await runtime.start(lifecycle_start_payload(payload)) + first = await runtime.invoke(lifecycle_invocation(payload)) + payload["runtime_context"]["invocation_id"] = "inv-2" + payload["request"]["input"] = "continue" + second = await runtime.invoke(lifecycle_invocation(payload)) + await runtime.stop() + + assert fake_relay["integration_adds"] == 1 + assert fake_relay["plugin_enters"] == 2 + assert fake_relay["plugin_exits"] == 2 + assert fake_relay["scopes"] == [ + ("deepagents-request", "agent"), + ("deepagents-request", "agent"), + ] + assert first["thread_id"] == second["thread_id"] + assert first["relay_artifacts"] == second["relay_artifacts"] == artifacts + assert fake_sdks["saver_exits"] == 1 async def test_stream_requests_subgraphs(tmp_path, make_payload, fake_sdks): # Streaming must opt into subgraphs so delegated (subagent) steps are visible # for usage aggregation. - await adapter.run_deepagents(make_payload(tmp_path)) + await invoke_once(make_payload(tmp_path)) assert fake_sdks["subgraphs"] is True @@ -727,16 +858,23 @@ async def test_subagents_are_gated_by_blocked_tools(tmp_path, make_payload): settings = payload["config"]["harness"]["settings"] create_kwargs = await adapter.build_agent_kwargs(payload, MagicMock(), settings) - assert create_kwargs["middleware"], "main agent blocked-tools middleware not attached" + assert create_kwargs["middleware"], ( + "main agent blocked-tools middleware not attached" + ) subagents = create_kwargs["subagents"] - assert [subagent["name"] for subagent in subagents] == ["general-purpose", "researcher"] + assert [subagent["name"] for subagent in subagents] == [ + "general-purpose", + "researcher", + ] assert all(subagent["middleware"] for subagent in subagents) async def handler(_request: types.SimpleNamespace) -> str: return "executed" def request(name: str) -> types.SimpleNamespace: - return types.SimpleNamespace(tool_call={"name": name, "id": "call-1", "args": {}}) + return types.SimpleNamespace( + tool_call={"name": name, "id": "call-1", "args": {}} + ) gates = [create_kwargs["middleware"][-1]] gates.extend(subagent["middleware"][-1] for subagent in subagents) @@ -745,7 +883,10 @@ def request(name: str) -> types.SimpleNamespace: blocked = await middleware.awrap_tool_call(request("write_file"), handler) assert isinstance(blocked, ToolMessage) assert blocked.status == "error" - assert await middleware.awrap_tool_call(request("read_file"), handler) == "executed" + assert ( + await middleware.awrap_tool_call(request("read_file"), handler) + == "executed" + ) @pytest.mark.usefixtures("use_real_langgraph") @@ -756,12 +897,18 @@ async def test_default_subagent_is_gated_by_blocked_tools(tmp_path, make_payload settings = payload["config"]["harness"]["settings"] create_kwargs = await adapter.build_agent_kwargs(payload, MagicMock(), settings) - assert [subagent["name"] for subagent in create_kwargs["subagents"]] == ["general-purpose"] + assert [subagent["name"] for subagent in create_kwargs["subagents"]] == [ + "general-purpose" + ] assert create_kwargs["subagents"][0]["middleware"] -@pytest.mark.parametrize("unsupported", [{"graph_id": "remote"}, {"runnable": "compiled"}]) -async def test_blocked_tools_reject_unenforceable_subagents(tmp_path, make_payload, unsupported): +@pytest.mark.parametrize( + "unsupported", [{"graph_id": "remote"}, {"runnable": "compiled"}] +) +async def test_blocked_tools_reject_unenforceable_subagents( + tmp_path, make_payload, unsupported +): payload = make_payload(tmp_path) payload["config"]["tools"] = {"blocked": ["write_file"]} payload["config"]["harness"]["settings"]["deepagents"] = { @@ -793,26 +940,32 @@ def test_gated_subagents_reject_invalid_configuration(subagents, message): assert str(error.value) == message -async def test_deepagents_passthrough_forwards_supported_options(tmp_path, make_payload, fake_sdks): +async def test_deepagents_passthrough_forwards_supported_options( + tmp_path, make_payload, fake_sdks +): # Documented JSON-serializable options reach create_deep_agent unchanged. payload = make_payload(tmp_path) - payload["config"]["harness"]["settings"]["deepagents"] = {"interrupt_on": {"write_file": True}} + payload["config"]["harness"]["settings"]["deepagents"] = { + "interrupt_on": {"write_file": True} + } - await adapter.run_deepagents(payload) + await invoke_once(payload) assert fake_sdks["create_kwargs"]["interrupt_on"] == {"write_file": True} -async def test_deepagents_passthrough_cannot_override_fabric_owned_keys(tmp_path, make_payload): +async def test_deepagents_passthrough_cannot_override_fabric_owned_keys( + tmp_path, make_payload +): # Overriding a Fabric-owned key (here backend) would defeat workspace confinement; # it must fail loudly rather than silently replacing the derived value. payload = make_payload(tmp_path) - payload["config"]["harness"]["settings"]["deepagents"] = {"backend": {"root_dir": "/etc"}} - - output = await adapter.run_deepagents(payload) + payload["config"]["harness"]["settings"]["deepagents"] = { + "backend": {"root_dir": "/etc"} + } - assert output["failed"] is True - assert "backend" in output["error"] + with pytest.raises(adapter.AdapterConfigError, match="backend"): + await adapter.DeepAgentsRuntime().start(lifecycle_start_payload(payload)) async def test_deepagents_passthrough_rejects_unknown_option(tmp_path, make_payload): @@ -822,10 +975,8 @@ async def test_deepagents_passthrough_rejects_unknown_option(tmp_path, make_payl "interupt_on": {} # note the typo } - output = await adapter.run_deepagents(payload) - - assert output["failed"] is True - assert "interupt_on" in output["error"] + with pytest.raises(adapter.AdapterConfigError, match="interupt_on"): + await adapter.DeepAgentsRuntime().start(lifecycle_start_payload(payload)) async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): @@ -859,7 +1010,7 @@ async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): agent.astream = astream monkeypatch.setattr(deepagents, "create_deep_agent", MagicMock(return_value=agent)) - output = await adapter.run_deepagents(make_payload(tmp_path)) + output = await invoke_once(make_payload(tmp_path)) # subagent (10/20/30) + main (2/3/5), the duplicate subagent message counted once assert output["usage"] == { @@ -871,30 +1022,34 @@ async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): assert any(evt.get("subgraph") == "task:researcher" for evt in output["events"]) -async def test_bad_mcp_transport_is_normalized_failure(tmp_path, make_payload): +async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): # A misconfigured MCP server must fail loudly, not be silently dropped. payload = make_payload(tmp_path) payload["capability_plan"] = { - "native": {"mcp_servers": {"bad": {"transport": "carrier-pigeon", "url": "http://x/mcp"}}} + "native": { + "mcp_servers": { + "bad": {"transport": "carrier-pigeon", "url": "http://x/mcp"} + } + } } - output = await adapter.run_deepagents(payload) + with pytest.raises(adapter.AdapterConfigError, match="transport"): + await adapter.DeepAgentsRuntime().start(lifecycle_start_payload(payload)) - assert output["failed"] is True - assert "transport" in output["error"] - -async def test_empty_mcp_url_is_normalized_failure(tmp_path, make_payload): +async def test_empty_mcp_url_fails_runtime_start(tmp_path, make_payload): payload = make_payload(tmp_path) - payload["capability_plan"] = {"native": {"mcp_servers": {"bad": {"transport": "streamable_http", "url": ""}}}} - - output = await adapter.run_deepagents(payload) + payload["capability_plan"] = { + "native": {"mcp_servers": {"bad": {"transport": "streamable_http", "url": ""}}} + } - assert output["failed"] is True - assert "url" in output["error"] + with pytest.raises(adapter.AdapterConfigError, match="url"): + await adapter.DeepAgentsRuntime().start(lifecycle_start_payload(payload)) -async def test_unknown_provider_requires_api_key_env(tmp_path, make_payload, monkeypatch): +async def test_unknown_provider_requires_api_key_env( + tmp_path, make_payload, monkeypatch +): # An unknown provider with no explicit api_key_env must fail loudly rather than # defaulting to NVIDIA_API_KEY and sending the wrong key to the endpoint. monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-x") @@ -904,13 +1059,13 @@ async def test_unknown_provider_requires_api_key_env(tmp_path, make_payload, mon "model": "claude-x", } - output = await adapter.run_deepagents(payload) + with pytest.raises(adapter.AdapterConfigError, match="api_key_env"): + await adapter.DeepAgentsRuntime().start(lifecycle_start_payload(payload)) - assert output["failed"] is True - assert "api_key_env" in output["error"] - -async def test_openai_provider_defaults_to_openai_key(tmp_path, make_payload, monkeypatch, fake_sdks): +async def test_openai_provider_defaults_to_openai_key( + tmp_path, make_payload, monkeypatch, fake_sdks +): # provider openai with no explicit api_key_env defaults to OPENAI_API_KEY, never # NVIDIA_API_KEY, and keeps ChatOpenAI's own endpoint. monkeypatch.delenv("NVIDIA_API_KEY", raising=False) @@ -921,7 +1076,7 @@ async def test_openai_provider_defaults_to_openai_key(tmp_path, make_payload, mo "model": "gpt-4o", } - output = await adapter.run_deepagents(payload) + output = await invoke_once(payload) assert output["failed"] is False, output["error"] assert output["base_url"] is None @@ -937,7 +1092,14 @@ async def test_openai_compatible_provider_requires_api_key_env(tmp_path, make_pa "model": "some/model", } - output = await adapter.run_deepagents(payload) + with pytest.raises(adapter.AdapterConfigError, match="api_key_env"): + await adapter.DeepAgentsRuntime().start(lifecycle_start_payload(payload)) + + +def test_main_serves_persistent_runtime(monkeypatch): + serve = MagicMock() + monkeypatch.setattr(adapter.lifecycle, "serve", serve) + + adapter.main() - assert output["failed"] is True - assert "api_key_env" in output["error"] + serve.assert_called_once_with(adapter.DeepAgentsRuntime) diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 0124ab8f..710b4af9 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -11,7 +11,7 @@ import os import sys from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace from unittest.mock import MagicMock import pytest @@ -41,17 +41,126 @@ def test_validate_hermes_telemetry_provider_accepts_relay( def test_validate_hermes_telemetry_provider_rejects_native(): payload = {"telemetry_plan": {"providers": ["native"], "relay_enabled": False}} - with pytest.raises(ValueError, match="only relay telemetry is supported for Hermes"): + with pytest.raises( + ValueError, match="only relay telemetry is supported for Hermes" + ): adapter.validate_hermes_telemetry_provider(payload) def test_validate_hermes_telemetry_provider_rejects_mixed_native_and_relay(): - payload = {"telemetry_plan": {"providers": ["relay", "native"], "relay_enabled": True}} + payload = { + "telemetry_plan": {"providers": ["relay", "native"], "relay_enabled": True} + } - with pytest.raises(ValueError, match="only relay telemetry is supported for Hermes"): + with pytest.raises( + ValueError, match="only relay telemetry is supported for Hermes" + ): adapter.validate_hermes_telemetry_provider(payload) +def test_finalize_relay_session_flushes_before_artifact_collection(monkeypatch): + calls: list[str] = [] + invoke_hook = MagicMock(side_effect=lambda *args, **kwargs: calls.append("hook")) + runtime = adapter.HermesRuntime() + runtime._relay_plugin_config = {"components": []} + runtime._agent = SimpleNamespace( + session_id="runtime-1", + model="test-model", + platform="fabric", + ) + runtime._invoke_hook = invoke_hook + runtime._relay_session_pending = True + + from nemo_relay import subscribers + + monkeypatch.setattr(subscribers, "flush", lambda: calls.append("flush")) + + runtime._finalize_relay_session() + + assert calls == ["hook", "flush"] + invoke_hook.assert_called_once_with( + "on_session_finalize", + session_id="runtime-1", + model="test-model", + platform="fabric", + ) + + +async def test_stop_does_not_refinalize_completed_relay_turn(monkeypatch): + invoke_hook = MagicMock() + runtime = adapter.HermesRuntime() + runtime._started = True + runtime._relay_plugin_config = {"components": []} + agent = MagicMock( + session_id="runtime-1", + model="test-model", + platform="fabric", + ) + session_db = MagicMock() + runtime._agent = agent + runtime._session_db = session_db + runtime._invoke_hook = invoke_hook + runtime._relay_session_pending = True + + from nemo_relay import subscribers + + flush = MagicMock() + monkeypatch.setattr(subscribers, "flush", flush) + + runtime._finalize_relay_session() + await runtime.stop() + + invoke_hook.assert_called_once_with( + "on_session_finalize", + session_id="runtime-1", + model="test-model", + platform="fabric", + ) + flush.assert_called_once_with() + agent.close.assert_called_once_with() + session_db.close.assert_called_once_with() + assert runtime._started is False + assert runtime._relay_session_pending is False + assert runtime._relay_finalize_hook_invoked is False + + +async def test_stop_retries_failed_relay_flush_without_refinalizing(monkeypatch): + invoke_hook = MagicMock() + runtime = adapter.HermesRuntime() + runtime._started = True + runtime._relay_plugin_config = {"components": []} + runtime._agent = MagicMock( + session_id="runtime-1", + model="test-model", + platform="fabric", + ) + runtime._session_db = MagicMock() + runtime._invoke_hook = invoke_hook + runtime._relay_session_pending = True + + from nemo_relay import subscribers + + flush = MagicMock(side_effect=[RuntimeError("flush failed"), None]) + monkeypatch.setattr(subscribers, "flush", flush) + + with pytest.raises(RuntimeError, match="flush failed"): + runtime._finalize_relay_session() + + assert runtime._relay_session_pending is True + assert runtime._relay_finalize_hook_invoked is True + await runtime.stop() + + invoke_hook.assert_called_once_with( + "on_session_finalize", + session_id="runtime-1", + model="test-model", + platform="fabric", + ) + assert flush.call_count == 2 + assert runtime._relay_session_pending is False + assert runtime._relay_finalize_hook_invoked is False + + def test_build_hermes_config_maps_fabric_config_to_hermes_config(): os.environ["MCP_URL"] = "http://localhost:9000/mcp" payload = { @@ -70,26 +179,26 @@ def test_build_hermes_config_maps_fabric_config_to_hermes_config(): } }, "config": { - "harness": { - "settings": { - "model": "review", - "max_iterations": 4, - "disabled_toolsets": ["browser"], - "terminal_backend": "local", - "terminal_timeout": 90, - "enabled_toolsets": "git", - "toolset_platform": "cli", - "plugins_enabled": ["custom/plugin"], - } - }, - "tools": {"blocked": ["shell", "browser"]}, - "models": { - "review": { - "provider": "nvidia", - "model": "nvidia/review-model", - "settings": {"base_url": "https://model.example/v1"}, - } - }, + "harness": { + "settings": { + "model": "review", + "max_iterations": 4, + "disabled_toolsets": ["browser"], + "terminal_backend": "local", + "terminal_timeout": 90, + "enabled_toolsets": "git", + "toolset_platform": "cli", + "plugins_enabled": ["custom/plugin"], + } + }, + "tools": {"blocked": ["shell", "browser"]}, + "models": { + "review": { + "provider": "nvidia", + "model": "nvidia/review-model", + "settings": {"base_url": "https://model.example/v1"}, + } + }, }, } @@ -134,7 +243,9 @@ def test_default_max_iterations_matches_hermes_library_default(): # multi-step tasks while the trial still reports success. assert adapter.DEFAULT_MAX_ITERATIONS > 1 - hermes_default = inspect.signature(AIAgent.__init__).parameters["max_iterations"].default + hermes_default = ( + inspect.signature(AIAgent.__init__).parameters["max_iterations"].default + ) assert adapter.DEFAULT_MAX_ITERATIONS == hermes_default @@ -143,8 +254,8 @@ def test_build_hermes_config_omits_max_turns_when_max_iterations_unset(): # absent so Hermes applies its own default rather than a starving override. payload = { "config": { - "harness": {"settings": {}}, - "models": {"default": {"provider": "nvidia", "model": "nvidia/test-model"}}, + "harness": {"settings": {}}, + "models": {"default": {"provider": "nvidia", "model": "nvidia/test-model"}}, } } @@ -158,8 +269,8 @@ def test_build_hermes_config_omits_max_turns_when_max_iterations_null(): # omitted so Hermes applies its own default instead of a starving override. payload = { "config": { - "harness": {"settings": {"max_iterations": None}}, - "models": {"default": {"provider": "nvidia", "model": "nvidia/test-model"}}, + "harness": {"settings": {"max_iterations": None}}, + "models": {"default": {"provider": "nvidia", "model": "nvidia/test-model"}}, } } @@ -216,20 +327,20 @@ def test_hermes_config_variation_matrix_surfaces_supported_capabilities( "agent_name": "matrix-agent", "base_dir": str(tmp_path), "config": { - "harness": { - "settings": { - "model": "review", - "enabled_toolsets": ["git", "shell"], - "toolset_platform": "cli", - "terminal_backend": "local", - } - }, - "models": { - "review": { - "provider": "nvidia", - "model": "nvidia/review-model", - } - }, + "harness": { + "settings": { + "model": "review", + "enabled_toolsets": ["git", "shell"], + "toolset_platform": "cli", + "terminal_backend": "local", + } + }, + "models": { + "review": { + "provider": "nvidia", + "model": "nvidia/review-model", + } + }, }, } @@ -258,8 +369,12 @@ def test_hermes_config_variation_matrix_surfaces_supported_capabilities( } assert config["platform_toolsets"] == {"cli": ["git", "shell"]} assert config["plugins"]["enabled"] == ["observability/nemo_relay"] - assert observability["atof"]["output_directory"] == str(tmp_path / "relay" / "atof" / "runtime-matrix") - assert observability["atif"]["output_directory"] == str(tmp_path / "relay" / "atif" / "runtime-matrix") + assert observability["atof"]["output_directory"] == str( + tmp_path / "relay" / "atof" / "runtime-matrix" + ) + assert observability["atif"]["output_directory"] == str( + tmp_path / "relay" / "atif" / "runtime-matrix" + ) assert observability["atif"]["agent_name"] == "matrix-agent" assert observability["atif"]["model_name"] == "nvidia/review-model" @@ -267,8 +382,8 @@ def test_hermes_config_variation_matrix_surfaces_supported_capabilities( def test_write_hermes_config_writes_file(tmp_path: Path): payload = { "config": { - "harness": {"settings": {}}, - "models": {"default": {"provider": "nvidia", "model": "nvidia/test-model"}}, + "harness": {"settings": {}}, + "models": {"default": {"provider": "nvidia", "model": "nvidia/test-model"}}, } } @@ -340,36 +455,50 @@ def test_summarize_hermes_config(): } -async def test_hermes_rejects_native_telemetry(): +async def test_runtime_start_rejects_native_telemetry(): payload = {"telemetry_plan": {"providers": ["native"], "relay_enabled": False}} - with pytest.raises(ValueError, match="only relay telemetry is supported for Hermes"): - await adapter.run_hermes(payload) + with pytest.raises( + ValueError, match="only relay telemetry is supported for Hermes" + ): + await adapter.HermesRuntime().start(payload) -async def test_fabric_runtime_id_drives_hermes_session_id_and_db_history( +async def test_persistent_runtime_reuses_hermes_agent_session_and_history( monkeypatch, tmp_path: Path, ): - db_history = [{"role": "user", "content": "from hermes db"}] - mock_session_db = MagicMock(spec=SessionDB) - mock_session_db.get_session.return_value = {"id": "runtime-resolved-456"} - mock_session_db.resolve_resume_session_id.return_value = "runtime-resolved-456" - mock_session_db.get_messages_as_conversation.return_value = db_history mock_session_db_type = MagicMock(spec=SessionDB, return_value=mock_session_db) mock_ai_agent = MagicMock(spec=AIAgent) mock_ai_agent.session_id = "runtime-fabric-123" mock_ai_agent.model = "test-model" mock_ai_agent.platform = "fabric" - mock_ai_agent.run_conversation.__signature__ = inspect.signature(AIAgent.run_conversation) - mock_ai_agent.run_conversation.return_value = { - "response": "ok", - "completed": True, - "failed": False, - "messages": [{"role": "assistant", "content": "ok"}], - } + mock_ai_agent.run_conversation.__signature__ = inspect.signature( + AIAgent.run_conversation + ) + first_messages = [ + {"role": "assistant", "content": "first response"}, + ] + second_messages = [ + *first_messages, + {"role": "assistant", "content": "second response"}, + ] + mock_ai_agent.run_conversation.side_effect = [ + { + "response": "first response", + "completed": True, + "failed": False, + "messages": first_messages, + }, + { + "response": "second response", + "completed": True, + "failed": False, + "messages": second_messages, + }, + ] mock_ai_agent_type = MagicMock(spec=AIAgent, return_value=mock_ai_agent) monkeypatch.setattr( mock_ai_agent_type.__init__.__func__, @@ -400,22 +529,22 @@ async def test_fabric_runtime_id_drives_hermes_session_id_and_db_history( "agent_name": "demo", "base_dir": str(tmp_path), "config": { - "harness": { - "settings": { - "hermes_home": "./hermes-home", - "enabled_toolsets": [], - "system_prompt": "system", - # Explicit null must resolve to DEFAULT_MAX_ITERATIONS (not int(None)). - "max_iterations": None, - } - }, - "models": { - "default": { - "provider": "test-provider", - "model": "test-model", - "api_key_env": "TEST_API_KEY", - } - }, + "harness": { + "settings": { + "hermes_home": "./hermes-home", + "enabled_toolsets": [], + "system_prompt": "system", + # Explicit null must resolve to DEFAULT_MAX_ITERATIONS (not int(None)). + "max_iterations": None, + } + }, + "models": { + "default": { + "provider": "test-provider", + "model": "test-model", + "api_key_env": "TEST_API_KEY", + } + }, }, "runtime_context": { "runtime_id": "runtime-fabric-123", @@ -428,12 +557,27 @@ async def test_fabric_runtime_id_drives_hermes_session_id_and_db_history( "capability_plan": {"native": {}}, } - output = await adapter.run_hermes(payload) + start_payload = {key: value for key, value in payload.items() if key != "request"} + runtime = adapter.HermesRuntime() + + await runtime.start(start_payload) + first = await runtime.invoke( + { + "runtime_context": payload["runtime_context"], + "request": payload["request"], + } + ) + payload["runtime_context"]["invocation_id"] = "invocation-2" + payload["request"]["input"] = "continue" + second = await runtime.invoke( + { + "runtime_context": payload["runtime_context"], + "request": payload["request"], + } + ) + await runtime.stop() mock_session_db_type.assert_called_once_with() - mock_session_db.resolve_resume_session_id.assert_called_once_with("runtime-fabric-123") - mock_session_db.get_session.assert_called_once_with("runtime-resolved-456") - mock_session_db.get_messages_as_conversation.assert_called_once_with("runtime-resolved-456") mock_ai_agent_type.assert_called_once_with( base_url=None, api_key="secret", @@ -452,12 +596,36 @@ async def test_fabric_runtime_id_drives_hermes_session_id_and_db_history( session_id="runtime-fabric-123", session_db=mock_session_db, ) - mock_ai_agent.run_conversation.assert_called_once_with( - "hello", - system_message="system", - conversation_history=db_history, - ) - assert "session_id" not in output - assert Path(output["hermes_home"]) == ( + assert mock_ai_agent.run_conversation.call_count == 2 + first_call, second_call = mock_ai_agent.run_conversation.call_args_list + assert first_call.args == ("hello",) + assert first_call.kwargs == { + "system_message": "system", + "conversation_history": None, + } + assert second_call.args == ("continue",) + assert second_call.kwargs == { + "system_message": "system", + "conversation_history": first_messages, + } + mock_ai_agent.close.assert_called_once_with() + mock_session_db.close.assert_called_once_with() + assert runtime._agent is None + assert runtime._session_db is None + assert runtime._start_payload is None + assert runtime._conversation_history is None + assert first["response"] == "first response" + assert second["response"] == "second response" + assert "session_id" not in second + assert Path(second["hermes_home"]) == ( tmp_path / "hermes-home" / "runtimes" / "runtime-fabric-123" ) + + +def test_main_serves_persistent_runtime(monkeypatch): + serve = MagicMock() + monkeypatch.setattr(adapter.lifecycle, "serve", serve) + + adapter.main() + + serve.assert_called_once_with(adapter.HermesRuntime) diff --git a/tests/e2e/test_claude.py b/tests/e2e/test_claude.py index ee5e5eca..c3eb2810 100644 --- a/tests/e2e/test_claude.py +++ b/tests/e2e/test_claude.py @@ -130,7 +130,7 @@ def fabric_config( return config -async def test_fabric_session_launches_fresh_processes_and_resumes(tmp_path): +async def test_fabric_session_reuses_persistent_claude_runtime(tmp_path): config = fabric_config(tmp_path, cli_path=MOCK_CLAUDE_CLI) async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: @@ -146,14 +146,18 @@ async def test_fabric_session_launches_fresh_processes_and_resumes(tmp_path): assert first.output["usage"] == {"input_tokens": 1, "output_tokens": 2} assert first.output["cost_usd"] == 0.001 assert [event["type"] for event in first.output["events"]] == ["AssistantMessage"] - arguments = [json.loads(line) for line in (tmp_path / "claude-args.jsonl").read_text().splitlines()] - assert len(arguments) == 2 + assert first.metadata["adapter_runner"] == "persistent_local_host" + assert first.metadata["host_pid"] == second.metadata["host_pid"] + arguments = [ + json.loads(line) + for line in (tmp_path / "claude-args.jsonl").read_text().splitlines() + ] + assert len(arguments) == 1 assert "--resume" not in arguments[0] - assert arguments[1][arguments[1].index("--resume") + 1] == SESSION_ID assert all("--mcp-config" in args for args in arguments) assert all("--plugin-dir" in args for args in arguments) plugin_paths = [args[args.index("--plugin-dir") + 1] for args in arguments] - assert plugin_paths[0] == plugin_paths[1] + assert len(plugin_paths) == 1 assert not any(artifact.kind == "stderr" for artifact in second.artifacts.artifacts) @@ -226,17 +230,19 @@ async def test_fabric_claude_accepts_real_relay_gateway_with_mock_claude(tmp_pat os.environ.get("RUN_FABRIC_CLAUDE_INTEGRATION") != "1", reason="set RUN_FABRIC_CLAUDE_INTEGRATION=1 to run Claude Agent SDK integration", ) -async def test_live_claude_one_shot_and_session(tmp_path): +async def test_live_claude_single_invocation_and_runtime(tmp_path): fabric = Fabric() - one_shot = await fabric.run( - fabric_config(tmp_path / "oneshot"), - base_dir=tmp_path / "oneshot", + single = await fabric.run( + fabric_config(tmp_path / "single"), + base_dir=tmp_path / "single", input="Reply only with: FABRIC_CLAUDE_OK", ) - assert one_shot.status == "succeeded" + assert single.status == "succeeded" session_root = tmp_path / "session" - async with await fabric.start_runtime(fabric_config(session_root), base_dir=session_root) as session: + async with await fabric.start_runtime( + fabric_config(session_root), base_dir=session_root + ) as session: first = await session.invoke(input="Remember token FABRIC-CONTINUITY-7") second = await session.invoke( input="Reply only with the token I asked you to remember" @@ -267,3 +273,34 @@ async def test_live_claude_relay_one_shot(tmp_path): result.output, "FABRIC_CLAUDE_RELAY_OK", ) + + +@pytest.mark.skipif( + os.environ.get("RUN_FABRIC_CLAUDE_RELAY_INTEGRATION") != "1", + reason="set RUN_FABRIC_CLAUDE_RELAY_INTEGRATION=1 to run Claude with NeMo Relay", +) +async def test_live_claude_relay_session(tmp_path): + relay_command = os.environ.get("FABRIC_TEST_NEMO_RELAY_COMMAND") + config = fabric_config( + tmp_path, + relay=True, + nemo_relay_command=relay_command, + ) + + async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: + first = await runtime.invoke(input="Remember token FABRIC-CLAUDE-RELAY-7") + second = await runtime.invoke( + input="Reply only with the token I asked you to remember" + ) + + results = (first.to_mapping(), second.to_mapping()) + assert first.status == second.status == "succeeded", results + assert first.output["session_id"] == second.output["session_id"], results + assert first.metadata["host_pid"] == second.metadata["host_pid"], results + assert "FABRIC-CLAUDE-RELAY-7" in second.output["response"], results + for turn in (first, second): + assert turn.telemetry[0].provider == "relay", turn.to_mapping() + assert {artifact["kind"] for artifact in turn.output["relay_artifacts"]} == { + "atof", + "atif", + }, turn.to_mapping() diff --git a/tests/e2e/test_cli_scaffold.py b/tests/e2e/test_cli_scaffold.py index f36521ff..dddc35e8 100644 --- a/tests/e2e/test_cli_scaffold.py +++ b/tests/e2e/test_cli_scaffold.py @@ -9,6 +9,7 @@ ROOT = Path(__file__).parents[2] +SUBPROCESS_TIMEOUT_SECONDS = 120 def generate_scaffold(destination: Path, language: str) -> None: @@ -31,6 +32,7 @@ def generate_scaffold(destination: Path, language: str) -> None: check=True, capture_output=True, text=True, + timeout=SUBPROCESS_TIMEOUT_SECONDS, ) @@ -43,6 +45,7 @@ def test_generated_python_scaffold_installs_editable(tmp_path: Path): check=True, capture_output=True, text=True, + timeout=SUBPROCESS_TIMEOUT_SECONDS, ) python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") environment = os.environ.copy() @@ -56,6 +59,7 @@ def test_generated_python_scaffold_installs_editable(tmp_path: Path): check=True, capture_output=True, text=True, + timeout=SUBPROCESS_TIMEOUT_SECONDS, ) @@ -80,6 +84,7 @@ def test_generated_rust_scaffold_builds(tmp_path: Path): check=True, capture_output=True, text=True, + timeout=SUBPROCESS_TIMEOUT_SECONDS, ) @@ -105,7 +110,11 @@ def test_failed_cli_run_returns_nonzero_status(): check=False, capture_output=True, text=True, + timeout=SUBPROCESS_TIMEOUT_SECONDS, ) assert result.returncode != 0 - assert json.loads(result.stdout)["status"] == "failed" + failure = json.loads(result.stdout) + assert failure["status"] == "failed" + assert failure["error"]["code"] == "runtime_error" + assert "adapter lifecycle start failed" in result.stderr diff --git a/tests/e2e/test_codex.py b/tests/e2e/test_codex.py index 8c4da773..ac059d4d 100644 --- a/tests/e2e/test_codex.py +++ b/tests/e2e/test_codex.py @@ -56,17 +56,17 @@ async def _run() -> None: config = _select_codex_runtime(codex_config()) nonce = f"fabric-{uuid.uuid4().hex[:8]}" client = Fabric() - oneshot = await client.run( + single = await client.run( config, base_dir=BASE_DIR, - input="Reply with exactly: FABRIC_CODEX_ONESHOT_OK", + input="Reply with exactly: FABRIC_CODEX_SINGLE_INVOCATION_OK", ) - assert oneshot["status"] == "succeeded", oneshot.to_mapping() - assert "fabric_codex_oneshot_ok" in oneshot["output"]["response"].lower(), ( - oneshot.to_mapping() - ) - assert oneshot["output"]["adapter"] == "sdk", oneshot.to_mapping() - assert "command" not in oneshot["output"], oneshot.to_mapping() + assert single["status"] == "succeeded", single.to_mapping() + assert ( + "fabric_codex_single_invocation_ok" in single["output"]["response"].lower() + ), single.to_mapping() + assert single["output"]["adapter"] == "sdk", single.to_mapping() + assert "command" not in single["output"], single.to_mapping() async with await client.start_runtime( config, @@ -81,6 +81,8 @@ async def _run() -> None: assert first["status"] == second["status"] == "succeeded", results assert first["output"]["thread_id"] == second["output"]["thread_id"], results assert nonce in second["output"]["response"], second.to_mapping() + assert first["metadata"]["adapter_runner"] == "persistent_local_host", results + assert first["metadata"]["host_pid"] == second["metadata"]["host_pid"], results assert first["output"]["events"], first.to_mapping() assert second["output"]["usage"] is not None, second.to_mapping() @@ -120,6 +122,8 @@ async def _run_relay(relay_command: str) -> None: assert first["status"] == second["status"] == "succeeded", results assert first["output"]["thread_id"] == second["output"]["thread_id"], results assert nonce in second["output"]["response"], second.to_mapping() + assert first["metadata"]["adapter_runner"] == "persistent_local_host", results + assert first["metadata"]["host_pid"] == second["metadata"]["host_pid"], results for turn in (first, second): assert turn["output"]["relay_runtime"]["enabled"] is True, turn.to_mapping() assert {item["kind"] for item in turn["output"]["relay_artifacts"]} >= { diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 964d5001..5a90d3d0 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Opt-in real Deep Agents smoke for Fabric one-shot and multi-turn runtimes. +"""Opt-in real Deep Agents smoke for single-invocation and multi-turn runtimes. RUN_FABRIC_DEEPAGENTS_INTEGRATION=1 NVIDIA_API_KEY=... \ pytest tests/e2e/test_deepagents.py @@ -16,12 +16,89 @@ import pytest +@pytest.mark.usefixtures("mock_nvidia_api_key") +async def test_deepagents_persistent_host_with_mock_model(api_server, tmp_path): + pytest.importorskip("deepagents") + from examples.code_review_agent import deepagents_config + from nemo_fabric import EnvironmentConfig, Fabric, RuntimeConfig + + config = deepagents_config() + config.harness.settings["base_url"] = f"{api_server}/v1" + config.harness.settings["workspace"] = str(tmp_path) + config.environment = EnvironmentConfig( + provider="local", + workspace=tmp_path, + artifacts=tmp_path / "artifacts", + ) + config.runtime = RuntimeConfig( + input_schema="chat", + output_schema="message", + artifacts=tmp_path / "artifacts", + ) + + async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: + first = await runtime.invoke(input="first") + second = await runtime.invoke(input="second") + + results = (first.to_mapping(), second.to_mapping()) + assert first["status"] == second["status"] == "succeeded", results + assert first["metadata"]["adapter_runner"] == "persistent_local_host", results + assert first["metadata"]["host_pid"] == second["metadata"]["host_pid"], results + assert first["output"]["thread_id"] == second["output"]["thread_id"], results + assert first["output"]["resumed"] is False, results + assert second["output"]["resumed"] is True, results + assert "user_count=2" in second["output"]["response"], results + + +@pytest.mark.usefixtures("mock_nvidia_api_key", "nemo_relay") +async def test_deepagents_persistent_host_with_relay_and_mock_model( + api_server, tmp_path +): + pytest.importorskip("deepagents") + from examples.code_review_agent import deepagents_config, with_relay + from nemo_fabric import EnvironmentConfig, Fabric, RuntimeConfig + + config = with_relay(deepagents_config()) + config.harness.settings["base_url"] = f"{api_server}/v1" + config.harness.settings["workspace"] = str(tmp_path) + config.environment = EnvironmentConfig( + provider="local", + workspace=tmp_path, + artifacts=tmp_path / "artifacts", + ) + config.runtime = RuntimeConfig( + input_schema="chat", + output_schema="message", + artifacts=tmp_path / "artifacts", + ) + + async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: + first = await runtime.invoke(input="first") + second = await runtime.invoke(input="second") + + results = (first.to_mapping(), second.to_mapping()) + assert first["status"] == second["status"] == "succeeded", results + assert first["metadata"]["host_pid"] == second["metadata"]["host_pid"], results + assert first["output"]["thread_id"] == second["output"]["thread_id"], results + assert first["output"]["resumed"] is False, results + assert second["output"]["resumed"] is True, results + assert "user_count=2" in second["output"]["response"], results + for turn in (first, second): + assert turn.telemetry[0].provider == "relay", turn.to_mapping() + assert {artifact["kind"] for artifact in turn["output"]["relay_artifacts"]} >= { + "atof", + "atif", + }, turn.to_mapping() + + @pytest.fixture(name="_require_integration") def _require_integration_fixture() -> None: if os.environ.get("RUN_FABRIC_DEEPAGENTS_INTEGRATION") != "1": pytest.skip("set RUN_FABRIC_DEEPAGENTS_INTEGRATION=1 to run") if importlib.util.find_spec("deepagents") is None: - pytest.fail("the deepagents package is required (pip install -e '.[deepagents]')") + pytest.fail( + "the deepagents package is required (pip install -e '.[deepagents]')" + ) if importlib.util.find_spec("nemo_fabric._native") is None: pytest.fail("the nemo_fabric native extension is required (pip install -e .)") if not os.environ.get("NVIDIA_API_KEY"): @@ -29,20 +106,19 @@ def _require_integration_fixture() -> None: @pytest.mark.usefixtures("_require_integration") -async def test_deepagents_oneshot(): +async def test_deepagents_single_invocation(): from examples.code_review_agent import BASE_DIR, deepagents_config from nemo_fabric import Fabric client = Fabric() - oneshot = await client.run( + single = await client.run( deepagents_config(), base_dir=BASE_DIR, - input="Reply with exactly: FABRIC_DEEPAGENTS_ONESHOT_OK", + input="Reply with exactly: FABRIC_DEEPAGENTS_SINGLE_INVOCATION_OK", ) - assert oneshot["status"] == "succeeded", oneshot.to_mapping() - assert oneshot["output"]["response"], oneshot.to_mapping() - # each one-shot run gets a fresh runtime, so it is never a resume - assert oneshot["output"]["resumed"] is False, oneshot.to_mapping() + assert single["status"] == "succeeded", single.to_mapping() + assert single["output"]["response"], single.to_mapping() + assert single["output"]["resumed"] is False, single.to_mapping() @pytest.mark.usefixtures("_require_integration") @@ -53,7 +129,9 @@ async def test_deepagents_multi_turn(): client = Fabric() nonce = f"fabric-{uuid.uuid4().hex[:8]}" - async with await client.start_runtime(deepagents_config(), base_dir=BASE_DIR) as runtime: + async with await client.start_runtime( + deepagents_config(), base_dir=BASE_DIR + ) as runtime: first = await runtime.invoke(input=f"Remember this value: {nonce}") second = await runtime.invoke( input="Reply with only the value I asked you to remember." @@ -63,10 +141,42 @@ async def test_deepagents_multi_turn(): assert first["status"] == second["status"] == "succeeded", results # one started runtime keeps a stable LangGraph thread across turns assert first["output"]["thread_id"] == second["output"]["thread_id"], results - # the first turn opens the runtime; the second resumes and recalls turn one + assert first["metadata"]["adapter_runner"] == "persistent_local_host", results + assert first["metadata"]["host_pid"] == second["metadata"]["host_pid"], results + # The second turn continues the live runtime and recalls turn one. + assert first["output"]["resumed"] is False, results + assert second["output"]["resumed"] is True, results + assert nonce in second["output"]["response"], second.to_mapping() + + +@pytest.mark.usefixtures("_require_integration", "nemo_relay") +async def test_deepagents_multi_turn_with_relay(): + from examples.code_review_agent import BASE_DIR, deepagents_config, with_relay + from nemo_fabric import Fabric + + client = Fabric() + nonce = f"fabric-relay-{uuid.uuid4().hex[:8]}" + config = with_relay(deepagents_config()) + + async with await client.start_runtime(config, base_dir=BASE_DIR) as runtime: + first = await runtime.invoke(input=f"Remember this value: {nonce}") + second = await runtime.invoke( + input="Reply with only the value I asked you to remember." + ) + + results = (first.to_mapping(), second.to_mapping()) + assert first["status"] == second["status"] == "succeeded", results + assert first["output"]["thread_id"] == second["output"]["thread_id"], results + assert first["metadata"]["host_pid"] == second["metadata"]["host_pid"], results assert first["output"]["resumed"] is False, results assert second["output"]["resumed"] is True, results assert nonce in second["output"]["response"], second.to_mapping() + for turn in (first, second): + assert turn.telemetry[0].provider == "relay", turn.to_mapping() + assert {artifact["kind"] for artifact in turn["output"]["relay_artifacts"]} >= { + "atof", + "atif", + }, turn.to_mapping() @pytest.mark.usefixtures("_require_integration") diff --git a/tests/e2e/test_hermes_e2e.py b/tests/e2e/test_hermes_e2e.py index 2ef94a5e..eb63e298 100644 --- a/tests/e2e/test_hermes_e2e.py +++ b/tests/e2e/test_hermes_e2e.py @@ -21,12 +21,71 @@ pytestmark = pytest.mark.usefixtures("requires_hermes_agent") +@pytest.mark.usefixtures("mock_nvidia_api_key") +async def test_hermes_persistent_host_reuses_native_session( + code_review_agent_dir: Path, + api_server: str, +): + os.environ["ADAPTER_PYTHON"] = sys.executable + config = hermes_config() + config.harness.settings["base_url"] = f"{api_server}/v1" + + async with await Fabric().start_runtime( + config, base_dir=code_review_agent_dir + ) as runtime: + first = await runtime.invoke(input="first") + second = await runtime.invoke(input="second") + + results = (first.to_mapping(), second.to_mapping()) + assert first["status"] == second["status"] == "succeeded", results + assert first["metadata"]["adapter_runner"] == "persistent_local_host", results + assert first["metadata"]["host_pid"] == second["metadata"]["host_pid"], results + assert "user_count=2" in second["output"]["response"], results + + +@pytest.mark.usefixtures("mock_nvidia_api_key", "nemo_relay") +async def test_hermes_persistent_host_with_relay( + code_review_agent_dir: Path, + api_server: str, +): + os.environ["ADAPTER_PYTHON"] = sys.executable + config = with_relay(hermes_config()) + config.harness.settings["base_url"] = f"{api_server}/v1" + + async with await Fabric().start_runtime( + config, base_dir=code_review_agent_dir + ) as runtime: + first = await runtime.invoke(input="first") + second = await runtime.invoke(input="second") + + results = (first.to_mapping(), second.to_mapping()) + assert first["status"] == second["status"] == "succeeded", results + assert first["metadata"]["host_pid"] == second["metadata"]["host_pid"], results + assert "user_count=2" in second["output"]["response"], results + for turn in (first, second): + assert turn.telemetry[0].provider == "relay", turn.to_mapping() + assert {artifact["kind"] for artifact in turn["output"]["relay_artifacts"]} >= { + "atof", + "atif", + }, turn.to_mapping() + + atof_path = next( + Path(artifact["path"]) + for artifact in second["output"]["relay_artifacts"] + if artifact["kind"] == "atof" + ) + atof_records = [ + json.loads(line) for line in atof_path.read_text(encoding="utf-8").splitlines() + ] + assert sum(record["name"] == "hermes.session.end" for record in atof_records) == 2 + + class TestHermesE2E: """End-to-end Hermes relay assertions.""" config_builder = staticmethod(hermes_config) adapter_kind = "python" - adapter_runner = "python" + adapter_runner = "persistent_local_host" output_adapter = "python" mode = "hermes" artifact_dir = "hermes" @@ -62,7 +121,6 @@ async def run_hermes_with_relay( ).resolve() self.relay_artifacts = self.output["relay_artifacts"] - async def test_artifacts(self): assert self.result["status"] == "succeeded" assert self.result["adapter_kind"] == self.adapter_kind @@ -79,9 +137,9 @@ async def test_artifacts(self): assert output["base_url"] == f"{self.api_server}/v1" assert output["error"] is None assert output["relay_runtime"]["enabled"] is True - assert output["relay_runtime"]["emitter"] == "hermes.observability/nemo_relay" + assert output["relay_runtime"]["emitter"] == "hermes.observability/nemo_relay" assert output["failed"] is False - + assert "echo user_count=" in output["response"] hermes_home = Path(output["hermes_home"]).resolve() @@ -105,8 +163,7 @@ async def test_artifacts(self): assert self.artifact_root.is_dir() artifact_by_name = { - artifact["name"]: artifact - for artifact in self.artifacts["artifacts"] + artifact["name"]: artifact for artifact in self.artifacts["artifacts"] } assert "relay_config" in artifact_by_name assert "stdout" in artifact_by_name @@ -114,12 +171,12 @@ async def test_artifacts(self): relay_config_path = Path(artifact_by_name["relay_config"]["path"]).resolve() assert relay_config_path.is_file() assert relay_config_path.is_relative_to(self.artifact_root) - + relay_config = json.loads(relay_config_path.read_text(encoding="utf-8")) assert relay_config["schema_version"] == "fabric.relay/v1alpha1" assert relay_config["relay"]["enabled"] is True assert relay_config["fabric"]["agent_name"] == "code-review-agent" - + async def test_atof_artifacts(self): kinds = {artifact["kind"] for artifact in self.relay_artifacts} assert "atof" in kinds @@ -131,13 +188,10 @@ async def test_atof_artifacts(self): ] assert atof_paths assert all(path.exists() for path in atof_paths) - assert all( - path.is_relative_to(self.relay_artifact_root) for path in atof_paths - ) + assert all(path.is_relative_to(self.relay_artifact_root) for path in atof_paths) atof_records = [ - json.loads(line) - for line in atof_paths[0].read_text().strip().splitlines() + json.loads(line) for line in atof_paths[0].read_text().strip().splitlines() ] expected_atof_fields = { "atof_version", @@ -156,17 +210,16 @@ async def test_atof_artifacts(self): assert actual_atof_fields.issuperset(expected_atof_fields) assert len(atof_records) == 7 - + assert all( record["metadata"]["model"] == "nvidia/nemotron-3-nano-30b-a3b" and record["metadata"]["platform"] == self.atof_platform for record in atof_records ) - + assert atof_records[-2]["name"] == "hermes.session.end" assert atof_records[-1]["scope_category"] == "end" - async def test_atif_artifacts(self): kinds = {artifact["kind"] for artifact in self.relay_artifacts} assert "atif" in kinds @@ -178,9 +231,7 @@ async def test_atif_artifacts(self): ] assert atif_paths assert all(path.exists() for path in atif_paths) - assert all( - path.is_relative_to(self.relay_artifact_root) for path in atif_paths - ) + assert all(path.is_relative_to(self.relay_artifact_root) for path in atif_paths) trajectory = json.loads(atif_paths[0].read_text()) assert trajectory["agent"]["name"] in {"code-review-agent", "Hermes Agent"} diff --git a/tests/e2e/test_hermes_runtime.py b/tests/e2e/test_hermes_runtime.py index 173fcfce..4f0e57a0 100644 --- a/tests/e2e/test_hermes_runtime.py +++ b/tests/e2e/test_hermes_runtime.py @@ -23,7 +23,20 @@ import pytest + async def test_hermes_runtime(): + _require_hermes_integration() + await _run(relay=False) + + +async def test_hermes_runtime_with_relay(): + _require_hermes_integration() + if importlib.util.find_spec("nemo_relay") is None: + pytest.fail("the nemo-relay Python package is required") + await _run(relay=True) + + +def _require_hermes_integration() -> None: if os.environ.get("RUN_FABRIC_HERMES_INTEGRATION") != "1": pytest.skip("set RUN_FABRIC_HERMES_INTEGRATION=1 to run") if not os.environ.get("NVIDIA_API_KEY"): @@ -54,20 +67,24 @@ async def test_hermes_runtime(): ) python_bin = Path(sys.executable).resolve().parent os.environ["PATH"] = f"{python_bin}{os.pathsep}{os.environ.get('PATH', '')}" - await _run() -async def _run() -> None: - from examples.code_review_agent import BASE_DIR, hermes_config +async def _run(*, relay: bool) -> None: + from examples.code_review_agent import BASE_DIR, hermes_config, with_relay from nemo_fabric import Fabric, RuntimeStatus + config = hermes_config() + if relay: + config = with_relay(config) async with await Fabric().start_runtime( - hermes_config(), + config, base_dir=BASE_DIR, ) as runtime: assert runtime.status is RuntimeStatus.ACTIVE, runtime.status - r1 = await runtime.invoke(input="My name is Robin. Please remember it for later.") + r1 = await runtime.invoke( + input="My name is Robin. Please remember it for later." + ) assert r1["status"] == "succeeded", r1 after_turn1 = runtime.messages assert len(after_turn1) >= 2, after_turn1 @@ -75,10 +92,19 @@ async def _run() -> None: r2 = await runtime.invoke(input="What is my name? Reply with just the name.") assert r2["status"] == "succeeded", r2 assert r2["runtime_id"] == r1["runtime_id"], (r1, r2) + assert r1["metadata"]["adapter_runner"] == "persistent_local_host", (r1, r2) + assert r1["metadata"]["host_pid"] == r2["metadata"]["host_pid"], (r1, r2) # Hermes should return a transcript that includes the prior turn. assert len(runtime.messages) > len(after_turn1), runtime.messages # And the model must recall the name supplied in turn 1. response = (r2["output"].get("response") or "").lower() assert "robin" in response, response + if relay: + for result in (r1, r2): + assert result.telemetry[0].provider == "relay", result.to_mapping() + assert { + artifact["kind"] + for artifact in result["output"]["relay_artifacts"] + } >= {"atof", "atif"}, result.to_mapping() assert runtime.status is RuntimeStatus.STOPPED, runtime.status diff --git a/tests/fixtures/claude/mock-claude-cli.py b/tests/fixtures/claude/mock-claude-cli.py index 164ca7f4..f453d5a6 100755 --- a/tests/fixtures/claude/mock-claude-cli.py +++ b/tests/fixtures/claude/mock-claude-cli.py @@ -77,4 +77,3 @@ ), flush=True, ) - break diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/fabric-adapter.json b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/fabric-adapter.json index b5c7a26c..3df5e70c 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/fabric-adapter.json +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/fabric-adapter.json @@ -5,7 +5,6 @@ "adapter_kind": "python", "runner": { "module": "nemo_fabric_test_adapters.hermes_shim.adapter", - "callable": "run", "cwd": ".", "env": { "PYTHONPATH": "adapters/hermes-shim/src" diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py index 8aee6c9b..d89373e1 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py @@ -6,24 +6,38 @@ from __future__ import annotations -import json -import sys from pathlib import Path from typing import Any +from nemo_fabric_adapters.common import lifecycle + def main() -> None: - payload = json.load(sys.stdin) - output = run(payload) - print(json.dumps(output, sort_keys=True)) - if output.get("failed"): - raise SystemExit(2) + lifecycle.serve(ShimRuntime) + + +class ShimRuntime: + def __init__(self) -> None: + self._start_payload: dict[str, Any] | None = None + async def start(self, payload: dict[str, Any]) -> None: + self._start_payload = payload -def run(payload: dict[str, Any]) -> dict[str, Any]: - """Test adapter entrypoint used by SDK smoke tests.""" + async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]: + if self._start_payload is None: + raise lifecycle.LifecycleError( + "hermes_runtime_not_started", + "shim runtime is not started", + ) + payload = { + **self._start_payload, + "runtime_context": invocation.get("runtime_context"), + "request": invocation.get("request"), + } + return run_selected_mode(payload) - return run_selected_mode(payload) + async def stop(self) -> None: + self._start_payload = None def fabric_config(payload: dict[str, Any]) -> dict[str, Any]: @@ -39,11 +53,13 @@ def request_payload(payload: dict[str, Any]) -> dict[str, Any]: def environment_payload(payload: dict[str, Any]) -> dict[str, Any]: - return runtime_context(payload).get("environment") or payload.get("environment") or {} + return ( + runtime_context(payload).get("environment") or payload.get("environment") or {} + ) def settings_payload(payload: dict[str, Any]) -> dict[str, Any]: - harness = (fabric_config(payload).get("harness") or {}) + harness = fabric_config(payload).get("harness") or {} return harness.get("settings") or payload.get("settings") or {} @@ -73,9 +89,15 @@ def run_shim(payload: dict[str, Any]) -> dict[str, Any]: "runtime_id": context.get("runtime_id"), "workspace": environment.get("workspace") or settings.get("workspace"), "native_skill_paths": (capabilities.get("native") or {}).get("skill_paths", []), - "native_mcp_servers": sorted((capabilities.get("native") or {}).get("mcp_servers", {}).keys()), - "managed_skill_paths": (capabilities.get("managed") or {}).get("skill_paths", []), - "managed_mcp_servers": sorted((capabilities.get("managed") or {}).get("mcp_servers", {}).keys()), + "native_mcp_servers": sorted( + (capabilities.get("native") or {}).get("mcp_servers", {}).keys() + ), + "managed_skill_paths": (capabilities.get("managed") or {}).get( + "skill_paths", [] + ), + "managed_mcp_servers": sorted( + (capabilities.get("managed") or {}).get("mcp_servers", {}).keys() + ), "capability_routes": capabilities.get("routes", []), "telemetry": payload.get("telemetry"), } diff --git a/tests/python/test_environment_handle.py b/tests/python/test_environment_handle.py index 22e6f9c5..a56fc46a 100644 --- a/tests/python/test_environment_handle.py +++ b/tests/python/test_environment_handle.py @@ -6,15 +6,18 @@ from __future__ import annotations import os +from pathlib import Path -from examples.code_review_agent import BASE_DIR, base_config +from examples.code_review_agent import base_config from nemo_fabric import Fabric -async def test_environment_handle(): +async def test_environment_handle(hermes_shim_agent_dir: Path): + config = base_config() + config.harness.adapter_id = "test.fabric.hermes_shim" runtime = await Fabric().start_runtime( - base_config(), - base_dir=BASE_DIR, + config, + base_dir=hermes_shim_agent_dir, ) try: workspace = runtime.handle["environment"]["workspace"] @@ -22,4 +25,4 @@ async def test_environment_handle(): await runtime.stop() assert os.path.isabs(workspace), f"workspace not absolute: {workspace}" - assert workspace == str((BASE_DIR / "repos" / "my-service").resolve()) + assert workspace == str((hermes_shim_agent_dir / "repos" / "my-service").resolve()) diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index ab9576ed..cda9e6be 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -170,7 +170,7 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert result["status"] == "succeeded" assert result.harness == "hermes" assert result["adapter_kind"] == "python" - assert result["metadata"]["adapter_runner"] == "python" + assert result["metadata"]["adapter_runner"] == "persistent_local_host" assert result["output"]["received"] == "hello native" assert result.output["native_mcp_servers"] == ["github"] assert any(artifact.name == "stdout" for artifact in result.artifacts.artifacts) diff --git a/tests/python/test_typed_config.py b/tests/python/test_typed_config.py index 49946049..8dc8446b 100644 --- a/tests/python/test_typed_config.py +++ b/tests/python/test_typed_config.py @@ -121,7 +121,7 @@ async def runs_with_typed_config_and_adapter_directory(client: Fabric) -> None: assert result["status"] == "succeeded", result.get("status") assert result.request_id == "typed-request-1" assert result["adapter_kind"] == "python" - assert result["metadata"]["adapter_runner"] == "python" + assert result["metadata"]["adapter_runner"] == "persistent_local_host" assert result["output"]["received"] == "hello typed"