diff --git a/adapters/common/src/nemo_fabric_adapters/common/utils.py b/adapters/common/src/nemo_fabric_adapters/common/utils.py index e8dad5d3..cf8433b8 100644 --- a/adapters/common/src/nemo_fabric_adapters/common/utils.py +++ b/adapters/common/src/nemo_fabric_adapters/common/utils.py @@ -183,6 +183,9 @@ def merge_unique(*values: Any) -> list[str]: merged.append(item) return merged +def without_none(mapping: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in mapping.items() if value is not None} + def dump_yaml(value: dict[str, Any]) -> str: try: diff --git a/examples/harbor/calculator/README.md b/examples/harbor/calculator/README.md index 26d48dbd..a95887c4 100644 --- a/examples/harbor/calculator/README.md +++ b/examples/harbor/calculator/README.md @@ -78,7 +78,7 @@ Expected Harbor summary: one trial, zero exceptions, and mean reward `1.000`. ```bash : "${NVIDIA_API_KEY:?Export NVIDIA_API_KEY before running Hermes Agent}" -uv run --extra runtime --extra harbor harbor run \ +uv run --extra runtime --extra harbor --extra hermes-agent harbor run \ --path "$TASK_DIR" \ --agent nemo_fabric.integrations.harbor:FabricAgent \ --model nvidia/nemotron-3-nano-30b-a3b \ @@ -102,7 +102,7 @@ typed config. The API key is passed separately as a task credential. ```bash : "${NVIDIA_API_KEY:?Export NVIDIA_API_KEY before running Hermes Agent}" -uv run --extra runtime --extra harbor harbor run \ +uv run --extra runtime --extra harbor --extra hermes-agent --extra relay harbor run \ --path "$TASK_DIR" \ --agent nemo_fabric.integrations.harbor:FabricAgent \ --model nvidia/nemotron-3-nano-30b-a3b \ @@ -142,7 +142,7 @@ Claude authentication variables selected by the adapter; this command uses ```bash : "${ANTHROPIC_API_KEY:?Export ANTHROPIC_API_KEY before running Claude}" -uv run --extra runtime --extra harbor harbor run \ +uv run --extra runtime --extra harbor --extra claude harbor run \ --path "$TASK_DIR" \ --agent nemo_fabric.integrations.harbor:FabricAgent \ --model anthropic/claude-sonnet-4-5 \ diff --git a/examples/harbor/prepare_swebench.sh b/examples/harbor/prepare_swebench.sh deleted file mode 100755 index 5930013d..00000000 --- a/examples/harbor/prepare_swebench.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# TEMPORARY SOURCE-CHECKOUT BOOTSTRAP — remove when Fabric is released. -# -# The wheel-building path exists only because the Fabric distributions are not -# yet available from PyPI. Once they are published, the Harbor README should use -# a pinned PyPI requirement directly and this script's wheelhouse, -# .fabric-package file, and PIP_FIND_LINKS plumbing should be deleted. -# -# Claude Relay also needs the standalone nemo-relay executable. If that binary -# is not distributed or discovered automatically when the wheels ship, retain -# only the Relay CLI preparation below until its packaging is resolved. - -set -euo pipefail - -repo_root="$(git rev-parse --show-toplevel)" -bundle_dir="$repo_root/examples/harbor/swebench" -wheelhouse="$bundle_dir/.wheelhouse" -relay_root="$bundle_dir/.relay" -relay_cli_version="0.6.0" - -if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then - echo "This SWE-Bench example currently requires an x86_64 Linux host." >&2 - exit 1 -fi - -mkdir -p "$wheelhouse" "$relay_root" -find "$wheelhouse" -maxdepth 1 -type f -name 'nemo_fabric*.whl' -delete - -# TEMP: Build the unpublished Python packages from this source checkout. -for project in adapters/common adapters/claude adapters/hermes .; do - uv build --wheel --out-dir "$wheelhouse" "$repo_root/$project" -done - -docker run --rm \ - -e CARGO_TARGET_DIR=/tmp/target \ - -e HOST_UID="$(id -u)" \ - -e HOST_GID="$(id -g)" \ - -v "$repo_root:/io:ro" \ - -v "$wheelhouse:/out" \ - -w /io/python \ - --entrypoint /bin/bash \ - ghcr.io/pyo3/maturin:v1.9.6 \ - -lc 'maturin build --release --locked --out /out --compatibility manylinux2014 && chown "$HOST_UID:$HOST_GID" /out/*.whl' - -# TEMP: Keep only this block if PyPI wheels land before Relay CLI distribution. -relay_version="$({ "$relay_root/bin/nemo-relay" --version 2>/dev/null || true; })" -if [[ "$relay_version" != "nemo-relay $relay_cli_version" ]]; then - docker run --rm \ - --user "$(id -u):$(id -g)" \ - -e CARGO_HOME=/tmp/cargo \ - -e CARGO_TARGET_DIR=/tmp/target \ - -v "$relay_root:/out" \ - rust:1.94-bullseye \ - cargo install nemo-relay-cli --version "$relay_cli_version" --locked --force --root /out -fi - -# TEMP: Record the source-built meta-wheel path for installation in the task. -fabric_wheel="$(find "$wheelhouse" -maxdepth 1 -type f -name 'nemo_fabric-*-py3-none-any.whl' -printf '%f\n' | sort | tail -n 1)" -if [[ -z "$fabric_wheel" ]]; then - echo "The nemo-fabric wheel was not created." >&2 - exit 1 -fi - -printf '%s\n' \ - "nemo-fabric[claude,harbor,hermes,hermes-agent,relay,runtime] @ file:///tmp/nemo-fabric-config/.wheelhouse/$fabric_wheel" \ - > "$bundle_dir/.fabric-package" - -echo "Prepared $bundle_dir" -echo "Fabric requirement: $(< "$bundle_dir/.fabric-package")" diff --git a/examples/harbor/swebench/.gitignore b/examples/harbor/swebench/.gitignore index c4984b47..831f5c07 100644 --- a/examples/harbor/swebench/.gitignore +++ b/examples/harbor/swebench/.gitignore @@ -2,5 +2,3 @@ # SPDX-License-Identifier: Apache-2.0 .relay/ -.wheelhouse/ -.fabric-package diff --git a/examples/harbor/swebench/README.md b/examples/harbor/swebench/README.md index 8102eb01..f6eaff2e 100644 --- a/examples/harbor/swebench/README.md +++ b/examples/harbor/swebench/README.md @@ -19,34 +19,30 @@ then continue in the same shell. Export `NVIDIA_API_KEY` for Hermes Agent runs o ## Prepare the Task Bundle -Build the NeMo Fabric wheels and Relay executable that will be uploaded into the -isolated task container: +Build the standalone Relay executable that will be uploaded into the isolated +task container for the Claude walkthrough: -> **TEMP — remove when NeMo Fabric is released:** This source-checkout bootstrap is -> needed only while NeMo Fabric wheels are unavailable from PyPI. At release, the -> docs writer should replace `FABRIC_PACKAGE` with a pinned PyPI requirement and -> remove `prepare_swebench.sh`, `.fabric-package`, `.wheelhouse`, -> `FABRIC_FIND_LINKS`, and every `PIP_FIND_LINKS` argument. If the Claude Relay -> executable is not yet distributed or discovered automatically, retain only -> the Relay CLI preparation until that is resolved. +> **TEMP — replace the pre-release alpha tag with a stable release once available.** ```bash cd "$(git rev-parse --show-toplevel)" -./examples/harbor/prepare_swebench.sh export FABRIC_AGENT='nemo_fabric.integrations.harbor:FabricAgent' export FABRIC_BUNDLE="$PWD/examples/harbor/swebench" -export FABRIC_PACKAGE="$(< "$FABRIC_BUNDLE/.fabric-package")" -export FABRIC_FIND_LINKS='/tmp/nemo-fabric-config/.wheelhouse' +export FABRIC_PACKAGE='nemo-fabric[claude,harbor,hermes-agent,relay,runtime]==0.1.0a20260724' export RUNS_DIR="$PWD/.tmp/harbor/fabric-swebench" + +curl -fsSL https://raw.githubusercontent.com/NVIDIA/NeMo-Relay/main/install.sh | + NEMO_RELAY_VERSION=0.6.0 sh -s -- \ + --install-dir "$FABRIC_BUNDLE/.relay/bin" ``` -The preparation script builds the native runtime in a manylinux2014 container, -builds the standalone Relay 0.6.0 CLI in Debian Bullseye, and writes the exact -task-local package requirement to `.fabric-package`. The generated files are -ignored by Git. The uploaded bundle contains adapter descriptors, the example -MCP server, generated wheels, and Relay; it does not contain a persisted -`FabricConfig`. +The curl command downloads and installs the standalone NeMo Relay 0.6.0 CLI tool +and verifies its checksum. For other installation methods, refer to the +[NeMo Relay installation instructions](../../../docs/getting-started/install.mdx#install-nemo-relay). + +The generated files are ignored by Git. The uploaded bundle contains the example +MCP server and Relay; it does not contain a persisted `FabricConfig`. Keep this shell open for the commands below. @@ -61,7 +57,6 @@ uv run --extra runtime --extra harbor harbor run \ --ak fabric_adapter_id=nvidia.fabric.hermes \ --ak fabric_config_bundle="$FABRIC_BUNDLE" \ --ak "fabric_package=$FABRIC_PACKAGE" \ - --ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \ --install-only \ --job-name django-13741-install \ --jobs-dir "$RUNS_DIR" \ @@ -84,7 +79,6 @@ uv run --extra runtime --extra harbor harbor run \ --ak fabric_adapter_id=nvidia.fabric.hermes \ --ak fabric_config_bundle="$FABRIC_BUNDLE" \ --ak "fabric_package=$FABRIC_PACKAGE" \ - --ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \ --ae "NVIDIA_API_KEY=$NVIDIA_API_KEY" \ --job-name django-13741-hermes \ --jobs-dir "$RUNS_DIR" \ @@ -115,7 +109,6 @@ uv run --extra runtime --extra harbor harbor run \ --ak fabric_telemetry=relay \ --ak 'fabric_harness_settings={"nemo_relay_command":"/tmp/nemo-fabric-config/.relay/bin/nemo-relay"}' \ --ak "fabric_package=$FABRIC_PACKAGE" \ - --ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \ --ae "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" \ --job-name django-13741-claude \ --jobs-dir "$RUNS_DIR" \ @@ -154,7 +147,6 @@ uv run --extra runtime --extra harbor harbor run \ --ak fabric_config_bundle="$FABRIC_BUNDLE" \ --ak fabric_telemetry=relay \ --ak "fabric_package=$FABRIC_PACKAGE" \ - --ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \ --ae "NVIDIA_API_KEY=$NVIDIA_API_KEY" \ --job-name django-13741-hermes-skill \ --jobs-dir "$RUNS_DIR" \ @@ -245,7 +237,6 @@ uv run --extra runtime --extra harbor harbor run \ --ak fabric_config_bundle="$FABRIC_BUNDLE" \ --ak fabric_telemetry=relay \ --ak "fabric_package=$FABRIC_PACKAGE" \ - --ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \ --ae "NVIDIA_API_KEY=$NVIDIA_API_KEY" \ --job-name swebench-verified-hermes-5 \ --jobs-dir "$RUNS_DIR" \ diff --git a/examples/harbor/swebench/sample-artifacts/README.md b/examples/harbor/swebench/sample-artifacts/README.md index bf46bb1e..c5751608 100644 --- a/examples/harbor/swebench/sample-artifacts/README.md +++ b/examples/harbor/swebench/sample-artifacts/README.md @@ -24,5 +24,5 @@ ATOF and ATIF output before running the example yourself. The checked-in trajectories record Relay 0.5.0 and the earlier `nvidia/nemotron-3-nano` model identifier because they are historical evidence from the original runs. New runs use the compatible Relay CLI version pinned by -[`prepare_swebench.sh`](../../prepare_swebench.sh) and the model selected in the -current walkthrough; do not treat sample metadata as installation guidance. +the [task-bundle setup](../README.md#prepare-the-task-bundle) in the walkthrough +and the selected model; do not treat sample metadata as installation guidance. diff --git a/pyproject.toml b/pyproject.toml index 0986794b..00a547ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,7 @@ nemo-fabric-adapters-hermes = { path = "adapters/hermes", editable = true } [tool.pytest.ini_options] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" +testpaths = ["tests"] [tool.ruff.lint] # flake8-bugbear: enforce only the cached-instance-method rule (B019). diff --git a/tests/e2e/test_hermes_e2e.py b/tests/e2e/test_hermes_e2e.py index eb63e298..96331d65 100644 --- a/tests/e2e/test_hermes_e2e.py +++ b/tests/e2e/test_hermes_e2e.py @@ -236,13 +236,19 @@ async def test_atif_artifacts(self): trajectory = json.loads(atif_paths[0].read_text()) assert trajectory["agent"]["name"] in {"code-review-agent", "Hermes Agent"} steps = trajectory["steps"] - assert len(steps) == 5 + assert len(steps) == 2 first_step = steps[0] - assert first_step["message"] == "hermes.turn.start" - assert first_step["extra"]["event_payload"]["is_first_turn"] is True + assert first_step["source"] == "user" + assert first_step["message"] == "Reply with exactly: relay ok" + assert ( + first_step["extra"]["llm_request"]["model"] + == "nvidia/nemotron-3-nano-30b-a3b" + ) last_step = steps[-1] - assert last_step["message"] == "hermes.session.end" + assert last_step["source"] == "agent" + assert last_step["message"] == self.output["response"] + assert last_step["model_name"] == "nvidia/nemotron-3-nano-30b-a3b" assert last_step["extra"]["invocation"]["framework"] == "nemo_relay" assert last_step["extra"]["invocation"]["status"] == "completed" diff --git a/tests/integrations/test_harbor_runner.py b/tests/integrations/test_harbor_runner.py index 3e8f239e..d1d99f0b 100644 --- a/tests/integrations/test_harbor_runner.py +++ b/tests/integrations/test_harbor_runner.py @@ -17,7 +17,6 @@ CALCULATOR_FABRIC_ROOT = CALCULATOR_ROOT / "task" / "environment" / "fabric" SWEBENCH_ROOT = ROOT / "examples" / "harbor" / "swebench" SWEBENCH_README = SWEBENCH_ROOT / "README.md" -SWEBENCH_PREPARE = ROOT / "examples" / "harbor" / "prepare_swebench.sh" SWEBENCH_MCP_CONFIG = SWEBENCH_ROOT / "mcp" / "repo-inspector.mcp.json" INTEGRATION_README = ROOT / "examples" / "harbor" / "README.md" SDK_INTEGRATION_README = ( @@ -287,7 +286,7 @@ def test_harbor_calculator_documents_explicit_cli_commands(): swebench = SWEBENCH_README.read_text(encoding="utf-8") assert "run.sh" not in calculator - assert calculator.count("uv run --extra runtime --extra harbor harbor run") == 4 + assert calculator.count(" harbor run \\") == 4 assert landing.count("uv run --extra runtime --extra harbor harbor run") == 0 assert swebench.count("uv run --extra runtime --extra harbor harbor run") == 5 assert "--agent-import-path" not in landing + calculator + swebench @@ -305,9 +304,13 @@ def test_harbor_calculator_documents_explicit_cli_commands(): assert "swebench/README.md" in landing assert "fabric_adapter_id" in landing assert 'export TMPDIR="$HOME/harbor-tmp"' in landing - assert "./examples/harbor/prepare_swebench.sh" in swebench - assert 'FABRIC_PACKAGE="$(< "$FABRIC_BUNDLE/.fabric-package")"' in swebench - assert '--ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS"' in swebench + assert "raw.githubusercontent.com/NVIDIA/NeMo-Relay/main/install.sh" in swebench + assert ( + "FABRIC_PACKAGE=" + "'nemo-fabric[claude,harbor,hermes-agent,relay,runtime]==0.1.0a20260724'" + in swebench + ) + assert "PIP_FIND_LINKS" not in swebench assert "--dataset swe-bench/swe-bench-verified" in swebench for flag in ( "--path", @@ -340,17 +343,14 @@ def test_harbor_generated_paths_are_ignored(): assert "task/environment/.vendor.*/" in ignore -def test_swebench_bootstrap_pins_a_supported_relay_cli(): +def test_swebench_setup_pins_a_supported_relay_cli(): from nemo_fabric_adapters.common.relay_gateway import RELAY_MINIMUM_VERSION - prepare = SWEBENCH_PREPARE.read_text(encoding="utf-8") + swebench = SWEBENCH_README.read_text(encoding="utf-8") minimum = ".".join(str(part) for part in RELAY_MINIMUM_VERSION) - assert f'relay_cli_version="{minimum}"' in prepare - assert '"nemo-relay $relay_cli_version"' in prepare - assert 'nemo-relay-cli --version "$relay_cli_version" --locked' in prepare - assert '"$(uname -s)" != "Linux"' in prepare - assert '"$(uname -m)" != "x86_64"' in prepare + assert f"NEMO_RELAY_VERSION={minimum}" in swebench + assert '--install-dir "$FABRIC_BUNDLE/.relay/bin"' in swebench def test_harbor_calculator_setup_and_solution_fail_fast(): diff --git a/tests/python/test_installed_adapter_discovery.py b/tests/python/test_installed_adapter_discovery.py index 61771b90..b2a02a50 100644 --- a/tests/python/test_installed_adapter_discovery.py +++ b/tests/python/test_installed_adapter_discovery.py @@ -98,7 +98,7 @@ def _python_sysconfig_path(python: Path, name: str) -> Path: @pytest.fixture(name="adapter_python") def adapter_python_fixture(tmp_path: Path) -> tuple[Path, Path]: adapter_env = tmp_path / "adapter-env" - venv.EnvBuilder(with_pip=False).create(adapter_env) + venv.EnvBuilder(with_pip=False, symlinks=os.name != "nt").create(adapter_env) python = adapter_env / ("Scripts/python.exe" if os.name == "nt" else "bin/python") data_root = _python_sysconfig_path(python, "data") descriptor = _write_descriptor(data_root, "configured.adapter") @@ -162,7 +162,7 @@ def test_adapter_python_data_directory_replaces_current_data_directory( patch_sysconfig_data(current_data_root) adapter_env = tmp_path / "adapter-env" - venv.EnvBuilder(with_pip=False).create(adapter_env) + venv.EnvBuilder(with_pip=False, symlinks=os.name != "nt").create(adapter_env) adapter_python = adapter_env / ( "Scripts/python.exe" if os.name == "nt" else "bin/python" ) @@ -235,7 +235,7 @@ def test_unset_harness_python_env_uses_sdk_interpreter( def test_adapter_python_data_path_query_times_out(tmp_path: Path): adapter_env = tmp_path / "slow-adapter-env" - venv.EnvBuilder(with_pip=False).create(adapter_env) + venv.EnvBuilder(with_pip=False, symlinks=os.name != "nt").create(adapter_env) adapter_python = adapter_env / ( "Scripts/python.exe" if os.name == "nt" else "bin/python" )