Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions adapters/common/src/nemo_fabric_adapters/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,9 @@ def merge_unique(*values: Any) -> list[str]:
merged.append(item)
return merged

def without_none(mapping: dict[str, Any]) -> dict[str, Any]:
return {key: value for key, value in mapping.items() if value is not None}


def dump_yaml(value: dict[str, Any]) -> str:
try:
Expand Down
6 changes: 3 additions & 3 deletions examples/harbor/calculator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ Expected Harbor summary: one trial, zero exceptions, and mean reward `1.000`.
```bash
: "${NVIDIA_API_KEY:?Export NVIDIA_API_KEY before running Hermes Agent}"

uv run --extra runtime --extra harbor harbor run \
uv run --extra runtime --extra harbor --extra hermes-agent harbor run \
--path "$TASK_DIR" \
--agent nemo_fabric.integrations.harbor:FabricAgent \
--model nvidia/nemotron-3-nano-30b-a3b \
Expand All @@ -102,7 +102,7 @@ typed config. The API key is passed separately as a task credential.
```bash
: "${NVIDIA_API_KEY:?Export NVIDIA_API_KEY before running Hermes Agent}"

uv run --extra runtime --extra harbor harbor run \
uv run --extra runtime --extra harbor --extra hermes-agent --extra relay harbor run \
--path "$TASK_DIR" \
--agent nemo_fabric.integrations.harbor:FabricAgent \
--model nvidia/nemotron-3-nano-30b-a3b \
Expand Down Expand Up @@ -142,7 +142,7 @@ Claude authentication variables selected by the adapter; this command uses
```bash
: "${ANTHROPIC_API_KEY:?Export ANTHROPIC_API_KEY before running Claude}"

uv run --extra runtime --extra harbor harbor run \
uv run --extra runtime --extra harbor --extra claude harbor run \
--path "$TASK_DIR" \
--agent nemo_fabric.integrations.harbor:FabricAgent \
--model anthropic/claude-sonnet-4-5 \
Expand Down
72 changes: 0 additions & 72 deletions examples/harbor/prepare_swebench.sh

This file was deleted.

2 changes: 0 additions & 2 deletions examples/harbor/swebench/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,3 @@
# SPDX-License-Identifier: Apache-2.0

.relay/
.wheelhouse/
.fabric-package
37 changes: 14 additions & 23 deletions examples/harbor/swebench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,34 +19,30 @@ then continue in the same shell. Export `NVIDIA_API_KEY` for Hermes Agent runs o

## Prepare the Task Bundle

Build the NeMo Fabric wheels and Relay executable that will be uploaded into the
isolated task container:
Build the standalone Relay executable that will be uploaded into the isolated
task container for the Claude walkthrough:
Comment thread
coderabbitai[bot] marked this conversation as resolved.

> **TEMP — remove when NeMo Fabric is released:** This source-checkout bootstrap is
> needed only while NeMo Fabric wheels are unavailable from PyPI. At release, the
> docs writer should replace `FABRIC_PACKAGE` with a pinned PyPI requirement and
> remove `prepare_swebench.sh`, `.fabric-package`, `.wheelhouse`,
> `FABRIC_FIND_LINKS`, and every `PIP_FIND_LINKS` argument. If the Claude Relay
> executable is not yet distributed or discovered automatically, retain only
> the Relay CLI preparation until that is resolved.
> **TEMP — replace the pre-release alpha tag with a stable release once available.**
Comment thread
dagardner-nv marked this conversation as resolved.

```bash
cd "$(git rev-parse --show-toplevel)"
./examples/harbor/prepare_swebench.sh

export FABRIC_AGENT='nemo_fabric.integrations.harbor:FabricAgent'
export FABRIC_BUNDLE="$PWD/examples/harbor/swebench"
export FABRIC_PACKAGE="$(< "$FABRIC_BUNDLE/.fabric-package")"
export FABRIC_FIND_LINKS='/tmp/nemo-fabric-config/.wheelhouse'
export FABRIC_PACKAGE='nemo-fabric[claude,harbor,hermes-agent,relay,runtime]==0.1.0a20260724'
Comment thread
dagardner-nv marked this conversation as resolved.
export RUNS_DIR="$PWD/.tmp/harbor/fabric-swebench"

curl -fsSL https://raw.githubusercontent.com/NVIDIA/NeMo-Relay/main/install.sh |
NEMO_RELAY_VERSION=0.6.0 sh -s -- \
--install-dir "$FABRIC_BUNDLE/.relay/bin"
Comment thread
dagardner-nv marked this conversation as resolved.
```

The preparation script builds the native runtime in a manylinux2014 container,
builds the standalone Relay 0.6.0 CLI in Debian Bullseye, and writes the exact
task-local package requirement to `.fabric-package`. The generated files are
ignored by Git. The uploaded bundle contains adapter descriptors, the example
MCP server, generated wheels, and Relay; it does not contain a persisted
`FabricConfig`.
The curl command downloads and installs the standalone NeMo Relay 0.6.0 CLI tool
and verifies its checksum. For other installation methods, refer to the
[NeMo Relay installation instructions](../../../docs/getting-started/install.mdx#install-nemo-relay).

The generated files are ignored by Git. The uploaded bundle contains the example
MCP server and Relay; it does not contain a persisted `FabricConfig`.

Keep this shell open for the commands below.

Expand All @@ -61,7 +57,6 @@ uv run --extra runtime --extra harbor harbor run \
--ak fabric_adapter_id=nvidia.fabric.hermes \
--ak fabric_config_bundle="$FABRIC_BUNDLE" \
--ak "fabric_package=$FABRIC_PACKAGE" \
--ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \
--install-only \
--job-name django-13741-install \
--jobs-dir "$RUNS_DIR" \
Expand All @@ -84,7 +79,6 @@ uv run --extra runtime --extra harbor harbor run \
--ak fabric_adapter_id=nvidia.fabric.hermes \
--ak fabric_config_bundle="$FABRIC_BUNDLE" \
--ak "fabric_package=$FABRIC_PACKAGE" \
--ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \
--ae "NVIDIA_API_KEY=$NVIDIA_API_KEY" \
--job-name django-13741-hermes \
--jobs-dir "$RUNS_DIR" \
Expand Down Expand Up @@ -115,7 +109,6 @@ uv run --extra runtime --extra harbor harbor run \
--ak fabric_telemetry=relay \
--ak 'fabric_harness_settings={"nemo_relay_command":"/tmp/nemo-fabric-config/.relay/bin/nemo-relay"}' \
--ak "fabric_package=$FABRIC_PACKAGE" \
--ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \
--ae "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" \
--job-name django-13741-claude \
--jobs-dir "$RUNS_DIR" \
Expand Down Expand Up @@ -154,7 +147,6 @@ uv run --extra runtime --extra harbor harbor run \
--ak fabric_config_bundle="$FABRIC_BUNDLE" \
--ak fabric_telemetry=relay \
--ak "fabric_package=$FABRIC_PACKAGE" \
--ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \
--ae "NVIDIA_API_KEY=$NVIDIA_API_KEY" \
--job-name django-13741-hermes-skill \
--jobs-dir "$RUNS_DIR" \
Expand Down Expand Up @@ -245,7 +237,6 @@ uv run --extra runtime --extra harbor harbor run \
--ak fabric_config_bundle="$FABRIC_BUNDLE" \
--ak fabric_telemetry=relay \
--ak "fabric_package=$FABRIC_PACKAGE" \
--ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS" \
--ae "NVIDIA_API_KEY=$NVIDIA_API_KEY" \
--job-name swebench-verified-hermes-5 \
--jobs-dir "$RUNS_DIR" \
Expand Down
4 changes: 2 additions & 2 deletions examples/harbor/swebench/sample-artifacts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,5 @@ ATOF and ATIF output before running the example yourself.
The checked-in trajectories record Relay 0.5.0 and the earlier
`nvidia/nemotron-3-nano` model identifier because they are historical evidence
from the original runs. New runs use the compatible Relay CLI version pinned by
[`prepare_swebench.sh`](../../prepare_swebench.sh) and the model selected in the
current walkthrough; do not treat sample metadata as installation guidance.
the [task-bundle setup](../README.md#prepare-the-task-bundle) in the walkthrough
and the selected model; do not treat sample metadata as installation guidance.
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ nemo-fabric-adapters-hermes = { path = "adapters/hermes", editable = true }
[tool.pytest.ini_options]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "session"
testpaths = ["tests"]

[tool.ruff.lint]
# flake8-bugbear: enforce only the cached-instance-method rule (B019).
Expand Down
14 changes: 10 additions & 4 deletions tests/e2e/test_hermes_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,13 +236,19 @@ async def test_atif_artifacts(self):
trajectory = json.loads(atif_paths[0].read_text())
assert trajectory["agent"]["name"] in {"code-review-agent", "Hermes Agent"}
steps = trajectory["steps"]
assert len(steps) == 5
assert len(steps) == 2

first_step = steps[0]
assert first_step["message"] == "hermes.turn.start"
assert first_step["extra"]["event_payload"]["is_first_turn"] is True
assert first_step["source"] == "user"
assert first_step["message"] == "Reply with exactly: relay ok"
assert (
first_step["extra"]["llm_request"]["model"]
== "nvidia/nemotron-3-nano-30b-a3b"
)

last_step = steps[-1]
assert last_step["message"] == "hermes.session.end"
assert last_step["source"] == "agent"
assert last_step["message"] == self.output["response"]
assert last_step["model_name"] == "nvidia/nemotron-3-nano-30b-a3b"
assert last_step["extra"]["invocation"]["framework"] == "nemo_relay"
assert last_step["extra"]["invocation"]["status"] == "completed"
24 changes: 12 additions & 12 deletions tests/integrations/test_harbor_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
CALCULATOR_FABRIC_ROOT = CALCULATOR_ROOT / "task" / "environment" / "fabric"
SWEBENCH_ROOT = ROOT / "examples" / "harbor" / "swebench"
SWEBENCH_README = SWEBENCH_ROOT / "README.md"
SWEBENCH_PREPARE = ROOT / "examples" / "harbor" / "prepare_swebench.sh"
SWEBENCH_MCP_CONFIG = SWEBENCH_ROOT / "mcp" / "repo-inspector.mcp.json"
INTEGRATION_README = ROOT / "examples" / "harbor" / "README.md"
SDK_INTEGRATION_README = (
Expand Down Expand Up @@ -287,7 +286,7 @@ def test_harbor_calculator_documents_explicit_cli_commands():
swebench = SWEBENCH_README.read_text(encoding="utf-8")

assert "run.sh" not in calculator
assert calculator.count("uv run --extra runtime --extra harbor harbor run") == 4
assert calculator.count(" harbor run \\") == 4
assert landing.count("uv run --extra runtime --extra harbor harbor run") == 0
assert swebench.count("uv run --extra runtime --extra harbor harbor run") == 5
assert "--agent-import-path" not in landing + calculator + swebench
Expand All @@ -305,9 +304,13 @@ def test_harbor_calculator_documents_explicit_cli_commands():
assert "swebench/README.md" in landing
assert "fabric_adapter_id" in landing
assert 'export TMPDIR="$HOME/harbor-tmp"' in landing
assert "./examples/harbor/prepare_swebench.sh" in swebench
assert 'FABRIC_PACKAGE="$(< "$FABRIC_BUNDLE/.fabric-package")"' in swebench
assert '--ae "PIP_FIND_LINKS=$FABRIC_FIND_LINKS"' in swebench
assert "raw.githubusercontent.com/NVIDIA/NeMo-Relay/main/install.sh" in swebench
assert (
"FABRIC_PACKAGE="
"'nemo-fabric[claude,harbor,hermes-agent,relay,runtime]==0.1.0a20260724'"
in swebench
)
assert "PIP_FIND_LINKS" not in swebench
assert "--dataset swe-bench/swe-bench-verified" in swebench
for flag in (
"--path",
Expand Down Expand Up @@ -340,17 +343,14 @@ def test_harbor_generated_paths_are_ignored():
assert "task/environment/.vendor.*/" in ignore


def test_swebench_bootstrap_pins_a_supported_relay_cli():
def test_swebench_setup_pins_a_supported_relay_cli():
from nemo_fabric_adapters.common.relay_gateway import RELAY_MINIMUM_VERSION

prepare = SWEBENCH_PREPARE.read_text(encoding="utf-8")
swebench = SWEBENCH_README.read_text(encoding="utf-8")
minimum = ".".join(str(part) for part in RELAY_MINIMUM_VERSION)

assert f'relay_cli_version="{minimum}"' in prepare
assert '"nemo-relay $relay_cli_version"' in prepare
assert 'nemo-relay-cli --version "$relay_cli_version" --locked' in prepare
assert '"$(uname -s)" != "Linux"' in prepare
assert '"$(uname -m)" != "x86_64"' in prepare
assert f"NEMO_RELAY_VERSION={minimum}" in swebench
assert '--install-dir "$FABRIC_BUNDLE/.relay/bin"' in swebench


def test_harbor_calculator_setup_and_solution_fail_fast():
Expand Down
6 changes: 3 additions & 3 deletions tests/python/test_installed_adapter_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def _python_sysconfig_path(python: Path, name: str) -> Path:
@pytest.fixture(name="adapter_python")
def adapter_python_fixture(tmp_path: Path) -> tuple[Path, Path]:
adapter_env = tmp_path / "adapter-env"
venv.EnvBuilder(with_pip=False).create(adapter_env)
venv.EnvBuilder(with_pip=False, symlinks=os.name != "nt").create(adapter_env)
python = adapter_env / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
data_root = _python_sysconfig_path(python, "data")
descriptor = _write_descriptor(data_root, "configured.adapter")
Expand Down Expand Up @@ -162,7 +162,7 @@ def test_adapter_python_data_directory_replaces_current_data_directory(
patch_sysconfig_data(current_data_root)

adapter_env = tmp_path / "adapter-env"
venv.EnvBuilder(with_pip=False).create(adapter_env)
venv.EnvBuilder(with_pip=False, symlinks=os.name != "nt").create(adapter_env)
adapter_python = adapter_env / (
"Scripts/python.exe" if os.name == "nt" else "bin/python"
)
Expand Down Expand Up @@ -235,7 +235,7 @@ def test_unset_harness_python_env_uses_sdk_interpreter(

def test_adapter_python_data_path_query_times_out(tmp_path: Path):
adapter_env = tmp_path / "slow-adapter-env"
venv.EnvBuilder(with_pip=False).create(adapter_env)
venv.EnvBuilder(with_pip=False, symlinks=os.name != "nt").create(adapter_env)
adapter_python = adapter_env / (
"Scripts/python.exe" if os.name == "nt" else "bin/python"
)
Expand Down
Loading