Feature/tracing

.env.example

@@ -22,46 +22,20 @@ NETOPSBENCH_INFLUXDB_TOKEN=replace-me
 NETOPSBENCH_INFLUXDB_ORG=netopsbench
 NETOPSBENCH_INFLUXDB_BUCKET=netopsbench
 
-# --- Grafana ---
-# NETOPSBENCH_GRAFANA_URL=http://localhost:3000
-# NETOPSBENCH_GRAFANA_PASSWORD=admin
-# NETOPSBENCH_GRAFANA_INFLUXDB_URL=http://influxdb:8086
-# NETOPSBENCH_GRAFANA_DEFAULT_BUCKET=netopsbench
-
-# --- Topology ---
+# --- Manual topology / thin runtime wrappers (optional) ---
 # NETOPSBENCH_TOPOLOGY_DIR=lab-topology/generated_topology_xs
-# NETOPSBENCH_TOPOLOGY_ID=dcn
 # NETOPSBENCH_LAB_NAME=dcn
 # NETOPSBENCH_MGMT_SUBNET=                     # Override management subnet (e.g. 172.20.20.0/24)
 # NETOPSBENCH_MGMT_NETWORK=                    # Override Docker management network name
 
-# --- SONiC / gNMI Telemetry ---
-# SONIC_GNMI_PORT=50051
-# SONIC_GNMI_USERNAME=admin
-# SONIC_GNMI_PASSWORD=                          # Set the SONiC gNMI password for telemetry collection
-# SONIC_GNMI_ENCODING=json_ietf
-# SONIC_GNMI_TARGET=COUNTERS_DB
-# SONIC_GNMI_SUBSCRIPTION_MODE=on_change       # on_change | sample | target_defined
-# SONIC_GNMI_SAMPLE_INTERVAL=10s               # Only used when subscription_mode=sample
-
-# --- Telemetry collectors (sFlow / syslog) ---
-# NETOPSBENCH_SFLOW_COLLECTOR=                 # sFlow collector IP (defaults to management IP)
-# NETOPSBENCH_SYSLOG_COLLECTOR=               # Syslog collector IP (defaults to management IP)
-# NETOPSBENCH_SFLOW_PORT=6343
-# NETOPSBENCH_SFLOW_POLLING_INTERVAL=20
-# NETOPSBENCH_SFLOW_SAMPLE_RATE=1000
-# NETOPSBENCH_SFLOW_SAMPLE_DIRECTION=ingress
-
-# --- Pingmesh ---
-# PINGMESH_CYCLE_INTERVAL=5                    # Probe cycle interval in seconds
-
 # --- Runtime / debug ---
 # NETOPSBENCH_LOG_LEVEL=INFO                   # DEBUG | INFO | WARNING | ERROR
 # NETOPSBENCH_NO_SUDO=0                        # Set to 1 in CI/test environments without sudo
-
-# --- Worker Pool ---
-# NETOPSBENCH_WORKER_DEPLOY_JOBS=              # Parallel worker deploy jobs (default: num_workers)
-# NETOPSBENCH_SONIC_WAIT_TRIES=180             # Max SONiC readiness retries (5s each, default: 15 min)
-# NETOPSBENCH_AGENT_TIMEOUT_SECONDS=300
-# NETOPSBENCH_WORKER_AGENT_TIMEOUT_SECONDS=600
-# NETOPSBENCH_WORKER_DISABLE_LANGSMITH=false
+# NETOPSBENCH_PYTHON=/path/to/python3          # Interpreter used by repository shell wrappers
+# NETOPSBENCH_TRAFFIC_PARALLELISM=32           # Parallel docker exec workers for benchmark traffic start/stop
+
+# --- Optional semantic fault-type judge ---
+# NETOPSBENCH_FAULT_TYPE_JUDGE_ENABLED=0
+# NETOPSBENCH_FAULT_TYPE_JUDGE_MODEL=gpt-4o-mini
+# NETOPSBENCH_FAULT_TYPE_JUDGE_API_KEY=
+# NETOPSBENCH_FAULT_TYPE_JUDGE_BASE_URL=

.gitignore

@@ -30,6 +30,8 @@ lab-topology/
 .worktrees/
 .agents/
 .pytest_cache/
+build/
+dist/
 tmp/
 .netopsbench*/
 
@@ -42,6 +44,7 @@ observability/telegraf.conf
 .coverage
 plot_*
 !scripts/plot_results.py
+examples/agents/minimal_deepagent/conversation_history/
 
 # Fumadocs / Next.js site artifacts
 docs/node_modules/

AGENTS.md

@@ -30,8 +30,9 @@ asks for a breaking change.
   experience; keep commands, docs, and imports aligned.
 - `scenarios/generated/<scale>/`: generated benchmark scenarios used by
   examples and suite runs.
-- `observability/` and `scripts/runtime/`: Docker, Containerlab, Telegraf,
-  Pingmesh, and BGP runtime support.
+- `netopsbench/platform/observability/assets/`: packaged Grafana, Telegraf,
+  and Docker Compose assets.
+- `scripts/runtime/`: documented thin wrappers around the Python runtime CLI.
 - `tests/`: lightweight unit and contract tests. Real Containerlab tests are
   usually marked `real` or named `*_real`.
 
@@ -136,9 +137,9 @@ Pingmesh, and BGP snapshots. Preserve these behaviors:
 
 Relevant files:
 
-- `scripts/observability/start_worker_telegraf.sh`
-- `scripts/runtime/run_bgp_collector.py`
-- `observability/telegraf.conf.template`
+- `netopsbench/platform/observability/lifecycle.py`
+- `netopsbench/platform/observability/bgp_collector.py`
+- `netopsbench/platform/observability/assets/telegraf.conf.template`
 - `tests/test_bgp_collector.py`
 - `tests/test_runtime_config_consistency.py`
 

docs/content/docs/architecture/system-overview.mdx

@@ -11,7 +11,7 @@ A benchmark run is a closed loop: resolve a scenario, provision a topology, inje
 
 | Stage | What happens |
 |---|---|
-| Scenario and topology | A generated scenario selects topology scale, traffic profile, fault type, target device, and target interface when applicable. |
+| Scenario and topology | A generated scenario selects topology scale, fault type, target device, and target interface. All scenarios use the canonical standard background traffic model. |
 | Runtime provisioning | The SDK provisions a Linux / Containerlab runtime and starts observability services. |
 | Fault episode | The scenario executor applies traffic generation, injects the selected fault, waits through the observation window, and records symptoms. |
 | Diagnosis context | The platform assembles topology, symptoms, Pingmesh summaries, runtime metadata, and optional tools into `DiagnosticContext`. |

docs/content/docs/build-your-agent/python-api-guide.mdx

@@ -69,9 +69,9 @@ When `artifacts_dir` is omitted, session artifacts are written under the workspa
 
 The `scenario_summaries[*].raw_result_path` fields point to raw JSON files for case-level debugging.
 
-Agent traces are saved by default and can be disabled for a run with `trace=False` or by setting `NETOPSBENCH_TRACE=0`. Disabling trace prevents private runtime trace collection and sidecar artifact creation. Ground truth and score details are written to `traces/results.jsonl`, not into the agent trajectory.
+Agent traces are saved by default and can be disabled for a run with `trace=False`. Disabling trace prevents private runtime trace collection and sidecar artifact creation. Ground truth and score details are written to `traces/results.jsonl`, not into the agent trajectory.
 
-NetOpsBench stores visible prompts, model messages, tool calls, and observations with secret redaction and per-field truncation. The bundled `MinimalDeepAgent` attaches `context.trace.langchain_callback()` to its LangChain-compatible runtime so private LLM messages and tool events flow into the same recorder. Non-LangChain agents can use the advanced manual recorder methods, such as `context.trace.record_llm_request(...)` and `context.trace.record_llm_response(...)`, when they need to capture private model calls. Set `NETOPSBENCH_TRACE_MAX_FIELD_CHARS` to tune truncation.
+NetOpsBench stores visible prompts, model messages, tool calls, and observations with secret redaction and a fixed per-field safety limit. The bundled `MinimalDeepAgent` attaches `context.trace.langchain_callback()` to its LangChain-compatible runtime so private LLM messages and tool events flow into the same recorder. Non-LangChain agents can use the advanced manual recorder methods, such as `context.trace.record_llm_request(...)` and `context.trace.record_llm_response(...)`, when they need to capture private model calls.
 
 Open a completed run directly in the Harbor viewer:
 

docs/content/docs/debug-operate/observability.mdx

@@ -97,14 +97,9 @@ Common overrides:
 
 ```bash
 export NETOPSBENCH_MGMT_SUBNET=172.31.250.0/24
-export SONIC_GNMI_PORT=50051
-export SONIC_GNMI_USERNAME=admin
-export SONIC_GNMI_PASSWORD=<your-password>
-export NETOPSBENCH_SYSLOG_COLLECTOR=172.20.20.200
-export NETOPSBENCH_SFLOW_COLLECTOR=172.20.20.200
 ```
 
-The default SONiC image is `yyyyyt123/netopsbench-sonic-vs-202505-telemetry:202505-telemetry`.
+The default SONiC image is `yyyyyt123/netopsbench-sonic-vs-202505-telemetry:202505-telemetry`. Its gNMI contract is fixed by the benchmark image and generated Telegraf configuration (`50051`, `admin`, `json_ietf`, `COUNTERS_DB`, `on_change`) so benchmark telemetry does not vary with process environment.
 
 ## Cleanup
 

docs/content/docs/run-benchmarks/methodology.mdx

@@ -47,7 +47,7 @@ Each scale combines fault cases with healthy negative samples:
 | Medium | 24 | 4 | 28 |
 | Large | 48 | 4 | 52 |
 
-Scenarios are generated from `scenarios/specs/fault_campaign.yaml`. Running `netopsbench benchmark prepare --scales <scale>` materializes YAML files under `scenarios/generated/<scale>/`.
+Scenarios are generated from the packaged canonical campaign spec. Running `netopsbench benchmark prepare --scales <scale>` materializes YAML files under `scenarios/generated/<scale>/`; `--spec` can select a custom campaign.
 
 ## Scoring dimensions
 

docs/content/docs/run-benchmarks/run-scenario-vs-suite.mdx

@@ -101,9 +101,9 @@ Useful environment overrides:
 |---|---|
 | `BENCH_VENDOR` | Provider passed to `examples/03_run_scale_benchmark.py`. |
 | `BENCH_SCALES` | Space-separated scale list such as `xs small`. |
+| `BENCH_WORKERS_<SCALE>` | Worker count override for one scale, for example `BENCH_WORKERS_LARGE=2`. |
 | `BENCH_CLEAN_RUNS=1` | Explicitly remove previous `.netopsbench/runs/` artifacts before the batch. By default, previous reports and traces are preserved. |
-| `NETOPSBENCH_WORKER_DEPLOY_JOBS` | Worker deployment parallelism override. |
-| `NETOPSBENCH_WORKER_HEALTH_RETRIES` | Health-check retry budget for larger fabrics. |
+| `NETOPSBENCH_TRAFFIC_PARALLELISM` | Bounded client traffic start/stop concurrency; defaults to 32. |
 
 The script writes logs under `scenario_results/benchmark_logs_<timestamp>/`, records the run ids for the batch in `benchmark_runs_<timestamp>.jsonl`, and writes a CSV summary under `scenario_results/benchmark_summary_<timestamp>.csv`. Run artifacts use timestamp ids such as `run-20260605T124040Z`; use `netopsbench trace view` to sync trace-enabled runs into the local Harbor viewer cache and inspect saved traces.
 

examples/03_run_scale_benchmark.py

@@ -25,10 +25,10 @@
     wait_and_print_run,
 )
 from examples.agents import MinimalDeepAgent
-from netopsbench.sdk import NetOpsBench
+from netopsbench.sdk import NetOpsBench, supported_scales
 
 DEFAULT_SCALE = "xs"
-SCALE_CHOICES = ["xs", "small", "medium", "large"]
+SCALE_CHOICES = list(supported_scales())
 
 
 def main(

examples/agents/minimal_deepagent/skills/network-diagnosis/SKILL.md

@@ -0,0 +1,41 @@
+---
+name: network-diagnosis
+description: Example-local troubleshooting guidance for the MinimalDeepAgent DeepAgents demo.
+allowed-tools: get_pingmesh_summary get_pingmesh_hotspots get_topology get_device_interfaces get_interface_metrics query_bgp_events get_bgp_neighbors get_bgp_neighbor get_bgp_rib get_route_table get_device_config get_device_logs ping_test traceroute
+---
+
+# Network Diagnosis Skill
+
+Use this skill for the public DeepAgent example when you need to diagnose a likely fabric fault from MCP evidence.
+
+## Preferred flow
+
+1. Start with `get_pingmesh_summary`, `get_pingmesh_hotspots`, and `get_topology`.
+2. Before declaring the network healthy, call `query_bgp_events` for the episode window.
+3. Pick one likely owner device when possible and validate with the smallest useful set of device-level checks:
+   - `get_device_interfaces`
+   - `get_interface_metrics`
+   - For each event, confirm only the affected device with `get_bgp_neighbors`, then inspect the peer with `get_bgp_neighbor`.
+   - Use `get_bgp_rib`, `get_route_table`, or `get_device_config` only when route impact or configuration needs confirmation.
+   - Report the canonical fault label `bgp_neighbor_misconfig`; do not substitute `bgp_down` or `link_flap`.
+   - For an AS mismatch, set `location.device` to the device whose configured remote-AS disagrees with the peer's actual local AS. Do not report the correctly configured peer.
+   - Separate a control-plane session event from confirmed data-plane impact. Historical events outside the episode window are not current faults.
+   - `get_device_logs`
+4. Use `ping_test` and `traceroute` only when they are likely to clarify the fault.
+
+For a symmetric Pingmesh leaf pair, do not infer the faulty side from probe direction alone: an echoed reply can
+cross a discard route in either direction. Inspect the exact affected client IP on both attached switches with
+`get_route_table`; prefer structured `is_discard`/`discard_interface` evidence over interpreting the route legend.
+When the selected route is a discard/Null0 route, report the canonical label `blackhole_route`. Reserve
+`static_route_misconfig` for a non-discard static route whose selected next hop or egress path is incorrect.
+An exact client route is sufficient to close a route diagnosis when one endpoint selects that static route, the
+other endpoint has no matching static override, and BGP remains established. Return the diagnosis at that point;
+do not fan out into aggregation switches, broad route-table dumps, logs, ACLs, or traceroute unless the exact route
+evidence is missing or contradictory.
+
+## Output guidance
+
+- Prefer a single device and at most one interface.
+- Return benchmark-style fault labels when the evidence is clear.
+- If the evidence is weak, return `inconclusive` instead of guessing.
+- Keep the final reasoning short and tied to tool output.

netopsbench/agents/_trace_utils.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import os
 import re
 from pathlib import Path
 from typing import Any
@@ -12,8 +11,7 @@
     re.compile(r"\bsk-[A-Za-z0-9_\-]{12,}\b"),
     re.compile(r"\bBearer\s+[A-Za-z0-9_\-\.]{12,}\b", re.IGNORECASE),
 )
-_DEFAULT_MAX_FIELD_CHARS = 200_000
-_MAX_FIELD_CHARS_ENV = "NETOPSBENCH_TRACE_MAX_FIELD_CHARS"
+MAX_FIELD_CHARS = 200_000
 
 
 def jsonable(value: Any) -> Any:
@@ -65,17 +63,10 @@ def redact_secret_values(value: str) -> str:
 
 
 def truncate_text(value: str) -> str:
-    limit = max_field_chars()
+    limit = MAX_FIELD_CHARS
     if limit <= 0 or len(value) <= limit:
         return value
     return value[:limit] + f"...<truncated {len(value) - limit} chars>"
 
 
-def max_field_chars() -> int:
-    try:
-        return int(os.environ.get(_MAX_FIELD_CHARS_ENV, str(_DEFAULT_MAX_FIELD_CHARS)))
-    except ValueError:
-        return _DEFAULT_MAX_FIELD_CHARS
-
-
 __all__ = ["jsonable"]

netopsbench/cli/main.py

@@ -10,11 +10,12 @@
 
 from netopsbench.cli.trace import add_trace_subparser, cmd_trace
 from netopsbench.logging_utils import configure_logging
+from netopsbench.models.profiles import supported_scales
 from netopsbench.platform.scenario import generator as scenario_generator
 from netopsbench.platform.topology.generator import generate_topology
 from netopsbench.sdk import NetOpsBench
 
-SUPPORTED_SCALES = ("xs", "small", "medium", "large")
+SUPPORTED_SCALES = supported_scales()
 
 
 def build_parser() -> argparse.ArgumentParser:
@@ -48,9 +49,7 @@ def build_parser() -> argparse.ArgumentParser:
     scenario_validate.add_argument("file", help="Scenario YAML file")
     scenario_generate = scenario_sub.add_parser("generate", help="Generate scenario YAML for one scale")
     scenario_generate.add_argument("--scale", required=True, choices=SUPPORTED_SCALES, help="Topology scale")
-    scenario_generate.add_argument(
-        "--spec", help="Scenario generation spec file (default: scenarios/specs/fault_campaign.yaml)"
-    )
+    scenario_generate.add_argument("--spec", help="Scenario generation spec file (default: packaged campaign)")
     scenario_generate.add_argument(
         "--topology-dir", help="Generated topology directory (default: lab-topology/generated_topology_<scale>)"
     )
@@ -76,9 +75,7 @@ def build_parser() -> argparse.ArgumentParser:
         default=",".join(SUPPORTED_SCALES),
         help="Comma-separated scale list (default: xs,small,medium,large)",
     )
-    benchmark_prepare.add_argument(
-        "--spec", help="Scenario generation spec file (default: scenarios/specs/fault_campaign.yaml)"
-    )
+    benchmark_prepare.add_argument("--spec", help="Scenario generation spec file (default: packaged campaign)")
     benchmark_prepare.add_argument("--seed", type=int, default=42, help="Random seed for reproducible generation")
 
     return parser
@@ -134,8 +131,8 @@ def _cmd_runtime(bench: NetOpsBench, args: argparse.Namespace) -> int:
     raise AssertionError(f"unhandled runtime action: {args.runtime_action}")
 
 
-def _default_scenario_spec(workspace: Path) -> Path:
-    return workspace / "scenarios" / "specs" / "fault_campaign.yaml"
+def _default_scenario_spec() -> Path:
+    return scenario_generator.default_campaign_spec()
 
 
 def _default_topology_output_dir(workspace: Path, scale: str) -> Path:
@@ -221,7 +218,7 @@ def _cmd_scenario(bench: NetOpsBench, args: argparse.Namespace) -> int:
         print(f"valid: {scenario_file}")
         return 0
     if args.scenario_action == "generate":
-        spec = Path(args.spec) if args.spec else _default_scenario_spec(bench.workspace)
+        spec = Path(args.spec) if args.spec else _default_scenario_spec()
         topology_dir = Path(args.topology_dir) if args.topology_dir else None
         out = Path(args.out) if args.out else None
         return _generate_scenarios(
@@ -279,7 +276,7 @@ def _cmd_result(bench: NetOpsBench, args: argparse.Namespace) -> int:
 def _cmd_benchmark(bench: NetOpsBench, args: argparse.Namespace) -> int:
     if args.benchmark_action == "prepare":
         scales = _parse_scales(args.scales)
-        spec = Path(args.spec) if args.spec else _default_scenario_spec(bench.workspace)
+        spec = Path(args.spec) if args.spec else _default_scenario_spec()
         print("Preparing benchmark assets")
         print(f"  workspace: {bench.workspace}")
         print(f"  scales: {', '.join(scales)}")