diff --git a/.env.example b/.env.example index 1b4ba23..15d1ae2 100644 --- a/.env.example +++ b/.env.example @@ -22,46 +22,20 @@ NETOPSBENCH_INFLUXDB_TOKEN=replace-me NETOPSBENCH_INFLUXDB_ORG=netopsbench NETOPSBENCH_INFLUXDB_BUCKET=netopsbench -# --- Grafana --- -# NETOPSBENCH_GRAFANA_URL=http://localhost:3000 -# NETOPSBENCH_GRAFANA_PASSWORD=admin -# NETOPSBENCH_GRAFANA_INFLUXDB_URL=http://influxdb:8086 -# NETOPSBENCH_GRAFANA_DEFAULT_BUCKET=netopsbench - -# --- Topology --- +# --- Manual topology / thin runtime wrappers (optional) --- # NETOPSBENCH_TOPOLOGY_DIR=lab-topology/generated_topology_xs -# NETOPSBENCH_TOPOLOGY_ID=dcn # NETOPSBENCH_LAB_NAME=dcn # NETOPSBENCH_MGMT_SUBNET= # Override management subnet (e.g. 172.20.20.0/24) # NETOPSBENCH_MGMT_NETWORK= # Override Docker management network name -# --- SONiC / gNMI Telemetry --- -# SONIC_GNMI_PORT=50051 -# SONIC_GNMI_USERNAME=admin -# SONIC_GNMI_PASSWORD= # Set the SONiC gNMI password for telemetry collection -# SONIC_GNMI_ENCODING=json_ietf -# SONIC_GNMI_TARGET=COUNTERS_DB -# SONIC_GNMI_SUBSCRIPTION_MODE=on_change # on_change | sample | target_defined -# SONIC_GNMI_SAMPLE_INTERVAL=10s # Only used when subscription_mode=sample - -# --- Telemetry collectors (sFlow / syslog) --- -# NETOPSBENCH_SFLOW_COLLECTOR= # sFlow collector IP (defaults to management IP) -# NETOPSBENCH_SYSLOG_COLLECTOR= # Syslog collector IP (defaults to management IP) -# NETOPSBENCH_SFLOW_PORT=6343 -# NETOPSBENCH_SFLOW_POLLING_INTERVAL=20 -# NETOPSBENCH_SFLOW_SAMPLE_RATE=1000 -# NETOPSBENCH_SFLOW_SAMPLE_DIRECTION=ingress - -# --- Pingmesh --- -# PINGMESH_CYCLE_INTERVAL=5 # Probe cycle interval in seconds - # --- Runtime / debug --- # NETOPSBENCH_LOG_LEVEL=INFO # DEBUG | INFO | WARNING | ERROR # NETOPSBENCH_NO_SUDO=0 # Set to 1 in CI/test environments without sudo - -# --- Worker Pool --- -# NETOPSBENCH_WORKER_DEPLOY_JOBS= # Parallel worker deploy jobs (default: num_workers) -# NETOPSBENCH_SONIC_WAIT_TRIES=180 # Max SONiC readiness retries (5s each, default: 15 min) -# NETOPSBENCH_AGENT_TIMEOUT_SECONDS=300 -# NETOPSBENCH_WORKER_AGENT_TIMEOUT_SECONDS=600 -# NETOPSBENCH_WORKER_DISABLE_LANGSMITH=false +# NETOPSBENCH_PYTHON=/path/to/python3 # Interpreter used by repository shell wrappers +# NETOPSBENCH_TRAFFIC_PARALLELISM=32 # Parallel docker exec workers for benchmark traffic start/stop + +# --- Optional semantic fault-type judge --- +# NETOPSBENCH_FAULT_TYPE_JUDGE_ENABLED=0 +# NETOPSBENCH_FAULT_TYPE_JUDGE_MODEL=gpt-4o-mini +# NETOPSBENCH_FAULT_TYPE_JUDGE_API_KEY= +# NETOPSBENCH_FAULT_TYPE_JUDGE_BASE_URL= diff --git a/.gitignore b/.gitignore index b6a37a4..b0759ba 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,8 @@ lab-topology/ .worktrees/ .agents/ .pytest_cache/ +build/ +dist/ tmp/ .netopsbench*/ @@ -42,6 +44,7 @@ observability/telegraf.conf .coverage plot_* !scripts/plot_results.py +examples/agents/minimal_deepagent/conversation_history/ # Fumadocs / Next.js site artifacts docs/node_modules/ diff --git a/AGENTS.md b/AGENTS.md index 7cc5094..8580448 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,8 +30,9 @@ asks for a breaking change. experience; keep commands, docs, and imports aligned. - `scenarios/generated//`: generated benchmark scenarios used by examples and suite runs. -- `observability/` and `scripts/runtime/`: Docker, Containerlab, Telegraf, - Pingmesh, and BGP runtime support. +- `netopsbench/platform/observability/assets/`: packaged Grafana, Telegraf, + and Docker Compose assets. +- `scripts/runtime/`: documented thin wrappers around the Python runtime CLI. - `tests/`: lightweight unit and contract tests. Real Containerlab tests are usually marked `real` or named `*_real`. @@ -136,9 +137,9 @@ Pingmesh, and BGP snapshots. Preserve these behaviors: Relevant files: -- `scripts/observability/start_worker_telegraf.sh` -- `scripts/runtime/run_bgp_collector.py` -- `observability/telegraf.conf.template` +- `netopsbench/platform/observability/lifecycle.py` +- `netopsbench/platform/observability/bgp_collector.py` +- `netopsbench/platform/observability/assets/telegraf.conf.template` - `tests/test_bgp_collector.py` - `tests/test_runtime_config_consistency.py` diff --git a/docs/content/docs/architecture/system-overview.mdx b/docs/content/docs/architecture/system-overview.mdx index e2eb017..641dc4b 100644 --- a/docs/content/docs/architecture/system-overview.mdx +++ b/docs/content/docs/architecture/system-overview.mdx @@ -11,7 +11,7 @@ A benchmark run is a closed loop: resolve a scenario, provision a topology, inje | Stage | What happens | |---|---| -| Scenario and topology | A generated scenario selects topology scale, traffic profile, fault type, target device, and target interface when applicable. | +| Scenario and topology | A generated scenario selects topology scale, fault type, target device, and target interface. All scenarios use the canonical standard background traffic model. | | Runtime provisioning | The SDK provisions a Linux / Containerlab runtime and starts observability services. | | Fault episode | The scenario executor applies traffic generation, injects the selected fault, waits through the observation window, and records symptoms. | | Diagnosis context | The platform assembles topology, symptoms, Pingmesh summaries, runtime metadata, and optional tools into `DiagnosticContext`. | diff --git a/docs/content/docs/build-your-agent/python-api-guide.mdx b/docs/content/docs/build-your-agent/python-api-guide.mdx index cb4107b..bad2f01 100644 --- a/docs/content/docs/build-your-agent/python-api-guide.mdx +++ b/docs/content/docs/build-your-agent/python-api-guide.mdx @@ -69,9 +69,9 @@ When `artifacts_dir` is omitted, session artifacts are written under the workspa The `scenario_summaries[*].raw_result_path` fields point to raw JSON files for case-level debugging. -Agent traces are saved by default and can be disabled for a run with `trace=False` or by setting `NETOPSBENCH_TRACE=0`. Disabling trace prevents private runtime trace collection and sidecar artifact creation. Ground truth and score details are written to `traces/results.jsonl`, not into the agent trajectory. +Agent traces are saved by default and can be disabled for a run with `trace=False`. Disabling trace prevents private runtime trace collection and sidecar artifact creation. Ground truth and score details are written to `traces/results.jsonl`, not into the agent trajectory. -NetOpsBench stores visible prompts, model messages, tool calls, and observations with secret redaction and per-field truncation. The bundled `MinimalDeepAgent` attaches `context.trace.langchain_callback()` to its LangChain-compatible runtime so private LLM messages and tool events flow into the same recorder. Non-LangChain agents can use the advanced manual recorder methods, such as `context.trace.record_llm_request(...)` and `context.trace.record_llm_response(...)`, when they need to capture private model calls. Set `NETOPSBENCH_TRACE_MAX_FIELD_CHARS` to tune truncation. +NetOpsBench stores visible prompts, model messages, tool calls, and observations with secret redaction and a fixed per-field safety limit. The bundled `MinimalDeepAgent` attaches `context.trace.langchain_callback()` to its LangChain-compatible runtime so private LLM messages and tool events flow into the same recorder. Non-LangChain agents can use the advanced manual recorder methods, such as `context.trace.record_llm_request(...)` and `context.trace.record_llm_response(...)`, when they need to capture private model calls. Open a completed run directly in the Harbor viewer: diff --git a/docs/content/docs/debug-operate/observability.mdx b/docs/content/docs/debug-operate/observability.mdx index 43eee41..2a54e76 100644 --- a/docs/content/docs/debug-operate/observability.mdx +++ b/docs/content/docs/debug-operate/observability.mdx @@ -97,14 +97,9 @@ Common overrides: ```bash export NETOPSBENCH_MGMT_SUBNET=172.31.250.0/24 -export SONIC_GNMI_PORT=50051 -export SONIC_GNMI_USERNAME=admin -export SONIC_GNMI_PASSWORD= -export NETOPSBENCH_SYSLOG_COLLECTOR=172.20.20.200 -export NETOPSBENCH_SFLOW_COLLECTOR=172.20.20.200 ``` -The default SONiC image is `yyyyyt123/netopsbench-sonic-vs-202505-telemetry:202505-telemetry`. +The default SONiC image is `yyyyyt123/netopsbench-sonic-vs-202505-telemetry:202505-telemetry`. Its gNMI contract is fixed by the benchmark image and generated Telegraf configuration (`50051`, `admin`, `json_ietf`, `COUNTERS_DB`, `on_change`) so benchmark telemetry does not vary with process environment. ## Cleanup diff --git a/docs/content/docs/run-benchmarks/methodology.mdx b/docs/content/docs/run-benchmarks/methodology.mdx index 06a1c75..0a01693 100644 --- a/docs/content/docs/run-benchmarks/methodology.mdx +++ b/docs/content/docs/run-benchmarks/methodology.mdx @@ -47,7 +47,7 @@ Each scale combines fault cases with healthy negative samples: | Medium | 24 | 4 | 28 | | Large | 48 | 4 | 52 | -Scenarios are generated from `scenarios/specs/fault_campaign.yaml`. Running `netopsbench benchmark prepare --scales ` materializes YAML files under `scenarios/generated//`. +Scenarios are generated from the packaged canonical campaign spec. Running `netopsbench benchmark prepare --scales ` materializes YAML files under `scenarios/generated//`; `--spec` can select a custom campaign. ## Scoring dimensions diff --git a/docs/content/docs/run-benchmarks/run-scenario-vs-suite.mdx b/docs/content/docs/run-benchmarks/run-scenario-vs-suite.mdx index 3809a2f..59aa2c7 100644 --- a/docs/content/docs/run-benchmarks/run-scenario-vs-suite.mdx +++ b/docs/content/docs/run-benchmarks/run-scenario-vs-suite.mdx @@ -101,9 +101,9 @@ Useful environment overrides: |---|---| | `BENCH_VENDOR` | Provider passed to `examples/03_run_scale_benchmark.py`. | | `BENCH_SCALES` | Space-separated scale list such as `xs small`. | +| `BENCH_WORKERS_` | Worker count override for one scale, for example `BENCH_WORKERS_LARGE=2`. | | `BENCH_CLEAN_RUNS=1` | Explicitly remove previous `.netopsbench/runs/` artifacts before the batch. By default, previous reports and traces are preserved. | -| `NETOPSBENCH_WORKER_DEPLOY_JOBS` | Worker deployment parallelism override. | -| `NETOPSBENCH_WORKER_HEALTH_RETRIES` | Health-check retry budget for larger fabrics. | +| `NETOPSBENCH_TRAFFIC_PARALLELISM` | Bounded client traffic start/stop concurrency; defaults to 32. | The script writes logs under `scenario_results/benchmark_logs_/`, records the run ids for the batch in `benchmark_runs_.jsonl`, and writes a CSV summary under `scenario_results/benchmark_summary_.csv`. Run artifacts use timestamp ids such as `run-20260605T124040Z`; use `netopsbench trace view` to sync trace-enabled runs into the local Harbor viewer cache and inspect saved traces. diff --git a/examples/03_run_scale_benchmark.py b/examples/03_run_scale_benchmark.py index 05ba36e..e6179eb 100644 --- a/examples/03_run_scale_benchmark.py +++ b/examples/03_run_scale_benchmark.py @@ -25,10 +25,10 @@ wait_and_print_run, ) from examples.agents import MinimalDeepAgent -from netopsbench.sdk import NetOpsBench +from netopsbench.sdk import NetOpsBench, supported_scales DEFAULT_SCALE = "xs" -SCALE_CHOICES = ["xs", "small", "medium", "large"] +SCALE_CHOICES = list(supported_scales()) def main( diff --git a/examples/agents/minimal_deepagent/skills/network-diagnosis/SKILL.md b/examples/agents/minimal_deepagent/skills/network-diagnosis/SKILL.md new file mode 100644 index 0000000..1ae1664 --- /dev/null +++ b/examples/agents/minimal_deepagent/skills/network-diagnosis/SKILL.md @@ -0,0 +1,41 @@ +--- +name: network-diagnosis +description: Example-local troubleshooting guidance for the MinimalDeepAgent DeepAgents demo. +allowed-tools: get_pingmesh_summary get_pingmesh_hotspots get_topology get_device_interfaces get_interface_metrics query_bgp_events get_bgp_neighbors get_bgp_neighbor get_bgp_rib get_route_table get_device_config get_device_logs ping_test traceroute +--- + +# Network Diagnosis Skill + +Use this skill for the public DeepAgent example when you need to diagnose a likely fabric fault from MCP evidence. + +## Preferred flow + +1. Start with `get_pingmesh_summary`, `get_pingmesh_hotspots`, and `get_topology`. +2. Before declaring the network healthy, call `query_bgp_events` for the episode window. +3. Pick one likely owner device when possible and validate with the smallest useful set of device-level checks: + - `get_device_interfaces` + - `get_interface_metrics` + - For each event, confirm only the affected device with `get_bgp_neighbors`, then inspect the peer with `get_bgp_neighbor`. + - Use `get_bgp_rib`, `get_route_table`, or `get_device_config` only when route impact or configuration needs confirmation. + - Report the canonical fault label `bgp_neighbor_misconfig`; do not substitute `bgp_down` or `link_flap`. + - For an AS mismatch, set `location.device` to the device whose configured remote-AS disagrees with the peer's actual local AS. Do not report the correctly configured peer. + - Separate a control-plane session event from confirmed data-plane impact. Historical events outside the episode window are not current faults. + - `get_device_logs` +4. Use `ping_test` and `traceroute` only when they are likely to clarify the fault. + +For a symmetric Pingmesh leaf pair, do not infer the faulty side from probe direction alone: an echoed reply can +cross a discard route in either direction. Inspect the exact affected client IP on both attached switches with +`get_route_table`; prefer structured `is_discard`/`discard_interface` evidence over interpreting the route legend. +When the selected route is a discard/Null0 route, report the canonical label `blackhole_route`. Reserve +`static_route_misconfig` for a non-discard static route whose selected next hop or egress path is incorrect. +An exact client route is sufficient to close a route diagnosis when one endpoint selects that static route, the +other endpoint has no matching static override, and BGP remains established. Return the diagnosis at that point; +do not fan out into aggregation switches, broad route-table dumps, logs, ACLs, or traceroute unless the exact route +evidence is missing or contradictory. + +## Output guidance + +- Prefer a single device and at most one interface. +- Return benchmark-style fault labels when the evidence is clear. +- If the evidence is weak, return `inconclusive` instead of guessing. +- Keep the final reasoning short and tied to tool output. diff --git a/examples/agents/minimal_deepagent/skills/network_diagnosis/SKILL.md b/examples/agents/minimal_deepagent/skills/network_diagnosis/SKILL.md deleted file mode 100644 index 6fac2ef..0000000 --- a/examples/agents/minimal_deepagent/skills/network_diagnosis/SKILL.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: network_diagnosis -description: Example-local troubleshooting guidance for the MinimalDeepAgent DeepAgents demo. -allowed-tools: get_pingmesh_summary get_pingmesh_hotspots get_topology get_device_interfaces get_interface_metrics get_bgp_neighbors get_all_bgp_status get_route_table get_device_logs ping_test traceroute ---- - -# Network Diagnosis Skill - -Use this skill for the public DeepAgent example when you need to diagnose a likely fabric fault from MCP evidence. - -## Preferred flow - -1. Start with `get_pingmesh_summary`, `get_pingmesh_hotspots`, and `get_topology`. -2. Pick one likely owner device when possible. -3. Validate with the smallest useful set of device-level checks: - - `get_device_interfaces` - - `get_interface_metrics` - - `get_bgp_neighbors` or `get_all_bgp_status` - - `get_route_table` - - `get_device_logs` -4. Use `ping_test` and `traceroute` only when they are likely to clarify the fault. - -## Output guidance - -- Prefer a single device and at most one interface. -- Return benchmark-style fault labels when the evidence is clear. -- If the evidence is weak, return `inconclusive` instead of guessing. -- Keep the final reasoning short and tied to tool output. diff --git a/netopsbench/agents/_trace_utils.py b/netopsbench/agents/_trace_utils.py index 9a2ddc9..a858234 100644 --- a/netopsbench/agents/_trace_utils.py +++ b/netopsbench/agents/_trace_utils.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os import re from pathlib import Path from typing import Any @@ -12,8 +11,7 @@ re.compile(r"\bsk-[A-Za-z0-9_\-]{12,}\b"), re.compile(r"\bBearer\s+[A-Za-z0-9_\-\.]{12,}\b", re.IGNORECASE), ) -_DEFAULT_MAX_FIELD_CHARS = 200_000 -_MAX_FIELD_CHARS_ENV = "NETOPSBENCH_TRACE_MAX_FIELD_CHARS" +MAX_FIELD_CHARS = 200_000 def jsonable(value: Any) -> Any: @@ -65,17 +63,10 @@ def redact_secret_values(value: str) -> str: def truncate_text(value: str) -> str: - limit = max_field_chars() + limit = MAX_FIELD_CHARS if limit <= 0 or len(value) <= limit: return value return value[:limit] + f"..." -def max_field_chars() -> int: - try: - return int(os.environ.get(_MAX_FIELD_CHARS_ENV, str(_DEFAULT_MAX_FIELD_CHARS))) - except ValueError: - return _DEFAULT_MAX_FIELD_CHARS - - __all__ = ["jsonable"] diff --git a/netopsbench/cli/main.py b/netopsbench/cli/main.py index 50d6ef8..8d6e85c 100644 --- a/netopsbench/cli/main.py +++ b/netopsbench/cli/main.py @@ -10,11 +10,12 @@ from netopsbench.cli.trace import add_trace_subparser, cmd_trace from netopsbench.logging_utils import configure_logging +from netopsbench.models.profiles import supported_scales from netopsbench.platform.scenario import generator as scenario_generator from netopsbench.platform.topology.generator import generate_topology from netopsbench.sdk import NetOpsBench -SUPPORTED_SCALES = ("xs", "small", "medium", "large") +SUPPORTED_SCALES = supported_scales() def build_parser() -> argparse.ArgumentParser: @@ -48,9 +49,7 @@ def build_parser() -> argparse.ArgumentParser: scenario_validate.add_argument("file", help="Scenario YAML file") scenario_generate = scenario_sub.add_parser("generate", help="Generate scenario YAML for one scale") scenario_generate.add_argument("--scale", required=True, choices=SUPPORTED_SCALES, help="Topology scale") - scenario_generate.add_argument( - "--spec", help="Scenario generation spec file (default: scenarios/specs/fault_campaign.yaml)" - ) + scenario_generate.add_argument("--spec", help="Scenario generation spec file (default: packaged campaign)") scenario_generate.add_argument( "--topology-dir", help="Generated topology directory (default: lab-topology/generated_topology_)" ) @@ -76,9 +75,7 @@ def build_parser() -> argparse.ArgumentParser: default=",".join(SUPPORTED_SCALES), help="Comma-separated scale list (default: xs,small,medium,large)", ) - benchmark_prepare.add_argument( - "--spec", help="Scenario generation spec file (default: scenarios/specs/fault_campaign.yaml)" - ) + benchmark_prepare.add_argument("--spec", help="Scenario generation spec file (default: packaged campaign)") benchmark_prepare.add_argument("--seed", type=int, default=42, help="Random seed for reproducible generation") return parser @@ -134,8 +131,8 @@ def _cmd_runtime(bench: NetOpsBench, args: argparse.Namespace) -> int: raise AssertionError(f"unhandled runtime action: {args.runtime_action}") -def _default_scenario_spec(workspace: Path) -> Path: - return workspace / "scenarios" / "specs" / "fault_campaign.yaml" +def _default_scenario_spec() -> Path: + return scenario_generator.default_campaign_spec() def _default_topology_output_dir(workspace: Path, scale: str) -> Path: @@ -221,7 +218,7 @@ def _cmd_scenario(bench: NetOpsBench, args: argparse.Namespace) -> int: print(f"valid: {scenario_file}") return 0 if args.scenario_action == "generate": - spec = Path(args.spec) if args.spec else _default_scenario_spec(bench.workspace) + spec = Path(args.spec) if args.spec else _default_scenario_spec() topology_dir = Path(args.topology_dir) if args.topology_dir else None out = Path(args.out) if args.out else None return _generate_scenarios( @@ -279,7 +276,7 @@ def _cmd_result(bench: NetOpsBench, args: argparse.Namespace) -> int: def _cmd_benchmark(bench: NetOpsBench, args: argparse.Namespace) -> int: if args.benchmark_action == "prepare": scales = _parse_scales(args.scales) - spec = Path(args.spec) if args.spec else _default_scenario_spec(bench.workspace) + spec = Path(args.spec) if args.spec else _default_scenario_spec() print("Preparing benchmark assets") print(f" workspace: {bench.workspace}") print(f" scales: {', '.join(scales)}") diff --git a/netopsbench/config.py b/netopsbench/config.py index 05e3de4..6750d08 100644 --- a/netopsbench/config.py +++ b/netopsbench/config.py @@ -1,58 +1,17 @@ """Centralized runtime defaults for NetOpsBench.""" +from __future__ import annotations + import os from dataclasses import dataclass, field -from pathlib import Path - - -def repo_root() -> Path: - """Return the repository root directory. - - Resolution order: - 1. ``NETOPSBENCH_REPO_ROOT`` environment variable (if set). - 2. Walk up from this file: ``netopsbench/config.py`` → repo root (parents[1]). - """ - env = os.environ.get("NETOPSBENCH_REPO_ROOT") - if env: - return Path(env).resolve() - return Path(__file__).resolve().parents[1] - -DEFAULT_GRAFANA_URL = "http://localhost:3000" -DEFAULT_GRAFANA_USER = "admin" -DEFAULT_GRAFANA_PASSWORD = "admin" DEFAULT_INFLUXDB_URL = "http://localhost:8086" DEFAULT_INFLUXDB_TOKEN = "replace-me" DEFAULT_INFLUXDB_ORG = "netopsbench" DEFAULT_INFLUXDB_BUCKET = "netopsbench" -DEFAULT_AGENT_TIMEOUT_SECONDS = 300 -DEFAULT_TELEGRAF_RELOAD_WAIT_SECONDS = 3.0 -DEFAULT_WORKER_HEALTH_RETRIES = 12 -DEFAULT_WORKER_HEALTH_DELAY_SECONDS = 5 -DEFAULT_ACTIVE_INTERFACE_COVERAGE_MIN_RATIO = 0.5 -DEFAULT_PINGMESH_INFLUXDB_URL = "http://influxdb:8086" -DEFAULT_SONIC_WAIT_TRIES = 180 DEFAULT_FAULT_TYPE_JUDGE_MODEL = "gpt-4o-mini" -def _parse_int(value: str | None, default: int) -> int: - if value is None: - return default - try: - return max(1, int(str(value).strip())) - except (ValueError, AttributeError): - return default - - -def _parse_float(value: str | None, default: float) -> float: - if value is None: - return default - try: - return float(str(value).strip()) - except (ValueError, AttributeError): - return default - - def _parse_bool(value: str | None, default: bool) -> bool: if value is None: return default @@ -85,19 +44,13 @@ class FaultTypeJudgeConfig: @dataclass class NetOpsBenchConfig: - """Single source of truth for ``NETOPSBENCH_*`` environment variables. + """External service configuration read at the process boundary. - Use the module-level :data:`config` singleton everywhere instead of calling - ``os.environ.get("NETOPSBENCH_…")`` from individual modules. Tests can - construct a fresh instance to override values without touching the - process environment. + Runtime tuning belongs to scale profiles or subsystem constants. Tests can + construct a fresh instance to override external service values without + changing platform domain models. """ - grafana_url: str = field(default_factory=lambda: os.environ.get("NETOPSBENCH_GRAFANA_URL", DEFAULT_GRAFANA_URL)) - grafana_user: str = field(default_factory=lambda: os.environ.get("NETOPSBENCH_GRAFANA_USER", DEFAULT_GRAFANA_USER)) - grafana_password: str = field( - default_factory=lambda: os.environ.get("NETOPSBENCH_GRAFANA_PASSWORD", DEFAULT_GRAFANA_PASSWORD) - ) influxdb_url: str = field(default_factory=lambda: os.environ.get("NETOPSBENCH_INFLUXDB_URL", DEFAULT_INFLUXDB_URL)) influxdb_token: str = field( default_factory=lambda: os.environ.get("NETOPSBENCH_INFLUXDB_TOKEN", DEFAULT_INFLUXDB_TOKEN) @@ -107,58 +60,13 @@ class NetOpsBenchConfig: default_factory=lambda: os.environ.get("NETOPSBENCH_INFLUXDB_BUCKET", DEFAULT_INFLUXDB_BUCKET) ) topology_dir: str | None = field(default_factory=lambda: os.environ.get("NETOPSBENCH_TOPOLOGY_DIR")) - topology_id: str | None = field(default_factory=lambda: os.environ.get("NETOPSBENCH_TOPOLOGY_ID")) - workspace: str | None = field(default_factory=lambda: os.environ.get("NETOPSBENCH_WORKSPACE")) - pingmesh_start_time: str | None = field(default_factory=lambda: os.environ.get("NETOPSBENCH_PINGMESH_START_TIME")) - pingmesh_end_time: str | None = field(default_factory=lambda: os.environ.get("NETOPSBENCH_PINGMESH_END_TIME")) - agent_timeout_seconds: int = field( - default_factory=lambda: _parse_int( - os.environ.get("NETOPSBENCH_AGENT_TIMEOUT_SECONDS"), DEFAULT_AGENT_TIMEOUT_SECONDS - ) - ) - telegraf_reload_wait_seconds: float = field( - default_factory=lambda: _parse_float( - os.environ.get("NETOPSBENCH_TELEGRAF_RELOAD_WAIT_SECONDS"), DEFAULT_TELEGRAF_RELOAD_WAIT_SECONDS - ) - ) - skip_observability_refresh: bool = field( - default_factory=lambda: _parse_bool(os.environ.get("NETOPSBENCH_SKIP_OBSERVABILITY_REFRESH"), False) - ) - run_id_suffix: str = field(default_factory=lambda: os.environ.get("NETOPSBENCH_RUN_ID_SUFFIX", "")) - worker_health_retries: int = field( - default_factory=lambda: _parse_int( - os.environ.get("NETOPSBENCH_WORKER_HEALTH_RETRIES"), DEFAULT_WORKER_HEALTH_RETRIES - ) - ) - worker_health_delay_seconds: int = field( - default_factory=lambda: _parse_int( - os.environ.get("NETOPSBENCH_WORKER_HEALTH_DELAY_SECONDS"), DEFAULT_WORKER_HEALTH_DELAY_SECONDS - ) - ) - active_interface_coverage_min_ratio: float = field( - default_factory=lambda: _parse_float( - os.environ.get("NETOPSBENCH_ACTIVE_INTERFACE_COVERAGE_MIN_RATIO"), - DEFAULT_ACTIVE_INTERFACE_COVERAGE_MIN_RATIO, - ) - ) - pingmesh_influxdb_url: str = field( - default_factory=lambda: os.environ.get("NETOPSBENCH_PINGMESH_INFLUXDB_URL", DEFAULT_PINGMESH_INFLUXDB_URL) - ) - sonic_wait_tries: int = field( - default_factory=lambda: _parse_int(os.environ.get("NETOPSBENCH_SONIC_WAIT_TRIES"), DEFAULT_SONIC_WAIT_TRIES) - ) fault_type_judge_config: FaultTypeJudgeConfig = field(default_factory=FaultTypeJudgeConfig) # ------------------------------------------------------------------ # Convenience accessors # ------------------------------------------------------------------ - @property - def grafana_auth(self) -> tuple: - """Return ``(user, password)`` tuple for Grafana basic auth.""" - return (self.grafana_user, self.grafana_password) - - def reload(self) -> "NetOpsBenchConfig": + def reload(self) -> NetOpsBenchConfig: """Re-read all values from the current process environment. Returns ``self`` for chaining. Useful when test fixtures patch diff --git a/netopsbench/evaluator/scorer.py b/netopsbench/evaluator/scorer.py index 7f84e86..4723915 100644 --- a/netopsbench/evaluator/scorer.py +++ b/netopsbench/evaluator/scorer.py @@ -8,7 +8,7 @@ import json from dataclasses import asdict, dataclass, field -from datetime import datetime +from datetime import UTC, datetime from typing import Any from netopsbench.evaluator.fault_type_judge import FaultTypeJudge, canonicalize_fault_type, judge_fault_type_match @@ -410,8 +410,8 @@ def generate_report( detection_macro_f1 = None report = { - "benchmark_id": f"netopsbench-{datetime.now().strftime('%Y%m%d-%H%M%S')}", - "timestamp": datetime.now().isoformat(), + "benchmark_id": f"netopsbench-{datetime.now(UTC).strftime('%Y%m%d-%H%M%S')}", + "timestamp": datetime.now(UTC).isoformat(), "agent_name": agent_name, "topology_scale": topology_scale, "summary": { diff --git a/netopsbench/models/__init__.py b/netopsbench/models/__init__.py new file mode 100644 index 0000000..7b8bc98 --- /dev/null +++ b/netopsbench/models/__init__.py @@ -0,0 +1,33 @@ +"""Canonical persisted schemas shared across NetOpsBench layers.""" + +from .profiles import SCALE_PROFILES, ScaleProfile, get_scale_profile, supported_scales +from .runtime import RuntimeIdentity, safe_runtime_label +from .topology import ( + SCHEMA_VERSION, + Collector, + Device, + DeviceRole, + Link, + LinkEndpoint, + Management, + PingmeshPolicy, + TopologyManifest, +) + +__all__ = [ + "SCHEMA_VERSION", + "Collector", + "Device", + "DeviceRole", + "Link", + "LinkEndpoint", + "Management", + "PingmeshPolicy", + "RuntimeIdentity", + "SCALE_PROFILES", + "ScaleProfile", + "TopologyManifest", + "get_scale_profile", + "safe_runtime_label", + "supported_scales", +] diff --git a/netopsbench/models/profiles.py b/netopsbench/models/profiles.py new file mode 100644 index 0000000..04ad50d --- /dev/null +++ b/netopsbench/models/profiles.py @@ -0,0 +1,227 @@ +"""Neutral benchmark scale profile registry. + +This module intentionally lives below :mod:`netopsbench.models` so public SDK +code, CLI code, and platform internals can share scale facts without importing +platform implementation modules. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + + +@dataclass(frozen=True, slots=True) +class ScaleProfile: + """Resource and topology parameters for one supported benchmark scale.""" + + name: str + family: Literal["clos", "fat-tree"] + num_spines: int | None + num_leafs: int | None + fat_tree_k: int | None + num_cores: int | None + num_aggs: int | None + num_edges: int | None + clients_per_attached_switch: int + management_prefix: int + management_subnet_base: int + pingmesh_destination_batch_size: int | None + pingmesh_rtt_port_pool_size: int + pingmesh_rtt_ports_per_cycle: int + pingmesh_cycle_interval_seconds: int + traffic_max_pps_per_client: int + deploy_timeout_seconds: int + worker_deploy_parallelism: int + health_timeout_seconds: int + containerlab_max_workers: int | None = None + + @property + def clients_per_leaf(self) -> int | None: + return self.clients_per_attached_switch if self.family == "clos" else None + + @property + def clients_per_edge(self) -> int | None: + return self.clients_per_attached_switch if self.family == "fat-tree" else None + + @property + def total_clients(self) -> int: + if self.family == "clos": + return int(self.num_leafs or 0) * self.clients_per_attached_switch + return int(self.num_edges or 0) * self.clients_per_attached_switch + + @property + def total_switches(self) -> int: + if self.family == "clos": + return int(self.num_spines or 0) + int(self.num_leafs or 0) + return int(self.num_cores or 0) + int(self.num_aggs or 0) + int(self.num_edges or 0) + + +SCALE_PROFILES: dict[str, ScaleProfile] = { + "xs": ScaleProfile( + name="xs", + family="clos", + num_spines=2, + num_leafs=2, + fat_tree_k=None, + num_cores=None, + num_aggs=None, + num_edges=None, + clients_per_attached_switch=1, + management_prefix=24, + management_subnet_base=100, + pingmesh_destination_batch_size=None, + pingmesh_rtt_port_pool_size=16, + pingmesh_rtt_ports_per_cycle=8, + pingmesh_cycle_interval_seconds=1, + traffic_max_pps_per_client=250, + deploy_timeout_seconds=1800, + worker_deploy_parallelism=2, + health_timeout_seconds=60, + ), + "small": ScaleProfile( + name="small", + family="clos", + num_spines=2, + num_leafs=4, + fat_tree_k=None, + num_cores=None, + num_aggs=None, + num_edges=None, + clients_per_attached_switch=2, + management_prefix=24, + management_subnet_base=120, + pingmesh_destination_batch_size=None, + pingmesh_rtt_port_pool_size=16, + pingmesh_rtt_ports_per_cycle=8, + pingmesh_cycle_interval_seconds=1, + traffic_max_pps_per_client=250, + deploy_timeout_seconds=1800, + worker_deploy_parallelism=2, + health_timeout_seconds=60, + ), + "medium": ScaleProfile( + name="medium", + family="clos", + num_spines=4, + num_leafs=8, + fat_tree_k=None, + num_cores=None, + num_aggs=None, + num_edges=None, + clients_per_attached_switch=2, + management_prefix=24, + management_subnet_base=140, + pingmesh_destination_batch_size=None, + pingmesh_rtt_port_pool_size=16, + pingmesh_rtt_ports_per_cycle=6, + pingmesh_cycle_interval_seconds=1, + traffic_max_pps_per_client=200, + deploy_timeout_seconds=1800, + worker_deploy_parallelism=2, + health_timeout_seconds=60, + ), + "large": ScaleProfile( + name="large", + family="clos", + num_spines=4, + num_leafs=16, + fat_tree_k=None, + num_cores=None, + num_aggs=None, + num_edges=None, + clients_per_attached_switch=4, + management_prefix=24, + management_subnet_base=160, + pingmesh_destination_batch_size=None, + pingmesh_rtt_port_pool_size=16, + pingmesh_rtt_ports_per_cycle=4, + pingmesh_cycle_interval_seconds=1, + traffic_max_pps_per_client=150, + deploy_timeout_seconds=2700, + worker_deploy_parallelism=1, + health_timeout_seconds=180, + ), + "xlarge": ScaleProfile( + name="xlarge", + family="clos", + num_spines=16, + num_leafs=128, + fat_tree_k=None, + num_cores=None, + num_aggs=None, + num_edges=None, + clients_per_attached_switch=1, + management_prefix=23, + management_subnet_base=180, + pingmesh_destination_batch_size=16, + pingmesh_rtt_port_pool_size=16, + pingmesh_rtt_ports_per_cycle=4, + pingmesh_cycle_interval_seconds=2, + traffic_max_pps_per_client=100, + deploy_timeout_seconds=3600, + worker_deploy_parallelism=1, + health_timeout_seconds=240, + containerlab_max_workers=16, + ), + "fat-tree-k8": ScaleProfile( + name="fat-tree-k8", + family="fat-tree", + num_spines=None, + num_leafs=None, + fat_tree_k=8, + num_cores=16, + num_aggs=32, + num_edges=32, + clients_per_attached_switch=4, + management_prefix=24, + management_subnet_base=200, + pingmesh_destination_batch_size=16, + pingmesh_rtt_port_pool_size=16, + pingmesh_rtt_ports_per_cycle=4, + pingmesh_cycle_interval_seconds=2, + traffic_max_pps_per_client=100, + deploy_timeout_seconds=3600, + worker_deploy_parallelism=1, + health_timeout_seconds=240, + containerlab_max_workers=1, + ), + "fat-tree-k12": ScaleProfile( + name="fat-tree-k12", + family="fat-tree", + num_spines=None, + num_leafs=None, + fat_tree_k=12, + num_cores=36, + num_aggs=72, + num_edges=72, + clients_per_attached_switch=2, + management_prefix=23, + management_subnet_base=220, + pingmesh_destination_batch_size=16, + pingmesh_rtt_port_pool_size=16, + pingmesh_rtt_ports_per_cycle=4, + pingmesh_cycle_interval_seconds=2, + traffic_max_pps_per_client=50, + deploy_timeout_seconds=5400, + worker_deploy_parallelism=1, + health_timeout_seconds=300, + containerlab_max_workers=1, + ), +} + + +def supported_scales() -> tuple[str, ...]: + """Return scale names in their stable CLI and suite order.""" + return tuple(SCALE_PROFILES) + + +def get_scale_profile(name: str) -> ScaleProfile: + """Return one profile or raise a focused error for unsupported scales.""" + try: + return SCALE_PROFILES[name] + except KeyError as exc: + raise ValueError(f"Unknown scale: {name}") from exc + + +__all__ = ["SCALE_PROFILES", "ScaleProfile", "get_scale_profile", "supported_scales"] diff --git a/netopsbench/models/runtime.py b/netopsbench/models/runtime.py new file mode 100644 index 0000000..05c283f --- /dev/null +++ b/netopsbench/models/runtime.py @@ -0,0 +1,69 @@ +"""Canonical identity for a runtime worker and its isolated resources.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +def safe_runtime_label(value: str) -> str: + """Return a bucket-safe runtime label compatible with existing worker labels.""" + text = (value or "unknown").strip().lower() + return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in text) or "unknown" + + +def _default_topology_id(data: dict[str, Any]) -> str: + return str(data["lab_name"]) + + +def _default_bucket(data: dict[str, Any]) -> str: + return f"network_data_{safe_runtime_label(str(data['runtime_id']))}_w{int(data['worker_index']):02d}" + + +class RuntimeIdentity(BaseModel): + """Persisted worker identity with deterministic resource names.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["3"] = "3" + runtime_id: str + worker_id: str + worker_index: int = Field(ge=1) + lab_name: str + topology_id: str = Field(default_factory=_default_topology_id) + topology_dir: Path + bucket: str = Field(default_factory=_default_bucket) + mgmt_subnet: str + mgmt_network: str + + @classmethod + def create( + cls, + *, + runtime_id: str, + worker_id: str, + worker_index: int, + lab_name: str, + topology_dir: str | Path, + mgmt_subnet: str, + mgmt_network: str, + topology_id: str | None = None, + bucket: str | None = None, + ) -> RuntimeIdentity: + """Create an identity with deterministic topology and bucket defaults.""" + return cls( + runtime_id=runtime_id, + worker_id=worker_id, + worker_index=worker_index, + lab_name=lab_name, + topology_id=topology_id if topology_id is not None else lab_name, + topology_dir=Path(topology_dir), + bucket=bucket or f"network_data_{safe_runtime_label(runtime_id)}_w{worker_index:02d}", + mgmt_subnet=mgmt_subnet, + mgmt_network=mgmt_network, + ) + + +__all__ = ["RuntimeIdentity", "safe_runtime_label"] diff --git a/netopsbench/models/topology.py b/netopsbench/models/topology.py new file mode 100644 index 0000000..e3a8cfc --- /dev/null +++ b/netopsbench/models/topology.py @@ -0,0 +1,299 @@ +"""Canonical, persisted topology schemas and the legacy agent projection.""" + +from __future__ import annotations + +from enum import StrEnum +from math import ceil +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +SCHEMA_VERSION: Literal["3"] = "3" +DEFAULT_LINK_MTU = 9232 +DEFAULT_SONIC_PORT_MTU = 9100 + + +class DeviceRole(StrEnum): + """Supported canonical device roles.""" + + SPINE = "spine" + LEAF = "leaf" + CORE = "core" + AGG = "agg" + EDGE = "edge" + CLIENT = "client" + + +class _PersistedModel(BaseModel): + """Strict base model for versioned data written to disk.""" + + model_config = ConfigDict(extra="forbid") + + +class Device(_PersistedModel): + name: str + role: DeviceRole + mgmt_ip: str | None = None + data_ip: str | None = None + attached_switch: str | None = None + asn: int | None = None + router_id: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class LinkEndpoint(_PersistedModel): + device: str + interface: str + + +class Link(_PersistedModel): + kind: str + endpoints: tuple[LinkEndpoint, LinkEndpoint] + mtu: int = Field(default=DEFAULT_LINK_MTU, gt=0) + + +class Management(_PersistedModel): + network: str + ipv4_subnet: str + + +class Collector(_PersistedModel): + ipv4: str + + +class PingmeshPolicy(_PersistedModel): + destination_batch_size: int | None = Field(default=None, ge=1) + rtt_port_pool_size: int = Field(default=16, ge=1) + rtt_ports_per_cycle: int = Field(default=4, ge=1) + cycle_interval_seconds: int = Field(default=1, ge=1) + df_payload_size: int = Field(default=DEFAULT_SONIC_PORT_MTU - 28, ge=1) + + def destination_batch_count(self, client_count: int) -> int: + destinations = max(0, int(client_count) - 1) + batch_size = self.destination_batch_size or max(1, destinations) + return max(1, ceil(destinations / batch_size)) + + def port_batch_count(self) -> int: + return max(1, ceil(self.rtt_port_pool_size / self.rtt_ports_per_cycle)) + + def coverage_epoch_cycles(self, client_count: int) -> int: + return self.destination_batch_count(client_count) * self.port_batch_count() + + def coverage_epoch_seconds(self, client_count: int) -> int: + return self.coverage_epoch_cycles(client_count) * self.cycle_interval_seconds + + +class TopologyDefaults(_PersistedModel): + link_mtu: int = Field(default=DEFAULT_LINK_MTU, gt=0) + sonic_port_mtu: int = Field(default=DEFAULT_SONIC_PORT_MTU, gt=0) + + +class TopologyFacts(_PersistedModel): + """Typed scale facts shared by CLOS and fat-tree manifests.""" + + num_spines: int = Field(default=0, ge=0) + num_leafs: int = Field(default=0, ge=0) + num_cores: int = Field(default=0, ge=0) + num_aggs: int = Field(default=0, ge=0) + num_edges: int = Field(default=0, ge=0) + num_pods: int = Field(default=0, ge=0) + clients_per_attached_switch: int = Field(ge=1) + total_clients: int = Field(ge=0) + total_switches: int = Field(ge=0) + fat_tree_k: int | None = Field(default=None, ge=2) + full_density_clients_per_attached_switch: int | None = Field(default=None, ge=1) + host_density: Literal["standard", "sparse"] | None = None + + +class RoutingMetadata(_PersistedModel): + protocol: Literal["BGP"] = "BGP" + spine_asn: int | None = None + leaf_asn_range: str | None = None + core_asn_range: str | None = None + agg_asn_range: str | None = None + edge_asn_range: str | None = None + ecmp_hash_policy_by_role: dict[DeviceRole, Literal[0, 1]] + ecmp: bool = True + bfd: bool = True + + +class TopologyManifest(_PersistedModel): + """Versioned canonical topology, independent from runtime metadata formats.""" + + schema_version: Literal["3"] = SCHEMA_VERSION + topology_id: str + name: str + scale: str + family: Literal["clos", "fat-tree"] + management: Management + collector: Collector + defaults: TopologyDefaults + facts: TopologyFacts + devices: list[Device] + links: list[Link] + routing: RoutingMetadata + pingmesh: PingmeshPolicy = Field(default_factory=PingmeshPolicy) + + @model_validator(mode="after") + def _validate_topology_references(self) -> TopologyManifest: + names = [device.name for device in self.devices] + if len(names) != len(set(names)): + raise ValueError("device names must be unique") + + devices_by_name = {device.name: device for device in self.devices} + switch_roles = {device.role for device in self.switches()} + policy_roles = set(self.routing.ecmp_hash_policy_by_role) + if policy_roles != switch_roles: + missing = sorted(role.value for role in switch_roles - policy_roles) + unexpected = sorted(role.value for role in policy_roles - switch_roles) + details = [] + if missing: + details.append(f"missing roles: {', '.join(missing)}") + if unexpected: + details.append(f"unexpected roles: {', '.join(unexpected)}") + raise ValueError("ecmp_hash_policy_by_role must match switch roles (" + "; ".join(details) + ")") + + for link in self.links: + for endpoint in link.endpoints: + if endpoint.device not in devices_by_name: + raise ValueError(f"link endpoint references unknown device: {endpoint.device}") + + for client in self.clients(): + if not client.attached_switch: + raise ValueError(f"client {client.name} must define attached_switch") + attached = devices_by_name.get(client.attached_switch) + if attached is None: + raise ValueError(f"client attached_switch references unknown device: {client.attached_switch}") + if attached.role is DeviceRole.CLIENT: + raise ValueError("client attached_switch must reference a non-client device") + return self + + def device(self, name: str) -> Device | None: + """Return a device by name, or ``None`` when the manifest has no match.""" + return next((device for device in self.devices if device.name == name), None) + + def devices_by_role(self, role: DeviceRole | str) -> list[Device]: + """Return devices in their persisted order for one canonical role.""" + normalized = DeviceRole(role) + return [device for device in self.devices if device.role is normalized] + + def switches(self) -> list[Device]: + return [device for device in self.devices if device.role is not DeviceRole.CLIENT] + + def routing_devices(self) -> list[Device]: + return self.switches() + + def edge_devices(self) -> list[Device]: + edges = self.devices_by_role(DeviceRole.EDGE) + return edges if edges else self.devices_by_role(DeviceRole.LEAF) + + def clients(self) -> list[Device]: + return self.devices_by_role(DeviceRole.CLIENT) + + def client_attached_devices(self) -> list[Device]: + attached_names = {client.attached_switch for client in self.clients() if client.attached_switch} + return [device for device in self.switches() if device.name in attached_names] + + def to_agent_topology(self) -> dict[str, Any]: + """Adapt canonical data to the grouped topology metadata used by agents today.""" + groups = { + "spines": self.devices_by_role(DeviceRole.SPINE), + "leafs": self.devices_by_role(DeviceRole.LEAF), + "cores": self.devices_by_role(DeviceRole.CORE), + "aggs": self.devices_by_role(DeviceRole.AGG), + "edges": self.devices_by_role(DeviceRole.EDGE), + "clients": self.clients(), + } + if self.family == "fat-tree": + groups["spines"] = groups["cores"] + groups["leafs"] = groups["edges"] + + projected: dict[str, Any] = { + "name": self.name, + "topology_id": self.topology_id, + "topology_scale": self.scale, + "topology_type": self.family, + "management": self.management.model_dump(mode="json"), + "collector": self.collector.model_dump(mode="json"), + "defaults": self.defaults.model_dump(mode="json"), + "mtu_semantics": { + "link_mtu_scope": "containerlab/client link MTU", + "sonic_port_mtu_scope": "SONiC front-panel interface MTU", + "note": ( + "Do not compare link_mtu 9232 directly against healthy SONiC port MTU 9100 " + "when diagnosing faults." + ), + }, + "scale": self._agent_scale_facts(), + "devices": { + group: [self._device_to_agent_entry(device) for device in devices] for group, devices in groups.items() + }, + "links": [ + { + "type": link.kind, + "endpoints": [endpoint.device for endpoint in link.endpoints], + } + for link in self.links + ], + "routing": self.routing.model_dump(mode="json", exclude_none=True), + } + projected["pingmesh"] = { + **self.pingmesh.model_dump(mode="json"), + "destination_batch_count": self.pingmesh.destination_batch_count(self.facts.total_clients), + "port_batch_count": self.pingmesh.port_batch_count(), + "coverage_epoch_cycles": self.pingmesh.coverage_epoch_cycles(self.facts.total_clients), + "coverage_epoch_seconds": self.pingmesh.coverage_epoch_seconds(self.facts.total_clients), + } + if self.family == "fat-tree": + projected["fat_tree_k"] = self.facts.fat_tree_k + return projected + + def _agent_scale_facts(self) -> dict[str, Any]: + if self.family == "clos": + return { + "name": self.scale, + "num_spines": self.facts.num_spines, + "num_leafs": self.facts.num_leafs, + "clients_per_leaf": self.facts.clients_per_attached_switch, + "total_clients": self.facts.total_clients, + "total_devices": self.facts.total_switches, + } + return { + "name": self.scale, + "num_core": self.facts.num_cores, + "num_agg": self.facts.num_aggs, + "num_edge": self.facts.num_edges, + "num_pods": self.facts.num_pods, + "clients_per_edge": self.facts.clients_per_attached_switch, + "full_density_clients_per_edge": self.facts.full_density_clients_per_attached_switch, + "host_density": self.facts.host_density, + "total_clients": self.facts.total_clients, + "total_devices": self.facts.total_switches, + "num_spines": self.facts.num_cores, + "num_leafs": self.facts.num_edges, + } + + def _device_to_agent_entry(self, device: Device) -> dict[str, Any]: + entry = device.model_dump(exclude={"role", "metadata"}, exclude_none=True, mode="json") + entry.update({key: value for key, value in device.metadata.items() if key not in Device.model_fields}) + if device.role is DeviceRole.CLIENT and device.attached_switch: + entry["leaf"] = device.attached_switch + if self.family == "fat-tree": + entry["edge"] = device.attached_switch + return entry + + +__all__ = [ + "SCHEMA_VERSION", + "Collector", + "Device", + "DeviceRole", + "Link", + "LinkEndpoint", + "Management", + "PingmeshPolicy", + "RoutingMetadata", + "TopologyDefaults", + "TopologyFacts", + "TopologyManifest", +] diff --git a/netopsbench/platform/faults/__init__.py b/netopsbench/platform/faults/__init__.py index 562d700..2c42aa4 100644 --- a/netopsbench/platform/faults/__init__.py +++ b/netopsbench/platform/faults/__init__.py @@ -1,6 +1 @@ -"""Fault injection module for NetOpsBench.""" - -from .injector import FaultInjector -from .scenario_execution import inject_fault, recover_fault - -__all__ = ["FaultInjector", "inject_fault", "recover_fault"] +"""Internal fault injection subsystem.""" diff --git a/netopsbench/platform/faults/builtin/__init__.py b/netopsbench/platform/faults/builtin/__init__.py index f07b53c..ecca850 100644 --- a/netopsbench/platform/faults/builtin/__init__.py +++ b/netopsbench/platform/faults/builtin/__init__.py @@ -1,15 +1 @@ -"""Builtin fault spec families grouped by domain.""" - -from .acl_specs import build_acl_fault_specs -from .impairment_specs import build_impairment_fault_specs -from .link_specs import build_link_fault_specs -from .routing_specs import build_routing_fault_specs -from .system_specs import build_system_fault_specs - -__all__ = [ - "build_acl_fault_specs", - "build_impairment_fault_specs", - "build_link_fault_specs", - "build_routing_fault_specs", - "build_system_fault_specs", -] +"""Builtin fault specifications grouped by domain.""" diff --git a/netopsbench/platform/faults/builtin/acl_specs.py b/netopsbench/platform/faults/builtin/acl_specs.py index d9566a8..c0bb6de 100644 --- a/netopsbench/platform/faults/builtin/acl_specs.py +++ b/netopsbench/platform/faults/builtin/acl_specs.py @@ -2,7 +2,7 @@ from __future__ import annotations -from ..specs import FaultSpec +from ..models import FaultSpec from .common import episode_param diff --git a/netopsbench/platform/faults/builtin/impairment_specs.py b/netopsbench/platform/faults/builtin/impairment_specs.py index 018ddcf..428b77b 100644 --- a/netopsbench/platform/faults/builtin/impairment_specs.py +++ b/netopsbench/platform/faults/builtin/impairment_specs.py @@ -2,7 +2,7 @@ from __future__ import annotations -from ..specs import FaultSpec +from ..models import FaultSpec from .common import episode_param diff --git a/netopsbench/platform/faults/builtin/link_specs.py b/netopsbench/platform/faults/builtin/link_specs.py index 654bcf0..aeddb67 100644 --- a/netopsbench/platform/faults/builtin/link_specs.py +++ b/netopsbench/platform/faults/builtin/link_specs.py @@ -2,7 +2,7 @@ from __future__ import annotations -from ..specs import FaultSpec +from ..models import FaultSpec from .common import episode_param, recover_background_process_fault diff --git a/netopsbench/platform/faults/builtin/routing_specs.py b/netopsbench/platform/faults/builtin/routing_specs.py index 62a9d90..1b29f3d 100644 --- a/netopsbench/platform/faults/builtin/routing_specs.py +++ b/netopsbench/platform/faults/builtin/routing_specs.py @@ -2,7 +2,7 @@ from __future__ import annotations -from ..specs import FaultSpec +from ..models import FaultSpec from .common import episode_param diff --git a/netopsbench/platform/faults/builtin/system_specs.py b/netopsbench/platform/faults/builtin/system_specs.py index 3c31c7f..c46d88a 100644 --- a/netopsbench/platform/faults/builtin/system_specs.py +++ b/netopsbench/platform/faults/builtin/system_specs.py @@ -2,7 +2,7 @@ from __future__ import annotations -from ..specs import FaultSpec +from ..models import FaultSpec def _inject_device_down_episode(injector, episode): diff --git a/netopsbench/platform/faults/builtin_specs.py b/netopsbench/platform/faults/builtin_specs.py deleted file mode 100644 index a8d749c..0000000 --- a/netopsbench/platform/faults/builtin_specs.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Backward-compatible re-export shim. - -The builtin spec aggregator now lives in :mod:`netopsbench.platform.faults.specs`. -This module is kept solely so that any external code importing -``register_builtin_fault_specs`` from the historic location continues to work. -""" - -from __future__ import annotations - -from .specs import register_builtin_fault_specs - -__all__ = ["register_builtin_fault_specs"] diff --git a/netopsbench/platform/faults/context.py b/netopsbench/platform/faults/context.py index 8731a88..3975c55 100644 --- a/netopsbench/platform/faults/context.py +++ b/netopsbench/platform/faults/context.py @@ -1,53 +1,46 @@ -"""Shared topology context for fault injection components.""" +"""Canonical topology context shared by fault services and handlers.""" from __future__ import annotations -import copy -from dataclasses import dataclass, field +from dataclasses import dataclass +from pathlib import Path from typing import Any -from netopsbench.platform.topology.topology_utils import TopologyState +from netopsbench.models.topology import TopologyManifest +from netopsbench.platform.topology.topology_utils import clab_container_name -@dataclass +@dataclass(frozen=True, slots=True) class FaultContext: - """Shared topology and runtime state passed to all fault services and handlers.""" - - topology_name: str = "dcn" - container_names: dict[str, str] = field(default_factory=dict) - topology_metadata: dict[str, Any] = field(default_factory=dict) - device_mgmt_ips: dict[str, str] = field(default_factory=dict) - clients: list[dict[str, Any]] = field(default_factory=list) - clients_by_leaf: dict[str, list[dict[str, Any]]] = field(default_factory=dict) - clab_dir: str = "" - scenarios_dir: str = "" - - @classmethod - def from_topology_state( - cls, - state: TopologyState, - clab_dir: str = "", - scenarios_dir: str = "", - ) -> FaultContext: - return cls( - topology_name=state.topology_name, - container_names=dict(state.container_names), - topology_metadata=copy.deepcopy(state.topology_metadata), - device_mgmt_ips=dict(state.device_mgmt_ips), - clients=[copy.deepcopy(c) for c in state.clients], - clients_by_leaf={ - leaf: [copy.deepcopy(c) for c in clients] for leaf, clients in state.clients_by_leaf.items() - }, - clab_dir=clab_dir, - scenarios_dir=scenarios_dir, - ) - - def update_from_topology_state(self, state: TopologyState) -> None: - self.topology_name = state.topology_name - self.container_names = dict(state.container_names) - self.topology_metadata = copy.deepcopy(state.topology_metadata) - self.device_mgmt_ips = dict(state.device_mgmt_ips) - self.clients = [copy.deepcopy(c) for c in state.clients] - self.clients_by_leaf = { - leaf: [copy.deepcopy(c) for c in clients] for leaf, clients in state.clients_by_leaf.items() - } + """The manifest and artifact directory for one fault-injection runtime.""" + + manifest: TopologyManifest + clab_dir: Path + + @property + def topology_name(self) -> str: + return self.manifest.name + + @property + def topology_metadata(self) -> dict[str, Any]: + return self.manifest.model_dump(mode="json") + + @property + def container_names(self) -> dict[str, str]: + return {device.name: clab_container_name(self.manifest.name, device.name) for device in self.manifest.devices} + + @property + def clients(self) -> list[dict[str, Any]]: + return list(self.manifest.to_agent_topology()["devices"]["clients"]) + + @property + def clients_by_leaf(self) -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for client in self.clients: + attached_switch = str(client.get("leaf") or "").strip() + if attached_switch: + grouped.setdefault(attached_switch, []).append(client) + return grouped + + +__all__ = ["FaultContext"] diff --git a/netopsbench/platform/faults/handlers/__init__.py b/netopsbench/platform/faults/handlers/__init__.py index d302ab8..07adcb5 100644 --- a/netopsbench/platform/faults/handlers/__init__.py +++ b/netopsbench/platform/faults/handlers/__init__.py @@ -1,19 +1 @@ -"""Composable fault handler classes for NetOpsBench.""" - -from .acl import AclHandler -from .impairment import ImpairmentHandler -from .link import LinkHandler -from .routing_bgp import BgpHandler -from .routing_policy import RoutePolicyHandler -from .routing_static import StaticRouteHandler -from .system import SystemHandler - -__all__ = [ - "AclHandler", - "LinkHandler", - "ImpairmentHandler", - "BgpHandler", - "StaticRouteHandler", - "RoutePolicyHandler", - "SystemHandler", -] +"""Fault handlers grouped by failure domain.""" diff --git a/netopsbench/platform/faults/handlers/system.py b/netopsbench/platform/faults/handlers/system.py index 00a69d9..c60f20d 100644 --- a/netopsbench/platform/faults/handlers/system.py +++ b/netopsbench/platform/faults/handlers/system.py @@ -5,7 +5,7 @@ import time from typing import TYPE_CHECKING, Any -from netopsbench.platform.utils.proc import sudo_prefix +from netopsbench.platform.utils.proc import docker_prefix if TYPE_CHECKING: from ..context import FaultContext @@ -82,7 +82,7 @@ def recover_device_down(self, device: str, interfaces: list[str] | None = None) restored_runtime_config = False if not container_started: - start_result = self._cmd.run_cmd([*sudo_prefix(), "docker", "start", container], timeout=60) + start_result = self._cmd.run_cmd([*docker_prefix(), "docker", "start", container], timeout=60) if start_result.returncode != 0: return { "type": "device_down", diff --git a/netopsbench/platform/faults/injector.py b/netopsbench/platform/faults/injector.py index fb50136..9c71ba2 100644 --- a/netopsbench/platform/faults/injector.py +++ b/netopsbench/platform/faults/injector.py @@ -4,11 +4,9 @@ Supports various fault types for DCN troubleshooting benchmark. """ -import json -import os +from pathlib import Path from typing import Any -from netopsbench.config import config from netopsbench.platform.faults.context import FaultContext from netopsbench.platform.faults.handlers.acl import AclHandler from netopsbench.platform.faults.handlers.impairment import ImpairmentHandler @@ -17,20 +15,17 @@ from netopsbench.platform.faults.handlers.routing_policy import RoutePolicyHandler from netopsbench.platform.faults.handlers.routing_static import StaticRouteHandler from netopsbench.platform.faults.handlers.system import SystemHandler -from netopsbench.platform.faults.services import ( - CommandRunner, - FaultTracker, - InterfaceRuntime, - RoutingRuntime, - SonicRuntime, - TopologyRuntime, -) -from netopsbench.platform.faults.specs import get_fault_spec +from netopsbench.platform.faults.services.command_runner import CommandRunner +from netopsbench.platform.faults.services.interface_runtime import InterfaceRuntime +from netopsbench.platform.faults.services.routing_runtime import RoutingRuntime +from netopsbench.platform.faults.services.sonic_runtime import SonicRuntime +from netopsbench.platform.faults.services.topology_runtime import TopologyRuntime +from netopsbench.platform.faults.services.tracking import FaultTracker +from netopsbench.platform.faults.specs import FaultSpecRegistry, create_fault_registry from netopsbench.platform.topology.topology_utils import ( - build_topology_state_from_metadata, - discover_topology_dir, + coerce_topology_manifest, + load_topology_manifest, ) -from netopsbench.platform.utils.result import OperationResult class FaultInjector: @@ -41,30 +36,25 @@ class FaultInjector: Uses composition: each concern lives in a dedicated service or handler class. """ - SONIC_MIN_INTERFACE_MTU = 68 - SONIC_MAX_INTERFACE_MTU = 9216 - SONIC_DEFAULT_INTERFACE_MTU = 9100 - def __init__( self, - scenarios_dir: str | None = None, clab_dir: str | None = None, topology_metadata: dict[str, Any] | None = None, - fault_scripts_dir: str | None = None, + fault_registry: FaultSpecRegistry | None = None, ): - # Resolve workspace root - package_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - workspace_root = self._resolve_workspace_root(package_root) - resolved_scenarios_dir = scenarios_dir or os.path.join(workspace_root, "scenarios") - resolved_clab_dir = clab_dir or self._discover_topology_dir(workspace_root) - - # Build topology context - topo_state = self._load_topology_state(topology_metadata, resolved_clab_dir) - self._ctx = FaultContext.from_topology_state( - topo_state, - clab_dir=resolved_clab_dir, - scenarios_dir=resolved_scenarios_dir, - ) + self.fault_registry = fault_registry or create_fault_registry() + resolved_clab_dir = Path(clab_dir or ".").resolve() + if topology_metadata is not None: + manifest = coerce_topology_manifest(topology_metadata) + else: + metadata_file = resolved_clab_dir / "topology.json" + if not metadata_file.is_file(): + raise FileNotFoundError( + f"Topology metadata not found: {metadata_file}. " + "Pass topology_metadata or a generated topology directory." + ) + manifest = load_topology_manifest(metadata_file) + self._ctx = FaultContext(manifest=manifest, clab_dir=resolved_clab_dir) # Build services (order matters — each layer depends on previous ones) self._cmd = CommandRunner() @@ -83,40 +73,6 @@ def __init__( self._system = SystemHandler(self._cmd, self._sonic, self._iface, self._topo_rt, self._tracker, self._ctx) self._acl = AclHandler(self._cmd, self._sonic, self._routing, self._tracker, self._ctx) - # ------------------------------------------------------------------ - # Bootstrap helpers - # ------------------------------------------------------------------ - - def _discover_topology_dir(self, base_dir: str) -> str: - return discover_topology_dir(base_dir) - - @staticmethod - def _resolve_workspace_root(package_root: str) -> str: - env_root = config.workspace - if env_root and os.path.isdir(env_root): - return env_root - repo_root = os.path.dirname(package_root) - if os.path.isdir(os.path.join(repo_root, "scripts")) or os.path.isdir(os.path.join(repo_root, "scenarios")): - return repo_root - return package_root - - @staticmethod - def _load_topology_state(topology_metadata, clab_dir): - if topology_metadata: - return build_topology_state_from_metadata(topology_metadata) - metadata_file = os.path.join(clab_dir, "topology.json") - if os.path.exists(metadata_file): - with open(metadata_file, encoding="utf-8") as handle: - return build_topology_state_from_metadata(json.load(handle)) - raise FileNotFoundError( - f"Topology metadata not found: {metadata_file}. " - "Pass topology_metadata=... or set NETOPSBENCH_TOPOLOGY_DIR to a generated topology directory." - ) - - # ------------------------------------------------------------------ - # Topology attribute properties (backward-compatible) - # ------------------------------------------------------------------ - @property def topology_name(self) -> str: return self._ctx.topology_name @@ -125,33 +81,13 @@ def topology_name(self) -> str: def container_names(self) -> dict[str, str]: return self._ctx.container_names - @container_names.setter - def container_names(self, value: dict[str, str]): - self._ctx.container_names = value - @property def topology_metadata(self) -> dict[str, Any]: return self._ctx.topology_metadata - @property - def device_mgmt_ips(self) -> dict[str, str]: - return self._ctx.device_mgmt_ips - - @property - def clients(self) -> list[dict[str, Any]]: - return self._ctx.clients - - @property - def clients_by_leaf(self) -> dict[str, list[dict[str, Any]]]: - return self._ctx.clients_by_leaf - - @property - def scenarios_dir(self) -> str: - return self._ctx.scenarios_dir - @property def clab_dir(self) -> str: - return self._ctx.clab_dir + return str(self._ctx.clab_dir) @property def active_faults(self) -> list: @@ -161,142 +97,86 @@ def active_faults(self) -> list: def active_faults(self, value: list): self._tracker.active_faults = value - # ------------------------------------------------------------------ - # Topology reload - # ------------------------------------------------------------------ - - def reload_topology(self, topology_metadata: dict[str, Any] = None, metadata_file: str = None) -> OperationResult: - try: - if topology_metadata: - state = build_topology_state_from_metadata(topology_metadata) - elif metadata_file: - with open(metadata_file) as f: - state = build_topology_state_from_metadata(json.load(f)) - else: - return OperationResult( - success=False, error="Either topology_metadata or metadata_file must be provided" - ) - - self._ctx.update_from_topology_state(state) - return OperationResult( - success=True, - data={ - "topology_name": self._ctx.topology_name, - "devices": list(self._ctx.container_names.keys()), - "total_devices": len(self._ctx.container_names), - }, - ) - except Exception as e: - return OperationResult(success=False, error=str(e)) - - @staticmethod - def _normalize_fault_result(payload: dict[str, Any] | OperationResult) -> OperationResult: - if isinstance(payload, OperationResult): - return payload - data = dict(payload or {}) - if "success" in data: - success = bool(data.pop("success")) - elif "recovered" in data: - success = bool(data.get("recovered")) - else: - success = not data.get("error") - error = data.pop("error", None) - return OperationResult(success=success, error=error, data=data) - - @classmethod - def _legacy_fault_result(cls, payload: dict[str, Any] | OperationResult) -> dict[str, Any]: - return cls._normalize_fault_result(payload).to_dict() - # ------------------------------------------------------------------ # Link fault delegation # ------------------------------------------------------------------ def inject_link_down(self, device: str, interface: str) -> dict[str, Any]: - return self._legacy_fault_result(self._link.inject_link_down(device, interface)) + return self._link.inject_link_down(device, interface) def recover_link_down(self, device: str, interface: str) -> dict[str, Any]: - return self._legacy_fault_result(self._link.recover_link_down(device, interface)) + return self._link.recover_link_down(device, interface) def inject_link_flapping(self, device: str, interface: str, **kwargs) -> dict[str, Any]: - return self._legacy_fault_result(self._link.inject_link_flapping(device, interface, **kwargs)) + return self._link.inject_link_flapping(device, interface, **kwargs) # ------------------------------------------------------------------ # Impairment fault delegation # ------------------------------------------------------------------ def inject_mtu_mismatch(self, device: str, interface: str, mtu: int | None = None) -> dict[str, Any]: - return self._legacy_fault_result(self._impairment.inject_mtu_mismatch(device, interface, mtu=mtu)) + return self._impairment.inject_mtu_mismatch(device, interface, mtu=mtu) def recover_mtu_mismatch(self, device: str, interface: str, original_mtu: int | None = None) -> dict[str, Any]: - return self._legacy_fault_result( - self._impairment.recover_mtu_mismatch(device, interface, original_mtu=original_mtu) - ) + return self._impairment.recover_mtu_mismatch(device, interface, original_mtu=original_mtu) def inject_packet_corruption(self, device: str, interface: str, corruption_pct: float = 5.0) -> dict[str, Any]: - return self._legacy_fault_result( - self._impairment.inject_packet_corruption(device, interface, corruption_pct=corruption_pct) - ) + return self._impairment.inject_packet_corruption(device, interface, corruption_pct=corruption_pct) def inject_packet_loss(self, device: str, interface: str, loss_pct: float = 10.0) -> dict[str, Any]: - return self._legacy_fault_result(self._impairment.inject_packet_loss(device, interface, loss_pct=loss_pct)) + return self._impairment.inject_packet_loss(device, interface, loss_pct=loss_pct) def inject_high_latency(self, device: str, interface: str, latency_ms: float = 100.0) -> dict[str, Any]: - return self._legacy_fault_result(self._impairment.inject_high_latency(device, interface, latency_ms=latency_ms)) + return self._impairment.inject_high_latency(device, interface, latency_ms=latency_ms) def recover_tc_rules(self, device: str, interface: str) -> dict[str, Any]: - return self._legacy_fault_result(self._impairment.recover_tc_rules(device, interface)) + return self._impairment.recover_tc_rules(device, interface) # ------------------------------------------------------------------ # BGP fault delegation # ------------------------------------------------------------------ def inject_bgp_neighbor_misconfig(self, device: str, **kwargs) -> dict[str, Any]: - return self._legacy_fault_result(self._bgp.inject_bgp_neighbor_misconfig(device, **kwargs)) + return self._bgp.inject_bgp_neighbor_misconfig(device, **kwargs) def recover_bgp_neighbor_misconfig( self, device: str, peer_ip: str, misconfig_kind: str, **kwargs ) -> dict[str, Any]: - return self._legacy_fault_result( - self._bgp.recover_bgp_neighbor_misconfig(device, peer_ip, misconfig_kind, **kwargs) - ) + return self._bgp.recover_bgp_neighbor_misconfig(device, peer_ip, misconfig_kind, **kwargs) # ------------------------------------------------------------------ # Static route fault delegation # ------------------------------------------------------------------ def inject_blackhole_route(self, device: str, target_prefix: str) -> dict[str, Any]: - return self._legacy_fault_result(self._static_route.inject_blackhole_route(device, target_prefix)) + return self._static_route.inject_blackhole_route(device, target_prefix) def recover_blackhole_route(self, device: str, target_prefix: str) -> dict[str, Any]: - return self._legacy_fault_result(self._static_route.recover_blackhole_route(device, target_prefix)) + return self._static_route.recover_blackhole_route(device, target_prefix) def inject_static_route_misconfig( self, device: str, target_ip: str | None = None, wrong_nexthop: str | None = None ) -> dict[str, Any]: - return self._legacy_fault_result( - self._static_route.inject_static_route_misconfig(device, target_ip=target_ip, wrong_nexthop=wrong_nexthop) + return self._static_route.inject_static_route_misconfig( + device, target_ip=target_ip, wrong_nexthop=wrong_nexthop ) def recover_static_route_misconfig( self, device: str, target_ip: str, wrong_nexthop: str | None = None ) -> dict[str, Any]: - return self._legacy_fault_result( - self._static_route.recover_static_route_misconfig(device, target_ip, wrong_nexthop=wrong_nexthop) - ) + return self._static_route.recover_static_route_misconfig(device, target_ip, wrong_nexthop=wrong_nexthop) # ------------------------------------------------------------------ # Route policy fault delegation # ------------------------------------------------------------------ def inject_route_policy_misconfig(self, device: str, **kwargs) -> dict[str, Any]: - return self._legacy_fault_result(self._route_policy.inject_route_policy_misconfig(device, **kwargs)) + return self._route_policy.inject_route_policy_misconfig(device, **kwargs) def recover_route_policy_misconfig( self, device: str, target_prefix: str, misconfig_kind: str, **kwargs ) -> dict[str, Any]: - return self._legacy_fault_result( - self._route_policy.recover_route_policy_misconfig(device, target_prefix, misconfig_kind, **kwargs) - ) + return self._route_policy.recover_route_policy_misconfig(device, target_prefix, misconfig_kind, **kwargs) # ------------------------------------------------------------------ # ACL fault delegation @@ -305,10 +185,8 @@ def recover_route_policy_misconfig( def inject_acl_misconfig( self, device: str, target_prefix: str | None = None, interface: str | None = None, direction: str = "in" ) -> dict[str, Any]: - return self._legacy_fault_result( - self._acl.inject_acl_misconfig( - device, target_prefix=target_prefix, interface=interface, direction=direction - ) + return self._acl.inject_acl_misconfig( + device, target_prefix=target_prefix, interface=interface, direction=direction ) def recover_acl_misconfig( @@ -319,10 +197,8 @@ def recover_acl_misconfig( direction: str = "in", acl_name: str | None = None, ) -> dict[str, Any]: - return self._legacy_fault_result( - self._acl.recover_acl_misconfig( - device, target_prefix, interface=interface, direction=direction, acl_name=acl_name - ) + return self._acl.recover_acl_misconfig( + device, target_prefix, interface=interface, direction=direction, acl_name=acl_name ) # ------------------------------------------------------------------ @@ -330,10 +206,10 @@ def recover_acl_misconfig( # ------------------------------------------------------------------ def inject_device_down(self, device: str) -> dict[str, Any]: - return self._legacy_fault_result(self._system.inject_device_down(device)) + return self._system.inject_device_down(device) def recover_device_down(self, device: str, interfaces: list[str] | None = None) -> dict[str, Any]: - return self._legacy_fault_result(self._system.recover_device_down(device, interfaces=interfaces)) + return self._system.recover_device_down(device, interfaces=interfaces) # ------------------------------------------------------------------ # Recovery / query @@ -347,13 +223,11 @@ def recover_all(self) -> list[dict[str, Any]]: for fault in list(self.active_faults): try: fault_type = fault["type"] - spec = get_fault_spec(fault_type) + spec = self.fault_registry.get(fault_type) if spec is None or spec.recover_active_fault is None: result = {"type": fault_type, "recovered": False, "error": "Unknown fault type"} else: result = spec.recover_active_fault(self, fault) - result = self._legacy_fault_result(result) - results.append(result) if not result.get("recovered", False): remaining_faults.append(fault) @@ -364,7 +238,3 @@ def recover_all(self) -> list[dict[str, Any]]: self.active_faults = remaining_faults return results - - def get_active_faults(self) -> list[dict[str, Any]]: - """Get list of currently active faults.""" - return self._tracker.active_fault_dicts() diff --git a/netopsbench/platform/faults/models.py b/netopsbench/platform/faults/models.py index 07983af..73e233f 100644 --- a/netopsbench/platform/faults/models.py +++ b/netopsbench/platform/faults/models.py @@ -1,9 +1,10 @@ -"""Structured models for fault runtime state.""" +"""Fault extension contracts independent from registry and builtin packs.""" from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any +from typing import Any, Protocol @dataclass @@ -38,8 +39,7 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, payload: dict[str, Any]) -> ActiveFault: - payload = dict(payload or {}) - metadata = dict(payload) + metadata = dict(payload or {}) fault_type = metadata.pop("type") device = metadata.pop("device", None) interface = metadata.pop("interface", None) @@ -53,3 +53,49 @@ def from_dict(cls, payload: dict[str, Any]) -> ActiveFault: error=error, metadata=metadata, ) + + +@dataclass +class FaultSpec: + name: str + inject_episode: Callable[[Any, Any], dict[str, Any]] | None = None + recover_active_fault: Callable[[Any, dict[str, Any]], dict[str, Any]] | None = None + requires_interface: bool = False + requires_prefix: bool = False + required_parameters: tuple[str, ...] = () + episode_validator: Callable[[Any], list[str]] | None = None + aliases: list[str] = field(default_factory=list) + scenario_supported: bool = True + + def validate_episode(self, episode: Any, episode_index: int | None = None) -> list[str]: + errors: list[str] = [] + prefix = f"Episode {episode_index}: " if episode_index is not None else "" + if self.requires_interface and not getattr(episode, "target_interface", None): + errors.append(f"{prefix}{self.name} requires target_interface") + if self.requires_prefix and not getattr(episode, "target_prefix", None): + errors.append(f"{prefix}{self.name} requires target_prefix") + parameters = getattr(episode, "parameters", {}) or {} + metadata = getattr(episode, "metadata", {}) or {} + for key in self.required_parameters: + value = parameters.get(key, metadata.get(key)) + if value in (None, "", [], {}, ()): + errors.append(f"{prefix}{self.name} requires parameter '{key}'") + if self.episode_validator is not None: + errors.extend(self.episode_validator(episode)) + return errors + + +class FaultExecutor(Protocol): + def inject(self, context: Any) -> Any: ... + + def recover(self, context: Any) -> Any: ... + + +class FaultPack(Protocol): + name: str + version: str | None + + def register(self, registry: Any) -> None: ... + + +__all__ = ["ActiveFault", "FaultExecutor", "FaultPack", "FaultSpec"] diff --git a/netopsbench/platform/faults/scenario_execution.py b/netopsbench/platform/faults/scenario_execution.py index 261b849..382de05 100644 --- a/netopsbench/platform/faults/scenario_execution.py +++ b/netopsbench/platform/faults/scenario_execution.py @@ -3,8 +3,6 @@ from __future__ import annotations from netopsbench.logging_utils import get_logger -from netopsbench.platform.faults.specs import get_fault_spec -from netopsbench.platform.utils.events import emit as _emit logger = get_logger(__name__) @@ -13,25 +11,25 @@ def inject_fault(runner, episode) -> dict: """Inject a fault via the centralized fault registry.""" if episode.fault_type == "none": - _emit("\n[Fault Injection] No fault - baseline episode") + logger.info("\n[Fault Injection] No fault - baseline episode") return {"success": True, "fault_type": "none", "message": "Baseline episode"} - _emit(f"\n[Fault Injection] {episode.fault_type} on {episode.target_device}") + logger.info(f"\n[Fault Injection] {episode.fault_type} on {episode.target_device}") - spec = get_fault_spec(episode.fault_type) + spec = runner.fault_registry.get(episode.fault_type) if spec is None or spec.inject_episode is None: raise ValueError(f"Unsupported fault type: {episode.fault_type}") try: result = spec.inject_episode(runner.injector, episode) except RuntimeError as exc: - _emit(f" ✗ Fault injection failed: {exc}") + logger.info(f" ✗ Fault injection failed: {exc}") return {"success": False, "fault_type": episode.fault_type, "error": str(exc)} if result.get("success"): - _emit(" ✓ Fault injected successfully") + logger.info(" ✓ Fault injected successfully") else: - _emit(f" ✗ Fault injection failed: {result.get('error')}") + logger.info(f" ✗ Fault injection failed: {result.get('error')}") return result @@ -39,7 +37,7 @@ def inject_fault(runner, episode) -> dict: def recover_fault(runner): """Recover all active faults.""" - _emit("\n[Recovery] Recovering from faults...") + logger.info("\n[Recovery] Recovering from faults...") results = runner.injector.recover_all() - _emit(f" ✓ Recovered from {len(results)} faults") + logger.info(f" ✓ Recovered from {len(results)} faults") return results diff --git a/netopsbench/platform/faults/services/__init__.py b/netopsbench/platform/faults/services/__init__.py index 290e2b8..5df3ebe 100644 --- a/netopsbench/platform/faults/services/__init__.py +++ b/netopsbench/platform/faults/services/__init__.py @@ -1,17 +1 @@ -"""Composable fault injector service classes.""" - -from .command_runner import CommandRunner -from .interface_runtime import InterfaceRuntime -from .routing_runtime import RoutingRuntime -from .sonic_runtime import SonicRuntime -from .topology_runtime import TopologyRuntime -from .tracking import FaultTracker - -__all__ = [ - "CommandRunner", - "SonicRuntime", - "InterfaceRuntime", - "TopologyRuntime", - "RoutingRuntime", - "FaultTracker", -] +"""Low-level fault runtime operations.""" diff --git a/netopsbench/platform/faults/services/command_runner.py b/netopsbench/platform/faults/services/command_runner.py index 162adee..160b89c 100644 --- a/netopsbench/platform/faults/services/command_runner.py +++ b/netopsbench/platform/faults/services/command_runner.py @@ -6,7 +6,7 @@ import signal import subprocess -from netopsbench.platform.utils.proc import safe_run, sudo_prefix +from netopsbench.platform.utils.proc import docker_prefix, safe_run class CommandRunner: @@ -38,13 +38,15 @@ def terminate_process(self, pid: int | None, *, sig: int = signal.SIGKILL) -> bo return False def docker_exec(self, container: str, cmd_args: list[str], timeout: int = 30) -> subprocess.CompletedProcess: - return self.run_cmd([*sudo_prefix(), "docker", "exec", container] + cmd_args, timeout) + return self.run_cmd([*docker_prefix(), "docker", "exec", container] + cmd_args, timeout) def docker_exec_detached( self, container: str, cmd_args: list[str], timeout: int = 10 ) -> subprocess.CompletedProcess: - return self.run_cmd([*sudo_prefix(), "docker", "exec", "-d", container] + cmd_args, timeout) + return self.run_cmd([*docker_prefix(), "docker", "exec", "-d", container] + cmd_args, timeout) def container_is_running(self, container: str) -> bool: - result = self.run_cmd([*sudo_prefix(), "docker", "inspect", "-f", "{{.State.Running}}", container], timeout=10) + result = self.run_cmd( + [*docker_prefix(), "docker", "inspect", "-f", "{{.State.Running}}", container], timeout=10 + ) return result.returncode == 0 and (result.stdout or "").strip() == "true" diff --git a/netopsbench/platform/faults/services/routing_runtime.py b/netopsbench/platform/faults/services/routing_runtime.py index 8468928..2b01a03 100644 --- a/netopsbench/platform/faults/services/routing_runtime.py +++ b/netopsbench/platform/faults/services/routing_runtime.py @@ -89,11 +89,9 @@ def get_bgp_config_snapshot(self, device: str) -> dict[str, Any]: } def get_device_asn(self, device: str) -> int | None: - devices = (self._ctx.topology_metadata or {}).get("devices", {}) - for group in ("spines", "leafs"): - for entry in devices.get(group, []): - if entry.get("name") == device and entry.get("asn") is not None: - return int(entry["asn"]) + for entry in self._ctx.manifest.routing_devices(): + if entry.name == device and entry.asn is not None: + return int(entry.asn) snapshot = self.get_bgp_config_snapshot(device) if snapshot.get("local_as") is not None: return int(snapshot["local_as"]) diff --git a/netopsbench/platform/faults/services/sonic_runtime.py b/netopsbench/platform/faults/services/sonic_runtime.py index b024355..f088f8b 100644 --- a/netopsbench/platform/faults/services/sonic_runtime.py +++ b/netopsbench/platform/faults/services/sonic_runtime.py @@ -37,16 +37,6 @@ def config_cmd(self, device: str, args: list[str]) -> subprocess.CompletedProces raise ValueError(f"Unknown device: {device}") return self._cmd.docker_exec(container, ["config"] + args) - def services_ready(self, device: str) -> bool: - result = self.vtysh(device, ["show ip bgp summary"]) - output = f"{result.stdout or ''}\n{result.stderr or ''}".lower() - return ( - result.returncode == 0 - and "bgpd is not running" not in output - and "failed to connect to any daemons" not in output - and "% bgp instance not found" not in output - ) - def reload_bgp_config(self, device: str) -> subprocess.CompletedProcess: container = self._ctx.container_names.get(device) if not container: diff --git a/netopsbench/platform/faults/services/topology_runtime.py b/netopsbench/platform/faults/services/topology_runtime.py index 063a51d..7308930 100644 --- a/netopsbench/platform/faults/services/topology_runtime.py +++ b/netopsbench/platform/faults/services/topology_runtime.py @@ -2,8 +2,9 @@ from __future__ import annotations -import os -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING + +from netopsbench.platform.topology.configdb_payload import interface_names_for_config if TYPE_CHECKING: from ..context import FaultContext @@ -19,50 +20,15 @@ def __init__(self, sonic: SonicRuntime, iface: InterfaceRuntime, ctx: FaultConte self._iface = iface self._ctx = ctx - def is_client_device(self, device: str) -> bool: - if not device: - return False - if device.startswith("client"): - return True - return any(c.get("name") == device for c in self._ctx.clients) - - def pick_client_pair(self) -> dict[str, Any] | None: - if not self._ctx.clients or len(self._ctx.clients) < 2: - return None - clients = sorted(self._ctx.clients, key=lambda c: c.get("name", "")) - for i, c1 in enumerate(clients): - for c2 in clients[i + 1 :]: - if c1.get("leaf") and c2.get("leaf") and c1.get("leaf") != c2.get("leaf"): - return {"client1": c1, "client2": c2} - return {"client1": clients[0], "client2": clients[1]} - def device_config_path(self, device: str) -> str: - return os.path.join(self._ctx.clab_dir, "configs", f"{device}.sh") + return str(self._ctx.clab_dir / "configs" / "sonic" / device / "config_db.json") def load_device_config_lines(self, device: str) -> list[str]: - config_path = self.device_config_path(device) - if os.path.exists(config_path): - with open(config_path, encoding="utf-8") as handle: - return handle.read().splitlines() - result = self._sonic.vtysh(device, ["show running-config"]) if result.returncode == 0: return (result.stdout or "").splitlines() return [] def configured_device_interfaces(self, device: str) -> list[str]: - interfaces: list[str] = [] - seen = set() - for raw_line in self.load_device_config_lines(device): - line = raw_line.strip() - parts = line.split() - if line.startswith("config interface startup") and len(parts) >= 4: - interface = self._iface.resolve_sonic(parts[3]) - elif line.startswith("config interface ip add") and len(parts) >= 5: - interface = self._iface.resolve_sonic(parts[4]) - else: - continue - if interface not in seen: - seen.add(interface) - interfaces.append(interface) - return interfaces + configdb_interfaces = interface_names_for_config(self.device_config_path(device)) + return [self._iface.resolve_sonic(interface) for interface in configdb_interfaces] diff --git a/netopsbench/platform/faults/services/tracking.py b/netopsbench/platform/faults/services/tracking.py index b954e0b..781848f 100644 --- a/netopsbench/platform/faults/services/tracking.py +++ b/netopsbench/platform/faults/services/tracking.py @@ -43,7 +43,3 @@ def stop_background(self, control_id: str, *, join_timeout: float = 1.0) -> bool def remove_faults(self, predicate: Callable[[ActiveFault], bool]) -> None: with self._lock: self.active_faults = [fault for fault in self.active_faults if not predicate(fault)] - - def active_fault_dicts(self) -> list[dict]: - with self._lock: - return [fault.to_dict() if isinstance(fault, ActiveFault) else dict(fault) for fault in self.active_faults] diff --git a/netopsbench/platform/faults/specs.py b/netopsbench/platform/faults/specs.py index 6a138c7..964ae43 100644 --- a/netopsbench/platform/faults/specs.py +++ b/netopsbench/platform/faults/specs.py @@ -3,71 +3,20 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass, field -from typing import Any, Protocol +from dataclasses import replace +from functools import cache +from types import MappingProxyType +from typing import Any + +from .models import FaultExecutor as FaultExecutor +from .models import FaultPack as FaultPack +from .models import FaultSpec as FaultSpec _CANONICAL_FAULT_ALIASES: dict[str, str] = { "static_route_misconfiguration": "static_route_misconfig", } -@dataclass -class FaultSpec: - name: str - inject_episode: Callable[[Any, Any], dict[str, Any]] | None = None - recover_active_fault: Callable[[Any, dict[str, Any]], dict[str, Any]] | None = None - requires_interface: bool = False - requires_prefix: bool = False - required_parameters: tuple[str, ...] = () - episode_validator: Callable[[Any], list[str]] | None = None - aliases: list[str] = field(default_factory=list) - scenario_supported: bool = True - - def validate_episode(self, episode: Any, episode_index: int | None = None) -> list[str]: - """Validate an episode against this fault spec.""" - errors: list[str] = [] - prefix = f"Episode {episode_index}: " if episode_index is not None else "" - - if self.requires_interface and not getattr(episode, "target_interface", None): - errors.append(f"{prefix}{self.name} requires target_interface") - - if self.requires_prefix and not getattr(episode, "target_prefix", None): - errors.append(f"{prefix}{self.name} requires target_prefix") - - if self.required_parameters: - parameters = getattr(episode, "parameters", {}) or {} - metadata = getattr(episode, "metadata", {}) or {} - for key in self.required_parameters: - value = parameters.get(key, metadata.get(key)) - if value in (None, "", [], {}, ()): - errors.append(f"{prefix}{self.name} requires parameter '{key}'") - - if self.episode_validator is not None: - errors.extend(self.episode_validator(episode)) - - return errors - - -class FaultExecutor(Protocol): - """Public protocol for fault injection/recovery executors.""" - - def inject(self, context: Any) -> Any: - """Inject a fault for the provided context.""" - - def recover(self, context: Any) -> Any: - """Recover a fault for the provided context.""" - - -class FaultPack(Protocol): - """Public protocol for grouping related fault registrations.""" - - name: str - version: str | None - - def register(self, registry: Any) -> None: - """Register faults into the provided registry.""" - - # --------------------------------------------------------------------------- # FaultSpecRegistry — encapsulates all mutable spec state # --------------------------------------------------------------------------- @@ -76,8 +25,7 @@ def register(self, registry: Any) -> None: class FaultSpecRegistry: """Registry holding fault specs, aliases, and schema defaults. - A default module-level instance powers the convenience functions below. - Tests can create isolated instances to avoid cross-test pollution. + Each runtime owns an instance so custom faults cannot leak across sessions. """ def __init__( @@ -104,9 +52,7 @@ def canonicalize(self, name: str | None) -> str: def register(self, spec: FaultSpec) -> FaultSpec: canonical_name = self.canonicalize(spec.name) or spec.name defaults = self._schema_defaults.get(canonical_name, {}) - for key, value in defaults.items(): - if getattr(spec, key) in (False, None, (), []): - setattr(spec, key, value) + values = {key: value for key, value in defaults.items() if getattr(spec, key) in (False, None, (), [])} normalized_aliases = sorted( { alias @@ -114,25 +60,17 @@ def register(self, spec: FaultSpec) -> FaultSpec: if alias and alias != canonical_name } ) - spec.name = canonical_name - spec.aliases = normalized_aliases - self._specs[canonical_name] = spec + normalized = replace( + spec, + name=canonical_name, + aliases=normalized_aliases, + **values, + ) + self._specs[canonical_name] = normalized self._aliases[canonical_name] = canonical_name for alias in normalized_aliases: self._aliases[alias] = canonical_name - return spec - - def unregister(self, name: str) -> None: - canonical_name = self.canonicalize(name) - spec = self._specs.pop(canonical_name, None) - if spec is None: - return - self._aliases.pop(canonical_name, None) - for alias in spec.aliases: - self._aliases.pop(alias, None) - for alias, target in list(_CANONICAL_FAULT_ALIASES.items()): - if target == canonical_name: - self._aliases[alias] = target + return normalized def get(self, name: str) -> FaultSpec | None: self.load_builtins() @@ -161,47 +99,6 @@ def get_builtin_specs(self) -> list[FaultSpec]: return [self._specs[name] for name in builtin_names if name in self._specs] -# --------------------------------------------------------------------------- -# Default module-level instance + backward-compatible convenience functions -# --------------------------------------------------------------------------- - -_default_registry = FaultSpecRegistry() - - -def canonicalize_fault_name(name: str | None) -> str: - return _default_registry.canonicalize(name) - - -def register_fault_spec(spec: FaultSpec) -> FaultSpec: - return _default_registry.register(spec) - - -def unregister_fault_spec(name: str) -> None: - _default_registry.unregister(name) - - -def get_fault_spec(name: str) -> FaultSpec | None: - return _default_registry.get(name) - - -def list_fault_specs() -> list[FaultSpec]: - return _default_registry.list_all() - - -def get_supported_scenario_faults() -> list[str]: - return _default_registry.supported_scenario_faults() - - -def get_builtin_fault_specs() -> list[FaultSpec]: - """Return the builtin fault specs that ship with NetOpsBench.""" - return _default_registry.get_builtin_specs() - - -def load_builtin_fault_specs() -> list[Any]: - """Ensure builtin fault specs are registered and return a snapshot.""" - return list(get_builtin_fault_specs()) - - # --------------------------------------------------------------------------- # Builtin spec aggregation (formerly netopsbench.platform.faults.builtin_specs) # --------------------------------------------------------------------------- @@ -213,13 +110,11 @@ def _build_builtin_fault_specs() -> list[FaultSpec]: Imported lazily inside the function to avoid an import cycle: ``builtin/__init__`` may transitively import ``specs`` for shared types. """ - from .builtin import ( - build_acl_fault_specs, - build_impairment_fault_specs, - build_link_fault_specs, - build_routing_fault_specs, - build_system_fault_specs, - ) + from .builtin.acl_specs import build_acl_fault_specs + from .builtin.impairment_specs import build_impairment_fault_specs + from .builtin.link_specs import build_link_fault_specs + from .builtin.routing_specs import build_routing_fault_specs + from .builtin.system_specs import build_system_fault_specs builders: tuple[Callable[[], list[FaultSpec]], ...] = ( build_link_fault_specs, @@ -234,12 +129,30 @@ def _build_builtin_fault_specs() -> list[FaultSpec]: return aggregated -def register_builtin_fault_specs() -> None: - """Register every builtin fault spec into the default registry. +def create_fault_registry() -> FaultSpecRegistry: + """Return an isolated registry populated with builtin fault specs.""" + registry = FaultSpecRegistry() + registry.load_builtins() + return registry - Equivalent to :func:`load_builtin_fault_specs` but does not return the - snapshot. Kept for backward compatibility with external callers that - imported the previous ``builtin_specs`` module. - """ + +@cache +def _builtin_alias_index() -> MappingProxyType: + aliases = dict(_CANONICAL_FAULT_ALIASES) for spec in _build_builtin_fault_specs(): - register_fault_spec(spec) + canonical = aliases.get(spec.name, spec.name) + aliases[canonical] = canonical + aliases[spec.name] = canonical + aliases.update({alias: canonical for alias in spec.aliases}) + return MappingProxyType(aliases) + + +def canonicalize_fault_name(name: str | None) -> str: + """Canonicalize one builtin fault name without exposing mutable registry state.""" + raw_name = str(name or "").strip() + return _builtin_alias_index().get(raw_name, raw_name) + + +def get_supported_scenario_faults() -> list[str]: + """Return builtin scenario fault names from an isolated registry.""" + return create_fault_registry().supported_scenario_faults() diff --git a/observability/docker-compose.yaml b/netopsbench/platform/observability/assets/docker-compose.yaml similarity index 80% rename from observability/docker-compose.yaml rename to netopsbench/platform/observability/assets/docker-compose.yaml index 51ab189..ac19e01 100644 --- a/observability/docker-compose.yaml +++ b/netopsbench/platform/observability/assets/docker-compose.yaml @@ -13,7 +13,7 @@ services: - DOCKER_INFLUXDB_INIT_USERNAME=admin - DOCKER_INFLUXDB_INIT_PASSWORD=adminpassword - DOCKER_INFLUXDB_INIT_ORG=${NETOPSBENCH_INFLUXDB_ORG:-netopsbench} - - DOCKER_INFLUXDB_INIT_BUCKET=${NETOPSBENCH_GRAFANA_DEFAULT_BUCKET:-netopsbench} + - DOCKER_INFLUXDB_INIT_BUCKET=${NETOPSBENCH_INFLUXDB_BUCKET:-netopsbench} - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${NETOPSBENCH_INFLUXDB_TOKEN:-replace-me} grafana: @@ -28,9 +28,9 @@ services: environment: - GF_SECURITY_ADMIN_USER=admin - GF_SECURITY_ADMIN_PASSWORD=admin - - NETOPSBENCH_GRAFANA_INFLUXDB_URL=${NETOPSBENCH_GRAFANA_INFLUXDB_URL:-http://influxdb:8086} + - NETOPSBENCH_INFLUXDB_URL=http://influxdb:8086 - NETOPSBENCH_INFLUXDB_ORG=${NETOPSBENCH_INFLUXDB_ORG:-netopsbench} - - NETOPSBENCH_GRAFANA_DEFAULT_BUCKET=${NETOPSBENCH_GRAFANA_DEFAULT_BUCKET:-netopsbench} + - NETOPSBENCH_INFLUXDB_BUCKET=${NETOPSBENCH_INFLUXDB_BUCKET:-netopsbench} - NETOPSBENCH_INFLUXDB_TOKEN=${NETOPSBENCH_INFLUXDB_TOKEN:-replace-me} depends_on: - influxdb @@ -39,4 +39,3 @@ volumes: influxdb-data: influxdb-config: grafana-data: - diff --git a/observability/grafana/dashboards/network_overview.json b/netopsbench/platform/observability/assets/grafana/dashboards/network_overview.json similarity index 100% rename from observability/grafana/dashboards/network_overview.json rename to netopsbench/platform/observability/assets/grafana/dashboards/network_overview.json diff --git a/observability/grafana/dashboards/pingmesh.json b/netopsbench/platform/observability/assets/grafana/dashboards/pingmesh.json similarity index 100% rename from observability/grafana/dashboards/pingmesh.json rename to netopsbench/platform/observability/assets/grafana/dashboards/pingmesh.json diff --git a/observability/grafana/provisioning/dashboards/default.yaml b/netopsbench/platform/observability/assets/grafana/provisioning/dashboards/default.yaml similarity index 100% rename from observability/grafana/provisioning/dashboards/default.yaml rename to netopsbench/platform/observability/assets/grafana/provisioning/dashboards/default.yaml diff --git a/observability/grafana/provisioning/datasources/default.yaml b/netopsbench/platform/observability/assets/grafana/provisioning/datasources/default.yaml similarity index 77% rename from observability/grafana/provisioning/datasources/default.yaml rename to netopsbench/platform/observability/assets/grafana/provisioning/datasources/default.yaml index b622604..9fefda8 100644 --- a/observability/grafana/provisioning/datasources/default.yaml +++ b/netopsbench/platform/observability/assets/grafana/provisioning/datasources/default.yaml @@ -9,13 +9,12 @@ datasources: uid: InfluxDB_v2 type: influxdb access: proxy - url: ${NETOPSBENCH_GRAFANA_INFLUXDB_URL} + url: ${NETOPSBENCH_INFLUXDB_URL} isDefault: true jsonData: version: Flux organization: ${NETOPSBENCH_INFLUXDB_ORG} - defaultBucket: ${NETOPSBENCH_GRAFANA_DEFAULT_BUCKET} + defaultBucket: ${NETOPSBENCH_INFLUXDB_BUCKET} tlsSkipVerify: true secureJsonData: token: ${NETOPSBENCH_INFLUXDB_TOKEN} - diff --git a/observability/telegraf.conf.template b/netopsbench/platform/observability/assets/telegraf.conf.template similarity index 86% rename from observability/telegraf.conf.template rename to netopsbench/platform/observability/assets/telegraf.conf.template index 97ff933..3fd1a63 100644 --- a/observability/telegraf.conf.template +++ b/netopsbench/platform/observability/assets/telegraf.conf.template @@ -5,13 +5,13 @@ [agent] interval = "10s" round_interval = true - metric_batch_size = 1000 - metric_buffer_limit = 10000 + metric_batch_size = 5000 + metric_buffer_limit = 200000 collection_jitter = "0s" - flush_interval = "10s" - flush_jitter = "0s" + flush_interval = "5s" + flush_jitter = "1s" precision = "" - debug = true + debug = false # --- OUTPUTS --- [[outputs.influxdb_v2]] @@ -54,9 +54,6 @@ def apply(metric): namepass = ["syslog"] # Drop logs tagged as drop_me -[[processors.printer]] - # This processor prints to stdout, useful for debugging if needed, but we mainly want to filter. - [[processors.regex]] namepass = ["syslog"] @@ -119,20 +116,7 @@ def apply(metric): result_key = "bgp_peer" # 4. gNMI - Interface Counters - DYNAMIC -[[inputs.gnmi]] - addresses = [ - {{GNMI_ADDRESSES}} - ] - username = "{{GNMI_USERNAME}}" - password = "{{GNMI_PASSWORD}}" - encoding = "{{GNMI_ENCODING}}" - redial = "10s" - path_guessing_strategy = "subscription" - tls_enable = false - insecure_skip_verify = true - target = "{{GNMI_TARGET}}" - -{{GNMI_SUBSCRIPTIONS}} +{{GNMI_INPUTS}} [[processors.regex]] namepass = ["interfaces"] @@ -178,11 +162,6 @@ def apply(metric): field = "SAI_PORT_STAT_IF_OUT_ERRORS" dest = "out_errors" -# 5. sFlow - Flow Samples - DYNAMIC -[[inputs.sflow]] - service_address = "{{SFLOW_SERVICE_ADDRESS}}" - - # --- PINGMESH INPUTS --- # Pingmesh metrics collection [[inputs.tail]] diff --git a/netopsbench/platform/observability/bgp_collector.py b/netopsbench/platform/observability/bgp_collector.py new file mode 100644 index 0000000..20c3418 --- /dev/null +++ b/netopsbench/platform/observability/bgp_collector.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Poll BGP summary from SONiC nodes and emit Influx line protocol snapshots.""" + +from __future__ import annotations + +import argparse +import signal +import subprocess +import sys +import threading +import time +from collections.abc import Iterable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from netopsbench.platform.observability.bgp_parser import parse_bgp_summary +from netopsbench.platform.topology.topology_utils import load_topology_manifest +from netopsbench.platform.utils.proc import docker_prefix + +DEFAULT_BGP_COLLECTOR_MAX_BYTES = 128 * 1024 * 1024 +DEFAULT_BGP_COLLECTOR_PARALLELISM = 16 +DEFAULT_BGP_POLL_INTERVAL_SECONDS = 10.0 + + +def _escape_tag(value: str) -> str: + return str(value).replace("\\", "\\\\").replace(",", "\\,").replace(" ", "\\ ").replace("=", "\\=") + + +def _escape_string_field(value: str) -> str: + return str(value).replace("\\", "\\\\").replace('"', '\\"') + + +def normalize_bgp_state(value: str | None) -> str: + if not value: + return "UNKNOWN" + return str(value).strip().upper() + + +def _int_field(name: str, value: object) -> str | None: + if value is None or value == "": + return None + if not isinstance(value, (str, bytes, bytearray, int, float)): + return None + return f"{name}={int(value)}i" + + +def build_bgp_lines(device: str, rows: Iterable[dict], timestamp_ns: int, topology_id: str = "") -> list[str]: + lines: list[str] = [] + source = _escape_tag(device) + topology_tag = _escape_tag(topology_id) + for row in rows: + neighbor = row.get("neighbor") + if not neighbor: + continue + tags = [f"source={source}", f"neighbor_address={_escape_tag(str(neighbor))}"] + if topology_tag: + tags.append(f"topology_id={topology_tag}") + fields = [f'session_state="{_escape_string_field(normalize_bgp_state(row.get("state")))}"'] + for key in ("asn", "prefixes_received", "msg_rcvd", "msg_sent", "in_q", "out_q"): + field = _int_field(key, row.get(key)) + if field: + fields.append(field) + up_down = row.get("up_down") + if up_down: + fields.append(f'up_down="{_escape_string_field(str(up_down))}"') + lines.append(f"bgp_neighbors,{','.join(tags)} {','.join(fields)} {timestamp_ns}") + return lines + + +def build_bgp_collection_line( + device: str, + timestamp_ns: int, + topology_id: str, + collection_ok: bool, + neighbor_count: int, + duration_ms: int, + error_type: str, +) -> str: + tags = [f"source={_escape_tag(device)}"] + if topology_id: + tags.append(f"topology_id={_escape_tag(topology_id)}") + fields = [ + f"collection_ok={'true' if collection_ok else 'false'}", + f"neighbor_count={max(0, int(neighbor_count))}i", + f"duration_ms={max(0, int(duration_ms))}i", + f'error_type="{_escape_string_field(error_type)}"', + ] + return f"bgp_collection,{','.join(tags)} {','.join(fields)} {timestamp_ns}" + + +def _read_topology(metadata_file: Path) -> tuple[str, list[str]]: + manifest = load_topology_manifest(metadata_file) + lab_name = manifest.name.strip() + names = [device.name for device in manifest.routing_devices()] + return lab_name, names + + +def _collect_device_bgp( + lab_name: str, + device: str, + docker_prefix: list[str], + timestamp_ns: int, + topology_id: str, +) -> list[str]: + container = f"clab-{lab_name}-{device}" # matches clab_container_name() convention + started = time.monotonic() + error_type = "" + rows: list[dict] = [] + try: + result = subprocess.run( + [*docker_prefix, "docker", "exec", container, "vtysh", "-c", "show ip bgp summary"], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + if result.returncode != 0: + error_type = "command_failed" + else: + rows = parse_bgp_summary(result.stdout) + if result.stdout.strip() and "Neighbor" not in result.stdout and not rows: + error_type = "parser_failed" + except subprocess.TimeoutExpired: + error_type = "timeout" + except Exception: + error_type = "collector_error" + duration_ms = round((time.monotonic() - started) * 1000) + lines = build_bgp_lines(device, rows, timestamp_ns, topology_id=topology_id) + lines.append( + build_bgp_collection_line( + device, + timestamp_ns, + topology_id, + not error_type, + len(rows), + duration_ms, + error_type, + ) + ) + return lines + + +def collect_bgp_lines( + metadata_file: Path, + timestamp_ns: int | None = None, + parallelism: int = 1, + topology_id: str | None = None, +) -> list[str]: + lab_name, devices = _read_topology(metadata_file) + resolved_topology_id = topology_id or lab_name + command_prefix = docker_prefix() + resolved_timestamp = time.time_ns() if timestamp_ns is None else int(timestamp_ns) + workers = max(1, min(int(parallelism), len(devices) or 1)) + + if workers == 1: + device_lines = [ + _collect_device_bgp(lab_name, device, command_prefix, resolved_timestamp, resolved_topology_id) + for device in devices + ] + else: + with ThreadPoolExecutor(max_workers=workers) as executor: + device_lines = list( + executor.map( + lambda device: _collect_device_bgp( + lab_name, + device, + command_prefix, + resolved_timestamp, + resolved_topology_id, + ), + devices, + ) + ) + + lines: list[str] = [] + for entries in device_lines: + lines.extend(entries) + return lines + + +def _collect_bgp_lines_paced( + metadata_file: Path, + interval_seconds: float, + parallelism: int, + stop_event: threading.Event, + topology_id: str | None = None, +) -> list[str]: + """Collect one fleet snapshot while spreading docker exec starts over the interval.""" + lab_name, devices = _read_topology(metadata_file) + if not devices: + return [] + + resolved_topology_id = topology_id or lab_name + command_prefix = docker_prefix() + workers = max(1, min(int(parallelism), len(devices))) + launch_spacing = max(0.0, float(interval_seconds)) / len(devices) + round_started = time.monotonic() + futures = [] + + with ThreadPoolExecutor(max_workers=workers) as executor: + for index, device in enumerate(devices): + launch_at = round_started + index * launch_spacing + wait_seconds = max(0.0, launch_at - time.monotonic()) + if wait_seconds and stop_event.wait(wait_seconds): + break + futures.append( + executor.submit( + _collect_device_bgp, + lab_name, + device, + command_prefix, + time.time_ns(), + resolved_topology_id, + ) + ) + + lines: list[str] = [] + for future in futures: + lines.extend(future.result()) + return lines + + +def _write_lines(output_file: Path, lines: list[str], max_bytes: int = DEFAULT_BGP_COLLECTOR_MAX_BYTES) -> None: + output_file.parent.mkdir(parents=True, exist_ok=True) + rendered = "\n".join(lines) + if rendered: + rendered += "\n" + mode = "a" + if max_bytes > 0 and output_file.exists(): + current_size = output_file.stat().st_size + if current_size + len(rendered.encode("utf-8")) > max_bytes: + mode = "w" + with output_file.open(mode, encoding="utf-8") as handle: + if rendered: + handle.write(rendered) + + +def run_once( + metadata_file: Path, + output_file: Path, + parallelism: int = 1, + max_bytes: int = DEFAULT_BGP_COLLECTOR_MAX_BYTES, + topology_id: str | None = None, +) -> int: + _write_lines( + output_file, + collect_bgp_lines(metadata_file, parallelism=parallelism, topology_id=topology_id), + max_bytes=max_bytes, + ) + return 0 + + +def run_loop( + metadata_file: Path, + output_file: Path, + interval_seconds: float, + parallelism: int = 1, + max_bytes: int = DEFAULT_BGP_COLLECTOR_MAX_BYTES, + topology_id: str | None = None, +) -> int: + stop_event = threading.Event() + + def _stop(_signum, _frame): + stop_event.set() + + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.touch(exist_ok=True) + + while not stop_event.is_set(): + iteration_started = time.monotonic() + try: + _write_lines( + output_file, + _collect_bgp_lines_paced( + metadata_file, + interval_seconds, + parallelism, + stop_event, + topology_id=topology_id, + ), + max_bytes=max_bytes, + ) + except Exception as exc: + print(f"WARN: bgp collector iteration failed: {exc}", file=sys.stderr) + elapsed = time.monotonic() - iteration_started + stop_event.wait(max(0.0, interval_seconds - elapsed)) + return 0 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Emit BGP neighbor snapshots as Influx line protocol") + parser.add_argument("metadata_file", help="Path to topology.json") + parser.add_argument("--output", required=True, help="Output line protocol file") + parser.add_argument( + "--interval", + type=float, + default=DEFAULT_BGP_POLL_INTERVAL_SECONDS, + help="Polling interval in seconds", + ) + parser.add_argument( + "--parallelism", + type=int, + default=DEFAULT_BGP_COLLECTOR_PARALLELISM, + help="Maximum concurrent docker exec workers; starts are spread over each polling interval", + ) + parser.add_argument( + "--max-bytes", + type=int, + default=DEFAULT_BGP_COLLECTOR_MAX_BYTES, + help="Maximum BGP line protocol file size before truncating to the latest snapshot; <=0 disables.", + ) + parser.add_argument("--once", action="store_true", help="Collect one snapshot and exit") + parser.add_argument("--topology-id", help="Explicit topology identity for emitted line protocol") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.once: + return run_once( + Path(args.metadata_file), + Path(args.output), + parallelism=args.parallelism, + max_bytes=args.max_bytes, + topology_id=args.topology_id, + ) + return run_loop( + Path(args.metadata_file), + Path(args.output), + args.interval, + parallelism=args.parallelism, + max_bytes=args.max_bytes, + topology_id=args.topology_id, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/netopsbench/platform/observability/bgp_parser.py b/netopsbench/platform/observability/bgp_parser.py new file mode 100644 index 0000000..f64c5b8 --- /dev/null +++ b/netopsbench/platform/observability/bgp_parser.py @@ -0,0 +1,65 @@ +"""Parse FRR BGP summary output for collectors and device tools.""" + +from __future__ import annotations + +import ipaddress +import re +from typing import Any + + +def _normalize_key(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_") + + +def _coerce_value(value: Any) -> Any: + if value is None: + return None + text = str(value).strip() + try: + return int(text) + except ValueError: + return text + + +def parse_bgp_summary(text: str) -> list[dict[str, Any]]: + """Return normalized neighbor rows from ``show ip bgp summary`` output.""" + lines = [line.rstrip() for line in text.splitlines() if line.strip()] + header_index = next( + (index for index, line in enumerate(lines) if line.startswith("Neighbor") and "State" in line), + None, + ) + if header_index is None: + return [] + + headers = lines[header_index].split() + rows: list[dict[str, Any]] = [] + for line in lines[header_index + 1 :]: + parts = line.split() + if len(parts) < 2: + continue + try: + ipaddress.ip_address(parts[0]) + except ValueError: + continue + if len(parts) > len(headers): + parts = parts[: len(headers) - 1] + [" ".join(parts[len(headers) - 1 :])] + row = {_normalize_key(key): value for key, value in zip(headers, parts, strict=False)} + state_value = row.get("state_pfxrcd") + established = state_value is not None and str(state_value).isdigit() + rows.append( + { + "neighbor": row.get("neighbor"), + "asn": _coerce_value(row.get("as")), + "state": "Established" if established else state_value, + "prefixes_received": int(state_value) if established else None, + "up_down": row.get("up_down"), + "msg_rcvd": _coerce_value(row.get("msgrcvd")), + "msg_sent": _coerce_value(row.get("msgsent")), + "in_q": _coerce_value(row.get("inq")), + "out_q": _coerce_value(row.get("outq")), + } + ) + return rows + + +__all__ = ["parse_bgp_summary"] diff --git a/netopsbench/platform/observability/influxdb.py b/netopsbench/platform/observability/influxdb.py index be24eab..dd36c09 100644 --- a/netopsbench/platform/observability/influxdb.py +++ b/netopsbench/platform/observability/influxdb.py @@ -1,19 +1,56 @@ -#!/usr/bin/env python3 -"""Create an InfluxDB bucket if it does not already exist.""" +"""Shared InfluxDB lifecycle and Flux query client.""" -import argparse import json import time import urllib.error import urllib.parse import urllib.request +from dataclasses import dataclass +from typing import Literal + +import requests -from netopsbench.config import config from netopsbench.logging_utils import get_logger logger = get_logger(__name__) +@dataclass(frozen=True) +class FluxQueryResult: + status: Literal["ok", "error"] + text: str = "" + error: str | None = None + + +def query_flux( + base_url: str, + token: str, + org: str, + query: str, + *, + timeout: int = 30, +) -> FluxQueryResult: + """Execute one Flux query without interpreting its domain-specific CSV.""" + headers = { + "Authorization": f"Token {token}", + "Content-Type": "application/vnd.flux", + "Accept": "application/csv", + } + try: + response = requests.post( + f"{base_url.rstrip('/')}/api/v2/query", + params={"org": org}, + headers=headers, + data=query, + timeout=timeout, + proxies={"http": "", "https": ""}, + ) + response.raise_for_status() + except requests.RequestException as exc: + return FluxQueryResult(status="error", error=f"{type(exc).__name__}: {exc}") + return FluxQueryResult(status="ok", text=response.text) + + def _make_url_opener(url: str): hostname = (urllib.parse.urlparse(url).hostname or "").strip().lower() if hostname in {"localhost", "127.0.0.1", "::1"}: @@ -74,23 +111,4 @@ def ensure_bucket(base_url: str, token: str, org: str, bucket: str, retries: int raise RuntimeError(f"Failed to ensure InfluxDB bucket '{bucket}': {last_error}") -def main() -> int: - parser = argparse.ArgumentParser(description="Create an InfluxDB bucket if needed") - parser.add_argument("--url", default="http://localhost:8086", help="InfluxDB base URL") - parser.add_argument("--token", default=config.influxdb_token, help="InfluxDB token") - parser.add_argument("--org", default=config.influxdb_org, help="InfluxDB organization") - parser.add_argument("--bucket", required=True, help="Bucket name") - parser.add_argument("--retries", type=int, default=20, help="Retry count while waiting for InfluxDB") - parser.add_argument("--delay", type=float, default=2.0, help="Delay between retries in seconds") - args = parser.parse_args() - - ensure_bucket(args.url, args.token, args.org, args.bucket, retries=args.retries, delay=args.delay) - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except Exception as exc: # pragma: no cover - CLI error path - logger.error("%s", exc) - raise SystemExit(1) from None +__all__ = ["FluxQueryResult", "ensure_bucket", "query_flux"] diff --git a/netopsbench/platform/observability/lifecycle.py b/netopsbench/platform/observability/lifecycle.py new file mode 100644 index 0000000..fe02d6f --- /dev/null +++ b/netopsbench/platform/observability/lifecycle.py @@ -0,0 +1,183 @@ +"""Idempotent observability lifecycle operations for runtime workers.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from importlib.resources import files +from pathlib import Path + +from netopsbench.config import config +from netopsbench.models.runtime import RuntimeIdentity +from netopsbench.platform.observability.influxdb import ensure_bucket +from netopsbench.platform.observability.telegraf import update_telegraf_config +from netopsbench.platform.utils.proc import docker_prefix, safe_run + +BGP_POLL_INTERVAL_SECONDS = 10 +BGP_COLLECTOR_PARALLELISM = 16 +INTERNAL_INFLUXDB_URL = "http://influxdb:8086" + + +def observability_asset_root() -> Path: + root = files("netopsbench.platform.observability").joinpath("assets") + if not root.is_dir(): + raise FileNotFoundError("Packaged observability assets are missing") + return Path(str(root)) + + +def ensure_observability_core() -> None: + root = observability_asset_root() + safe_run( + [ + *docker_prefix(), + "docker", + "compose", + "--project-name", + "observability", + "--project-directory", + str(root), + "-f", + str(root / "docker-compose.yaml"), + "up", + "-d", + "influxdb", + "grafana", + ], + cwd=root, + check=True, + timeout=600, + ) + + +def ensure_worker_observability(worker: RuntimeIdentity) -> None: + """Reconcile the shared core, worker collector, and Telegraf sidecar.""" + ensure_observability_core() + ensure_bucket( + config.influxdb_url, + config.influxdb_token, + config.influxdb_org, + worker.bucket, + ) + docker = [*docker_prefix(), "docker"] + safe_run([*docker, "inspect", "influxdb"], check=True, timeout=30) + safe_run( + [*docker, "network", "connect", "--alias", "influxdb", worker.mgmt_network, "influxdb"], + check=False, + timeout=60, + ) + ensure_worker_bgp_collector(worker) + ensure_worker_telegraf(worker) + + +def ensure_worker_telegraf(worker: RuntimeIdentity) -> None: + topology_dir = worker.topology_dir.resolve() + topology_file = topology_dir / "topology.json" + if not topology_file.is_file(): + raise FileNotFoundError(f"Topology metadata not found: {topology_file}") + + container_name = f"telegraf-{worker.lab_name}" + config_path = topology_dir / f"{container_name}.conf" + bgp_file = topology_dir / "bgp_neighbors.lp" + update_telegraf_config( + str(topology_file), + output_file=str(config_path), + influxdb_url=INTERNAL_INFLUXDB_URL, + influxdb_token=config.influxdb_token, + influxdb_org=config.influxdb_org, + influxdb_bucket=worker.bucket, + topology_id=worker.topology_id, + ) + bgp_file.touch(exist_ok=True) + topology_dir.chmod(0o755) + config_path.chmod(0o644) + bgp_file.chmod(0o644) + + docker = [*docker_prefix(), "docker"] + safe_run([*docker, "rm", "-f", container_name], check=False, timeout=60) + safe_run( + [ + *docker, + "run", + "-d", + "--name", + container_name, + "--restart", + "unless-stopped", + "--network", + worker.mgmt_network, + "--ip", + _collector_ip(topology_file), + "-v", + f"{config_path}:/etc/telegraf/telegraf.conf:ro", + "-v", + f"{topology_dir}:/var/lib/netopsbench:ro", + "telegraf:latest", + ], + check=True, + timeout=600, + ) + + +def ensure_worker_bgp_collector(worker: RuntimeIdentity) -> None: + topology_dir = worker.topology_dir + topology_file = topology_dir / "topology.json" + if not topology_file.is_file(): + raise FileNotFoundError(f"Topology metadata not found: {topology_file}") + + pid_file = topology_dir / "bgp_collector.pid" + output_file = topology_dir / "bgp_neighbors.lp" + log_file = topology_dir / "bgp_collector.log" + output_file.touch(exist_ok=True) + if _bgp_collector_is_running(pid_file, topology_file): + return + pid_file.unlink(missing_ok=True) + + command = [ + sys.executable, + "-m", + "netopsbench.platform.observability.bgp_collector", + str(topology_file), + "--output", + str(output_file), + "--interval", + str(BGP_POLL_INTERVAL_SECONDS), + "--parallelism", + str(BGP_COLLECTOR_PARALLELISM), + "--topology-id", + worker.topology_id, + ] + with log_file.open("a", encoding="utf-8") as log_handle: + process = subprocess.Popen( + command, + cwd=topology_dir, + stdout=log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + pid_file.write_text(f"{process.pid}\n", encoding="utf-8") + + +def _bgp_collector_is_running(pid_file: Path, topology_file: Path) -> bool: + try: + pid = int(pid_file.read_text(encoding="utf-8").strip()) + os.kill(pid, 0) + command_line = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode(errors="replace") + except (FileNotFoundError, OSError, ValueError): + return False + return "netopsbench.platform.observability.bgp_collector" in command_line and str(topology_file) in command_line + + +def _collector_ip(topology_file: Path) -> str: + from netopsbench.platform.topology.topology_utils import load_topology_manifest + + return load_topology_manifest(topology_file).collector.ipv4 + + +__all__ = [ + "ensure_observability_core", + "ensure_worker_bgp_collector", + "ensure_worker_observability", + "ensure_worker_telegraf", + "observability_asset_root", +] diff --git a/netopsbench/platform/observability/telegraf.py b/netopsbench/platform/observability/telegraf.py index 5f6411e..cd5ddfa 100644 --- a/netopsbench/platform/observability/telegraf.py +++ b/netopsbench/platform/observability/telegraf.py @@ -4,19 +4,118 @@ Generates telegraf.conf from template based on topology metadata """ -import argparse -import json -import os import re -import sys +from importlib.resources import files from pathlib import Path -from netopsbench.config import config, repo_root +from netopsbench.config import config from netopsbench.logging_utils import get_logger +from netopsbench.models.topology import Device, DeviceRole, TopologyManifest +from netopsbench.platform.topology.configdb_payload import interface_names_from_configdb +from netopsbench.platform.topology.topology_utils import load_topology_manifest logger = get_logger(__name__) -REPO_ROOT = repo_root() +GNMI_PORT = 50051 +GNMI_USERNAME = "admin" +GNMI_PASSWORD = "" +GNMI_ENCODING = "json_ietf" +GNMI_TARGET = "COUNTERS_DB" +GNMI_SUBSCRIPTION_MODE = "on_change" +INTERNAL_INFLUXDB_URL = "http://influxdb:8086" + + +def _port_key(name: str) -> int: + return int(name.replace("Ethernet", "")) if name.startswith("Ethernet") else 0 + + +def _device_config_interfaces(topology_dir: Path, device_name: str) -> list[str]: + config_path = topology_dir / "configs" / "sonic" / device_name / "config_db.json" + if not config_path.is_file(): + raise FileNotFoundError(f"Generated ConfigDB artifact not found: {config_path}") + interfaces = interface_names_from_configdb(config_path) + if not interfaces: + raise ValueError(f"Generated ConfigDB artifact has no interfaces: {config_path}") + return interfaces + + +def _role_port_names(topology_dir: Path, devices: list[dict]) -> list[str]: + port_names: set[str] = set() + for device in devices: + port_names.update(_device_config_interfaces(topology_dir, str(device.get("name", "")))) + return sorted(port_names, key=_port_key) + + +def _devices_for_role(raw_devices: list[Device], gnmi_port: int) -> list[dict]: + devices = [] + for device in raw_devices or []: + if not device.mgmt_ip: + continue + devices.append( + { + "name": device.name, + "mgmt_ip": str(device.mgmt_ip).split("/")[0], + "gnmi_port": gnmi_port, + } + ) + return devices + + +def _gnmi_addresses(devices: list[dict]) -> list[str]: + return [f'"{device["mgmt_ip"]}:{device["gnmi_port"]}"' for device in devices] + + +def _gnmi_role_groups(manifest: TopologyManifest, gnmi_port: int) -> list[tuple[str, list[dict]]]: + if manifest.family == "fat-tree": + return [ + ("core", _devices_for_role(manifest.devices_by_role(DeviceRole.CORE), gnmi_port)), + ("agg", _devices_for_role(manifest.devices_by_role(DeviceRole.AGG), gnmi_port)), + ("edge", _devices_for_role(manifest.devices_by_role(DeviceRole.EDGE), gnmi_port)), + ] + + return [ + ("spine", _devices_for_role(manifest.devices_by_role(DeviceRole.SPINE), gnmi_port)), + ("leaf", _devices_for_role(manifest.devices_by_role(DeviceRole.LEAF), gnmi_port)), + ] + + +def _render_gnmi_subscriptions(port_names: list[str]) -> str: + subscriptions = [] + for port in port_names: + subscription_lines = [ + " [[inputs.gnmi.subscription]]", + ' name = "interfaces"', + f' path = "COUNTERS/{port}"', + f' subscription_mode = "{GNMI_SUBSCRIPTION_MODE}"', + ] + subscriptions.append("\n".join(subscription_lines) + "\n") + return "\n".join(subscriptions) + + +def _render_gnmi_input( + role: str, + devices: list[dict], + port_names: list[str], +) -> str: + if not devices or not port_names: + return "" + addresses = ",\n ".join(_gnmi_addresses(devices)) + subscriptions = _render_gnmi_subscriptions(port_names) + return f"""# gNMI role: {role} +[[inputs.gnmi]] + addresses = [ + {addresses} + ] + username = "{GNMI_USERNAME}" + password = "{GNMI_PASSWORD}" + encoding = "{GNMI_ENCODING}" + redial = "10s" + path_guessing_strategy = "subscription" + tls_enable = false + insecure_skip_verify = true + target = "{GNMI_TARGET}" + +{subscriptions}""" def update_telegraf_config( @@ -41,123 +140,43 @@ def update_telegraf_config( topology_id: Optional topology identifier tag override Generates telegraf.conf with: - - {{GNMI_ADDRESSES}}: List of device management IPs for gNMI polling + - {{GNMI_INPUTS}}: Role-scoped SONiC gNMI inputs and subscriptions - {{IP_MAPPINGS}}: Processor rules to map IPs to hostnames """ # Read topology metadata topology_path = Path(topology_file) if not topology_path.exists(): - logger.error("Topology file not found: %s", topology_file) - sys.exit(1) - - with open(topology_path) as f: - topo = json.load(f) - - # gNMI settings (override with env vars if needed) - gnmi_port = int(os.getenv("SONIC_GNMI_PORT", os.getenv("GNMI_PORT", "50051"))) - gnmi_username = os.getenv("SONIC_GNMI_USERNAME", os.getenv("GNMI_USERNAME", "admin")) - gnmi_password = os.getenv("SONIC_GNMI_PASSWORD", os.getenv("GNMI_PASSWORD", "")) - if not gnmi_password: - logger.warning("SONIC_GNMI_PASSWORD is not set; gNMI telemetry collection will likely fail.") - gnmi_encoding = os.getenv("SONIC_GNMI_ENCODING", os.getenv("GNMI_ENCODING", "json_ietf")) - gnmi_target = os.getenv("SONIC_GNMI_TARGET", os.getenv("GNMI_TARGET", "COUNTERS_DB")) - gnmi_subscription_mode = ( - os.getenv( - "SONIC_GNMI_SUBSCRIPTION_MODE", - os.getenv("GNMI_SUBSCRIPTION_MODE", "on_change"), - ) - .strip() - .lower() - ) - if gnmi_subscription_mode not in {"on_change", "sample", "target_defined"}: - logger.error( - "unsupported gNMI subscription mode %r; expected one of on_change, sample, target_defined", - gnmi_subscription_mode, - ) - sys.exit(1) - gnmi_sample_interval = os.getenv("SONIC_GNMI_SAMPLE_INTERVAL", os.getenv("GNMI_SAMPLE_INTERVAL", "10s")).strip() - resolved_influxdb_url = influxdb_url or os.getenv("NETOPSBENCH_INFLUXDB_URL", "http://influxdb:8086") - resolved_influxdb_token = influxdb_token or os.getenv( - "NETOPSBENCH_INFLUXDB_TOKEN", - config.influxdb_token, - ) - resolved_influxdb_org = influxdb_org or os.getenv("NETOPSBENCH_INFLUXDB_ORG", config.influxdb_org) - resolved_influxdb_bucket = influxdb_bucket or os.getenv("NETOPSBENCH_INFLUXDB_BUCKET", config.influxdb_bucket) - resolved_topology_id = ( - topology_id - or os.getenv("NETOPSBENCH_TOPOLOGY_ID") - or str(topo.get("name") or "").strip() - or topology_path.parent.name - ) - - # sFlow settings (override with env vars if needed) - sflow_service_address = os.getenv("SONIC_SFLOW_SERVICE_ADDRESS", os.getenv("SFLOW_SERVICE_ADDRESS", "udp://:6343")) - if "://" not in sflow_service_address: - sflow_service_address = f"udp://{sflow_service_address}" - - # Extract device information - devices = [] + raise FileNotFoundError(f"Topology file not found: {topology_file}") - # Add spines - for spine in topo["devices"]["spines"]: - devices.append( - { - "name": spine["name"], - "mgmt_ip": spine["mgmt_ip"].split("/")[0], # Remove CIDR notation - "gnmi_port": gnmi_port, - } - ) + manifest = load_topology_manifest(topology_path) - # Add leafs - for leaf in topo["devices"]["leafs"]: - devices.append( - { - "name": leaf["name"], - "mgmt_ip": leaf["mgmt_ip"].split("/")[0], # Remove CIDR notation - "gnmi_port": gnmi_port, - } - ) + resolved_influxdb_url = influxdb_url or INTERNAL_INFLUXDB_URL + resolved_influxdb_token = influxdb_token or config.influxdb_token + resolved_influxdb_org = influxdb_org or config.influxdb_org + resolved_influxdb_bucket = influxdb_bucket or config.influxdb_bucket + resolved_topology_id = topology_id or manifest.topology_id + + topology_dir = topology_path.parent + role_groups = _gnmi_role_groups(manifest, GNMI_PORT) + devices = [device for _role, role_devices in role_groups for device in role_devices] logger.info("Found %d network devices:", len(devices)) for d in devices: logger.info(" - %s: %s", d["name"], d["mgmt_ip"]) - # Derive SONiC front-panel ports used by this topology (Ethernet0/4/8...) - scale = topo.get("scale", {}) - num_spines = int(scale.get("num_spines", 0)) - num_leafs = int(scale.get("num_leafs", 0)) - clients_per_leaf = int(scale.get("clients_per_leaf", 0)) - - port_set = set() - for leaf_idx in range(1, num_leafs + 1): - port_set.add(f"Ethernet{(leaf_idx - 1) * 4}") - for spine_idx in range(1, num_spines + 1): - port_set.add(f"Ethernet{(spine_idx - 1) * 4}") - for client_idx in range(1, clients_per_leaf + 1): - port_set.add(f"Ethernet{(num_spines + client_idx - 1) * 4}") - - def _port_key(name: str) -> int: - return int(name.replace("Ethernet", "")) if name.startswith("Ethernet") else 0 - - port_names = sorted(port_set, key=_port_key) - - # Generate gNMI address list - gnmi_addresses = [f'"{d["mgmt_ip"]}:{d["gnmi_port"]}"' for d in devices] - gnmi_addresses_str = ",\n ".join(gnmi_addresses) - - # Generate gNMI subscriptions (explicit per-port paths) - gnmi_subscriptions = [] - for port in port_names: - subscription_lines = [ - " [[inputs.gnmi.subscription]]", - ' name = "interfaces"', - f' path = "COUNTERS/{port}"', - f' subscription_mode = "{gnmi_subscription_mode}"', - ] - if gnmi_subscription_mode == "sample": - subscription_lines.append(f' sample_interval = "{gnmi_sample_interval}"') - gnmi_subscriptions.append("\n".join(subscription_lines) + "\n") - gnmi_subscriptions_str = "\n".join(gnmi_subscriptions) + rendered_inputs = [] + role_subscription_counts: dict[str, set[int]] = {} + for role, role_devices in role_groups: + port_names = _role_port_names(topology_dir, role_devices) + role_subscription_counts[role] = {len(port_names)} + rendered_inputs.append( + _render_gnmi_input( + role, + role_devices, + port_names, + ), + ) + gnmi_inputs_str = "\n".join(block for block in rendered_inputs if block) # Generate IP to hostname mappings (processor rules) ip_mappings = [] @@ -172,24 +191,13 @@ def _port_key(name: str) -> int: ip_mappings_str = "".join(ip_mappings) # Read template file - template_file = REPO_ROOT / "observability" / "telegraf.conf.template" - - if not template_file.exists(): - logger.error("Template file not found: %s", template_file) - logger.error("Please create telegraf.conf.template first") - sys.exit(1) - - with open(template_file, encoding="utf-8") as f: - template = f.read() + template_resource = files("netopsbench.platform.observability").joinpath("assets", "telegraf.conf.template") + if not template_resource.is_file(): + raise FileNotFoundError("Packaged Telegraf template is missing") + template = template_resource.read_text(encoding="utf-8") # Replace placeholders - rendered_config = template.replace("{{GNMI_ADDRESSES}}", gnmi_addresses_str) - rendered_config = rendered_config.replace("{{GNMI_USERNAME}}", gnmi_username) - rendered_config = rendered_config.replace("{{GNMI_PASSWORD}}", gnmi_password) - rendered_config = rendered_config.replace("{{GNMI_ENCODING}}", gnmi_encoding) - rendered_config = rendered_config.replace("{{GNMI_TARGET}}", gnmi_target) - rendered_config = rendered_config.replace("{{GNMI_SUBSCRIPTIONS}}", gnmi_subscriptions_str) - rendered_config = rendered_config.replace("{{SFLOW_SERVICE_ADDRESS}}", sflow_service_address) + rendered_config = template.replace("{{GNMI_INPUTS}}", gnmi_inputs_str) rendered_config = rendered_config.replace("{{IP_MAPPINGS}}", ip_mappings_str) rendered_config = rendered_config.replace("{{INFLUXDB_URL}}", resolved_influxdb_url) rendered_config = rendered_config.replace("{{INFLUXDB_TOKEN}}", resolved_influxdb_token) @@ -206,41 +214,17 @@ def _port_key(name: str) -> int: ) # Write output file - output_path = Path(output_file) if output_file else (REPO_ROOT / "observability" / "telegraf.conf") + output_path = Path(output_file) if output_file else (Path.cwd() / "observability" / "telegraf.conf") output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, "w", encoding="utf-8") as f: f.write(rendered_config) logger.info("Telegraf configuration updated: %s", output_path) logger.info(" - %d devices configured", len(devices)) - logger.info(" - gNMI targets: %d", len(gnmi_addresses)) - logger.info(" - gNMI subscription mode: %s", gnmi_subscription_mode) + logger.info(" - gNMI targets: %d", len(devices)) + for role, counts in role_subscription_counts.items(): + logger.info(" - gNMI %s subscriptions per target: %s", role, ", ".join(str(item) for item in sorted(counts))) + logger.info(" - gNMI subscription mode: %s", GNMI_SUBSCRIPTION_MODE) logger.info(" - IP mappings: %d", len(ip_mappings)) logger.info(" - InfluxDB bucket: %s", resolved_influxdb_bucket) return 0 - - -def main() -> int: - parser = argparse.ArgumentParser(description="Generate telegraf.conf from topology metadata") - parser.add_argument("topology_file", help="Path to topology.json") - parser.add_argument("--output", help="Output telegraf config path") - parser.add_argument("--influxdb-url", help="InfluxDB URL override") - parser.add_argument("--influxdb-token", help="InfluxDB token override") - parser.add_argument("--influxdb-org", help="InfluxDB organization override") - parser.add_argument("--bucket", help="InfluxDB bucket override") - parser.add_argument("--topology-id", help="Topology identifier override") - args = parser.parse_args() - - return update_telegraf_config( - args.topology_file, - output_file=args.output, - influxdb_url=args.influxdb_url, - influxdb_token=args.influxdb_token, - influxdb_org=args.influxdb_org, - influxdb_bucket=args.bucket, - topology_id=args.topology_id, - ) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/netopsbench/platform/observability/validation.py b/netopsbench/platform/observability/validation.py index 3f099a5..5e49894 100644 --- a/netopsbench/platform/observability/validation.py +++ b/netopsbench/platform/observability/validation.py @@ -1,48 +1,35 @@ -#!/usr/bin/env python3 """Validate the worker observability path through InfluxDB.""" from __future__ import annotations -import argparse import csv import re -import time -import urllib.error -import urllib.parse -import urllib.request from collections.abc import Callable, Sequence -from netopsbench.logging_utils import get_logger - -logger = get_logger(__name__) - - QueryRunner = Callable[[str], str] - - -def _make_url_opener(base_url: str): - hostname = (urllib.parse.urlparse(base_url).hostname or "").strip().lower() - if hostname in {"localhost", "127.0.0.1", "::1"}: - return urllib.request.build_opener(urllib.request.ProxyHandler({})) - return urllib.request.build_opener() - - -def run_query(base_url: str, token: str, org: str, query: str) -> str: - """Execute one Flux query against InfluxDB and return CSV output.""" - headers = { - "Authorization": f"Token {token}", - "Accept": "application/csv", - "Content-Type": "application/vnd.flux", - } - org_q = urllib.parse.quote(org, safe="") - req = urllib.request.Request( - f"{base_url.rstrip('/')}/api/v2/query?org={org_q}", - data=query.encode("utf-8"), - headers=headers, - method="POST", - ) - with _make_url_opener(base_url).open(req, timeout=20) as resp: - return resp.read().decode("utf-8") +_INTERFACE_COUNTER_FIELDS = { + "in_octets", + "out_octets", + "in_unicast_pkts", + "out_unicast_pkts", + "in_discarded_packets", + "out_discarded_packets", + "in_errors", + "out_errors", + "SAI_PORT_STAT_IF_IN_OCTETS", + "SAI_PORT_STAT_IF_OUT_OCTETS", + "SAI_PORT_STAT_IF_IN_UCAST_PKTS", + "SAI_PORT_STAT_IF_OUT_UCAST_PKTS", + "SAI_PORT_STAT_IF_IN_DISCARDS", + "SAI_PORT_STAT_IF_OUT_DISCARDS", + "SAI_PORT_STAT_IF_IN_ERRORS", + "SAI_PORT_STAT_IF_OUT_ERRORS", +} + + +def _interface_counter_field_filter() -> str: + predicates = " or ".join(f'r._field == "{field}"' for field in sorted(_INTERFACE_COUNTER_FIELDS)) + return f" |> filter(fn: (r) => {predicates})\n" def count_data_rows(csv_text: str) -> int: @@ -71,6 +58,11 @@ def extract_interface_names(csv_text: str) -> list[str]: for row in reader: if row.get("result") == "result": continue + if (row.get("_value") or "").strip() == "": + continue + field = (row.get("_field") or "").strip() + if field and field not in _INTERFACE_COUNTER_FIELDS: + continue name = (row.get("name") or "").strip() path = (row.get("path") or "").strip() if name and name != "name": @@ -116,7 +108,10 @@ def check_observability( f" |> range(start: -10m)\n" f' |> filter(fn: (r) => r._measurement == "pingmesh")\n' f"{topology_filter}" - f" |> limit(n: 5)\n" + f' |> filter(fn: (r) => r._field == "rtt_p99")\n' + f" |> last()\n" + f" |> group()\n" + f" |> limit(n: 1)\n" ) if count_data_rows(query_runner(pingmesh_query)) <= 0: errors.append("no recent pingmesh samples found in InfluxDB") @@ -130,29 +125,28 @@ def check_observability( f' |> filter(fn: (r) => r.source == "{bgp_device}")\n' f' |> filter(fn: (r) => r._field == "session_state")\n' f" |> last()\n" + f" |> group()\n" + f" |> limit(n: 1)\n" ) if count_data_rows(query_runner(bgp_query)) <= 0: errors.append(f"no recent bgp neighbor samples found for {bgp_device} in InfluxDB") - interface_query = ( + interface_identity_query = ( f'from(bucket: "{bucket}")\n' f" |> range(start: -10m)\n" f' |> filter(fn: (r) => r._measurement == "interfaces")\n' + f"{topology_filter}" f' |> filter(fn: (r) => r.source == "{obs_device}")\n' - f" |> limit(n: 5)\n" + f"{_interface_counter_field_filter()}" + f' |> group(columns: ["name", "path", "_field"])\n' + f" |> last()\n" + f' |> keep(columns: ["_time", "_field", "_value", "name", "path"])\n' ) - if count_data_rows(query_runner(interface_query)) <= 0: + interface_rows = query_runner(interface_identity_query) + if count_data_rows(interface_rows) <= 0: errors.append(f"no recent interface samples found for {obs_device} in InfluxDB") else: - interface_identity_query = ( - f'from(bucket: "{bucket}")\n' - f" |> range(start: -10m)\n" - f' |> filter(fn: (r) => r._measurement == "interfaces")\n' - f' |> filter(fn: (r) => r.source == "{obs_device}")\n' - f' |> keep(columns: ["name", "path"])\n' - f" |> limit(n: 2000)\n" - ) - observed_interfaces = extract_interface_names(query_runner(interface_identity_query)) + observed_interfaces = extract_interface_names(interface_rows) if active: observed_active = [name for name in active if name in observed_interfaces] coverage = len(observed_active) / len(active) @@ -175,76 +169,13 @@ def check_observability( f'from(bucket: "{bucket}")\n' f" |> range(start: -5m)\n" f' |> filter(fn: (r) => r._measurement == "syslog")\n' + f"{topology_filter}" f' |> filter(fn: (r) => r._field == "message")\n' f' |> filter(fn: (r) => strings.containsStr(v: r._value, substr: "{syslog_marker}"))\n' - f" |> limit(n: 5)\n" + f" |> group()\n" + f" |> limit(n: 1)\n" ) - found = False - for attempt in range(6): - if count_data_rows(query_runner(syslog_query)) > 0: - found = True - break - if attempt < 5: - time.sleep(2) - if not found: + if count_data_rows(query_runner(syslog_query)) <= 0: errors.append("syslog collector path check failed: marker not found in InfluxDB") return errors - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Validate worker observability data in InfluxDB") - parser.add_argument("--url", required=True, help="InfluxDB base URL") - parser.add_argument("--token", required=True, help="InfluxDB token") - parser.add_argument("--org", required=True, help="InfluxDB organization") - parser.add_argument("--bucket", required=True, help="InfluxDB bucket") - parser.add_argument("--obs-device", required=True, help="Device expected to emit interface metrics") - parser.add_argument("--bgp-device", default="", help="Device expected to emit BGP neighbor metrics") - parser.add_argument("--topology-id", default="", help="Optional topology identifier for Pingmesh filtering") - parser.add_argument("--syslog-marker", default="", help="Optional syslog marker to verify") - parser.add_argument( - "--active-interfaces", - default="", - help="Comma-separated active interfaces expected in the interface metrics", - ) - parser.add_argument( - "--min-active-coverage-ratio", - type=float, - default=0.5, - help="Minimum acceptable coverage ratio for active interface observability", - ) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - active_interfaces = [item for item in args.active_interfaces.split(",") if item] - try: - errors = check_observability( - lambda query: run_query(args.url, args.token, args.org, query), - bucket=args.bucket, - obs_device=args.obs_device, - bgp_device=args.bgp_device, - topology_id=args.topology_id, - syslog_marker=args.syslog_marker, - active_interfaces=active_interfaces, - min_active_coverage_ratio=args.min_active_coverage_ratio, - ) - except urllib.error.HTTPError as exc: - logger.error("InfluxDB query failed: HTTP %s", exc.code) - return 1 - except urllib.error.URLError as exc: - logger.error("InfluxDB query failed: %s", exc) - return 1 - except Exception as exc: - logger.error("observability query failed: %s", exc) - return 1 - - if errors: - logger.error("%s", "; ".join(errors)) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/netopsbench/platform/pingmesh/__init__.py b/netopsbench/platform/pingmesh/__init__.py index 01bcb37..a6ede0a 100644 --- a/netopsbench/platform/pingmesh/__init__.py +++ b/netopsbench/platform/pingmesh/__init__.py @@ -1,14 +1 @@ -"""Platform pingmesh subsystem.""" - -from .agent import PingmeshAgent -from .detector import Anomaly, AnomalyDetector -from .generator import PinglistGenerator, ProbeTask, generate_pinglist_from_topology - -__all__ = [ - "Anomaly", - "AnomalyDetector", - "PinglistGenerator", - "ProbeTask", - "PingmeshAgent", - "generate_pinglist_from_topology", -] +"""Internal Pingmesh subsystem.""" diff --git a/netopsbench/platform/pingmesh/_agent_influx.py b/netopsbench/platform/pingmesh/_agent_influx.py index 7b6400e..3f8f6b6 100644 --- a/netopsbench/platform/pingmesh/_agent_influx.py +++ b/netopsbench/platform/pingmesh/_agent_influx.py @@ -2,10 +2,14 @@ from __future__ import annotations -try: - from ._agent_support import logger, queue, requests, time -except ImportError: - from _agent_support import logger, queue, requests, time +import queue +import time + +import requests + +from netopsbench.logging_utils import get_logger + +logger = get_logger(__name__) class PingInfluxMixin: @@ -48,6 +52,11 @@ def _build_line_protocol(self, measurement: str, probe: dict, result: dict, time f"packet_loss={result['loss_pct']}", f"rtt_ports_active={int(result.get('rtt_ports_active', result.get('packets_sent', 0)))}i", f"rtt_ports_total={int(result.get('rtt_ports_total', result.get('packets_sent', 0)))}i", + f"probe_cycle={int(result.get('probe_cycle', 0))}i", + f"destination_batch_index={int(result.get('destination_batch_index', 0))}i", + f"port_batch_index={int(result.get('port_batch_index', 0))}i", + f"coverage_epoch={int(result.get('coverage_epoch', 0))}i", + f"coverage_epoch_cycles={int(result.get('coverage_epoch_cycles', 1))}i", f"df_success={int(result.get('df_success', 0))}i", f"df_loss_pct={result.get('df_loss_pct', 0.0)}", f"df_rtt_avg={result.get('df_rtt_avg', 0.0)}", diff --git a/netopsbench/platform/pingmesh/_agent_probe.py b/netopsbench/platform/pingmesh/_agent_probe.py index 9b7527c..18c541f 100644 --- a/netopsbench/platform/pingmesh/_agent_probe.py +++ b/netopsbench/platform/pingmesh/_agent_probe.py @@ -22,11 +22,9 @@ import struct import time -try: - from ._agent_support import logger -except ImportError: # standalone in-container deployment - from _agent_support import logger # type: ignore[no-redef] +from netopsbench.logging_utils import get_logger +logger = get_logger(__name__) # --------------------------------------------------------------------------- # Linux IP_MTU_DISCOVER constants (define locally so the module is portable @@ -82,16 +80,13 @@ class UdpProbeMixin: Required attributes on the host class: udp_dst_port (int) n_rtt_ports (int) - n_df_ports (int) rtt_ports_per_cycle (int, optional) - df_ports_per_cycle (int, optional) rtt_src_port_base (int) - df_src_port_base (int) df_payload_size (int) - burst_timeout_s (float) + interval (float) """ - def _open_burst_sockets( + def _open_probe_sockets( self, src_ip: str, src_port_base: int, @@ -133,23 +128,15 @@ def _close_socket_pool(self, sockets: list) -> None: def _close_udp_probe_sockets(self) -> None: """Close persistent UDP probe sockets owned by this agent.""" self._close_socket_pool(getattr(self, "_udp_rtt_sockets", [])) - self._close_socket_pool(getattr(self, "_udp_df_sockets", [])) self._udp_rtt_sockets = [] - self._udp_df_sockets = [] self._udp_probe_src_ip = None - self._udp_rtt_socket_cursor = 0 - self._udp_df_socket_cursor = 0 def _ensure_udp_probe_sockets( self, src_ip: str, include_rtt: bool = True, - include_df: bool | None = None, - ) -> tuple[list, list]: + ) -> list: """Create persistent source-port socket pools on first use.""" - if include_df is None: - include_df = bool(getattr(self, "enable_df_probe", True)) - bind_ip = src_ip or "" current_src_ip = getattr(self, "_udp_probe_src_ip", None) if current_src_ip is not None and current_src_ip != bind_ip: @@ -164,24 +151,22 @@ def _ensure_udp_probe_sockets( self._udp_probe_src_ip = bind_ip if not hasattr(self, "_udp_rtt_sockets"): self._udp_rtt_sockets = [] - if not hasattr(self, "_udp_df_sockets"): - self._udp_df_sockets = [] if include_rtt and not self._udp_rtt_sockets: - self._udp_rtt_sockets = self._open_burst_sockets( + self._udp_rtt_sockets = self._open_probe_sockets( src_ip=bind_ip, src_port_base=self.rtt_src_port_base, n_ports=self.n_rtt_ports, set_df=True, ) - if include_df and not self._udp_df_sockets: - self._udp_df_sockets = self._open_burst_sockets( - src_ip=bind_ip, - src_port_base=self.df_src_port_base, - n_ports=self.n_df_ports, - set_df=True, - ) - return self._udp_rtt_sockets, self._udp_df_sockets + if len(self._udp_rtt_sockets) != int(self.n_rtt_ports): + opened_ports = {src_port for src_port, _sock in self._udp_rtt_sockets} + expected_ports = set(range(self.rtt_src_port_base, self.rtt_src_port_base + self.n_rtt_ports)) + missing = sorted(expected_ports - opened_ports) + self._close_socket_pool(self._udp_rtt_sockets) + self._udp_rtt_sockets = [] + raise RuntimeError(f"Unable to bind the complete Pingmesh source-port pool; missing ports: {missing}") + return self._udp_rtt_sockets def _next_udp_probe_seq(self) -> int: seq = int(getattr(self, "_udp_probe_seq", 0)) & 0xFFFFFFFFFFFFFFFF @@ -200,91 +185,79 @@ def _probe_bind_src_ip(self, probes: list[dict]) -> str: break return src_ip - def _active_socket_count(self, sockets: list, attr_name: str, fallback_attr_name: str) -> int: - """Return the clamped number of sockets to use in this cycle.""" + def _socket_batch(self, sockets: list, batch_index: int) -> list: + """Select the configured, non-overlapping source-port batch.""" if not sockets: - return 0 - requested = getattr(self, attr_name, getattr(self, fallback_attr_name, len(sockets))) - try: - requested_int = int(requested) - except (TypeError, ValueError): - requested_int = len(sockets) - return max(0, min(requested_int, len(sockets))) - - def _select_rotating_sockets(self, sockets: list, active_count: int, cursor_attr: str) -> list: - """Select ``active_count`` sockets and advance a round-robin cursor.""" - if active_count <= 0 or not sockets: return [] - if active_count >= len(sockets): - return list(sockets) - cursor = int(getattr(self, cursor_attr, 0)) % len(sockets) - selected = [sockets[(cursor + offset) % len(sockets)] for offset in range(active_count)] - setattr(self, cursor_attr, (cursor + active_count) % len(sockets)) - return selected - - def _udp_fanout( + batch_size = max(1, int(self.rtt_ports_per_cycle)) + batch_count = max(1, (len(sockets) + batch_size - 1) // batch_size) + start = (int(batch_index) % batch_count) * batch_size + return sockets[start : start + batch_size] + + @staticmethod + def _new_probe_stats(probes: list[dict]) -> list[dict]: + return [{"sent": 0, "received": 0, "rtts": [], "mtu_drops": 0} for _probe in probes] + + @staticmethod + def _drain_socket(sock: socket.socket) -> None: + """Discard replies left by a completed cycle before sockets are reused.""" + while True: + try: + sock.recvfrom(65535) + except BlockingIOError: + return + except OSError: + return + + @staticmethod + def _build_send_queue(probes: list[dict], rtt_sockets: list, enable_df: bool) -> list: + """Build a stable queue where each DF probe reuses one active RTT tuple.""" + send_queue: list[tuple[str, int, socket.socket, str]] = [] + for probe_index, probe in enumerate(probes): + dst_ip = str(probe.get("dst_ip") or "") + if not dst_ip: + continue + for _src_port, rtt_socket in rtt_sockets: + send_queue.append(("rtt", probe_index, rtt_socket, dst_ip)) + if enable_df and rtt_sockets: + _src_port, df_socket = rtt_sockets[probe_index % len(rtt_sockets)] + send_queue.append(("df", probe_index, df_socket, dst_ip)) + return send_queue + + def _run_udp_cycle( self, probes: list[dict], - sockets: list, - payload_size: int, - timeout_s: float, - ) -> list[dict]: - """Fan out one payload per socket/probe pair and collect echoes.""" - stats = [ - { - "sent": 0, - "received": 0, - "rtts": [], - "mtu_drops": 0, - } - for _probe in probes - ] - if not probes or not sockets: - return stats - - payload_size = max(payload_size, _MIN_PAYLOAD_BYTES) - padding = b"\x00" * (payload_size - _HEADER_SIZE) - pending: dict[int, tuple[int, socket.socket, int]] = {} - socket_pending: dict[socket.socket, int] = {} - - for _src_port, sock in sockets: - for probe_index, probe in enumerate(probes): - dst_ip = probe.get("dst_ip") - if not dst_ip: - continue - seq = self._next_udp_probe_seq() - send_time_ns = time.time_ns() - payload = struct.pack(_HEADER_FMT, send_time_ns, seq) + padding - try: - sock.sendto(payload, (dst_ip, self.udp_dst_port)) - stats[probe_index]["sent"] += 1 - pending[seq] = (probe_index, sock, send_time_ns) - socket_pending[sock] = socket_pending.get(sock, 0) + 1 - except OSError as exc: - # EMSGSIZE (errno 90) means the kernel's cached PMTU - # cannot accommodate this packet with DF set; count it as - # a sent-but-dropped probe because it is the MTU signal. - if getattr(exc, "errno", None) == 90: - stats[probe_index]["sent"] += 1 - stats[probe_index]["mtu_drops"] += 1 - else: - logger.debug("sendto %s failed: %s", dst_ip, exc) - - deadline = time.monotonic() + timeout_s - while pending and socket_pending: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - try: - readable, _, _ = select.select(list(socket_pending), [], [], remaining) - except (OSError, ValueError): - break - if not readable: - break - for sock in readable: + rtt_sockets: list, + enable_df: bool, + ) -> tuple[list[dict], list[dict]]: + """Schedule RTT and DF datagrams in one non-blocking event loop.""" + rtt_stats = self._new_probe_stats(probes) + df_stats = self._new_probe_stats(probes) + active_sockets = [sock for _src_port, sock in rtt_sockets] + for sock in active_sockets: + self._drain_socket(sock) + + payloads = { + "rtt": b"\x00" * (max(_DEFAULT_RTT_PAYLOAD_BYTES, _MIN_PAYLOAD_BYTES) - _HEADER_SIZE), + "df": b"\x00" * (max(self.df_payload_size, _MIN_PAYLOAD_BYTES) - _HEADER_SIZE), + } + stats_by_kind = {"rtt": rtt_stats, "df": df_stats} + send_queue = self._build_send_queue(probes, rtt_sockets, enable_df) + + interval = max(0.001, float(self.interval)) + cycle_started = time.monotonic() + cycle_deadline = cycle_started + interval + send_window = interval / 2.0 + send_spacing = send_window / max(1, len(send_queue) - 1) + next_send_at = cycle_started + send_index = 0 + pending: dict[int, tuple[str, int, int]] = {} + + def drain_readable(readable: list[socket.socket]) -> None: + for readable_socket in readable: while True: try: - data, _addr = sock.recvfrom(65535) + data, _addr = readable_socket.recvfrom(65535) except BlockingIOError: break except OSError as exc: @@ -292,22 +265,49 @@ def _udp_fanout( break if len(data) < _HEADER_SIZE: continue - sent_ns, seq = struct.unpack(_HEADER_FMT, data[:_HEADER_SIZE]) + _payload_sent_ns, seq = struct.unpack(_HEADER_FMT, data[:_HEADER_SIZE]) pending_item = pending.pop(seq, None) if pending_item is None: continue - probe_index, pending_sock, _send_time_ns = pending_item - count = socket_pending.get(pending_sock, 0) - 1 - if count > 0: - socket_pending[pending_sock] = count - else: - socket_pending.pop(pending_sock, None) - rtt_ms = (time.time_ns() - sent_ns) / 1e6 + kind, probe_index, recorded_sent_ns = pending_item + rtt_ms = (time.time_ns() - recorded_sent_ns) / 1e6 if rtt_ms >= 0: - stats[probe_index]["received"] += 1 - stats[probe_index]["rtts"].append(rtt_ms) + stats_by_kind[kind][probe_index]["received"] += 1 + stats_by_kind[kind][probe_index]["rtts"].append(rtt_ms) - return stats + while time.monotonic() < cycle_deadline and (send_index < len(send_queue) or pending): + now = time.monotonic() + if send_index < len(send_queue) and now >= next_send_at: + kind, probe_index, sock, dst_ip = send_queue[send_index] + seq = self._next_udp_probe_seq() + send_time_ns = time.time_ns() + payload = struct.pack(_HEADER_FMT, send_time_ns, seq) + payloads[kind] + stats = stats_by_kind[kind][probe_index] + try: + sock.sendto(payload, (dst_ip, self.udp_dst_port)) + stats["sent"] += 1 + pending[seq] = (kind, probe_index, send_time_ns) + except OSError as exc: + if getattr(exc, "errno", None) == 90: + stats["sent"] += 1 + stats["mtu_drops"] += 1 + else: + logger.debug("sendto %s failed: %s", dst_ip, exc) + send_index += 1 + next_send_at = max(next_send_at + send_spacing, time.monotonic()) + + now = time.monotonic() + wake_at = min(next_send_at, cycle_deadline) if send_index < len(send_queue) else cycle_deadline + timeout = max(0.0, wake_at - now) + try: + readable, _, _ = select.select(active_sockets, [], [], timeout) + except (OSError, ValueError): + break + drain_readable(readable) + + for sock in active_sockets: + self._drain_socket(sock) + return rtt_stats, df_stats def _rtt_result_from_stats(self, stats: dict, expected_sent: int) -> dict: sent = int(stats["sent"]) @@ -362,50 +362,25 @@ def _df_result_from_stats(self, stats: dict, expected_sent: int) -> dict: "mtu_drops": mtu_drops, } - def udp_probe_cycle(self, probes: list[dict]) -> list[dict]: + def udp_probe_cycle(self, probes: list[dict], port_batch_index: int) -> list[dict]: """Probe all destinations once using persistent source-port sockets.""" if not probes: return [] src_ip = self._probe_bind_src_ip(probes) - rtt_sockets, df_sockets = self._ensure_udp_probe_sockets( + rtt_sockets = self._ensure_udp_probe_sockets( src_ip=src_ip, include_rtt=True, - include_df=bool(getattr(self, "enable_df_probe", True)), - ) - rtt_active_count = self._active_socket_count(rtt_sockets, "rtt_ports_per_cycle", "n_rtt_ports") - df_active_count = ( - self._active_socket_count(df_sockets, "df_ports_per_cycle", "n_df_ports") - if getattr(self, "enable_df_probe", True) - else 0 - ) - active_rtt_sockets = self._select_rotating_sockets( - rtt_sockets, - rtt_active_count, - "_udp_rtt_socket_cursor", - ) - active_df_sockets = self._select_rotating_sockets( - df_sockets, - df_active_count, - "_udp_df_socket_cursor", ) + active_rtt_sockets = self._socket_batch(rtt_sockets, port_batch_index) + enable_df = bool(getattr(self, "enable_df_probe", True)) try: - rtt_stats = self._udp_fanout( - probes=probes, - sockets=active_rtt_sockets, - payload_size=_DEFAULT_RTT_PAYLOAD_BYTES, - timeout_s=self.burst_timeout_s, + rtt_stats, df_stats = self._run_udp_cycle( + probes, + active_rtt_sockets, + enable_df, ) - if getattr(self, "enable_df_probe", True): - df_stats = self._udp_fanout( - probes=probes, - sockets=active_df_sockets, - payload_size=self.df_payload_size, - timeout_s=self.burst_timeout_s, - ) - else: - df_stats = [] except Exception as exc: logger.warning("UDP fanout probe cycle failed: %s", exc) return [{"success": False, "probe": probe, "error": str(exc)} for probe in probes] @@ -415,16 +390,18 @@ def udp_probe_cycle(self, probes: list[dict]) -> list[dict]: result = self._rtt_result_from_stats(rtt_stats[probe_index], len(active_rtt_sockets)) result["rtt_ports_active"] = len(active_rtt_sockets) result["rtt_ports_total"] = len(rtt_sockets) - if getattr(self, "enable_df_probe", True): - df_result = self._df_result_from_stats(df_stats[probe_index], len(active_df_sockets)) + result["port_batch_index"] = int(port_batch_index) + if enable_df: + expected_df_packets = 1 if active_rtt_sockets else 0 + df_result = self._df_result_from_stats(df_stats[probe_index], expected_df_packets) result["df_success"] = 1 if df_result.get("success") else 0 result["df_loss_pct"] = float(df_result.get("loss_pct", 100.0)) result["df_rtt_avg"] = float(df_result.get("rtt_avg", 0.0)) result["df_mtu_drops"] = int(df_result.get("mtu_drops", 0)) result["df_packets_sent"] = int(df_result.get("packets_sent", 0)) result["df_packets_lost"] = int(df_result.get("packets_lost", 0)) - result["df_ports_active"] = len(active_df_sockets) - result["df_ports_total"] = len(df_sockets) + result["df_ports_active"] = expected_df_packets + result["df_ports_total"] = len(rtt_sockets) else: result["df_success"] = 0 result["df_loss_pct"] = 0.0 @@ -436,49 +413,3 @@ def udp_probe_cycle(self, probes: list[dict]) -> list[dict]: result["df_ports_total"] = 0 cycle_results.append({"success": True, "probe": probe, "result": result}) return cycle_results - - def udp_rtt_burst(self, dst_ip: str, src_ip: str = None) -> dict: - """Run an RTT-oriented UDP burst: many small DF packets across src_ports.""" - try: - probe = {"src_ip": src_ip or "", "dst_ip": dst_ip} - rtt_sockets, _df_sockets = self._ensure_udp_probe_sockets( - src_ip=src_ip or "", - include_rtt=True, - include_df=False, - ) - stats = self._udp_fanout( - probes=[probe], - sockets=rtt_sockets, - payload_size=_DEFAULT_RTT_PAYLOAD_BYTES, - timeout_s=self.burst_timeout_s, - ) - except Exception as exc: - logger.warning("UDP RTT burst to %s failed: %s", dst_ip, exc) - return _empty_rtt_result(self.n_rtt_ports) - - return self._rtt_result_from_stats(stats[0], self.n_rtt_ports) - - def udp_df_burst(self, dst_ip: str, src_ip: str = None) -> dict: - """Run a DF-oriented UDP burst: a few large DF packets across src_ports. - - A failure rate >0 here indicates either path packet loss OR an - MTU mismatch on at least one ECMP path between src and dst. - """ - try: - probe = {"src_ip": src_ip or "", "dst_ip": dst_ip} - _rtt_sockets, df_sockets = self._ensure_udp_probe_sockets( - src_ip=src_ip or "", - include_rtt=False, - include_df=True, - ) - stats = self._udp_fanout( - probes=[probe], - sockets=df_sockets, - payload_size=self.df_payload_size, - timeout_s=self.burst_timeout_s, - ) - except Exception as exc: - logger.warning("UDP DF burst to %s failed: %s", dst_ip, exc) - return _empty_df_result(self.n_df_ports) - - return self._df_result_from_stats(stats[0], self.n_df_ports) diff --git a/netopsbench/platform/pingmesh/_agent_responder.py b/netopsbench/platform/pingmesh/_agent_responder.py index 3635939..dfc43d1 100644 --- a/netopsbench/platform/pingmesh/_agent_responder.py +++ b/netopsbench/platform/pingmesh/_agent_responder.py @@ -13,11 +13,9 @@ import socket import threading -try: - from ._agent_support import logger -except ImportError: # standalone in-container deployment - from _agent_support import logger # type: ignore[no-redef] +from netopsbench.logging_utils import get_logger +logger = get_logger(__name__) _RECV_BUF_SIZE = 65535 _SOCK_RCVBUF_BYTES = 4 * 1024 * 1024 # 4 MB; absorbs bursty traffic diff --git a/netopsbench/platform/pingmesh/_agent_runtime.py b/netopsbench/platform/pingmesh/_agent_runtime.py index 2baf557..b10dcf1 100644 --- a/netopsbench/platform/pingmesh/_agent_runtime.py +++ b/netopsbench/platform/pingmesh/_agent_runtime.py @@ -2,10 +2,17 @@ from __future__ import annotations -try: - from ._agent_support import logger, time -except ImportError: - from _agent_support import logger, time +import time + +from netopsbench.logging_utils import get_logger + +logger = get_logger(__name__) + + +def _next_cycle_deadline(previous_deadline: float, target_interval: float, now: float) -> float: + """Advance a fixed-rate cadence while avoiding unbounded catch-up bursts.""" + deadline = previous_deadline + target_interval + return now if deadline < now - target_interval else deadline class PingRuntimeMixin: @@ -25,12 +32,17 @@ def run(self): if startup_jitter > 0: logger.info(" Startup jitter sleep: %.3fs", startup_jitter) time.sleep(startup_jitter) + next_cycle_at = time.monotonic() while True: - start = time.time() + start = time.monotonic() completed = 0 failed = 0 try: - cycle_results = self.udp_probe_cycle(self.tasks) + tasks, cycle_metadata = self.next_probe_batch() + cycle_results = self.udp_probe_cycle(tasks, cycle_metadata["port_batch_index"]) + for item in cycle_results: + if item.get("success") and isinstance(item.get("result"), dict): + item["result"].update(cycle_metadata) except Exception as exc: logger.warning("Unexpected probe cycle error: %s", exc) cycle_results = [] @@ -57,9 +69,11 @@ def run(self): item.get("error", "unknown error"), ) failed += 1 - elapsed = time.time() - start + elapsed = time.monotonic() - start target_interval = min(max(self.min_interval, elapsed * 1.5), self.max_interval) - sleep_time = max(0, target_interval - elapsed) + now = time.monotonic() + next_cycle_at = _next_cycle_deadline(next_cycle_at, target_interval, now) + sleep_time = max(0, next_cycle_at - now) logger.info( "Cycle complete: %s succeeded, %s failed, %.1fs elapsed, sleeping %.1fs", completed, diff --git a/netopsbench/platform/pingmesh/_agent_support.py b/netopsbench/platform/pingmesh/_agent_support.py deleted file mode 100644 index 08f44c8..0000000 --- a/netopsbench/platform/pingmesh/_agent_support.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Shared bootstrap/runtime dependencies for Pingmesh agent.""" - -from __future__ import annotations - -import json -import os -import queue -import re -import subprocess -import threading -import time - -try: - import requests -except ImportError: - requests = None - -try: - from netopsbench.logging_utils import get_logger -except ModuleNotFoundError: - import logging - - def get_logger(name: str): - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - return logging.getLogger(name) - - -try: - from netopsbench.config import config -except ModuleNotFoundError: - - class _StandaloneConfig: - influxdb_url = os.environ.get("NETOPSBENCH_INFLUXDB_URL", "http://influxdb:8086") - influxdb_token = os.environ.get("NETOPSBENCH_INFLUXDB_TOKEN", "replace-me") - influxdb_org = os.environ.get("NETOPSBENCH_INFLUXDB_ORG", "netopsbench") - influxdb_bucket = os.environ.get("NETOPSBENCH_INFLUXDB_BUCKET", "netopsbench") - topology_id = os.environ.get("NETOPSBENCH_TOPOLOGY_ID", "") - topology_dir = os.environ.get("NETOPSBENCH_TOPOLOGY_DIR", "") - pingmesh_influxdb_url = os.environ.get("NETOPSBENCH_PINGMESH_INFLUXDB_URL", "http://influxdb:8086") - - config = _StandaloneConfig() - -logger = get_logger(__name__) - -__all__ = [ - "config", - "json", - "logger", - "os", - "queue", - "re", - "requests", - "subprocess", - "threading", - "time", -] diff --git a/netopsbench/platform/pingmesh/_detector_analysis.py b/netopsbench/platform/pingmesh/_detector_analysis.py index 6d6e9f8..f415fc2 100644 --- a/netopsbench/platform/pingmesh/_detector_analysis.py +++ b/netopsbench/platform/pingmesh/_detector_analysis.py @@ -1,127 +1,110 @@ -"""Analysis helpers for Pingmesh anomaly detector.""" +"""Pure snapshot analysis helpers for the Pingmesh anomaly detector.""" from __future__ import annotations import statistics +from dataclasses import dataclass from datetime import UTC, datetime +from math import ceil - -def _utcnow_iso() -> str: - """Return current UTC time in ISO-8601 format with Z suffix.""" - return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + "Z" - - -# Absolute minimum latency increase (ms) to avoid false positives on low-baseline paths. _MIN_LATENCY_ABS_THRESHOLD_MS = 2.0 - -# Samples at/above this loss percentage are treated as "unconverged / unreachable" -# and are excluded from the baseline mean. Without this filter, BGP startup samples -# (routing not yet established → 100% loss) pollute the baseline and push the -# anomaly threshold so high that later real faults (e.g. packet_corruption at -# 10-25% loss) become undetectable. 95 is a conservative cutoff: normal noise -# rarely exceeds ~70%, while un-routed paths are always 100%. +_HIGH_IMPACT_LATENCY_INCREASE_MS = 20.0 +_SUSTAINED_SAMPLE_FRACTION = 0.75 _BASELINE_UNREACHABLE_LOSS_PCT = 95.0 -def _filter_converged_loss(points: list[dict]) -> list[dict]: - """Drop baseline samples that reflect pre-BGP-convergence state. - - A sample with ``loss_pct >= 95`` almost certainly means the path was - unreachable (routing not yet installed), not that the link is bad — - so it should not influence the baseline mean used to compute the - packet-loss detection threshold. - """ - return [p for p in points if p.get("value", 0.0) < _BASELINE_UNREACHABLE_LOSS_PCT] - - -def _loss_samples( - points: list[dict], - *, - pct_field: str, - sent_field: str, - lost_field: str, -) -> list[dict]: - samples_by_time: dict[str, dict] = {} - for index, point in enumerate(points): - ts = str(point.get("_time") or point.get("time") or f"__row_{index}") - field = str(point.get("_field") or pct_field) - sample = samples_by_time.setdefault(ts, {"meta": point}) - value = point.get("value") - if field == pct_field: - sample["pct"] = value - elif field == sent_field: - sample["sent"] = value - elif field == lost_field: - sample["lost"] = value - - samples = [] - for sample in samples_by_time.values(): - sent = sample.get("sent") - lost = sample.get("lost") - pct = sample.get("pct") - try: - sent_value = float(sent) - lost_value = float(lost) - except (TypeError, ValueError): - sent_value = 0.0 - lost_value = 0.0 - if sent_value > 0: - sample["value"] = (lost_value / sent_value) * 100.0 - sample["sent"] = sent_value - sample["lost"] = lost_value - sample["has_counts"] = True - samples.append(sample) - continue - try: - sample["value"] = float(pct) - except (TypeError, ValueError): - continue - sample["has_counts"] = False - samples.append(sample) - return samples - - -def _aggregate_loss_pct( - points: list[dict], - *, - pct_field: str, - sent_field: str, - lost_field: str, - drop_unreachable: bool = False, -) -> float: - samples = _loss_samples(points, pct_field=pct_field, sent_field=sent_field, lost_field=lost_field) - if drop_unreachable: - samples = [sample for sample in samples if sample.get("value", 0.0) < _BASELINE_UNREACHABLE_LOSS_PCT] - if not samples: - return 0.0 - counted = [sample for sample in samples if sample.get("has_counts")] - sent = sum(float(sample.get("sent", 0.0) or 0.0) for sample in counted) - if sent > 0: - lost = sum(float(sample.get("lost", 0.0) or 0.0) for sample in counted) - return (lost / sent) * 100.0 - return statistics.mean([sample["value"] for sample in samples]) +def _utcnow_iso() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + "Z" -def _filter_converged_rtt(points: list[dict]) -> list[dict]: - """Drop baseline samples where ``rtt_avg == 0`` (UDP burst returned no - successful probes — a pre-convergence sentinel, not a legitimate RTT).""" - return [p for p in points if p.get("value", 0.0) > 0.0] +def _as_float(value, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _group_by_path(rows: list[dict]) -> dict[tuple[str, str], list[dict]]: + paths: dict[tuple[str, str], list[dict]] = {} + for row in rows: + key = (str(row.get("src_ip", "")), str(row.get("dst_ip", ""))) + paths.setdefault(key, []).append(row) + for points in paths.values(): + points.sort(key=lambda point: str(point.get("_time") or point.get("time") or "")) + return paths + + +def _loss_stats(rows: list[dict], *, prefix: str = "", drop_unreachable: bool = False) -> dict | None: + sent_field = f"{prefix}packets_sent" + lost_field = f"{prefix}packets_lost" + pct_field = "packet_loss" if not prefix else f"{prefix}loss_pct" + counted: list[tuple[float, float]] = [] + percentages: list[float] = [] + sample_count = 0 + mtu_drops = 0.0 + + for row in rows: + sent = _as_float(row.get(sent_field)) + lost = _as_float(row.get(lost_field)) + if sent > 0: + loss_pct = (lost / sent) * 100.0 + if drop_unreachable and loss_pct >= _BASELINE_UNREACHABLE_LOSS_PCT: + continue + counted.append((sent, lost)) + sample_count += 1 + elif row.get(pct_field) is not None: + loss_pct = _as_float(row.get(pct_field)) + if drop_unreachable and loss_pct >= _BASELINE_UNREACHABLE_LOSS_PCT: + continue + percentages.append(loss_pct) + sample_count += 1 + if prefix: + mtu_drops += _as_float(row.get("df_mtu_drops")) + + if not counted and not percentages: + return None + total_sent = sum(sent for sent, _lost in counted) + total_lost = sum(lost for _sent, lost in counted) + loss_pct = (total_lost / total_sent) * 100.0 if total_sent > 0 else statistics.mean(percentages) + return { + "loss_pct": loss_pct, + "sent": total_sent, + "lost": total_lost, + "samples": sample_count, + "has_counts": bool(counted), + "mtu_drops": mtu_drops, + } + + +@dataclass +class SnapshotAnalysis: + anomalies: list + quality: dict[str, int] class DetectorAnalysisMixin: - @staticmethod - def _safe_flux_string(value: str) -> str: - """Escape a string for safe interpolation into a Flux query literal.""" - return str(value).replace("\\", "\\\\").replace('"', '\\"') - - def _group_by_path(self, data: list[dict]) -> dict[tuple, list[dict]]: - paths = {} - for row in data: - key = (row.get("src_ip", ""), row.get("dst_ip", "")) - paths.setdefault(key, []).append(row) - return paths + def _resolve_leaf(self, leaf_tag: str, client_name: str) -> str: + if isinstance(leaf_tag, str) and leaf_tag: + return leaf_tag + if isinstance(client_name, str) and client_name in self.client_to_leaf: + return self.client_to_leaf[client_name] + return "unknown" - def _calculate_severity(self, value: float, threshold: float) -> str: + @staticmethod + def _signal_severity(anomaly_type: str, value: float, baseline: float, threshold: float) -> str: + increase = max(0.0, value - baseline) + if anomaly_type == "latency_spike": + if increase >= 50.0: + return "high" + if increase >= 10.0: + return "medium" + return "low" + if anomaly_type == "jitter_spike": + if increase >= 20.0: + return "high" + if increase >= 5.0: + return "medium" + return "low" if threshold == 0: return "high" ratio = value / threshold @@ -131,303 +114,222 @@ def _calculate_severity(self, value: float, threshold: float) -> str: return "medium" return "low" - def _resolve_leaf(self, leaf_tag: str, client_name: str) -> str: - if isinstance(leaf_tag, str) and leaf_tag: - return leaf_tag - if isinstance(client_name, str) and client_name in self.client_to_leaf: - return self.client_to_leaf[client_name] - return "unknown" + def _new_anomaly( + self, + anomaly_type: str, + point: dict, + *, + value: float, + baseline: float, + threshold: float, + severity: str | None = None, + samples_sent: int = 0, + samples_lost: int = 0, + sample_count: int = 0, + ): + return self._anomaly_type( + type=anomaly_type, + src_ip=str(point.get("src_ip", "")), + src_name=str(point.get("src_name", "")), + dst_ip=str(point.get("dst_ip", "")), + dst_name=str(point.get("dst_name", "")), + src_leaf=str(point.get("src_leaf", "")), + dst_leaf=str(point.get("dst_leaf", "")), + value=value, + baseline=baseline, + threshold=threshold, + severity=severity or self._signal_severity(anomaly_type, value, baseline, threshold), + timestamp=_utcnow_iso(), + samples_sent=samples_sent, + samples_lost=samples_lost, + sample_count=sample_count, + ) - def detect_latency_anomalies(self, baseline_start: str, baseline_end: str, current_start: str, current_end: str): - topology_filter = self._topology_filter() - _b = self._safe_flux_string(self.bucket) - _bs = self._safe_flux_string(baseline_start) - _be = self._safe_flux_string(baseline_end) - _cs = self._safe_flux_string(current_start) - _ce = self._safe_flux_string(current_end) - baseline_query = f'\nfrom(bucket: "{_b}")\n |> range(start: time(v: "{_bs}"), stop: time(v: "{_be}"))\n |> filter(fn: (r) => r._measurement == "pingmesh")\n{topology_filter} |> filter(fn: (r) => r._field == "rtt_avg")\n |> group(columns: ["src_ip", "dst_ip", "src_name", "dst_name", "src_leaf", "dst_leaf"])\n' - current_query = f'\nfrom(bucket: "{_b}")\n |> range(start: time(v: "{_cs}"), stop: time(v: "{_ce}"))\n |> filter(fn: (r) => r._measurement == "pingmesh")\n{topology_filter} |> filter(fn: (r) => r._field == "rtt_avg")\n |> group(columns: ["src_ip", "dst_ip", "src_name", "dst_name", "src_leaf", "dst_leaf"])\n' - baseline_data = self._query_influxdb(baseline_query) - current_data = self._query_influxdb(current_query) - if not baseline_data or not current_data: - return [] - baseline_paths = self._group_by_path(baseline_data) - current_paths = self._group_by_path(current_data) + def _detect_latency_from_rows(self, baseline_rows: list[dict], current_rows: list[dict]) -> list: + baseline_paths = _group_by_path(baseline_rows) + current_paths = _group_by_path(current_rows) anomalies = [] for path_key, current_points in current_paths.items(): baseline_points = baseline_paths.get(path_key, []) - if len(current_points) < 1 or len(baseline_points) < 1: - continue - # Drop rtt_avg == 0 baseline samples: these are the sentinel written - # when a UDP burst had zero successful probes (pre-BGP-convergence - # or path briefly unreachable), not a real 0 ms RTT. Mixing them - # with healthy RTTs understates the baseline mean and inflates - # the stddev → threshold, masking real latency spikes. - clean_baseline = _filter_converged_rtt(baseline_points) - if not clean_baseline: + baseline_values = [_as_float(point.get("rtt_avg")) for point in baseline_points] + current_values = [_as_float(point.get("rtt_avg")) for point in current_points] + baseline_values = [value for value in baseline_values if value > 0] + current_values = [value for value in current_values if value > 0] + if not current_values or not baseline_values: continue - baseline_values = [point["value"] for point in clean_baseline] - current_values = [point["value"] for point in current_points] baseline = statistics.mean(baseline_values) - stddev = statistics.stdev(baseline_values) if len(baseline_values) > 1 else 0 + baseline_stddev = statistics.stdev(baseline_values) if len(baseline_values) > 1 else 0.0 current = statistics.mean(current_values) - # Keep a minimum absolute threshold to avoid false positives on low-baseline paths. - threshold = max(baseline + 3 * stddev, baseline + _MIN_LATENCY_ABS_THRESHOLD_MS) + threshold = max(baseline + 3 * baseline_stddev, baseline + _MIN_LATENCY_ABS_THRESHOLD_MS) min_multiplier = 1.3 min_abs_increase = 0.0 if len(baseline_values) < 3 or len(current_values) < 2: - # With 1 s probe cycles short-window cases are rare; keep a - # modest guard so single noisy samples cannot trigger. min_multiplier = 1.5 min_abs_increase = 2.0 - if current > threshold and current > baseline * min_multiplier and (current - baseline) >= min_abs_increase: - first_point = current_points[0] + elevated = [ + value + for value in current_values + if value > threshold and value > baseline * min_multiplier and value - baseline >= min_abs_increase + ] + required = ceil(len(current_values) * _SUSTAINED_SAMPLE_FRACTION) + sustained = len(elevated) >= required + peak = max(current_values) + high_impact = peak > threshold and peak - baseline >= _HIGH_IMPACT_LATENCY_INCREASE_MS + if (current > threshold and sustained) or high_impact: + value = peak if high_impact and not sustained else current anomalies.append( - self._anomaly_type( - type="latency_spike", - src_ip=first_point.get("src_ip", ""), - src_name=first_point.get("src_name", ""), - dst_ip=first_point.get("dst_ip", ""), - dst_name=first_point.get("dst_name", ""), - src_leaf=first_point.get("src_leaf", ""), - dst_leaf=first_point.get("dst_leaf", ""), - value=current, + self._new_anomaly( + "latency_spike", + current_points[0], + value=value, baseline=baseline, threshold=threshold, - severity=self._calculate_severity(current, threshold), - timestamp=_utcnow_iso(), + severity=( + self._signal_severity("latency_spike", value, baseline, threshold) + if len(elevated) == len(current_values) or high_impact + else "low" + ), + sample_count=len(current_values), ) ) return anomalies - def detect_jitter_anomalies(self, baseline_start: str, baseline_end: str, current_start: str, current_end: str): - """Detect paths where RTT variance increased significantly (e.g. packet corruption).""" - topology_filter = self._topology_filter() - # Query both rtt_min and rtt_max to compute per-probe jitter - fields_filter = '|> filter(fn: (r) => r._field == "rtt_avg" or r._field == "rtt_max" or r._field == "rtt_min")' - _b = self._safe_flux_string(self.bucket) - _bs = self._safe_flux_string(baseline_start) - _be = self._safe_flux_string(baseline_end) - _cs = self._safe_flux_string(current_start) - _ce = self._safe_flux_string(current_end) - baseline_query = f'\nfrom(bucket: "{_b}")\n |> range(start: time(v: "{_bs}"), stop: time(v: "{_be}"))\n |> filter(fn: (r) => r._measurement == "pingmesh")\n{topology_filter} {fields_filter}\n |> group(columns: ["src_ip", "dst_ip", "src_name", "dst_name", "src_leaf", "dst_leaf", "_field"])\n' - current_query = f'\nfrom(bucket: "{_b}")\n |> range(start: time(v: "{_cs}"), stop: time(v: "{_ce}"))\n |> filter(fn: (r) => r._measurement == "pingmesh")\n{topology_filter} {fields_filter}\n |> group(columns: ["src_ip", "dst_ip", "src_name", "dst_name", "src_leaf", "dst_leaf", "_field"])\n' - baseline_data = self._query_influxdb(baseline_query) - current_data = self._query_influxdb(current_query) - if not baseline_data or not current_data: - return [] - - def _build_jitter_map(data: list[dict]) -> dict[tuple, list[float]]: - """Group by path and compute per-sample jitter (rtt_max - rtt_min).""" - fields_by_path_time: dict[tuple, dict[str, dict[str, float]]] = {} - for row in data: - path_key = (row.get("src_ip", ""), row.get("dst_ip", "")) - ts = row.get("_time", row.get("time", "")) - field = row.get("_field", "") - fields_by_path_time.setdefault(path_key, {}).setdefault(ts, {})[field] = row.get("value", 0.0) - jitter_map: dict[tuple, list[float]] = {} - for path_key, timestamps in fields_by_path_time.items(): - jitters = [] - for _ts, fields in timestamps.items(): - rtt_max = fields.get("rtt_max", 0.0) - rtt_min = fields.get("rtt_min", 0.0) - if rtt_max > 0 and rtt_min > 0: - jitters.append(rtt_max - rtt_min) - if jitters: - jitter_map[path_key] = jitters - return jitter_map - - # Also collect metadata (src_name, dst_name, etc.) from the first point of each path - path_metadata: dict[tuple, dict] = {} - for row in current_data: - key = (row.get("src_ip", ""), row.get("dst_ip", "")) - if key not in path_metadata: - path_metadata[key] = row - - baseline_jitter = _build_jitter_map(baseline_data) - current_jitter = _build_jitter_map(current_data) - + def _detect_jitter_from_rows(self, baseline_rows: list[dict], current_rows: list[dict]) -> list: + baseline_paths = _group_by_path(baseline_rows) + current_paths = _group_by_path(current_rows) anomalies = [] - for path_key, current_jitters in current_jitter.items(): - baseline_jitters = baseline_jitter.get(path_key, []) - if not baseline_jitters or not current_jitters: + for path_key, current_points in current_paths.items(): + baseline_points = baseline_paths.get(path_key, []) + baseline_values = [ + _as_float(point.get("rtt_max")) - _as_float(point.get("rtt_min")) + for point in baseline_points + if _as_float(point.get("rtt_max")) > 0 and _as_float(point.get("rtt_min")) > 0 + ] + current_values = [ + _as_float(point.get("rtt_max")) - _as_float(point.get("rtt_min")) + for point in current_points + if _as_float(point.get("rtt_max")) > 0 and _as_float(point.get("rtt_min")) > 0 + ] + if not baseline_values or not current_values: continue - baseline_mean = statistics.mean(baseline_jitters) - current_mean = statistics.mean(current_jitters) - baseline_std = statistics.stdev(baseline_jitters) if len(baseline_jitters) > 1 else 0 - # Require: current jitter > baseline + 3*stddev, at least 2ms absolute increase, and 2x multiplier - threshold = max(baseline_mean + 3 * baseline_std, baseline_mean + 2.0) - if current_mean > threshold and current_mean > baseline_mean * 2.0: - meta = path_metadata.get(path_key, {}) + baseline = statistics.mean(baseline_values) + current = statistics.mean(current_values) + baseline_std = statistics.stdev(baseline_values) if len(baseline_values) > 1 else 0.0 + threshold = max(baseline + 3 * baseline_std, baseline + 2.0) + elevated = [value for value in current_values if value > threshold and value > baseline * 2.0] + required = ceil(len(current_values) * _SUSTAINED_SAMPLE_FRACTION) + if current > threshold and current > baseline * 2.0 and len(elevated) >= required: anomalies.append( - self._anomaly_type( - type="jitter_spike", - src_ip=meta.get("src_ip", ""), - src_name=meta.get("src_name", ""), - dst_ip=meta.get("dst_ip", ""), - dst_name=meta.get("dst_name", ""), - src_leaf=meta.get("src_leaf", ""), - dst_leaf=meta.get("dst_leaf", ""), - value=current_mean, - baseline=baseline_mean, + self._new_anomaly( + "jitter_spike", + current_points[0], + value=current, + baseline=baseline, threshold=threshold, - severity=self._calculate_severity(current_mean, threshold), - timestamp=_utcnow_iso(), + severity=( + self._signal_severity("jitter_spike", current, baseline, threshold) + if len(current_values) >= 4 and len(elevated) == len(current_values) + else "low" + ), + sample_count=len(current_values), ) ) return anomalies - def detect_packet_loss(self, baseline_start: str, baseline_end: str, current_start: str, current_end: str): - topology_filter = self._topology_filter() - _b = self._safe_flux_string(self.bucket) - _bs = self._safe_flux_string(baseline_start) - _be = self._safe_flux_string(baseline_end) - _cs = self._safe_flux_string(current_start) - _ce = self._safe_flux_string(current_end) - fields_filter = ( - '|> filter(fn: (r) => r._field == "packet_loss" or r._field == "packets_sent" ' - 'or r._field == "packets_lost")' - ) - baseline_query = f'\nfrom(bucket: "{_b}")\n |> range(start: time(v: "{_bs}"), stop: time(v: "{_be}"))\n |> filter(fn: (r) => r._measurement == "pingmesh")\n{topology_filter} {fields_filter}\n |> group(columns: ["src_ip", "dst_ip", "src_name", "dst_name", "src_leaf", "dst_leaf", "_field"])\n' - current_query = f'\nfrom(bucket: "{_b}")\n |> range(start: time(v: "{_cs}"), stop: time(v: "{_ce}"))\n |> filter(fn: (r) => r._measurement == "pingmesh")\n{topology_filter} {fields_filter}\n |> group(columns: ["src_ip", "dst_ip", "src_name", "dst_name", "src_leaf", "dst_leaf", "_field"])\n' - baseline_data = self._query_influxdb(baseline_query) - current_data = self._query_influxdb(current_query) - - baseline_paths = self._group_by_path(baseline_data) if baseline_data else {} - current_paths = self._group_by_path(current_data) if current_data else {} - + def _detect_loss_from_rows(self, baseline_rows: list[dict], current_rows: list[dict]) -> tuple[list, int]: + baseline_paths = _group_by_path(baseline_rows) + current_paths = _group_by_path(current_rows) anomalies = [] - - # Detect paths that had baseline data but disappeared in the current - # window, which indicates complete connectivity loss. - if baseline_paths: - for path_key, baseline_points in baseline_paths.items(): - if path_key in current_paths: - continue # path still exists, will be checked below - if len(baseline_points) < 1: - continue - first_point = baseline_points[0] + insufficient_baseline = 0 + for path_key, current_points in current_paths.items(): + current = _loss_stats(current_points) + if current is None or current["sent"] <= 0: + continue + baseline = _loss_stats(baseline_paths.get(path_key, []), drop_unreachable=True) + if baseline is None: + insufficient_baseline += 1 + continue + threshold = max(self.loss_pct_threshold, baseline["loss_pct"] + self.loss_pct_delta) + if current["has_counts"] and current["sent"] > 0 and current["lost"] >= current["sent"]: anomalies.append( - self._anomaly_type( - type="path_unreachable", - src_ip=first_point.get("src_ip", ""), - src_name=first_point.get("src_name", ""), - dst_ip=first_point.get("dst_ip", ""), - dst_name=first_point.get("dst_name", ""), - src_leaf=first_point.get("src_leaf", ""), - dst_leaf=first_point.get("dst_leaf", ""), + self._new_anomaly( + "path_unreachable", + current_points[0], value=100.0, - baseline=_aggregate_loss_pct( - baseline_points, - pct_field="packet_loss", - sent_field="packets_sent", - lost_field="packets_lost", - ), + baseline=baseline["loss_pct"], threshold=100.0, severity="high", - timestamp=_utcnow_iso(), + samples_sent=int(current["sent"]), + samples_lost=int(current["lost"]), + sample_count=int(current["samples"]), ) ) - - for path_key, current_points in current_paths.items(): - baseline_points = baseline_paths.get(path_key, []) - if len(current_points) < 1: - continue - # Drop unreachable (>=95%) samples from baseline: they represent - # pre-BGP-convergence state, not real path quality. Without this - # filter, 100% loss samples from startup inflate baseline_loss - # and raise `threshold` above the fault's real loss signal. - baseline_loss = _aggregate_loss_pct( - baseline_points, - pct_field="packet_loss", - sent_field="packets_sent", - lost_field="packets_lost", - drop_unreachable=True, - ) - current_loss = _aggregate_loss_pct( - current_points, - pct_field="packet_loss", - sent_field="packets_sent", - lost_field="packets_lost", - ) - threshold = max(self.loss_pct_threshold, baseline_loss + self.loss_pct_delta) - if current_loss >= threshold: - first_point = current_points[0] + elif current["loss_pct"] >= threshold: anomalies.append( - self._anomaly_type( - type="packet_loss", - src_ip=first_point.get("src_ip", ""), - src_name=first_point.get("src_name", ""), - dst_ip=first_point.get("dst_ip", ""), - dst_name=first_point.get("dst_name", ""), - src_leaf=first_point.get("src_leaf", ""), - dst_leaf=first_point.get("dst_leaf", ""), - value=current_loss, - baseline=baseline_loss, + self._new_anomaly( + "packet_loss", + current_points[0], + value=current["loss_pct"], + baseline=baseline["loss_pct"], threshold=threshold, - severity=self._calculate_severity(current_loss, threshold), - timestamp=_utcnow_iso(), + samples_sent=int(current["sent"]), + samples_lost=int(current["lost"]), + sample_count=int(current["samples"]), ) ) - return anomalies + return anomalies, insufficient_baseline - def detect_df_fragmentation_anomalies( - self, baseline_start: str, baseline_end: str, current_start: str, current_end: str - ): - topology_filter = self._topology_filter() - _b = self._safe_flux_string(self.bucket) - _bs = self._safe_flux_string(baseline_start) - _be = self._safe_flux_string(baseline_end) - _cs = self._safe_flux_string(current_start) - _ce = self._safe_flux_string(current_end) - fields_filter = ( - '|> filter(fn: (r) => r._field == "df_loss_pct" or r._field == "df_packets_sent" ' - 'or r._field == "df_packets_lost")' - ) - baseline_query = f'\nfrom(bucket: "{_b}")\n |> range(start: time(v: "{_bs}"), stop: time(v: "{_be}"))\n |> filter(fn: (r) => r._measurement == "pingmesh")\n{topology_filter} {fields_filter}\n |> group(columns: ["src_ip", "dst_ip", "src_name", "dst_name", "src_leaf", "dst_leaf", "_field"])\n' - current_query = f'\nfrom(bucket: "{_b}")\n |> range(start: time(v: "{_cs}"), stop: time(v: "{_ce}"))\n |> filter(fn: (r) => r._measurement == "pingmesh")\n{topology_filter} {fields_filter}\n |> group(columns: ["src_ip", "dst_ip", "src_name", "dst_name", "src_leaf", "dst_leaf", "_field"])\n' - baseline_data = self._query_influxdb(baseline_query) - current_data = self._query_influxdb(current_query) - if not baseline_data or not current_data: - return [] - baseline_paths = self._group_by_path(baseline_data) - current_paths = self._group_by_path(current_data) + def _detect_df_from_rows(self, baseline_rows: list[dict], current_rows: list[dict]) -> list: + baseline_paths = _group_by_path(baseline_rows) + current_paths = _group_by_path(current_rows) anomalies = [] for path_key, current_points in current_paths.items(): baseline_points = baseline_paths.get(path_key, []) - if len(current_points) < 1 or len(baseline_points) < 1: + baseline_df = _loss_stats(baseline_points, prefix="df_", drop_unreachable=True) + current_df = _loss_stats(current_points, prefix="df_") + baseline_rtt = _loss_stats(baseline_points, drop_unreachable=True) + current_rtt = _loss_stats(current_points) + if baseline_df is None or current_df is None or baseline_rtt is None or current_rtt is None: continue - # Same pre-convergence guard as packet-loss: drop 100% df_loss_pct - # startup samples so they don't inflate the baseline. - baseline_loss = _aggregate_loss_pct( - baseline_points, - pct_field="df_loss_pct", - sent_field="df_packets_sent", - lost_field="df_packets_lost", - drop_unreachable=True, - ) - current_loss = _aggregate_loss_pct( - current_points, - pct_field="df_loss_pct", - sent_field="df_packets_sent", - lost_field="df_packets_lost", + rtt_threshold = max(self.loss_pct_threshold, baseline_rtt["loss_pct"] + self.loss_pct_delta) + rtt_healthy = current_rtt["sent"] > 0 and current_rtt["loss_pct"] < rtt_threshold + df_loss_signal = ( + current_df["loss_pct"] >= 20.0 and (current_df["loss_pct"] - baseline_df["loss_pct"]) >= 15.0 ) - if current_loss >= 20.0 and (current_loss - baseline_loss) >= 15.0: - first_point = current_points[0] - anomalies.append( - self._anomaly_type( - type="mtu_or_fragmentation_suspect", - src_ip=first_point.get("src_ip", ""), - src_name=first_point.get("src_name", ""), - dst_ip=first_point.get("dst_ip", ""), - dst_name=first_point.get("dst_name", ""), - src_leaf=first_point.get("src_leaf", ""), - dst_leaf=first_point.get("dst_leaf", ""), - value=current_loss, - baseline=baseline_loss, - threshold=max(20.0, baseline_loss + 15.0), - severity="high" if current_loss >= 50.0 else "medium", - timestamp=_utcnow_iso(), - ) + if not rtt_healthy or not (df_loss_signal or current_df["mtu_drops"] > 0): + continue + anomalies.append( + self._new_anomaly( + "mtu_or_fragmentation_suspect", + current_points[0], + value=current_df["loss_pct"], + baseline=baseline_df["loss_pct"], + threshold=max(20.0, baseline_df["loss_pct"] + 15.0), + severity="high" if current_df["loss_pct"] >= 50.0 else "medium", + samples_sent=int(current_df["sent"]), + samples_lost=int(current_df["lost"]), + sample_count=int(current_df["samples"]), ) + ) return anomalies + + def analyze_snapshot_rows(self, baseline_rows: list[dict], current_rows: list[dict]) -> SnapshotAnalysis: + loss_anomalies, insufficient_baseline = self._detect_loss_from_rows(baseline_rows, current_rows) + anomalies = [ + *self._detect_latency_from_rows(baseline_rows, current_rows), + *loss_anomalies, + *self._detect_df_from_rows(baseline_rows, current_rows), + *self._detect_jitter_from_rows(baseline_rows, current_rows), + ] + baseline_paths = _group_by_path(baseline_rows) + current_paths = _group_by_path(current_rows) + anomalies.sort(key=lambda item: (item.type, item.src_ip, item.dst_ip)) + return SnapshotAnalysis( + anomalies=anomalies, + quality={ + "baseline_paths_observed": len(baseline_paths), + "current_paths_observed": len(current_paths), + "not_observed_paths": len(set(baseline_paths) - set(current_paths)), + "insufficient_baseline_paths": insufficient_baseline, + }, + ) diff --git a/netopsbench/platform/pingmesh/_detector_coverage.py b/netopsbench/platform/pingmesh/_detector_coverage.py new file mode 100644 index 0000000..5befb9a --- /dev/null +++ b/netopsbench/platform/pingmesh/_detector_coverage.py @@ -0,0 +1,119 @@ +"""Coverage quality audit for bounded Pingmesh destination rotation.""" + +from __future__ import annotations + +from collections import Counter +from math import ceil + + +class DetectorCoverageMixin: + """Audit schedule coverage without consulting fault targets.""" + + def summarize_coverage(self, rows: list[dict]) -> dict: + client_count = len(self._pingmesh_clients) + policy = self._pingmesh_policy + required_policy_fields = { + "destination_batch_size", + "rtt_port_pool_size", + "rtt_ports_per_cycle", + "coverage_epoch_cycles", + } + missing_policy_fields = sorted(required_policy_fields - set(policy)) + if missing_policy_fields: + return { + "status": "error", + "coverage_status": "error", + "error": "Pingmesh coverage policy is missing fields: " + ", ".join(missing_policy_fields), + } + destination_count = max(0, client_count - 1) + destination_batch_size = policy["destination_batch_size"] or max(1, destination_count) + port_pool_size = int(policy["rtt_port_pool_size"]) + ports_per_cycle = int(policy["rtt_ports_per_cycle"]) + expected_destination_batches = max(1, ceil(destination_count / destination_batch_size)) + expected_port_batches = max(1, ceil(port_pool_size / ports_per_cycle)) + expected_cycles = int(policy["coverage_epoch_cycles"]) + if expected_cycles != expected_destination_batches * expected_port_batches: + return { + "status": "error", + "coverage_status": "error", + "error": "Pingmesh coverage policy is missing or stale; regenerate the topology", + } + + cycle_rows = [row for row in rows if row.get("probe_cycle") is not None] + cycles = {int(row["probe_cycle"]) for row in cycle_rows} + destination_batches = { + int(row["destination_batch_index"]) for row in rows if row.get("destination_batch_index") is not None + } + port_batches = {int(row["port_batch_index"]) for row in rows if row.get("port_batch_index") is not None} + sources = {row.get("src_name") for row in cycle_rows if row.get("src_name")} + cycles_by_source: dict[str, set[int]] = {} + for row in cycle_rows: + source = row.get("src_name") + if source: + cycles_by_source.setdefault(source, set()).add(int(row["probe_cycle"])) + pair_counts = Counter( + (row.get("src_name"), row.get("dst_name")) + for row in cycle_rows + if row.get("src_name") and row.get("dst_name") + ) + pair_port_counts = Counter( + (row.get("src_name"), row.get("dst_name"), int(row["port_batch_index"])) + for row in cycle_rows + if row.get("src_name") and row.get("dst_name") and row.get("port_batch_index") is not None + ) + observed_cycle_span = max(cycles) - min(cycles) + 1 if cycles else 0 + source_cycle_spans = [ + max(source_cycles) - min(source_cycles) + 1 for source_cycles in cycles_by_source.values() if source_cycles + ] + min_source_cycle_span = min(source_cycle_spans, default=0) + missing_destination_batches = sorted(set(range(expected_destination_batches)) - destination_batches) + missing_port_batches = sorted(set(range(expected_port_batches)) - port_batches) + missing_sources = max(0, client_count - len(sources)) + invalid_socket_rows = sum( + 1 + for row in cycle_rows + if _field_int(row, "rtt_ports_total") != port_pool_size + or _field_int(row, "rtt_ports_active") != ports_per_cycle + ) + expected_pairs = client_count * destination_count + expected_pair_port_combinations = expected_pairs * expected_port_batches + complete = bool( + not missing_destination_batches + and not missing_port_batches + and missing_sources == 0 + and len(pair_counts) == expected_pairs + and len(pair_port_counts) == expected_pair_port_combinations + and invalid_socket_rows == 0 + ) + + return { + "status": "ok", + "coverage_status": "complete" if complete else "incomplete", + "expected_epoch_cycles": expected_cycles, + "observed_cycle_span": observed_cycle_span, + "min_source_cycle_span": min_source_cycle_span, + "source_clients_observed": len(sources), + "expected_source_clients": client_count, + "destination_pairs_observed": len(pair_counts), + "expected_destination_pairs": expected_pairs, + "pair_port_combinations_observed": len(pair_port_counts), + "expected_pair_port_combinations": expected_pair_port_combinations, + "missing_pair_port_combinations": max( + 0, + expected_pair_port_combinations - len(pair_port_counts), + ), + "min_samples_per_pair": min(pair_counts.values(), default=0), + "port_batches_observed": sorted(port_batches), + "destination_batches_observed": sorted(destination_batches), + "missing_port_batches": missing_port_batches, + "missing_destination_batches": missing_destination_batches, + "missing_source_clients": missing_sources, + "invalid_socket_rows": invalid_socket_rows, + } + + +def _field_int(row: dict, field: str) -> int | None: + try: + return int(row[field]) + except (KeyError, TypeError, ValueError): + return None diff --git a/netopsbench/platform/pingmesh/_detector_query.py b/netopsbench/platform/pingmesh/_detector_query.py index 0406a8b..a2cce8f 100644 --- a/netopsbench/platform/pingmesh/_detector_query.py +++ b/netopsbench/platform/pingmesh/_detector_query.py @@ -3,66 +3,97 @@ from __future__ import annotations import csv -import glob -import json -import os +from dataclasses import dataclass from io import StringIO -import requests +from netopsbench.platform.observability.influxdb import query_flux -from netopsbench.config import config +_SNAPSHOT_FIELDS = ( + "rtt_min", + "rtt_avg", + "rtt_max", + "packets_sent", + "packets_lost", + "df_packets_sent", + "df_packets_lost", + "df_mtu_drops", + "probe_cycle", + "destination_batch_index", + "port_batch_index", + "rtt_ports_active", + "rtt_ports_total", +) +_SNAPSHOT_QUERY_TIMEOUT_SECONDS = 60 -try: - from netopsbench.logging_utils import get_logger -except ModuleNotFoundError: - import logging - def get_logger(name: str): - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - return logging.getLogger(name) +@dataclass(frozen=True) +class SnapshotQueryResult: + status: str + rows: list[dict] + error: str | None = None -logger = get_logger(__name__) +def _endpoint_name(endpoint) -> str | None: + if isinstance(endpoint, dict): + raw = endpoint.get("device") or endpoint.get("name") + else: + raw = endpoint + if not isinstance(raw, str): + return None + return raw.split(":", 1)[0].strip() or None class DetectorQueryMixin: - def _query_influxdb(self, query: str) -> list[dict]: - headers = { - "Authorization": f"Token {self.token}", - "Content-Type": "application/vnd.flux", - "Accept": "application/csv", - } - try: - self.last_query_error = None - response = requests.post( - f"{self.influxdb_url}/api/v2/query?org={self.org}", - headers=headers, - data=query, - timeout=30, - proxies={"http": "", "https": ""}, - ) - response.raise_for_status() - results = [] - csv_data = StringIO(response.text) - reader = csv.DictReader(csv_data) - for row in reader: - raw_value = row.get("_value") - if row.get("result", "") == "result" or raw_value in (None, ""): + @staticmethod + def _safe_flux_string(value: str) -> str: + return str(value).replace("\\", "\\\\").replace('"', '\\"') + + @staticmethod + def _parse_snapshot_csv(text: str) -> list[dict]: + data_lines = [line for line in text.splitlines() if line and not line.startswith("#")] + if not data_lines: + return [] + rows = [] + for row in csv.DictReader(StringIO("\n".join(data_lines))): + if row.get("_time") in (None, "", "_time"): + continue + parsed = {key: value for key, value in row.items() if key and value not in (None, "")} + for field in _SNAPSHOT_FIELDS: + if field not in parsed: continue try: - row["value"] = float(raw_value) - except (ValueError, KeyError): - continue - results.append(row) - return results - except Exception as e: - self.last_query_error = str(e) - logger.error("InfluxDB query error: %s", e) - return [] + parsed[field] = float(parsed[field]) + except (TypeError, ValueError): + parsed.pop(field, None) + rows.append(parsed) + return rows + + def _query_snapshot(self, start_time: str, end_time: str) -> SnapshotQueryResult: + bucket = self._safe_flux_string(self.bucket) + start = self._safe_flux_string(start_time) + end = self._safe_flux_string(end_time) + fields = " or ".join(f'r._field == "{field}"' for field in _SNAPSHOT_FIELDS) + query = ( + f'from(bucket: "{bucket}")\n' + f' |> range(start: time(v: "{start}"), stop: time(v: "{end}"))\n' + ' |> filter(fn: (r) => r._measurement == "pingmesh")\n' + + self._topology_filter() + + f" |> filter(fn: (r) => {fields})\n" + ' |> keep(columns: ["_time", "_field", "_value", "src_ip", "dst_ip", ' + '"src_name", "dst_name", "src_leaf", "dst_leaf", "path_type"])\n' + ' |> pivot(rowKey: ["_time", "src_ip", "dst_ip", "src_name", "dst_name", ' + '"src_leaf", "dst_leaf", "path_type"], columnKey: ["_field"], valueColumn: "_value")\n' + ) + result = query_flux( + self.influxdb_url, + self.token, + self.org, + query, + timeout=_SNAPSHOT_QUERY_TIMEOUT_SECONDS, + ) + if result.status != "ok": + return SnapshotQueryResult(status="error", rows=[], error=result.error) + return SnapshotQueryResult(status="ok", rows=self._parse_snapshot_csv(result.text)) def _topology_filter(self) -> str: if not self.topology_id: @@ -79,60 +110,44 @@ def _load_topology_metadata(self, metadata: dict) -> None: if isinstance(name, str) and isinstance(leaf, str) and name and leaf: self.client_to_leaf[name] = leaf - # Build leaf → spines mapping from topology links or spine/leaf lists. + # Build leaf/edge → core/spine mapping from topology links or role lists. spines = devices.get("spines", []) if isinstance(devices, dict) else [] leafs = devices.get("leafs", []) if isinstance(devices, dict) else [] + cores = devices.get("cores", []) if isinstance(devices, dict) else [] + aggs = devices.get("aggs", []) if isinstance(devices, dict) else [] + edges = devices.get("edges", []) if isinstance(devices, dict) else [] spine_names = [s.get("name") for s in spines if isinstance(s, dict) and s.get("name")] - if not hasattr(self, "leaf_to_spines"): - self.leaf_to_spines = {} + core_names = [c.get("name") for c in cores if isinstance(c, dict) and c.get("name")] + agg_names = [a.get("name") for a in aggs if isinstance(a, dict) and a.get("name")] + edge_names = [e.get("name") for e in edges if isinstance(e, dict) and e.get("name")] links = metadata.get("links", []) if isinstance(metadata, dict) else [] if links: - spine_set = set(spine_names) + adjacency: dict[str, set[str]] = {} for link in links: if not isinstance(link, dict): continue endpoints = link.get("endpoints", []) if len(endpoints) != 2: continue - a, b = endpoints[0], endpoints[1] - a_name = a.get("device") if isinstance(a, dict) else None - b_name = b.get("device") if isinstance(b, dict) else None - if a_name in spine_set and b_name not in spine_set: - self.leaf_to_spines.setdefault(b_name, []) - if a_name not in self.leaf_to_spines[b_name]: - self.leaf_to_spines[b_name].append(a_name) - elif b_name in spine_set and a_name not in spine_set: - self.leaf_to_spines.setdefault(a_name, []) - if b_name not in self.leaf_to_spines[a_name]: - self.leaf_to_spines[a_name].append(b_name) - elif spine_names and leafs: - # Full-mesh assumption: every leaf connects to every spine. - for leaf in leafs: - leaf_name = leaf.get("name") if isinstance(leaf, dict) else None - if leaf_name: - self.leaf_to_spines[leaf_name] = list(spine_names) - - def _load_topology_metadata_from_disk(self) -> dict | None: - base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - env_dir = config.topology_dir - candidates = [] - if env_dir: - candidates.append(os.path.join(env_dir, "topology.json")) - generated_dirs = sorted( - glob.glob(os.path.join(base_dir, "lab-topology", "generated_topology_*")), - key=os.path.getmtime, - reverse=True, - ) - for candidate_dir in generated_dirs: - candidates.append(os.path.join(candidate_dir, "topology.json")) - candidates.append(os.path.join(base_dir, "lab-topology", "topology.json")) - for metadata_file in candidates: - if not os.path.exists(metadata_file): - continue - try: - with open(metadata_file, encoding="utf-8") as f: - return json.load(f) - except Exception: - logger.debug("failed to read topology metadata %s", metadata_file, exc_info=True) - continue - return None + a_name = _endpoint_name(endpoints[0]) + b_name = _endpoint_name(endpoints[1]) + if not a_name or not b_name: + continue + adjacency.setdefault(a_name, set()).add(b_name) + adjacency.setdefault(b_name, set()).add(a_name) + + if core_names and agg_names and edge_names: + core_set = set(core_names) + agg_set = set(agg_names) + for edge_name in edge_names: + reachable_cores: set[str] = set() + for agg_name in adjacency.get(edge_name, set()) & agg_set: + reachable_cores.update(adjacency.get(agg_name, set()) & core_set) + if reachable_cores: + self.leaf_to_spines[edge_name] = sorted(reachable_cores) + else: + spine_set = set(spine_names) + for leaf_name in [leaf.get("name") for leaf in leafs if isinstance(leaf, dict) and leaf.get("name")]: + peers = sorted(adjacency.get(leaf_name, set()) & spine_set) + if peers: + self.leaf_to_spines[leaf_name] = peers diff --git a/netopsbench/platform/pingmesh/agent.py b/netopsbench/platform/pingmesh/agent.py index cc242cd..03ddb5a 100644 --- a/netopsbench/platform/pingmesh/agent.py +++ b/netopsbench/platform/pingmesh/agent.py @@ -2,98 +2,34 @@ """Pingmesh probe agent library. Defines :class:`PingmeshAgent`. The CLI entrypoint lives in -:mod:`netopsbench.platform.pingmesh.cli` (also copied into each client -container by :mod:`netopsbench.platform.pingmesh.deploy`). +:mod:`netopsbench.platform.pingmesh.cli` (also staged and bind-mounted into +each client container by :mod:`netopsbench.platform.pingmesh.deploy`). """ from __future__ import annotations -try: - from netopsbench.platform.pingmesh._agent_influx import PingInfluxMixin - from netopsbench.platform.pingmesh._agent_probe import UdpProbeMixin - from netopsbench.platform.pingmesh._agent_responder import UdpEchoResponder - from netopsbench.platform.pingmesh._agent_runtime import PingRuntimeMixin - from netopsbench.platform.pingmesh._agent_support import ( - config, - json, - logger, - os, - queue, - requests, - threading, - time, - ) -except ImportError: # In-container deployment runs from /tmp/pingmesh/ flat files. - from _agent_influx import PingInfluxMixin # type: ignore[no-redef] - from _agent_probe import UdpProbeMixin # type: ignore[no-redef] - from _agent_responder import UdpEchoResponder # type: ignore[no-redef] - from _agent_runtime import PingRuntimeMixin # type: ignore[no-redef] - from _agent_support import ( # type: ignore[no-redef] - config, - json, - logger, - os, - queue, - requests, - threading, - time, - ) +import hashlib +import json +import os +import queue +import threading +import time +import requests -def _pinglist_client_count(data: dict) -> int: - clients = set() - for probe in data.get("probes", []): - src_name = str(probe.get("src_name") or "").strip() - dst_name = str(probe.get("dst_name") or "").strip() - if src_name: - clients.add(src_name) - if dst_name: - clients.add(dst_name) - return len(clients) +from netopsbench.config import config +from netopsbench.logging_utils import get_logger +from netopsbench.platform.pingmesh._agent_influx import PingInfluxMixin +from netopsbench.platform.pingmesh._agent_probe import UdpProbeMixin +from netopsbench.platform.pingmesh._agent_responder import UdpEchoResponder +from netopsbench.platform.pingmesh._agent_runtime import PingRuntimeMixin +logger = get_logger(__name__) -def _default_ports_per_cycle(client_count: int) -> tuple[int, int]: - if client_count <= 8: - return 8, 2 - if client_count <= 16: - return 6, 1 - return 4, 1 - -def _optional_int_env(name: str) -> int | None: - raw = str(os.environ.get(name, "") or "").strip() - if not raw: - return None - try: - return int(raw) - except ValueError: - logger.warning("Ignoring invalid %s=%r; expected integer", name, raw) - return None - - -def _resolve_ports_per_cycle( - *, - client_count: int, - n_rtt_ports: int, - n_df_ports: int, - enable_df_probe: bool, -) -> tuple[int, int]: - default_rtt, default_df = _default_ports_per_cycle(client_count) - rtt_override = _optional_int_env("PINGMESH_RTT_PORTS_PER_CYCLE") - df_override = _optional_int_env("PINGMESH_DF_PORTS_PER_CYCLE") - - if n_rtt_ports <= 0: - rtt_ports = 0 - else: - rtt_ports = rtt_override if rtt_override is not None else default_rtt - rtt_ports = max(1, min(int(rtt_ports), int(n_rtt_ports))) - - if not enable_df_probe or n_df_ports <= 0: - df_ports = 0 - else: - df_ports = df_override if df_override is not None else default_df - df_ports = max(0, min(int(df_ports), int(n_df_ports))) - return rtt_ports, df_ports +def _stable_host_seed(hostname: str) -> int: + digest = hashlib.blake2s(str(hostname or "").encode("utf-8"), digest_size=8).digest() + return int.from_bytes(digest, "big") def _deterministic_startup_jitter_seconds(hostname: str, interval: float) -> float: @@ -103,54 +39,73 @@ def _deterministic_startup_jitter_seconds(hostname: str, interval: float) -> flo interval_value = 0.0 if interval_value <= 0: return 0.0 - seed = sum((idx + 1) * ord(ch) for idx, ch in enumerate(str(hostname or ""))) + seed = _stable_host_seed(hostname) return (seed % 10_000) / 10_000.0 * interval_value +def _deterministic_phase(hostname: str, modulus: int) -> int: + if modulus <= 1: + return 0 + return _stable_host_seed(hostname) % modulus + + +def _rotating_destination_batch( + tasks: list[dict], + batch_size: int, + batch_index: int, + phase_offset: int = 0, +) -> list[dict]: + if not tasks: + return [] + size = max(1, min(int(batch_size), len(tasks))) + batch_count = max(1, (len(tasks) + size - 1) // size) + phase = int(phase_offset) % len(tasks) + ordered = tasks[phase:] + tasks[:phase] + return ordered[int(batch_index) % batch_count :: batch_count] + + +def _probe_batch_indices( + cycle: int, + destination_batch_count: int, + port_batch_count: int, + port_phase: int = 0, +) -> tuple[int, int]: + """Return one deterministic destination/port-batch Cartesian pair.""" + destination_count = max(1, int(destination_batch_count)) + port_count = max(1, int(port_batch_count)) + epoch_cycle = int(cycle) % (destination_count * port_count) + destination_batch = epoch_cycle % destination_count + port_round = epoch_cycle // destination_count + return destination_batch, (port_round + int(port_phase)) % port_count + + class PingmeshAgent(UdpProbeMixin, PingInfluxMixin, PingRuntimeMixin): """Pingmesh probe agent that runs periodic ping tests.""" def __init__( self, pinglist_file: str, - interval: float = 1.0, - influxdb_url: str = None, - influxdb_token: str = None, - influxdb_org: str = None, - influxdb_bucket: str = None, - # UDP burst probe params (replaces former ICMP ping_count / ping_interval) - n_rtt_ports: int = 16, - n_df_ports: int = 4, + influxdb_url: str | None = None, + influxdb_token: str | None = None, + influxdb_org: str | None = None, + influxdb_bucket: str | None = None, udp_dst_port: int = 33434, rtt_src_port_base: int = 33000, - df_src_port_base: int = 33100, - burst_timeout_s: float = 1.0, enable_df_probe: bool = True, - df_payload_size: int = 1400, - min_interval: float = 1.0, - max_interval: float = 1.0, batch_size: int = 50, batch_timeout: float = 2.0, max_retries: int = 3, retry_backoff_base: float = 0.5, ): - self.interval = interval self.my_name = os.environ.get("HOSTNAME", "unknown") - self.n_rtt_ports = n_rtt_ports - self.n_df_ports = n_df_ports self.udp_dst_port = udp_dst_port self.rtt_src_port_base = rtt_src_port_base - self.df_src_port_base = df_src_port_base - self.burst_timeout_s = burst_timeout_s - self.min_interval = min_interval - self.max_interval = max_interval self.enable_df_probe = enable_df_probe - self.df_payload_size = df_payload_size self.influxdb_url = influxdb_url or config.influxdb_url self.influxdb_token = influxdb_token or config.influxdb_token self.influxdb_org = influxdb_org or config.influxdb_org self.influxdb_bucket = influxdb_bucket or config.influxdb_bucket - self.use_influxdb = requests is not None + self.use_influxdb = True self.batch_size = batch_size self.batch_timeout = batch_timeout self.max_retries = max_retries @@ -158,24 +113,53 @@ def __init__( with open(pinglist_file, encoding="utf-8") as f: data = json.load(f) - self.topology_id = config.topology_id or data.get("topology_id") or "" - self.pinglist_client_count = _pinglist_client_count(data) - self.rtt_ports_per_cycle, self.df_ports_per_cycle = _resolve_ports_per_cycle( - client_count=self.pinglist_client_count, - n_rtt_ports=self.n_rtt_ports, - n_df_ports=self.n_df_ports, - enable_df_probe=self.enable_df_probe, - ) + policy = dict(data.get("pingmesh_policy") or {}) + required_policy = { + "rtt_port_pool_size", + "rtt_ports_per_cycle", + "cycle_interval_seconds", + "destination_batch_count", + "port_batch_count", + "coverage_epoch_cycles", + "coverage_epoch_seconds", + "df_payload_size", + } + missing_policy = sorted(required_policy - policy.keys()) + if missing_policy: + raise ValueError( + "Pingmesh policy is incomplete; regenerate the topology. " + f"Missing fields: {', '.join(missing_policy)}" + ) + configured_destination_batch_size = policy.get("destination_batch_size") + self.n_rtt_ports = int(policy["rtt_port_pool_size"]) + self.rtt_ports_per_cycle = max(1, min(int(policy["rtt_ports_per_cycle"]), self.n_rtt_ports)) + self.interval = float(policy["cycle_interval_seconds"]) + self.min_interval = self.interval + self.max_interval = self.interval + self.coverage_epoch_cycles = max(1, int(policy["coverage_epoch_cycles"])) + self.coverage_epoch_seconds = max(1, int(policy["coverage_epoch_seconds"])) + self.df_payload_size = int(policy["df_payload_size"]) + self.topology_id = data.get("topology_id") or "" self.startup_jitter_s = _deterministic_startup_jitter_seconds(self.my_name, self.interval) - # Auto-derive DF payload size from topology MTU when using the default value. - if self.df_payload_size == 1400: - topo_mtu = self._infer_df_payload_from_topology() - if topo_mtu is not None: - self.df_payload_size = topo_mtu - logger.info(" DF payload auto-derived from topology MTU: %s bytes", self.df_payload_size) - self.tasks = [probe for probe in data["probes"] if probe["src_name"] == self.my_name] + self.destination_batch_size = int(configured_destination_batch_size or len(self.tasks) or 1) + self.destination_batch_count = max( + 1, (len(self.tasks) + self.destination_batch_size - 1) // self.destination_batch_size + ) + source_names = list(dict.fromkeys(str(probe.get("src_name")) for probe in data["probes"])) + self.destination_phase = source_names.index(self.my_name) if self.my_name in source_names else 0 + self.rtt_port_batch_count = max( + 1, (self.n_rtt_ports + self.rtt_ports_per_cycle - 1) // self.rtt_ports_per_cycle + ) + if int(policy["destination_batch_count"]) != self.destination_batch_count: + raise ValueError("Pingmesh destination batch count is stale; regenerate the topology") + if int(policy["port_batch_count"]) != self.rtt_port_batch_count: + raise ValueError("Pingmesh port batch count is stale; regenerate the topology") + if self.coverage_epoch_cycles != self.destination_batch_count * self.rtt_port_batch_count: + raise ValueError("Pingmesh coverage epoch is stale; regenerate the topology") + self.rtt_port_phase = _deterministic_phase(self.my_name, self.rtt_port_batch_count) + self.probe_cycle = 0 # Discover this host's data-plane IP from the pinglist so the UDP # responder binds to the fabric-facing interface (eth1), not mgmt. own_data_ip = next( @@ -208,30 +192,29 @@ def __init__( logger.info("Pingmesh Agent started: %s", self.my_name) logger.info(" Probe tasks: %s", len(self.tasks)) - active_port_count = self.rtt_ports_per_cycle + (self.df_ports_per_cycle if self.enable_df_probe else 0) + logger.info( + " Destination rotation: %s active, %s batches, phase=%s", + min(self.destination_batch_size, len(self.tasks)), + self.destination_batch_count, + self.destination_phase, + ) + logger.info(" Coverage epoch: %s cycles (%ss)", self.coverage_epoch_cycles, self.coverage_epoch_seconds) + active_port_count = self.rtt_ports_per_cycle logger.info(" Probe worker: 1") - logger.info(" Concurrent flows: %s", len(self.tasks) * active_port_count) + active_destinations = min(self.destination_batch_size, len(self.tasks)) + logger.info(" Concurrent flows: %s", active_destinations * active_port_count) logger.info( - " Port pool: RTT %s ports (src %s-%s), DF %s ports (src %s-%s)", + " Shared RTT/DF port pool: %s ports (src %s-%s)", self.n_rtt_ports, self.rtt_src_port_base, self.rtt_src_port_base + self.n_rtt_ports - 1, - self.n_df_ports if self.enable_df_probe else 0, - self.df_src_port_base, - self.df_src_port_base + self.n_df_ports - 1, ) logger.info( - " Active ports/cycle: RTT %s/%s, DF %s/%s", + " Active ports/cycle: RTT %s/%s; DF reuses one matching tuple per destination", self.rtt_ports_per_cycle, self.n_rtt_ports, - self.df_ports_per_cycle if self.enable_df_probe else 0, - self.n_df_ports if self.enable_df_probe else 0, - ) - logger.info( - " UDP destination: :%s, timeout=%ss", - self.udp_dst_port, - self.burst_timeout_s, ) + logger.info(" UDP destination: :%s", self.udp_dst_port) if self.enable_df_probe: logger.info( " DF payload: %s bytes", @@ -247,28 +230,26 @@ def __init__( logger.info(" Batch size: %s, timeout: %ss", self.batch_size, self.batch_timeout) logger.info(" InfluxDB mode: %s", "enabled" if self.use_influxdb else "disabled") - def _infer_df_payload_from_topology(self): - """Derive DF payload size from topology metadata MTU (minus IP+ICMP headers = 28 bytes).""" - topo_dir = config.topology_dir or "" - candidates = [] - if topo_dir: - candidates.append(os.path.join(topo_dir, "topology.json")) - # Fallback: look relative to project root - base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - import glob as _glob - - for d in sorted(_glob.glob(os.path.join(base_dir, "lab-topology", "generated_topology_*")), reverse=True): - candidates.append(os.path.join(d, "topology.json")) - for path in candidates: - try: - with open(path, encoding="utf-8") as f: - meta = json.load(f) - defaults = meta.get("defaults", {}) - # Prefer sonic_port_mtu (actual device MTU) over link_mtu (container MTU) - mtu = defaults.get("sonic_port_mtu") or defaults.get("link_mtu") - if isinstance(mtu, int) and mtu > 28: - return mtu - 28 # subtract IP + ICMP header overhead - except Exception: - logger.debug("failed to read MTU metadata from %s", path, exc_info=True) - continue - return None + def next_probe_batch(self) -> tuple[list[dict], dict[str, int]]: + cycle = self.probe_cycle + destination_batch_index, port_batch_index = _probe_batch_indices( + cycle, + self.destination_batch_count, + self.rtt_port_batch_count, + self.rtt_port_phase, + ) + tasks = _rotating_destination_batch( + self.tasks, + self.destination_batch_size, + destination_batch_index, + self.destination_phase, + ) + metadata = { + "probe_cycle": cycle, + "destination_batch_index": destination_batch_index, + "port_batch_index": port_batch_index, + "coverage_epoch": cycle // self.coverage_epoch_cycles, + "coverage_epoch_cycles": self.coverage_epoch_cycles, + } + self.probe_cycle += 1 + return tasks, metadata diff --git a/netopsbench/platform/pingmesh/cli.py b/netopsbench/platform/pingmesh/cli.py index fd1dc18..e66d614 100644 --- a/netopsbench/platform/pingmesh/cli.py +++ b/netopsbench/platform/pingmesh/cli.py @@ -2,10 +2,8 @@ """CLI entrypoint for the Pingmesh probe agent. Separated from :mod:`netopsbench.platform.pingmesh.agent` so the agent -class stays a pure library object. This module is also copied into each -client container by :mod:`netopsbench.platform.pingmesh.deploy`, so the -in-container ``python3 /tmp/pingmesh/run_pingmesh_agent.py ...`` invocation -keeps working when the package layout is not on ``sys.path``. +class stays a pure library object. The staged package is launched with +``python3 -m netopsbench.platform.pingmesh.cli`` inside each client. """ from __future__ import annotations @@ -14,10 +12,7 @@ class stays a pure library object. This module is also copied into each import sys from collections.abc import Sequence -try: - from netopsbench.platform.pingmesh.agent import PingmeshAgent -except ImportError: # In-container deployment runs from /tmp/pingmesh/ flat files. - from agent import PingmeshAgent # type: ignore[no-redef] +from netopsbench.platform.pingmesh.agent import PingmeshAgent def _build_parser() -> argparse.ArgumentParser: @@ -29,25 +24,13 @@ def _build_parser() -> argparse.ArgumentParser: "pinglist", help="Path to the pinglist JSON file (probes definition).", ) - parser.add_argument( - "interval", - nargs="?", - type=float, - default=1.0, - help="Probe cycle interval in seconds (default: 1.0).", - ) return parser def main(argv: Sequence[str] | None = None) -> int: """Run the Pingmesh agent until interrupted.""" args = _build_parser().parse_args(argv) - agent = PingmeshAgent( - args.pinglist, - interval=args.interval, - min_interval=args.interval, - max_interval=args.interval, - ) + agent = PingmeshAgent(args.pinglist) try: agent.run() except KeyboardInterrupt: diff --git a/netopsbench/platform/pingmesh/deploy.py b/netopsbench/platform/pingmesh/deploy.py index d1aac57..59432b5 100644 --- a/netopsbench/platform/pingmesh/deploy.py +++ b/netopsbench/platform/pingmesh/deploy.py @@ -1,12 +1,9 @@ """Deploy Pingmesh agents to all client containers for a topology. Replaces ``scripts/runtime/deploy_pingmesh.sh`` with a pure-Python -implementation that provides proper error handling, structured logging, -and direct integration with the topology metadata helpers. - -CLI usage (backward-compatible with the shell script):: - - python -m netopsbench.platform.pingmesh.deploy +implementation that stages the agent runtime once on the host, relies on the +Containerlab ``configs/pingmesh:/tmp/pingmesh:ro`` bind for clients, and starts +agents with bounded parallelism. Programmatic usage:: @@ -16,36 +13,45 @@ from __future__ import annotations -import json import os import shlex +import shutil import subprocess -import sys import time -from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field +from importlib.resources import files +from pathlib import Path + +import yaml -from netopsbench.config import config, repo_root +from netopsbench.config import config from netopsbench.logging_utils import get_logger from netopsbench.platform.pingmesh.generator import generate_pinglist_from_topology -from netopsbench.platform.topology.topology_utils import clab_container_name -from netopsbench.platform.utils.events import emit as _emit -from netopsbench.platform.utils.proc import safe_run, sudo_prefix +from netopsbench.platform.topology.topology_utils import clab_container_name, load_topology_manifest +from netopsbench.platform.utils.proc import docker_prefix, safe_run logger = get_logger(__name__) -# Agent files that must be copied into each client container. -_AGENT_FILES: list[str] = [ - "netopsbench/platform/pingmesh/agent.py", - "netopsbench/platform/pingmesh/cli.py", - "netopsbench/platform/pingmesh/_agent_support.py", - "netopsbench/platform/pingmesh/_agent_probe.py", - "netopsbench/platform/pingmesh/_agent_responder.py", - "netopsbench/platform/pingmesh/_agent_influx.py", - "netopsbench/platform/pingmesh/_agent_runtime.py", - "scripts/runtime/run_pingmesh_agent.py", -] +# Agent files staged once under configs/pingmesh and bind-mounted into clients. +_AGENT_MODULES: tuple[str, ...] = ( + "agent.py", + "cli.py", + "_agent_probe.py", + "_agent_responder.py", + "_agent_influx.py", + "_agent_runtime.py", +) +_PACKAGE_MODULES: tuple[str, ...] = ("config.py", "logging_utils.py") +_RUNTIME_SUBDIR = os.path.join("configs", "pingmesh") +_CONTAINER_RUNTIME_DIR = "/tmp/pingmesh" +_CONTAINER_PINGLIST = "/tmp/pingmesh/pinglist.json" +_CONTAINER_AGENT_MODULE = "netopsbench.platform.pingmesh.cli" +_CONTAINER_AGENT_SOURCE = "/tmp/pingmesh/netopsbench/platform/pingmesh/cli.py" +_PINGMESH_BIND = "configs/pingmesh:/tmp/pingmesh:ro" +_DEFAULT_DEPLOY_PARALLELISM = 32 +_INTERNAL_INFLUXDB_URL = "http://influxdb:8086" @dataclass @@ -61,7 +67,7 @@ def _docker(*args: str, check: bool = True, capture: bool = False, **kwargs) -> """Run a docker command, raising on failure when *check* is set.""" kwargs.setdefault("timeout", 60) return safe_run( - [*sudo_prefix(), "docker", *args], + [*docker_prefix(), "docker", *args], check=check, capture_output=capture, text=True, @@ -74,60 +80,196 @@ def _running_containers() -> list[str]: return result.stdout.strip().splitlines() if result.returncode == 0 else [] -def _load_topology_metadata(topology_dir: str) -> dict: +def _load_topology_metadata(topology_dir: str): metadata_path = os.path.join(topology_dir, "topology.json") if not os.path.isfile(metadata_path): raise FileNotFoundError(f"Topology metadata not found: {metadata_path}") - with open(metadata_path, encoding="utf-8") as fh: - return json.load(fh) + return load_topology_manifest(metadata_path) + + +def _runtime_dir(topology_dir: str) -> str: + return os.path.join(topology_dir, _RUNTIME_SUBDIR) + + +def _staged_pinglist_path(topology_dir: str) -> str: + return os.path.join(_runtime_dir(topology_dir), "pinglist.json") + + +def _copy_resource(source, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(source.read_bytes()) + + +def _stage_runtime(topology_dir: str, pinglist_file: str, topology_id: str) -> str: + runtime_dir = _runtime_dir(topology_dir) + os.makedirs(runtime_dir, exist_ok=True) + + staged_pinglist = _staged_pinglist_path(topology_dir) + metadata_file = os.path.join(topology_dir, "topology.json") + generate_pinglist_from_topology(metadata_file, staged_pinglist, topology_id=topology_id) + if not os.path.isfile(staged_pinglist): + raise RuntimeError("Pinglist generation failed") + + requested_pinglist = os.path.abspath(pinglist_file) + if requested_pinglist != os.path.abspath(staged_pinglist): + requested_dir = os.path.dirname(requested_pinglist) + if requested_dir: + os.makedirs(requested_dir, exist_ok=True) + shutil.copy2(staged_pinglist, requested_pinglist) + + staged_root = Path(runtime_dir) + package_dir = staged_root / "netopsbench" / "platform" / "pingmesh" + for package_init in ( + staged_root / "netopsbench" / "__init__.py", + staged_root / "netopsbench" / "platform" / "__init__.py", + package_dir / "__init__.py", + ): + package_init.parent.mkdir(parents=True, exist_ok=True) + package_init.write_text("", encoding="utf-8") + + pingmesh_resources = files("netopsbench.platform.pingmesh") + for module in _AGENT_MODULES: + source = pingmesh_resources.joinpath(module) + if not source.is_file(): + raise FileNotFoundError(f"Packaged Pingmesh runtime source not found: {module}") + _copy_resource(source, package_dir / module) + package_resources = files("netopsbench") + for module in _PACKAGE_MODULES: + source = package_resources.joinpath(module) + if not source.is_file(): + raise FileNotFoundError(f"Packaged Pingmesh dependency not found: {module}") + _copy_resource(source, staged_root / "netopsbench" / module) + + required = [staged_pinglist, str(package_dir / "cli.py")] + missing = [path for path in required if not os.path.isfile(path)] + if missing: + raise RuntimeError(f"Pingmesh runtime staging failed: missing {', '.join(missing)}") + return staged_pinglist + + +def _topology_yaml_path(topology_dir: str, lab_name: str) -> str: + candidate = os.path.join(topology_dir, f"{lab_name}.clab.yaml") + if os.path.isfile(candidate): + return candidate + matches = sorted( + os.path.join(topology_dir, name) for name in os.listdir(topology_dir) if name.endswith(".clab.yaml") + ) + return matches[0] if matches else candidate + + +def _validate_pingmesh_bind(topology_dir: str, lab_name: str) -> None: + yaml_path = _topology_yaml_path(topology_dir, lab_name) + if not os.path.isfile(yaml_path): + raise RuntimeError( + f"Containerlab topology YAML not found: {yaml_path}. Regenerate topology before deploying Pingmesh." + ) + + with open(yaml_path, encoding="utf-8") as fh: + topology = yaml.safe_load(fh) or {} + linux_kind = ((topology.get("topology") or {}).get("kinds") or {}).get("linux") or {} + binds = linux_kind.get("binds") or [] + if _PINGMESH_BIND not in binds: + raise RuntimeError( + f"Pingmesh bind missing from {yaml_path}: expected linux kind bind {_PINGMESH_BIND}. " + "Regenerate topology before deploying Pingmesh." + ) + + +def _env_assignment(name: str, value: str | int | float | None) -> str: + return f"{name}={shlex.quote(str(value or ''))}" + + +def _start_client_agent( + *, + client_name: str, + container: str, + topology_id: str, + influxdb_url: str, + influxdb_token: str, + influxdb_org: str, + influxdb_bucket: str, +) -> tuple[str, bool, str]: + env_values: dict[str, str | int] = { + "PYTHONPATH": f"{_CONTAINER_RUNTIME_DIR}:/tmp", + "NETOPSBENCH_TOPOLOGY_ID": topology_id, + "NETOPSBENCH_INFLUXDB_URL": influxdb_url, + "NETOPSBENCH_INFLUXDB_TOKEN": influxdb_token, + "NETOPSBENCH_INFLUXDB_ORG": influxdb_org, + "NETOPSBENCH_INFLUXDB_BUCKET": influxdb_bucket, + } + env_block = " ".join(_env_assignment(name, value) for name, value in env_values.items()) + command = ( + "set -e; " + "mkdir -p /var/log/pingmesh; " + f"test -r {_CONTAINER_AGENT_SOURCE}; " + f"test -r {_CONTAINER_PINGLIST}; " + f"for pid in $(pgrep -f {shlex.quote(_CONTAINER_AGENT_MODULE)} || true); do " + '[ "$pid" = "$$" ] && continue; ' + 'kill "$pid" >/dev/null 2>&1 || true; ' + "done; " + f"{env_block} nohup python3 -m {_CONTAINER_AGENT_MODULE} {_CONTAINER_PINGLIST} " + "> /var/log/pingmesh/agent.log 2>&1 DeployResult: """Deploy Pingmesh agents to every client container in *topology_dir*. Returns a :class:`DeployResult` with deployment counts and per-client verification status. """ - root = str(repo_root()) - - # --- resolve parameters from env with fallbacks --- - pinglist_file = pinglist_file or os.path.join(topology_dir, "pinglist.json") - cycle_interval = int(os.environ.get("PINGMESH_CYCLE_INTERVAL", cycle_interval)) - topology_id = topology_id or config.topology_id or os.path.basename(topology_dir) - influxdb_url = influxdb_url or config.pingmesh_influxdb_url + # --- resolve parameters from the canonical runtime topology --- + pinglist_file = pinglist_file or _staged_pinglist_path(topology_dir) + influxdb_url = influxdb_url or _INTERNAL_INFLUXDB_URL influxdb_token = influxdb_token or config.influxdb_token influxdb_org = influxdb_org or config.influxdb_org influxdb_bucket = influxdb_bucket or config.influxdb_bucket # --- load topology metadata --- topo = _load_topology_metadata(topology_dir) - lab_name = (topo.get("name") or "dcn").strip() + cycle_interval = topo.pingmesh.cycle_interval_seconds + topology_id = topology_id or topo.topology_id + lab_name = topo.name.strip() clients: list[str] = [] - for client in (topo.get("devices", {}) or {}).get("clients", []): - name = str(client.get("name") or "").strip() + for client in topo.clients(): + name = client.name.strip() if name: clients.append(name) if not clients: raise RuntimeError(f"No clients found in topology metadata: {topology_dir}/topology.json") + _validate_pingmesh_bind(topology_dir, lab_name) # --- validate running containers --- - _emit("=== Deploying Pingmesh Agents ===") - _emit(f"Topology: {topology_dir}") - _emit(f"Topology ID: {topology_id}") - _emit(f"InfluxDB: {influxdb_url} bucket={influxdb_bucket}") - _emit(f"Cycle interval: {cycle_interval}s") - _emit("") + logger.info("=== Deploying Pingmesh Agents ===") + logger.info(f"Topology: {topology_dir}") + logger.info(f"Topology ID: {topology_id}") + logger.info(f"InfluxDB: {influxdb_url} bucket={influxdb_bucket}") + logger.info(f"Cycle interval: {cycle_interval}s") + deploy_parallelism = max(1, int(parallelism)) + logger.info(f"Deploy parallelism: {deploy_parallelism}") + logger.info("") running = set(_running_containers()) expected = {clab_container_name(lab_name, c) for c in clients} @@ -142,81 +284,59 @@ def deploy_pingmesh( ", ".join(sorted(missing)), ) - _emit(f"[0/3] Validated {len(running_clients)}/{len(expected)} client containers") - _emit("") - - # --- generate pinglist --- - _emit("[1/3] Generating pinglist...") - metadata_file = os.path.join(topology_dir, "topology.json") - generate_pinglist_from_topology(metadata_file, pinglist_file, topology_id=topology_id) - if not os.path.isfile(pinglist_file): - raise RuntimeError("Pinglist generation failed") - _emit("") - - # --- deploy to each client --- - _emit("[2/3] Deploying agents to client containers...") + logger.info(f"[0/3] Validated {len(running_clients)}/{len(expected)} client containers") + logger.info("") + + # --- stage runtime files and pinglist --- + logger.info("[1/3] Staging Pingmesh runtime...") + stage_started = time.monotonic() + staged_pinglist = _stage_runtime(topology_dir, pinglist_file, topology_id) + stage_elapsed = time.monotonic() - stage_started + logger.info(f" Runtime: {_runtime_dir(topology_dir)}") + logger.info(f" Pinglist: {staged_pinglist}") + logger.info(f" Staged in {stage_elapsed:.1f}s") + logger.info("") + + # --- start agents in each client --- + logger.info("[2/3] Starting agents in client containers...") result = DeployResult() + start_started = time.monotonic() + outcomes: dict[str, tuple[bool, str]] = {} + futures = {} + with ThreadPoolExecutor(max_workers=deploy_parallelism) as executor: + for client_name in clients: + container = clab_container_name(lab_name, client_name) + if container not in running: + outcomes[client_name] = (False, "container is not running") + continue + futures[ + executor.submit( + _start_client_agent, + client_name=client_name, + container=container, + topology_id=topology_id, + influxdb_url=influxdb_url, + influxdb_token=influxdb_token, + influxdb_org=influxdb_org, + influxdb_bucket=influxdb_bucket, + ) + ] = client_name - agent_sources = [os.path.join(root, src) for src in _AGENT_FILES] + for future in as_completed(futures): + client_name, ok, message = future.result() + outcomes[client_name] = (ok, message) for client_name in clients: container = clab_container_name(lab_name, client_name) - if container not in running: - logger.warning("%s: not running, skipping", container) + ok, message = outcomes.get(client_name, (False, "not scheduled")) + if ok: + result.deployed += 1 + else: + logger.error("%s: %s", container, message) result.failed.append(client_name) - continue - - _emit(f" Deploying to {container}...") - # Create directories - ret = _docker("exec", container, "mkdir", "-p", "/tmp/pingmesh", "/var/log/pingmesh", check=False) - if ret.returncode != 0: - logger.error("%s: failed to create directories, skipping", container) - result.failed.append(client_name) - continue - - # Copy agent files - copy_ok = True - for src in agent_sources: - dst = f"{container}:/tmp/pingmesh/{os.path.basename(src)}" - ret = _docker("cp", src, dst, check=False) - if ret.returncode != 0: - logger.error( - "%s: failed to copy %s, skipping", - container, - os.path.basename(src), - ) - copy_ok = False - break - if not copy_ok: - result.failed.append(client_name) - continue - - # Copy pinglist - _docker("cp", pinglist_file, f"{container}:/tmp/pingmesh/pinglist.json", check=False) - - # Kill any existing agent, then start the new one - _docker("exec", container, "pkill", "-f", "/tmp/pingmesh/run_pingmesh_agent.py", check=False, capture=True) - - optional_env = "" - for env_name in ("PINGMESH_RTT_PORTS_PER_CYCLE", "PINGMESH_DF_PORTS_PER_CYCLE"): - if os.environ.get(env_name): - optional_env += f"{env_name}={shlex.quote(os.environ[env_name])} " - - env_block = ( - f"PYTHONPATH=/tmp " - f"NETOPSBENCH_TOPOLOGY_ID='{topology_id}' " - f"NETOPSBENCH_INFLUXDB_URL='{influxdb_url}' " - f"NETOPSBENCH_INFLUXDB_TOKEN='{influxdb_token}' " - f"NETOPSBENCH_INFLUXDB_ORG='{influxdb_org}' " - f"NETOPSBENCH_INFLUXDB_BUCKET='{influxdb_bucket}' " - f"{optional_env}" - f"nohup python3 /tmp/pingmesh/run_pingmesh_agent.py " - f"/tmp/pingmesh/pinglist.json {cycle_interval} " - f"> /var/log/pingmesh/agent.log 2>&1 &" - ) - _docker("exec", "-d", container, "sh", "-c", env_block, check=False) - result.deployed += 1 + start_elapsed = time.monotonic() - start_started + logger.info(f" Started {result.deployed}/{len(clients)} clients in {start_elapsed:.1f}s") if result.failed: logger.warning( @@ -224,11 +344,16 @@ def deploy_pingmesh( len(result.failed), ", ".join(result.failed), ) + if result.deployed == 0: + raise RuntimeError( + "Pingmesh deployment failed for all clients; /tmp/pingmesh bind or staged runtime is not readable. " + "Regenerate the topology before deploying Pingmesh." + ) # --- verify --- if verify: - _emit("") - _emit("[3/3] Verifying deployment...") + logger.info("") + logger.info("[3/3] Verifying deployment...") time.sleep(2) for client_name in clients[:3]: container = clab_container_name(lab_name, client_name) @@ -240,32 +365,13 @@ def deploy_pingmesh( check=False, capture=True, ) - agent_running = "run_pingmesh_agent.py" in (ret.stdout or "") + agent_running = _CONTAINER_AGENT_MODULE in (ret.stdout or "") result.verified[client_name] = agent_running status = "✓ running" if agent_running else "✗ NOT running" - _emit(f" {container}: Agent {status}") + logger.info(f" {container}: Agent {status}") - _emit("") - _emit("=== Pingmesh Deployment Complete ===") - _emit(f" Deployed to {result.deployed} clients") - _emit(f" Pinglist: {pinglist_file}") + logger.info("") + logger.info("=== Pingmesh Deployment Complete ===") + logger.info(f" Deployed to {result.deployed} clients") + logger.info(f" Pinglist: {staged_pinglist}") return result - - -def main(argv: Sequence[str] | None = None) -> int: - """CLI entry point (backward-compatible with shell script args).""" - args = list(argv if argv is not None else sys.argv[1:]) - - pinglist_file = args[0] if len(args) > 0 else None - topology_dir = args[1] if len(args) > 1 else (config.topology_dir or "generated_topology") - - try: - deploy_pingmesh(topology_dir=topology_dir, pinglist_file=pinglist_file) - except (FileNotFoundError, RuntimeError) as exc: - logger.error("%s", exc) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/netopsbench/platform/pingmesh/detector.py b/netopsbench/platform/pingmesh/detector.py index 6e16d4f..77131df 100644 --- a/netopsbench/platform/pingmesh/detector.py +++ b/netopsbench/platform/pingmesh/detector.py @@ -2,11 +2,15 @@ from __future__ import annotations -import json -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta +import time +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime + +from netopsbench.models.topology import TopologyManifest +from netopsbench.platform.topology.topology_utils import coerce_topology_manifest from ._detector_analysis import DetectorAnalysisMixin +from ._detector_coverage import DetectorCoverageMixin from ._detector_query import DetectorQueryMixin @@ -24,9 +28,14 @@ class Anomaly: threshold: float severity: str timestamp: str + samples_sent: int = 0 + samples_lost: int = 0 + sample_count: int = 0 + windows_observed: list[str] = field(default_factory=list) + persistence: str | None = None -class AnomalyDetector(DetectorQueryMixin, DetectorAnalysisMixin): +class AnomalyDetector(DetectorQueryMixin, DetectorCoverageMixin, DetectorAnalysisMixin): """Detect anomalies in Pingmesh data using statistical methods.""" _anomaly_type = Anomaly @@ -37,7 +46,7 @@ def __init__( token: str, org: str, bucket: str, - topology_metadata: dict | None = None, + topology_metadata: TopologyManifest | dict | None = None, topology_id: str | None = None, loss_pct_threshold: float = 5.0, loss_pct_delta: float = 5.0, @@ -46,18 +55,20 @@ def __init__( self.token = token self.org = org self.bucket = bucket - self.last_query_error: str | None = None self.client_to_leaf: dict[str, str] = {} self.leaf_to_spines: dict[str, list[str]] = {} - self.topology_id = topology_id self.loss_pct_threshold = float(loss_pct_threshold) self.loss_pct_delta = float(loss_pct_delta) - if isinstance(topology_metadata, dict): - self._load_topology_metadata(topology_metadata) - else: - auto_metadata = self._load_topology_metadata_from_disk() - if isinstance(auto_metadata, dict): - self._load_topology_metadata(auto_metadata) + self._pingmesh_clients: list[str] = [] + self._pingmesh_policy: dict = {} + if topology_metadata is None: + raise ValueError("Canonical topology_metadata is required for Pingmesh anomaly detection") + manifest = coerce_topology_manifest(topology_metadata) + self.topology_id = topology_id or manifest.topology_id + projected_topology = manifest.to_agent_topology() + self._pingmesh_clients = [device.name for device in manifest.clients()] + self._pingmesh_policy = dict(projected_topology["pingmesh"]) + self._load_topology_metadata(projected_topology) def _anomaly_to_dict(self, anomaly: Anomaly) -> dict: return asdict(anomaly) @@ -70,31 +81,22 @@ def _infer_spines_for_cross_rack(self, src_leaf: str, dst_leaf: str) -> list[str dst_spines = set(self.leaf_to_spines.get(dst_leaf, [])) return sorted(src_spines & dst_spines) if (src_spines and dst_spines) else sorted(src_spines | dst_spines) - def generate_anomaly_report( - self, baseline_start: str, baseline_end: str, current_start: str, current_end: str - ) -> dict: - latency_anomalies = self.detect_latency_anomalies(baseline_start, baseline_end, current_start, current_end) - loss_anomalies = self.detect_packet_loss(baseline_start, baseline_end, current_start, current_end) - df_anomalies = self.detect_df_fragmentation_anomalies(baseline_start, baseline_end, current_start, current_end) - jitter_anomalies = self.detect_jitter_anomalies(baseline_start, baseline_end, current_start, current_end) - all_anomalies = latency_anomalies + loss_anomalies + df_anomalies + jitter_anomalies - - # Separate path_unreachable from regular packet_loss for summary counts - path_unreachable = [a for a in loss_anomalies if a.type == "path_unreachable"] - regular_loss = [a for a in loss_anomalies if a.type == "packet_loss"] + @staticmethod + def _anomaly_family(anomaly: Anomaly) -> str: + if anomaly.type in {"packet_loss", "path_unreachable"}: + return "loss" + return anomaly.type + def _aggregate_anomalies(self, anomalies: list[Anomaly]) -> dict: by_src_leaf: dict[str, dict[str, int]] = {} by_dst_leaf: dict[str, dict[str, int]] = {} by_spine: dict[str, dict[str, int]] = {} - for anomaly in all_anomalies: + keys = ("drop_count", "latency_spikes", "jitter_spikes", "path_unreachable", "mtu_suspects") + for anomaly in anomalies: src_leaf = self._resolve_leaf(anomaly.src_leaf, anomaly.src_name) dst_leaf = self._resolve_leaf(anomaly.dst_leaf, anomaly.dst_name) - by_src_leaf.setdefault( - src_leaf, {"drop_count": 0, "latency_spikes": 0, "jitter_spikes": 0, "path_unreachable": 0} - ) - by_dst_leaf.setdefault( - dst_leaf, {"drop_count": 0, "latency_spikes": 0, "jitter_spikes": 0, "path_unreachable": 0} - ) + by_src_leaf.setdefault(src_leaf, dict.fromkeys(keys, 0)) + by_dst_leaf.setdefault(dst_leaf, dict.fromkeys(keys, 0)) if anomaly.type in ("packet_loss", "path_unreachable"): key = "path_unreachable" if anomaly.type == "path_unreachable" else "drop_count" by_src_leaf[src_leaf][key] += 1 @@ -102,6 +104,9 @@ def generate_anomaly_report( elif anomaly.type == "jitter_spike": by_src_leaf[src_leaf]["jitter_spikes"] += 1 by_dst_leaf[dst_leaf]["jitter_spikes"] += 1 + elif anomaly.type == "mtu_or_fragmentation_suspect": + by_src_leaf[src_leaf]["mtu_suspects"] += 1 + by_dst_leaf[dst_leaf]["mtu_suspects"] += 1 else: by_src_leaf[src_leaf]["latency_spikes"] += 1 by_dst_leaf[dst_leaf]["latency_spikes"] += 1 @@ -110,17 +115,37 @@ def generate_anomaly_report( if src_leaf != dst_leaf and src_leaf != "unknown" and dst_leaf != "unknown": inferred_spines = self._infer_spines_for_cross_rack(src_leaf, dst_leaf) for spine in inferred_spines: - bucket = by_spine.setdefault( - spine, {"drop_count": 0, "latency_spikes": 0, "jitter_spikes": 0, "path_unreachable": 0} - ) + bucket = by_spine.setdefault(spine, dict.fromkeys(keys, 0)) if anomaly.type in ("packet_loss", "path_unreachable"): key = "path_unreachable" if anomaly.type == "path_unreachable" else "drop_count" bucket[key] += 1 elif anomaly.type == "jitter_spike": bucket["jitter_spikes"] += 1 + elif anomaly.type == "mtu_or_fragmentation_suspect": + bucket["mtu_suspects"] += 1 else: bucket["latency_spikes"] += 1 + return {"by_src_leaf": by_src_leaf, "by_dst_leaf": by_dst_leaf, "by_spine": by_spine} + + def _build_report( + self, + *, + anomalies: list[Anomaly], + baseline_start: str, + baseline_end: str, + current_start: str, + current_end: str, + query_status: dict, + coverage: dict, + quality: dict, + ) -> dict: + latency = [item for item in anomalies if item.type == "latency_spike"] + regular_loss = [item for item in anomalies if item.type == "packet_loss"] + unreachable = [item for item in anomalies if item.type == "path_unreachable"] + mtu = [item for item in anomalies if item.type == "mtu_or_fragmentation_suspect"] + jitter = [item for item in anomalies if item.type == "jitter_spike"] + now_utc = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + "Z" return { "timestamp": now_utc, @@ -128,43 +153,162 @@ def generate_anomaly_report( "baseline": {"start": baseline_start, "end": baseline_end}, "current": {"start": current_start, "end": current_end}, }, - "query_status": {"ok": self.last_query_error is None, "error": self.last_query_error}, + "query_status": query_status, + "coverage": coverage, + "quality": quality, "summary": { - "total_anomalies": len(all_anomalies), - "latency_spikes": len(latency_anomalies), + "total_anomalies": len(anomalies), + "latency_spikes": len(latency), "packet_loss_events": len(regular_loss), - "path_unreachable_events": len(path_unreachable), - "mtu_or_fragmentation_events": len(df_anomalies), - "jitter_spikes": len(jitter_anomalies), - }, - "anomalies": [self._anomaly_to_dict(a) for a in all_anomalies], - "aggregated_anomalies": { - "by_src_leaf": by_src_leaf, - "by_dst_leaf": by_dst_leaf, - "by_spine": by_spine, + "path_unreachable_events": len(unreachable), + "mtu_or_fragmentation_events": len(mtu), + "jitter_spikes": len(jitter), }, + "anomalies": [self._anomaly_to_dict(item) for item in anomalies], + "returned_anomalies": len(anomalies), + "truncated": False, + "aggregated_anomalies": self._aggregate_anomalies(anomalies), } + @staticmethod + def _error_status(*results) -> dict: + errors = [result.error for result in results if result.status != "ok" and result.error] + failed = any(result.status != "ok" for result in results) + return { + "ok": not failed, + "error": "; ".join(dict.fromkeys(errors)) if errors else ("query_failed" if failed else None), + } + + @staticmethod + def _slice_rows(rows: list[dict], start_time: str, end_time: str) -> list[dict]: + start = datetime.fromisoformat(start_time.replace("Z", "+00:00")) + end = datetime.fromisoformat(end_time.replace("Z", "+00:00")) + selected = [] + for row in rows: + raw_time = str(row.get("_time") or "") + if not raw_time: + continue + try: + timestamp = datetime.fromisoformat(raw_time.replace("Z", "+00:00")) + except ValueError: + continue + if start <= timestamp < end: + selected.append(row) + return selected + + def _merge_window_anomalies(self, analyses: list[tuple[str, list[Anomaly]]]) -> list[Anomaly]: + merged: dict[tuple[str, str, str], Anomaly] = {} + seen_windows: dict[tuple[str, str, str], set[str]] = {} + statistical_types = {"latency_spike", "jitter_spike"} + full_statistical: dict[tuple[str, str, str], Anomaly] = {} + for window_name, anomalies in analyses: + if window_name != "full": + continue + for anomaly in anomalies: + if anomaly.type in statistical_types: + key = (self._anomaly_family(anomaly), anomaly.src_ip, anomaly.dst_ip) + full_statistical[key] = anomaly + severity_rank = {"low": 0, "medium": 1, "high": 2} + type_rank = {"packet_loss": 0, "path_unreachable": 1} + for window_name, anomalies in analyses: + for anomaly in anomalies: + key = (self._anomaly_family(anomaly), anomaly.src_ip, anomaly.dst_ip) + if anomaly.type in statistical_types: + if key not in full_statistical: + continue + seen_windows.setdefault(key, set()).add(window_name) + merged.setdefault(key, full_statistical[key]) + continue + seen_windows.setdefault(key, set()).add(window_name) + current = merged.get(key) + candidate_rank = ( + type_rank.get(anomaly.type, 0), + severity_rank.get(anomaly.severity, 0), + anomaly.value, + ) + current_rank = ( + ( + type_rank.get(current.type, 0), + severity_rank.get(current.severity, 0), + current.value, + ) + if current + else (-1, -1, -1.0) + ) + if current is None or candidate_rank > current_rank: + merged[key] = anomaly + for key, anomaly in merged.items(): + windows = sorted(seen_windows[key]) + anomaly.windows_observed = windows + if "early" in windows and "steady" in windows: + anomaly.persistence = "persistent" + elif "early" in windows: + anomaly.persistence = "early_only" + elif "steady" in windows: + anomaly.persistence = "steady_only" + else: + anomaly.persistence = "full_window" + return sorted(merged.values(), key=lambda item: (item.type, item.src_ip, item.dst_ip)) + + def generate_windowed_anomaly_report( + self, + *, + baseline_start: str, + baseline_end: str, + current_start: str, + current_end: str, + windows: list[dict], + ) -> dict: + baseline = self._query_snapshot(baseline_start, baseline_end) + current = self._query_snapshot(current_start, current_end) + query_status = self._error_status(baseline, current) + if query_status["ok"]: + initial_coverage = self.summarize_coverage(current.rows) + expected_seconds = int(initial_coverage.get("expected_epoch_cycles", 0)) * int( + self._pingmesh_policy.get("cycle_interval_seconds", 1) + ) + actual_seconds = max( + 0.0, + ( + datetime.fromisoformat(current_end.replace("Z", "+00:00")) + - datetime.fromisoformat(current_start.replace("Z", "+00:00")) + ).total_seconds(), + ) + for _attempt in range(2): + coverage = self.summarize_coverage(current.rows) + if actual_seconds < expected_seconds or coverage.get("coverage_status") == "complete": + break + time.sleep(max(1, int(self._pingmesh_policy.get("cycle_interval_seconds", 1)))) + current = self._query_snapshot(current_start, current_end) + query_status = self._error_status(baseline, current) + if not query_status["ok"]: + break + if not query_status["ok"]: + return self._build_report( + anomalies=[], + baseline_start=baseline_start, + baseline_end=baseline_end, + current_start=current_start, + current_end=current_end, + query_status=query_status, + coverage={"status": "error", "coverage_status": "error", "error": query_status["error"]}, + quality={}, + ) -def main() -> int: - from netopsbench.config import config - - detector = AnomalyDetector(config.influxdb_url, config.influxdb_token, config.influxdb_org, config.influxdb_bucket) - now = datetime.now(UTC) - - def _utc_iso(dt: datetime) -> str: - s = dt.replace(microsecond=0).isoformat() - return s[:-6] + "Z" if s.endswith("+00:00") else (s if s.endswith("Z") else s + "Z") - - baseline_end = _utc_iso(now) - baseline_start = _utc_iso(now - timedelta(minutes=10)) - current_start = _utc_iso(now - timedelta(minutes=5)) - current_end = baseline_end - report = detector.generate_anomaly_report( - baseline_start=baseline_start, - baseline_end=current_start, - current_start=current_start, - current_end=current_end, - ) - print(json.dumps(report, indent=2)) - return 0 + full_analysis = self.analyze_snapshot_rows(baseline.rows, current.rows) + window_analyses: list[tuple[str, list[Anomaly]]] = [("full", full_analysis.anomalies)] + for window in windows: + name = str(window.get("name") or "window") + rows = self._slice_rows(current.rows, str(window["start_time"]), str(window["end_time"])) + window_analyses.append((name, self.analyze_snapshot_rows(baseline.rows, rows).anomalies)) + anomalies = self._merge_window_anomalies(window_analyses) + return self._build_report( + anomalies=anomalies, + baseline_start=baseline_start, + baseline_end=baseline_end, + current_start=current_start, + current_end=current_end, + query_status=query_status, + coverage=self.summarize_coverage(current.rows), + quality=full_analysis.quality, + ) diff --git a/netopsbench/platform/pingmesh/generator.py b/netopsbench/platform/pingmesh/generator.py index 2011893..43ba1a3 100644 --- a/netopsbench/platform/pingmesh/generator.py +++ b/netopsbench/platform/pingmesh/generator.py @@ -5,9 +5,9 @@ import json from dataclasses import asdict, dataclass -from pathlib import Path from netopsbench.logging_utils import get_logger +from netopsbench.platform.topology.topology_utils import coerce_topology_manifest, load_topology_manifest logger = get_logger(__name__) @@ -28,27 +28,23 @@ class ProbeTask: class PinglistGenerator: - """Generates pinglist for all client-to-client pairs""" + """Generates pinglist tasks from client metadata.""" def generate(self, topology_metadata: dict) -> list[ProbeTask]: """ - Generate N×N probe tasks for all client pairs. + Generate probe tasks for client pairs. Args: topology_metadata: Topology metadata dict with devices.clients - Returns: - List of ProbeTask objects for all client pairs + List of ProbeTask objects. """ - clients = topology_metadata["devices"]["clients"] + manifest = coerce_topology_manifest(topology_metadata) + clients = manifest.to_agent_topology()["devices"]["clients"] tasks = [] - for src in clients: - for dst in clients: - if src["name"] == dst["name"]: - continue # Skip self-to-self - - # Determine path type + for src_idx, src in enumerate(clients): + for dst in self._destination_clients(clients, src_idx): path_type = "same_rack" if src["rack"] == dst["rack"] else "cross_rack" task = ProbeTask( @@ -66,11 +62,23 @@ def generate(self, topology_metadata: dict) -> list[ProbeTask]: return tasks + def _destination_clients( + self, + clients: list[dict], + src_idx: int, + ) -> list[dict]: + total_clients = len(clients) + if total_clients <= 1: + return [] + + return [clients[(src_idx + offset) % total_clients] for offset in range(1, total_clients)] + def save_pinglist( self, tasks: list[ProbeTask], output_file: str, topology_id: str | None = None, + pingmesh_policy: dict | None = None, ): """ Save pinglist to JSON file. @@ -82,6 +90,8 @@ def save_pinglist( data = {"total_probes": len(tasks), "probes": [asdict(t) for t in tasks]} if topology_id: data["topology_id"] = topology_id + if pingmesh_policy: + data["pingmesh_policy"] = pingmesh_policy with open(output_file, "w") as f: json.dump(data, f, indent=2) @@ -89,10 +99,6 @@ def save_pinglist( logger.info("Saved %s probe tasks to %s", len(tasks), output_file) -def _infer_topology_id(topology_file: str) -> str: - return Path(topology_file).resolve().parent.name - - def generate_pinglist_from_topology( topology_file: str, output_file: str = "pinglist.json", @@ -106,12 +112,24 @@ def generate_pinglist_from_topology( output_file: Output pinglist.json path topology_id: Explicit topology identifier; inferred from directory name if omitted """ - with open(topology_file) as f: - topology = json.load(f) + manifest = load_topology_manifest(topology_file) + topology = manifest.model_dump(mode="json") generator = PinglistGenerator() tasks = generator.generate(topology) - topology_id = topology_id or _infer_topology_id(topology_file) - generator.save_pinglist(tasks, output_file, topology_id=topology_id) + topology_id = topology_id or manifest.topology_id + policy = { + **manifest.pingmesh.model_dump(mode="json"), + "destination_batch_count": manifest.pingmesh.destination_batch_count(len(manifest.clients())), + "port_batch_count": manifest.pingmesh.port_batch_count(), + "coverage_epoch_cycles": manifest.pingmesh.coverage_epoch_cycles(len(manifest.clients())), + "coverage_epoch_seconds": manifest.pingmesh.coverage_epoch_seconds(len(manifest.clients())), + } + generator.save_pinglist( + tasks, + output_file, + topology_id=topology_id, + pingmesh_policy=policy, + ) return tasks diff --git a/netopsbench/platform/runtime/__init__.py b/netopsbench/platform/runtime/__init__.py index 3985a96..d68e8b1 100644 --- a/netopsbench/platform/runtime/__init__.py +++ b/netopsbench/platform/runtime/__init__.py @@ -1,12 +1 @@ -"""Platform runtime helpers.""" - -from .lab_lifecycle import deploy_lab, resolve_generated_topology_dir, teardown_lab -from .manager import RuntimeManager, RuntimePool - -__all__ = [ - "RuntimeManager", - "RuntimePool", - "deploy_lab", - "resolve_generated_topology_dir", - "teardown_lab", -] +"""Internal runtime lifecycle package.""" diff --git a/netopsbench/platform/runtime/apply_configs.py b/netopsbench/platform/runtime/apply_configs.py index 80b22e8..79d19cb 100644 --- a/netopsbench/platform/runtime/apply_configs.py +++ b/netopsbench/platform/runtime/apply_configs.py @@ -1,35 +1,28 @@ -"""Fast parallel SONiC configuration application. - -Replaces ``scripts/runtime/apply_configs_fast.sh`` with a pure-Python -implementation using :class:`concurrent.futures.ThreadPoolExecutor` -instead of GNU ``parallel``, removing that external dependency. - -CLI usage:: - - python -m netopsbench.platform.runtime.apply_configs [max_parallel] [lab_name] +"""Post-deploy SONiC activation for preseeded startup artifacts. Programmatic usage:: from netopsbench.platform.runtime.apply_configs import apply_configs - result = apply_configs("/path/to/topology_dir", max_parallel=4) + result = apply_configs("/path/to/topology_dir", max_parallel=32) """ from __future__ import annotations -import glob import json import os import subprocess -import sys import time -from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path -from netopsbench.config import config -from netopsbench.platform.topology.topology_utils import clab_container_name -from netopsbench.platform.utils.proc import safe_run, sudo_prefix +from netopsbench.models.topology import DeviceRole, TopologyManifest +from netopsbench.platform.topology.config import SONIC_PORT_COUNTER_INTERVAL_MS +from netopsbench.platform.topology.topology_utils import clab_container_name, load_topology_manifest +from netopsbench.platform.utils.proc import docker_prefix, safe_run + +_REQUIRED_SONIC_SERVICES = ("redis-server", "orchagent", "zebra") +SONIC_READINESS_MAX_TRIES = 180 @dataclass @@ -38,43 +31,167 @@ class ApplyResult: succeeded: list[str] = field(default_factory=list) failed: list[str] = field(default_factory=list) + durations: dict[str, float] = field(default_factory=dict) + readiness_durations: dict[str, float] = field(default_factory=dict) + activation_durations: dict[str, float] = field(default_factory=dict) elapsed_seconds: float = 0.0 -def _resolve_lab_name(topology_dir: str, lab_name: str | None) -> str: - """Resolve the lab name from argument, topology metadata, or clab YAML.""" - if lab_name: - return lab_name - - metadata_file = os.path.join(topology_dir, "topology.json") - if os.path.isfile(metadata_file): - with open(metadata_file, encoding="utf-8") as fh: - data = json.load(fh) - name = (data.get("name") or "").strip() - if name: - return name - - # Fall back to parsing the clab YAML header - clab_files = sorted(glob.glob(os.path.join(topology_dir, "*.clab.y*ml"))) - for clab_file in clab_files: - with open(clab_file, encoding="utf-8") as fh: - for line in fh: - if line.startswith("name:"): - name = line.split(":", 1)[1].strip() - if name: - return name - break - - return "dcn" - - -def _wait_for_sonic(container: str, max_tries: int | None = None, delay: int = 5) -> bool: - """Block until SONiC ``vtysh`` is responsive or *max_tries* is exceeded.""" - max_tries = max_tries or config.sonic_wait_tries - prefix = sudo_prefix() +def _preseed_config_file(topology_dir: str, device: str) -> str: + return os.path.join(topology_dir, "configs", "sonic", device, "config_db.json") + + +def _expected_port_count(config_file: str) -> int: + try: + with open(config_file, encoding="utf-8") as fh: + payload = json.load(fh) + except (OSError, json.JSONDecodeError): + return 0 + ports = payload.get("PORT") + return len(ports) if isinstance(ports, dict) else 0 + + +def _startup_wrapper_file(topology_dir: str) -> str: + return os.path.join(topology_dir, "configs", "sonic", "start.sh") + + +def _device_sort_key(device: str) -> tuple[int, int | str]: + for rank, prefix in enumerate(("core", "agg", "edge", "spine", "leaf")): + if device.startswith(prefix): + suffix = device[len(prefix) :] + return (rank, int(suffix) if suffix.isdigit() else suffix) + return (5, device) + + +def _discover_devices(topology_dir: str) -> list[str]: + preseed_root = Path(topology_dir) / "configs" / "sonic" + if not preseed_root.exists(): + return [] + devices = [ + device_dir.name + for device_dir in preseed_root.iterdir() + if device_dir.is_dir() and (device_dir / "config_db.json").is_file() + ] + return sorted(devices, key=_device_sort_key) + + +def _ecmp_hash_policies(manifest: TopologyManifest, devices: list[str]) -> dict[str, int]: + policies: dict[str, int] = {} + for device in devices: + manifest_device = manifest.device(device) + if manifest_device is None or manifest_device.role is DeviceRole.CLIENT: + raise RuntimeError(f"Preseeded SONiC device {device!r} is missing from topology manifest switches") + policies[device] = manifest.routing.ecmp_hash_policy_by_role[manifest_device.role] + return policies + + +def _configdb_ready(prefix: list[str], container: str) -> bool: + try: + configdb_ret = safe_run( + [ + *prefix, + "docker", + "exec", + container, + "sonic-cfggen", + "-d", + "-v", + "DEVICE_METADATA.localhost.hostname", + ], + capture_output=True, + text=True, + check=False, + timeout=15, + ) + except subprocess.TimeoutExpired: + return False + return configdb_ret.returncode == 0 + + +def _startup_status(prefix: list[str], container: str) -> str | None: + try: + ret = safe_run( + [*prefix, "docker", "exec", container, "supervisorctl", "status", "start.sh"], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except subprocess.TimeoutExpired: + return None + parts = str(ret.stdout or ret.stderr or "").split() + return parts[1].upper() if len(parts) >= 2 else None + + +def _required_services_ready(prefix: list[str], container: str) -> bool: + try: + ret = safe_run( + [*prefix, "docker", "exec", container, "supervisorctl", "status", *_REQUIRED_SONIC_SERVICES], + capture_output=True, + text=True, + check=False, + timeout=15, + ) + except subprocess.TimeoutExpired: + return False + output = str(ret.stdout or ret.stderr or "") + states: dict[str, str] = {} + for line in output.splitlines(): + parts = line.split() + if len(parts) >= 2: + states[parts[0]] = parts[1].upper() + return all(states.get(service) == "RUNNING" for service in _REQUIRED_SONIC_SERVICES) + + +def _countersdb_ready(prefix: list[str], container: str, expected_port_count: int) -> bool: + try: + ret = safe_run( + [ + *prefix, + "docker", + "exec", + container, + "redis-cli", + "-n", + "2", + "--raw", + "HLEN", + "COUNTERS_PORT_NAME_MAP", + ], + capture_output=True, + text=True, + check=False, + timeout=15, + ) + except subprocess.TimeoutExpired: + return False + if ret.returncode != 0: + return False + try: + return int(str(ret.stdout).strip()) >= expected_port_count + except ValueError: + return False + + +def _wait_for_sonic( + device: str, + container: str, + max_tries: int | None = None, + delay: int = 5, + expected_port_count: int | None = None, +) -> bool: + """Block until SONiC control-plane and ConfigDB commands are responsive.""" + max_tries = max_tries or SONIC_READINESS_MAX_TRIES + prefix = docker_prefix() for _ in range(max_tries): + _startup_status(prefix, container) + configdb_ready = _configdb_ready(prefix, container) + if not configdb_ready: + time.sleep(delay) + continue + try: - ret = safe_run( + vtysh_ret = safe_run( [*prefix, "docker", "exec", container, "vtysh", "-c", "show version"], capture_output=True, text=True, @@ -84,121 +201,204 @@ def _wait_for_sonic(container: str, max_tries: int | None = None, delay: int = 5 except subprocess.TimeoutExpired: time.sleep(delay) continue - if ret.returncode == 0: + + if vtysh_ret.returncode != 0: + time.sleep(delay) + continue + + if not _required_services_ready(prefix, container): + time.sleep(delay) + continue + + if _configdb_ready(prefix, container) and ( + expected_port_count is None or _countersdb_ready(prefix, container, expected_port_count) + ): return True time.sleep(delay) return False +def _activate_preseed_device(prefix: list[str], container: str, ecmp_hash_policy: int) -> bool: + command = ( + "set -e; " + f"sysctl -w net.ipv4.fib_multipath_hash_policy={ecmp_hash_policy} >/dev/null; " + f'test "$(sysctl -n net.ipv4.fib_multipath_hash_policy)" = "{ecmp_hash_policy}"; ' + "supervisorctl start bgpd >/dev/null 2>&1 || true; " + "if [ -s /etc/frr/frr.conf ]; then " + "vtysh -b >/tmp/netopsbench-vtysh-batch.log 2>&1; " + "fi; " + f"counterpoll port interval {SONIC_PORT_COUNTER_INTERVAL_MS} >/dev/null; " + "mkdir -p /var/run/telemetry /var/log/telemetry || true; " + "pkill -x telemetry >/dev/null 2>&1 || true; " + "for _ in $(seq 1 50); do " + "pgrep -x telemetry >/dev/null 2>&1 || break; sleep 0.1; " + "done; " + "if pgrep -x telemetry >/dev/null 2>&1; then exit 1; fi; " + "nohup /usr/sbin/telemetry -port 50051 -noTLS -client_auth none " + ">/var/log/telemetry/telemetry.log 2>&1 /dev/null 2>&1 && exit 0; sleep 0.1; " + "done; exit 1" + ) + ret = safe_run( + [*prefix, "docker", "exec", container, "bash", "-lc", command], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + return ret.returncode == 0 + + def _apply_single_device( device: str, topology_dir: str, lab_name: str, - max_attempts: int = 4, -) -> tuple[str, bool, str]: - """Apply config to one device. Returns ``(device, success, message)``.""" - config_file = os.path.join(topology_dir, "configs", f"{device}.sh") + ecmp_hash_policy: int, +) -> tuple[str, bool, str, float, float, float]: + """Activate one preseeded device after Containerlab has started it.""" + started_at = time.monotonic() container = clab_container_name(lab_name, device) - prefix = sudo_prefix() - - if not os.path.isfile(config_file): - return (device, False, f"config file not found: {config_file}") - - if not _wait_for_sonic(container): - return (device, False, "SONiC services not ready") - - # Copy config into container - safe_run( - [*prefix, "docker", "cp", config_file, f"{container}:/tmp/config.sh"], - capture_output=True, - check=False, - timeout=60, - ) + prefix = docker_prefix() + preseed_config = _preseed_config_file(topology_dir, device) + startup_wrapper = _startup_wrapper_file(topology_dir) + + if not os.path.isfile(preseed_config): + return (device, False, f"preseed config not found: {preseed_config}", time.monotonic() - started_at, 0.0, 0.0) + + if not os.path.isfile(startup_wrapper): + return ( + device, + False, + f"startup wrapper not found: {startup_wrapper}; regenerate topology before deploying", + time.monotonic() - started_at, + 0.0, + 0.0, + ) - for attempt in range(1, max_attempts + 1): - ret = safe_run( - [*prefix, "docker", "exec", container, "bash", "/tmp/config.sh"], - capture_output=True, - text=True, - check=False, - timeout=120, + expected_port_count = _expected_port_count(preseed_config) + if expected_port_count <= 0: + return ( + device, + False, + f"preseed config has no PORT entries: {preseed_config}", + time.monotonic() - started_at, + 0.0, + 0.0, ) - if ret.returncode == 0: - # Enable L4-aware ECMP hashing so that multi-flow probes - # (UDP/TCP with varying src_port) actually fan out across - # parallel uplinks. Without this, Linux defaults to L3-only - # hashing (src_ip, dst_ip) and every flow between the same - # endpoint pair pins to a single ECMP nexthop — causing - # link-level faults on alternate paths to be invisible to - # edge probes. See net.ipv4.fib_multipath_hash_policy(7). - safe_run( - [*prefix, "docker", "exec", container, "sysctl", "-w", "net.ipv4.fib_multipath_hash_policy=1"], - capture_output=True, - text=True, - check=False, - timeout=10, - ) - return (device, True, "config applied") - if attempt < max_attempts: - print(f" ! [{device}] attempt {attempt}/{max_attempts} failed; retrying...", flush=True) - time.sleep(5) - return (device, False, "failed to apply config after all attempts") + readiness_started = time.monotonic() + if not _wait_for_sonic(device, container, expected_port_count=expected_port_count): + readiness_elapsed = time.monotonic() - readiness_started + return ( + device, + False, + f"SONiC services not ready after {readiness_elapsed:.1f}s", + time.monotonic() - started_at, + readiness_elapsed, + 0.0, + ) + readiness_elapsed = time.monotonic() - readiness_started + + activation_started = time.monotonic() + if _activate_preseed_device(prefix, container, ecmp_hash_policy): + activation_elapsed = time.monotonic() - activation_started + return ( + device, + True, + "post-deploy activation", + time.monotonic() - started_at, + readiness_elapsed, + activation_elapsed, + ) + activation_elapsed = time.monotonic() - activation_started + return ( + device, + False, + "post-deploy activation failed", + time.monotonic() - started_at, + readiness_elapsed, + activation_elapsed, + ) def apply_configs( topology_dir: str, - max_parallel: int = 4, + max_parallel: int = 32, lab_name: str | None = None, ) -> ApplyResult: - """Apply SONiC configs for all spine/leaf devices in *topology_dir*. - - Uses :class:`ThreadPoolExecutor` for parallelism, replacing the - previous dependency on GNU ``parallel``. - """ - lab_name = _resolve_lab_name(topology_dir, lab_name) + """Activate all preseeded SONiC devices in *topology_dir*.""" + manifest = load_topology_manifest(topology_dir) + lab_name = lab_name or manifest.name # Discover devices - devices: list[str] = [] - for pattern in ("spine*.sh", "leaf*.sh"): - for cfg in sorted(glob.glob(os.path.join(topology_dir, "configs", pattern))): - devices.append(Path(cfg).stem) + devices = _discover_devices(topology_dir) if not devices: - raise RuntimeError(f"No config files found in {topology_dir}/configs/") + raise RuntimeError(f"No preseed config_db.json files found in {topology_dir}/configs/sonic/") + + ecmp_hash_policies = _ecmp_hash_policies(manifest, devices) - print("=== Fast SONiC Configuration Application ===") + print("=== SONiC Post-Deploy Activation ===") print(f"Topology directory: {topology_dir}") print(f"Max parallel: {max_parallel}") print(f"Lab name: {lab_name}") print() print(f"Found {len(devices)} devices to configure: {' '.join(devices)}") print() - print("[1/2] Applying configurations in parallel...") + print("[1/2] Activating devices in parallel...") start = time.monotonic() result = ApplyResult() with ThreadPoolExecutor(max_workers=max_parallel) as executor: - futures = {executor.submit(_apply_single_device, device, topology_dir, lab_name): device for device in devices} + futures = { + executor.submit( + _apply_single_device, + device, + topology_dir, + lab_name, + ecmp_hash_policies[device], + ): device + for device in devices + } for future in as_completed(futures): - device, success, msg = future.result() + device, success, msg, elapsed, readiness_elapsed, activation_elapsed = future.result() + result.durations[device] = elapsed + result.readiness_durations[device] = readiness_elapsed + result.activation_durations[device] = activation_elapsed if success: - print(f" ok [{device}] {msg}") + print( + f" ok [{device}] {msg}; ready={readiness_elapsed:.1f}s " + f"activate={activation_elapsed:.1f}s total={elapsed:.1f}s" + ) result.succeeded.append(device) else: - print(f" x [{device}] {msg}") + print( + f" x [{device}] {msg}; ready={readiness_elapsed:.1f}s " + f"activate={activation_elapsed:.1f}s total={elapsed:.1f}s" + ) result.failed.append(device) result.elapsed_seconds = time.monotonic() - start print() - print(f"Applied configs in {result.elapsed_seconds:.0f}s") + print(f"Activated devices in {result.elapsed_seconds:.0f}s") + if result.durations: + slowest = sorted(result.durations.items(), key=lambda item: item[1], reverse=True)[:10] + slowest_text = ", ".join(f"{device}={elapsed:.1f}s" for device, elapsed in slowest) + print(f"Slowest devices: {slowest_text}") + slowest_ready = sorted(result.readiness_durations.items(), key=lambda item: item[1], reverse=True)[:10] + slowest_ready_text = ", ".join(f"{device}={elapsed:.1f}s" for device, elapsed in slowest_ready) + print(f"Slowest readiness: {slowest_ready_text}") + slowest_activation = sorted(result.activation_durations.items(), key=lambda item: item[1], reverse=True)[:10] + slowest_activation_text = ", ".join(f"{device}={elapsed:.1f}s" for device, elapsed in slowest_activation) + print(f"Slowest activation: {slowest_activation_text}") print() # [2/2] Quick control-plane verification print("[2/2] Quick control-plane verification...") - prefix = sudo_prefix() - for role in ("spine", "leaf"): + prefix = docker_prefix() + for role in ("core", "agg", "edge", "spine", "leaf"): candidates = [d for d in devices if d.startswith(role)] if candidates: container = clab_container_name(lab_name, candidates[0]) @@ -211,29 +411,7 @@ def apply_configs( print(f" checked {candidates[0]}") print() - print("=== Configuration Application Complete ===") + print("=== SONiC Activation Complete ===") if result.failed: print(f"WARNING: {len(result.failed)} device(s) failed: {', '.join(result.failed)}") return result - - -def main(argv: Sequence[str] | None = None) -> int: - """CLI entry point (backward-compatible with shell script args).""" - args = list(argv if argv is not None else sys.argv[1:]) - - topology_dir = args[0] if len(args) > 0 else "generated_topology" - max_parallel_str = args[1] if len(args) > 1 else "4" - max_parallel = int(max_parallel_str) if max_parallel_str else 4 - lab_name = args[2] if len(args) > 2 else None - - try: - result = apply_configs(topology_dir, max_parallel, lab_name) - except (FileNotFoundError, RuntimeError) as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 1 - - return 1 if result.failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/netopsbench/platform/runtime/cli.py b/netopsbench/platform/runtime/cli.py new file mode 100644 index 0000000..7c90158 --- /dev/null +++ b/netopsbench/platform/runtime/cli.py @@ -0,0 +1,57 @@ +"""CLI boundary for Python-owned worker deployment.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence + +from netopsbench.platform.runtime.deployment import ( + deploy_worker_lab, + teardown_worker_lab, + worker_from_cli, + worker_from_topology, +) +from netopsbench.platform.runtime.lifecycle import ( + ensure_worker_observability, + ensure_worker_pingmesh, + validate_worker_health, +) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Deploy or teardown one NetOpsBench worker") + subparsers = parser.add_subparsers(dest="command", required=True) + deploy = subparsers.add_parser("deploy") + deploy.add_argument("scale") + deploy.add_argument("topology_dir") + deploy.add_argument("lab_name") + deploy.add_argument("mgmt_subnet") + deploy.add_argument("bucket", nargs="?", default="netopsbench") + deploy.add_argument("--mgmt-network") + teardown = subparsers.add_parser("teardown") + teardown.add_argument("topology_dir") + args = parser.parse_args(argv) + if args.command == "teardown": + teardown_worker_lab(worker_from_topology(args.topology_dir)) + return 0 + + worker = worker_from_cli( + scale=args.scale, + topology_dir=args.topology_dir, + lab_name=args.lab_name, + mgmt_subnet=args.mgmt_subnet, + bucket=args.bucket, + mgmt_network=args.mgmt_network, + ) + deploy_worker_lab(worker, args.scale) + ensure_worker_observability(worker) + ensure_worker_pingmesh(worker) + validate_worker_health(worker) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + + +__all__ = ["main"] diff --git a/netopsbench/platform/runtime/deployment.py b/netopsbench/platform/runtime/deployment.py new file mode 100644 index 0000000..c5d73c9 --- /dev/null +++ b/netopsbench/platform/runtime/deployment.py @@ -0,0 +1,275 @@ +"""Python-owned Containerlab worker deployment and teardown.""" + +from __future__ import annotations + +import fcntl +import ipaddress +import os +import signal +import tempfile +import time +from contextlib import contextmanager +from pathlib import Path + +from netopsbench.models.profiles import get_scale_profile +from netopsbench.models.runtime import RuntimeIdentity +from netopsbench.platform.runtime.apply_configs import apply_configs +from netopsbench.platform.topology.generator import generate_topology +from netopsbench.platform.topology.topology_utils import load_topology_manifest +from netopsbench.platform.utils.proc import docker_prefix, safe_run, sudo_prefix + +APPLY_CONFIG_PARALLELISM = 32 +LAB_REMOVAL_TIMEOUT_SECONDS = 120 +LAB_REMOVAL_POLL_SECONDS = 1.0 +RUNTIME_DEPLOY_LOCK_PATH = Path(tempfile.gettempdir()) / f"netopsbench-{os.getuid()}-runtime-deploy.lock" + + +def management_subnet_stride(scale: str) -> int: + prefix = get_scale_profile(scale).management_prefix + return 1 if prefix >= 24 else 2 ** (24 - prefix) + + +def management_subnet(scale: str, worker_index: int) -> str: + profile = get_scale_profile(scale) + stride = management_subnet_stride(scale) + offset = worker_index if profile.management_prefix == 24 else (worker_index - 1) * stride + third_octet = profile.management_subnet_base + offset + if third_octet + stride - 1 > 254: + raise RuntimeError(f"Worker index {worker_index} exceeds available management subnet range") + return f"172.31.{third_octet}.0/{profile.management_prefix}" + + +def _docker_management_subnets() -> set[str]: + docker = docker_prefix() + listed = safe_run( + [*docker, "docker", "network", "ls", "--format", "{{.ID}}"], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + if listed.returncode != 0: + details = (listed.stderr or listed.stdout or "no diagnostic output").strip() + raise RuntimeError(f"Unable to list Docker networks: {details}") + network_ids = [line.strip() for line in listed.stdout.splitlines() if line.strip()] + if not network_ids: + return set() + inspected = safe_run( + [ + *docker, + "docker", + "network", + "inspect", + "--format", + "{{range .IPAM.Config}}{{println .Subnet}}{{end}}", + *network_ids, + ], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + if inspected.returncode != 0: + details = (inspected.stderr or inspected.stdout or "no diagnostic output").strip() + raise RuntimeError(f"Unable to inspect Docker networks: {details}") + return {line.strip() for line in inspected.stdout.splitlines() if line.strip()} + + +def allocate_management_subnets(scale: str, worker_count: int) -> list[str]: + used_networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] + for subnet in _docker_management_subnets(): + try: + used_networks.append(ipaddress.ip_network(subnet, strict=False)) + except ValueError: + continue + + stride = management_subnet_stride(scale) + start = int(management_subnet(scale, 1).split(".")[2]) + selected: list[str] = [] + for octet in range(start, 255, stride): + candidate = f"172.31.{octet}.0/{get_scale_profile(scale).management_prefix}" + candidate_network = ipaddress.ip_network(candidate, strict=False) + selected_networks = [ipaddress.ip_network(item, strict=False) for item in selected] + if any( + candidate_network.version == network.version and candidate_network.overlaps(network) + for network in [*used_networks, *selected_networks] + ): + continue + selected.append(candidate) + if len(selected) == worker_count: + return selected + raise RuntimeError(f"Unable to allocate {worker_count} unique management subnets for scale {scale}") + + +@contextmanager +def runtime_deploy_lock(): + """Serialize host-side network allocation and Containerlab creation.""" + with RUNTIME_DEPLOY_LOCK_PATH.open("a+", encoding="utf-8") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def deploy_worker_lab(worker: RuntimeIdentity, scale: str) -> None: + """Generate, deploy, and activate one worker without observability side effects.""" + topology_dir = Path(worker.topology_dir) + topology_dir.mkdir(parents=True, exist_ok=True) + stale_paths = [topology_dir / "configs", *topology_dir.glob("clab-*")] + if stale_paths: + safe_run( + [*sudo_prefix(), "rm", "-rf", *(str(path) for path in stale_paths)], + check=True, + timeout=300, + ) + + generate_topology( + scale=scale, + output_dir=str(topology_dir), + name=worker.lab_name, + mgmt_subnet=worker.mgmt_subnet, + mgmt_network=worker.mgmt_network, + ) + topology_file = topology_dir / f"{worker.lab_name}.clab.yaml" + if not topology_file.is_file(): + raise FileNotFoundError(f"Generated Containerlab topology not found: {topology_file}") + + command = [*sudo_prefix(), "containerlab", "deploy", "-t", str(topology_file), "--reconfigure"] + profile = get_scale_profile(scale) + if profile.containerlab_max_workers is not None: + command.extend(["--max-workers", str(profile.containerlab_max_workers)]) + deploy_result = safe_run(command, cwd=topology_dir, check=False, timeout=profile.deploy_timeout_seconds) + if deploy_result.returncode != 0: + details = (deploy_result.stderr or deploy_result.stdout or "no diagnostic output").strip() + raise RuntimeError(f"Containerlab deploy failed ({deploy_result.returncode}): {details[-4000:]}") + + result = apply_configs(str(topology_dir), APPLY_CONFIG_PARALLELISM, worker.lab_name) + if result.failed: + raise RuntimeError(f"SONiC activation failed for: {', '.join(result.failed)}") + + +def teardown_worker_lab(worker: RuntimeIdentity) -> None: + """Remove one worker's collector, sidecar, Containerlab lab, and network.""" + topology_dir = Path(worker.topology_dir) + _stop_collector(topology_dir / "bgp_collector.pid") + docker = docker_prefix() + safe_run([*docker, "docker", "rm", "-f", f"telegraf-{worker.lab_name}"], check=False, timeout=60) + safe_run( + [*docker, "docker", "network", "disconnect", worker.mgmt_network, "influxdb"], + check=False, + timeout=60, + ) + + topology_file = topology_dir / f"{worker.lab_name}.clab.yaml" + profile = get_scale_profile(load_topology_manifest(topology_dir).scale) + command = ( + [*sudo_prefix(), "containerlab", "destroy", "-t", str(topology_file), "--cleanup"] + if topology_file.is_file() + else [*sudo_prefix(), "containerlab", "destroy", "--name", worker.lab_name, "--cleanup"] + ) + if profile.containerlab_max_workers is not None: + command.extend(["--max-workers", str(profile.containerlab_max_workers)]) + safe_run(command, cwd=topology_dir, check=False, timeout=600) + + names = _lab_container_names(docker, worker.lab_name) + if names: + safe_run([*docker, "docker", "rm", "-f", *names], check=False, timeout=300) + _wait_for_lab_removal(docker, worker.lab_name) + safe_run([*docker, "docker", "network", "rm", worker.mgmt_network], check=False, timeout=60) + + +def _lab_container_names(docker: list[str], lab_name: str) -> list[str]: + result = safe_run( + [ + *docker, + "docker", + "ps", + "-a", + "--filter", + f"name=clab-{lab_name}-", + "--format", + "{{.Names}}", + ], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def _wait_for_lab_removal( + docker: list[str], + lab_name: str, + *, + timeout: float = LAB_REMOVAL_TIMEOUT_SECONDS, +) -> None: + deadline = time.monotonic() + timeout + while True: + names = _lab_container_names(docker, lab_name) + if not names: + return + if time.monotonic() >= deadline: + preview = ", ".join(names[:8]) + raise RuntimeError(f"Timed out waiting for lab containers to be removed: {preview}") + safe_run([*docker, "docker", "rm", "-f", *names], check=False, timeout=60) + time.sleep(LAB_REMOVAL_POLL_SECONDS) + + +def worker_from_cli( + *, + scale: str, + topology_dir: str, + lab_name: str, + mgmt_subnet: str, + bucket: str, + mgmt_network: str | None = None, +) -> RuntimeIdentity: + root = Path(topology_dir).resolve() + profile = get_scale_profile(scale) + resolved_mgmt_subnet = mgmt_subnet or f"172.20.20.0/{profile.management_prefix}" + return RuntimeIdentity.create( + runtime_id=lab_name, + worker_id="worker-1", + worker_index=1, + lab_name=lab_name, + topology_dir=root, + mgmt_subnet=resolved_mgmt_subnet, + mgmt_network=mgmt_network or f"clab-mgmt-{lab_name}", + bucket=bucket, + ) + + +def worker_from_topology(topology_dir: str) -> RuntimeIdentity: + root = Path(topology_dir).expanduser().resolve() + manifest = load_topology_manifest(root) + return RuntimeIdentity.create( + runtime_id=manifest.name, + worker_id="worker-1", + worker_index=1, + lab_name=manifest.name, + topology_dir=root, + mgmt_subnet=manifest.management.ipv4_subnet, + mgmt_network=manifest.management.network, + ) + + +def _stop_collector(pid_file: Path) -> None: + try: + pid = int(pid_file.read_text(encoding="utf-8").strip()) + os.kill(pid, signal.SIGTERM) + except (FileNotFoundError, OSError, ValueError): + pass + pid_file.unlink(missing_ok=True) + + +__all__ = [ + "allocate_management_subnets", + "deploy_worker_lab", + "management_subnet", + "runtime_deploy_lock", + "teardown_worker_lab", + "worker_from_cli", + "worker_from_topology", +] diff --git a/netopsbench/platform/runtime/health.py b/netopsbench/platform/runtime/health.py new file mode 100644 index 0000000..2c67877 --- /dev/null +++ b/netopsbench/platform/runtime/health.py @@ -0,0 +1,383 @@ +"""Worker health check: container count, BGP convergence, connectivity, Pingmesh, observability. + +Replaces ``scripts/runtime/check_worker_health.sh`` with a pure-Python +implementation providing structured error reporting and retry logic. + +Programmatic usage:: + + from netopsbench.platform.runtime.health import check_worker_health + errors = check_worker_health(worker_identity) +""" + +from __future__ import annotations + +import os +import re +import subprocess +import time +from math import ceil + +from netopsbench.config import config +from netopsbench.logging_utils import get_logger +from netopsbench.models.profiles import get_scale_profile +from netopsbench.models.runtime import RuntimeIdentity +from netopsbench.models.topology import DeviceRole, TopologyManifest +from netopsbench.platform.topology.topology_utils import ( + clab_container_name, + coerce_topology_manifest, + load_topology_manifest, +) +from netopsbench.platform.utils.proc import docker_prefix, safe_run + +logger = get_logger(__name__) + +HEALTH_POLL_INTERVAL_SECONDS = 5 +ACTIVE_INTERFACE_COVERAGE_MIN_RATIO = 0.5 + + +def _docker_exec(container: str, *cmd: str, check: bool = False, capture: bool = True) -> subprocess.CompletedProcess: + prefix = docker_prefix() + return safe_run( + [*prefix, "docker", "exec", container, *cmd], + check=check, + capture_output=capture, + text=True, + timeout=60, + ) + + +def _running_container_count(lab_name: str) -> int: + result = safe_run( + [*docker_prefix(), "docker", "ps", "--filter", f"label=containerlab={lab_name}", "--format", "{{.Names}}"], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + return len([line for line in result.stdout.strip().splitlines() if line]) + + +def _load_topology_metadata(topology_dir: str) -> TopologyManifest: + path = os.path.join(topology_dir, "topology.json") + if not os.path.isfile(path): + raise FileNotFoundError(f"topology metadata not found: {path}") + return load_topology_manifest(path) + + +def _parse_bgp_established(bgp_output: str) -> int: + """Count established BGP sessions from ``vtysh -c 'show ip bgp summary'``.""" + count = 0 + for line in bgp_output.splitlines(): + parts = line.split() + if not parts: + continue + # First field is neighbor IP (a.b.c.d), 10th field is state/pfxrcd (int if established) + if re.match(r"^\d+\.\d+\.\d+\.\d+$", parts[0]) and len(parts) >= 10: + try: + int(parts[9]) + count += 1 + except ValueError: + pass + return count + + +def _parse_active_interfaces(show_interfaces_output: str) -> set[str]: + active: set[str] = set() + for line in show_interfaces_output.splitlines(): + parts = line.split() + if not parts or not parts[0].startswith("Ethernet"): + continue + status_tokens = [part.lower() for part in parts[1:]] + if status_tokens.count("up") >= 2: + active.add(parts[0]) + return active + + +def _expected_active_interface_count(topo: TopologyManifest | dict, device: str) -> int: + manifest = coerce_topology_manifest(topo) + target = manifest.device(device) + if target is None: + return 0 + clients = manifest.clients() + fat_tree_k = int(manifest.facts.fat_tree_k or 0) + + if fat_tree_k: + half = fat_tree_k // 2 + if target.role is DeviceRole.CORE: + return fat_tree_k + if target.role is DeviceRole.AGG: + return fat_tree_k + if target.role is DeviceRole.EDGE: + attached_clients = [client for client in clients if client.attached_switch == device] + clients_per_edge = len(attached_clients) or int(manifest.facts.clients_per_attached_switch) + return half + clients_per_edge + + if target.role is DeviceRole.SPINE: + return int(manifest.facts.num_leafs or len(manifest.devices_by_role(DeviceRole.LEAF))) + if target.role is DeviceRole.LEAF: + num_spines = int(manifest.facts.num_spines or len(manifest.devices_by_role(DeviceRole.SPINE))) + attached_clients = [client for client in clients if client.attached_switch == device] + clients_per_leaf = len(attached_clients) or int(manifest.facts.clients_per_attached_switch) + return num_spines + clients_per_leaf + return 0 + + +def _expected_bgp_neighbor_count(topo: TopologyManifest | dict, device: str) -> int: + manifest = coerce_topology_manifest(topo) + target = manifest.device(device) + if target is None: + return 0 + fat_tree_k = int(manifest.facts.fat_tree_k or 0) + if fat_tree_k: + if target.role in {DeviceRole.CORE, DeviceRole.AGG}: + return fat_tree_k + if target.role is DeviceRole.EDGE: + return fat_tree_k // 2 + if target.role is DeviceRole.SPINE: + return len(manifest.devices_by_role(DeviceRole.LEAF)) + if target.role is DeviceRole.LEAF: + return len(manifest.devices_by_role(DeviceRole.SPINE)) + return 0 + + +def _active_interface_coverage_error( + container: str, + device: str, + active_interfaces: set[str], + expected_count: int, +) -> str | None: + if expected_count <= 0 or len(active_interfaces) >= expected_count: + return None + return ( + f"active interface coverage too low on {container}: " + f"active={len(active_interfaces)} expected>={expected_count}" + ) + + +def check_worker_health( + worker: RuntimeIdentity, + influxdb_url: str | None = None, + influxdb_token: str | None = None, + influxdb_org: str | None = None, + health_retries: int | None = None, + health_delay: int | None = None, +) -> list[str]: + """Run all health checks and return a list of error messages (empty = healthy). + + Checks performed: + 1. Worker telegraf container is running + 2. Expected container count matches running containers + 3. BGP convergence on spine1 + 4. Client-to-client connectivity + Pingmesh agent + 5. InfluxDB observability path (via validation module) + """ + errors: list[str] = [] + topology_dir = str(worker.topology_dir) + topo = _load_topology_metadata(topology_dir) + projected_topo = topo.to_agent_topology() + lab_name = worker.lab_name + if topo.name != worker.lab_name or topo.topology_id != worker.topology_id: + raise RuntimeError( + "Runtime identity does not match topology manifest: " + f"identity=({worker.lab_name}, {worker.topology_id}) " + f"manifest=({topo.name}, {topo.topology_id})" + ) + devices = projected_topo.get("devices", {}) or {} + clients = devices.get("clients", []) or [] + switches = topo.switches() + routed = topo.routing_devices() + edge_switches = topo.edge_devices() + collector_ip = topo.collector.ipv4.strip() + + influxdb_url = influxdb_url or config.influxdb_url + influxdb_token = influxdb_token or config.influxdb_token + influxdb_org = influxdb_org or config.influxdb_org + + profile = get_scale_profile(topo.scale) + delay = HEALTH_POLL_INTERVAL_SECONDS if health_delay is None else health_delay + retries = health_retries or max(1, ceil(profile.health_timeout_seconds / max(1, delay))) + + if len(clients) < 2: + raise RuntimeError("need at least two clients in topology metadata for health check") + + logger.info("=== Worker Health Check ===") + logger.info("Lab name: %s", lab_name) + logger.info("Topology dir: %s", topology_dir) + + # [1/5] Worker telegraf + logger.info("[1/5] Checking worker telegraf...") + telegraf_container = f"telegraf-{lab_name}" + ret = safe_run( + [*docker_prefix(), "docker", "ps", "--format", "{{.Names}}"], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + if telegraf_container not in ret.stdout.strip().splitlines(): + errors.append(f"worker telegraf container is not running: {telegraf_container}") + # Return early — remaining checks depend on the infra + return errors + + # [2/5] Container count + logger.info("[2/5] Checking container count...") + expected_nodes = len(switches) + len(clients) + running_nodes = _running_container_count(lab_name) + if running_nodes < expected_nodes: + errors.append( + f"running node count mismatch for {lab_name}: " f"got {running_nodes} expected at least {expected_nodes}" + ) + return errors + + # [3/5] BGP convergence + bgp_device = routed[0].name if routed else "spine1" + logger.info("[3/5] Checking BGP convergence on %s...", bgp_device) + bgp_container = clab_container_name(lab_name, bgp_device) + expected_bgp = _expected_bgp_neighbor_count(topo, bgp_device) + established = 0 + for _ in range(retries): + ret = _docker_exec(bgp_container, "vtysh", "-c", "show ip bgp summary") + established = _parse_bgp_established(ret.stdout or "") + if established >= expected_bgp: + break + time.sleep(delay) + if established < expected_bgp: + errors.append(f"BGP not converged on {bgp_container}: " f"established={established} expected>={expected_bgp}") + return errors + + logger.info("[3b/5] Checking active interface coverage...") + coverage_targets = [bgp_device] + if topo.family == "fat-tree": + for role_devices in (topo.devices_by_role(DeviceRole.AGG), edge_switches): + if role_devices: + name = role_devices[0].name + if name and name not in coverage_targets: + coverage_targets.append(name) + if edge_switches: + last_edge = edge_switches[-1].name + if last_edge and last_edge not in coverage_targets: + coverage_targets.append(last_edge) + elif edge_switches: + first_edge = edge_switches[0].name + if first_edge and first_edge not in coverage_targets: + coverage_targets.append(first_edge) + last_edge = edge_switches[-1].name + if last_edge and last_edge not in coverage_targets: + coverage_targets.append(last_edge) + for device in coverage_targets: + expected_count = _expected_active_interface_count(topo, device) + if expected_count <= 0: + continue + container = clab_container_name(lab_name, device) + active_ifaces: set[str] = set() + for _ in range(retries): + ret = _docker_exec(container, "bash", "-lc", "show interfaces status") + active_ifaces = _parse_active_interfaces(ret.stdout or "") + if len(active_ifaces) >= expected_count: + break + time.sleep(delay) + coverage_error = _active_interface_coverage_error( + container=container, + device=device, + active_interfaces=active_ifaces, + expected_count=expected_count, + ) + if coverage_error: + errors.append(coverage_error) + return errors + + # [4/5] Client connectivity + Pingmesh agent + logger.info("[4/5] Checking client connectivity and Pingmesh agent...") + src_client = clients[0] + src_name = str(src_client.get("name", "")) + src_leaf = str(src_client.get("leaf", "")) + dst_ip = "" + # Prefer cross-rack destination + for other in clients[1:]: + other_ip = str(other.get("data_ip", "")).strip() + other_leaf = str(other.get("leaf", "")).strip() + if other_leaf != src_leaf and other_ip: + dst_ip = other_ip + break + if not dst_ip and other_ip: + dst_ip = other_ip + if not dst_ip: + errors.append("could not determine destination client IP for health check") + return errors + + src_container = clab_container_name(lab_name, src_name) + connectivity_ok = False + for _ in range(retries): + ret = _docker_exec(src_container, "ping", "-c", "1", "-W", "2", dst_ip) + if ret.returncode == 0: + connectivity_ok = True + break + time.sleep(delay) + if not connectivity_ok: + errors.append(f"client connectivity failed from {src_container} to {dst_ip}") + return errors + + agent_running = False + for _ in range(retries): + ret = _docker_exec(src_container, "ps", "aux") + if "netopsbench.platform.pingmesh.cli" in (ret.stdout or ""): + agent_running = True + break + time.sleep(delay) + if not agent_running: + errors.append(f"Pingmesh agent is not running in {src_container}") + return errors + + # [5/5] InfluxDB observability path + logger.info("[5/5] Checking InfluxDB observability path...") + obs_device = edge_switches[0].name if edge_switches else (routed[0].name if routed else bgp_device) + obs_container = clab_container_name(lab_name, obs_device) + + # Get active interfaces + ret = _docker_exec(obs_container, "bash", "-lc", "show interfaces status") + observed_active_interfaces = sorted(_parse_active_interfaces(ret.stdout or "")) + + # Send syslog marker + syslog_marker = f"NETOPSBENCH_HEALTH_{lab_name}_{int(time.time())}" + if collector_ip: + _docker_exec( + clab_container_name(lab_name, bgp_device), + "bash", + "-lc", + f"logger -n '{collector_ip}' -P 514 -d '{syslog_marker}'", + ) + + # Delegate to the observability validation module + from netopsbench.platform.observability.influxdb import query_flux + from netopsbench.platform.observability.validation import check_observability + + def query_runner(query: str) -> str: + result = query_flux(influxdb_url, influxdb_token, influxdb_org, query, timeout=20) + if result.status != "ok": + raise RuntimeError(result.error or "InfluxDB query failed") + return result.text + + obs_errors: list[str] = [] + for attempt in range(retries): + obs_errors = check_observability( + query_runner, + bucket=worker.bucket, + obs_device=obs_device, + bgp_device=bgp_device, + topology_id=worker.topology_id, + syslog_marker=syslog_marker, + active_interfaces=observed_active_interfaces, + min_active_coverage_ratio=ACTIVE_INTERFACE_COVERAGE_MIN_RATIO, + ) + if not obs_errors: + break + if attempt + 1 < retries: + time.sleep(delay) + errors.extend(obs_errors) + + if not errors: + logger.info("Worker health check passed: %s", lab_name) + return errors + + +__all__ = ["check_worker_health"] diff --git a/netopsbench/platform/runtime/lab_lifecycle.py b/netopsbench/platform/runtime/lab_lifecycle.py deleted file mode 100644 index 45a5be5..0000000 --- a/netopsbench/platform/runtime/lab_lifecycle.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Internal lab lifecycle helpers for runtime provisioning.""" - -from __future__ import annotations - -import os - -from netopsbench.config import repo_root -from netopsbench.platform.topology.topology_utils import resolve_topology_dir -from netopsbench.platform.utils.proc import safe_run - - -def _repo_root() -> str: - return str(repo_root()) - - -_SCRIPT_LOCATIONS = { - "deploy.sh": os.path.join("scripts", "runtime", "deploy.sh"), - "teardown.sh": os.path.join("scripts", "runtime", "teardown.sh"), -} - - -def _script_path(name: str) -> str: - rel_path = _SCRIPT_LOCATIONS.get(name, os.path.join("scripts", name)) - return os.path.join(_repo_root(), rel_path) - - -def resolve_generated_topology_dir(topology_dir: str | None, scale: str) -> str: - base = topology_dir or "lab-topology" - if base == "generated_topology": - return f"lab-topology/generated_topology_{scale}" - if any(base.endswith(f"generated_topology_{item}") for item in ("xs", "small", "medium", "large")): - return base - return os.path.join(base, f"generated_topology_{scale}") - - -def deploy_lab( - scale: str, - topology_dir: str | None, - lab_name: str | None = None, - mgmt_subnet: str | None = None, - mgmt_network: str | None = None, -) -> int: - script_path = _script_path("deploy.sh") - if not os.path.exists(script_path): - return 1 - topo_dir = topology_dir or "lab-topology" - env = os.environ.copy() - if lab_name: - env["NETOPSBENCH_LAB_NAME"] = lab_name - if mgmt_subnet: - env["NETOPSBENCH_MGMT_SUBNET"] = mgmt_subnet - if mgmt_network: - env["NETOPSBENCH_MGMT_NETWORK"] = mgmt_network - result = safe_run( - ["bash", script_path, scale, topo_dir], - check=False, - env=env, - timeout=1800, - ) - return result.returncode - - -def _can_teardown_topology_dir(topo_dir: str) -> bool: - has_topology_file = os.path.exists(os.path.join(topo_dir, "dcn.clab.yaml")) - has_any_clab_file = ( - any(name.endswith((".clab.yaml", ".clab.yml")) for name in os.listdir(topo_dir)) - if os.path.isdir(topo_dir) - else False - ) - return has_topology_file or has_any_clab_file - - -def teardown_lab(topology_dir: str | None) -> int: - topo_dir = resolve_topology_dir(topology_dir, "lab-topology") - if not _can_teardown_topology_dir(topo_dir): - return 1 - script_path = _script_path("teardown.sh") - if not os.path.exists(script_path): - return 1 - result = safe_run(["bash", script_path, topo_dir], check=False, timeout=600) - return result.returncode diff --git a/netopsbench/platform/runtime/lifecycle.py b/netopsbench/platform/runtime/lifecycle.py new file mode 100644 index 0000000..aaf6ea6 --- /dev/null +++ b/netopsbench/platform/runtime/lifecycle.py @@ -0,0 +1,235 @@ +"""Structured, idempotent runtime lifecycle orchestration.""" + +from __future__ import annotations + +import logging +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import UTC, datetime +from pathlib import Path +from time import monotonic +from typing import Any, Literal, Protocol + +from pydantic import BaseModel, ConfigDict, Field + +from netopsbench.config import config +from netopsbench.models.profiles import get_scale_profile +from netopsbench.models.runtime import RuntimeIdentity +from netopsbench.platform.observability.lifecycle import ensure_worker_observability +from netopsbench.platform.pingmesh.deploy import deploy_pingmesh +from netopsbench.platform.runtime.deployment import ( + allocate_management_subnets, + deploy_worker_lab, + runtime_deploy_lock, + teardown_worker_lab, +) +from netopsbench.platform.runtime.health import check_worker_health + +logger = logging.getLogger(__name__) + + +class RuntimePoolLike(Protocol): + scale: str + root_dir: Path + workers: list[RuntimeIdentity] + + @property + def size(self) -> int: ... + + +class LifecycleStageResult(BaseModel): + """Persisted result for one runtime lifecycle stage.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + stage: str + status: Literal["completed", "failed"] + started_at: datetime + ended_at: datetime + duration_seconds: float = Field(ge=0) + details: dict[str, Any] = Field(default_factory=dict) + error: str | None = None + + +class RuntimeLifecycleError(RuntimeError): + """Raised when a runtime lifecycle stage fails.""" + + def __init__(self, result: LifecycleStageResult): + self.result = result + super().__init__(f"Runtime lifecycle stage {result.stage!r} failed: {result.error}") + + +def _parallel_job_count(scale: str, total: int) -> int: + configured = get_scale_profile(scale).worker_deploy_parallelism + return max(1, min(total, configured)) + + +def _append_worker_log_header(path: Path, label: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "a", encoding="utf-8") as handle: + handle.write(f"\n=== {label} ===\n") + + +def _worker_deploy_log_path(worker: RuntimeIdentity, runtime_root: Path | None = None) -> Path: + if runtime_root is None: + return worker.topology_dir / "deploy.log" + return runtime_root / "logs" / f"worker_{worker.worker_index:02d}.deploy.log" + + +def deploy_workers(workers: Sequence[RuntimeIdentity], scale: str, runtime_root: Path | None = None) -> None: + if not workers: + return + job_count = _parallel_job_count(scale, len(workers)) + + def deploy(worker: RuntimeIdentity) -> None: + logger.info( + "[Worker Deploy %s/%s] %s subnet=%s", + worker.worker_index, + len(workers), + worker.lab_name, + worker.mgmt_subnet, + ) + _append_worker_log_header(_worker_deploy_log_path(worker, runtime_root), f"worker deploy {worker.lab_name}") + deploy_worker_lab(worker, scale) + + if job_count == 1: + for worker in workers: + deploy(worker) + return + + failures: list[tuple[RuntimeIdentity, Exception]] = [] + with ThreadPoolExecutor(max_workers=job_count) as executor: + future_map = {executor.submit(deploy, worker): worker for worker in workers} + for future in as_completed(future_map): + try: + future.result() + except Exception as exc: + failures.append((future_map[future], exc)) + if failures: + worker, error = failures[0] + logger.error( + "Worker deployment failed for %s; see %s", + worker.lab_name, + _worker_deploy_log_path(worker, runtime_root), + ) + raise error + + +def ensure_worker_pingmesh(worker: RuntimeIdentity) -> None: + deploy_pingmesh( + topology_dir=str(worker.topology_dir), + pinglist_file=str(Path(worker.topology_dir) / "configs" / "pingmesh" / "pinglist.json"), + influxdb_token=config.influxdb_token, + influxdb_org=config.influxdb_org, + influxdb_bucket=worker.bucket, + topology_id=worker.topology_id, + ) + + +def validate_worker_health(worker: RuntimeIdentity, runtime_root: Path | None = None) -> None: + log_path = _worker_deploy_log_path(worker, runtime_root) + _append_worker_log_header(log_path, "worker health validation") + errors = check_worker_health( + worker, + ) + if errors: + message = "; ".join(errors) + with open(log_path, "a", encoding="utf-8") as log_file: + log_file.write(f"Health check errors: {message}\n") + raise RuntimeError(f"Worker health check failed: {message}") + + +def teardown_workers(workers: Sequence[RuntimeIdentity]) -> None: + for worker in workers: + try: + teardown_worker_lab(worker) + except Exception: + logger.warning("worker teardown failed for %s", worker.lab_name, exc_info=True) + + +class RuntimeLifecycle: + """Execute the fixed runtime lifecycle stages.""" + + def run(self, stage: str, runtime: RuntimePoolLike) -> LifecycleStageResult: + operations = { + "deploy": self._deploy, + "observability": self._ensure_observability, + "pingmesh": self._ensure_pingmesh, + "warm": self._warm, + "teardown": self._teardown, + } + try: + operation = operations[stage] + except KeyError as exc: + raise ValueError(f"Unknown runtime lifecycle stage: {stage}") from exc + + started_at = datetime.now(UTC) + started_tick = monotonic() + try: + details = operation(runtime) or {} + except Exception as exc: + ended_at = datetime.now(UTC) + result = LifecycleStageResult( + stage=stage, + status="failed", + started_at=started_at, + ended_at=ended_at, + duration_seconds=monotonic() - started_tick, + error=f"{type(exc).__name__}: {exc}", + ) + raise RuntimeLifecycleError(result) from exc + + return LifecycleStageResult( + stage=stage, + status="completed", + started_at=started_at, + ended_at=datetime.now(UTC), + duration_seconds=monotonic() - started_tick, + details=details, + ) + + @staticmethod + def _deploy(runtime: RuntimePoolLike) -> dict[str, Any]: + with runtime_deploy_lock(): + subnets = allocate_management_subnets(runtime.scale, runtime.size) + runtime.workers = [ + worker.model_copy(update={"mgmt_subnet": subnets[index]}) + for index, worker in enumerate(runtime.workers) + ] + deploy_workers(runtime.workers, runtime.scale, runtime.root_dir) + return {"workers": runtime.size} + + @staticmethod + def _ensure_observability(runtime: RuntimePoolLike) -> dict[str, Any]: + for worker in runtime.workers: + ensure_worker_observability(worker) + return {"workers": runtime.size} + + @staticmethod + def _ensure_pingmesh(runtime: RuntimePoolLike) -> dict[str, Any]: + for worker in runtime.workers: + ensure_worker_pingmesh(worker) + return {"workers": runtime.size} + + @staticmethod + def _warm(runtime: RuntimePoolLike) -> dict[str, Any]: + for worker in runtime.workers: + validate_worker_health(worker, runtime.root_dir) + return {"workers": runtime.size, "health": "ready"} + + @staticmethod + def _teardown(runtime: RuntimePoolLike) -> dict[str, Any]: + teardown_workers(runtime.workers) + return {"workers": runtime.size} + + +__all__ = [ + "LifecycleStageResult", + "RuntimeLifecycle", + "RuntimeLifecycleError", + "deploy_workers", + "ensure_worker_observability", + "ensure_worker_pingmesh", + "teardown_workers", + "validate_worker_health", +] diff --git a/netopsbench/platform/runtime/manager.py b/netopsbench/platform/runtime/manager.py index 2b52764..452842a 100644 --- a/netopsbench/platform/runtime/manager.py +++ b/netopsbench/platform/runtime/manager.py @@ -4,32 +4,29 @@ import builtins import json -import re import shutil +import uuid from dataclasses import dataclass, field from pathlib import Path -from netopsbench.config import repo_root +from pydantic import ValidationError + from netopsbench.logging_utils import get_logger -from netopsbench.platform.utils.proc import safe_run, sudo_prefix -from netopsbench.platform.worker.lifecycle import deploy_workers, teardown_workers -from netopsbench.platform.worker.pool import WorkerSpec, _worker_bucket, _worker_mgmt_subnet +from netopsbench.models.runtime import RuntimeIdentity +from netopsbench.platform.runtime.deployment import management_subnet +from netopsbench.platform.runtime.lifecycle import ( + LifecycleStageResult, + RuntimeLifecycle, + RuntimeLifecycleError, + teardown_workers, +) +from netopsbench.platform.utils.proc import safe_run logger = get_logger(__name__) -def _runtime_subnet_base(scale: str, runtime_name: str) -> int: - base = _worker_mgmt_subnet(scale, 1).split(".")[2] - try: - base_value = int(base) - 1 - except ValueError: - base_value = 100 - match = re.search(r"(\d+)(?!.*\d)", runtime_name or "") - if match: - offset = int(match.group(1)) % 80 - else: - offset = sum(ord(ch) for ch in (runtime_name or scale)) % 80 - return min(254, base_value + offset) +class RuntimeMetadataError(ValueError): + """Raised when persisted runtime identity metadata is missing or invalid.""" @dataclass @@ -38,9 +35,10 @@ class RuntimePool: name: str scale: str root_dir: Path - workers: list[WorkerSpec] + workers: list[RuntimeIdentity] state: str = "created" metadata: dict[str, object] = field(default_factory=dict) + stage_results: dict[str, LifecycleStageResult] = field(default_factory=dict) @property def size(self) -> int: @@ -48,61 +46,59 @@ def size(self) -> int: def _payload(self) -> dict[str, object]: return { + "schema_version": "3", "id": self.id, "name": self.name, "scale": self.scale, "state": self.state, "metadata": dict(self.metadata), - "workers": [ - { - "id": worker.id, - "index": worker.index, - "name": worker.name, - "root_dir": str(worker.root_dir) if worker.root_dir is not None else None, - "lab_name": worker.lab_name, - "topology_dir": worker.topology_dir, - "mgmt_subnet": worker.mgmt_subnet, - "bucket": worker.bucket, - "shard_dir": worker.shard_dir, - "report_path": worker.report_path, - "log_path": worker.log_path, - "deploy_log_path": worker.deploy_log_path, - "metadata": dict(worker.metadata), - } - for worker in self.workers - ], + "stage_results": {stage: result.model_dump(mode="json") for stage, result in self.stage_results.items()}, + "workers": [worker.model_dump(mode="json") for worker in self.workers], } def _write_metadata(self) -> None: self.root_dir.mkdir(parents=True, exist_ok=True) (self.root_dir / "runtime.json").write_text(json.dumps(self._payload(), indent=2), encoding="utf-8") - def deploy(self) -> RuntimePool: - self.state = "deployed" + def _run_stage(self, stage: str, next_state: str) -> RuntimePool: + previous = self.stage_results.get(stage) + if previous is not None and previous.status == "completed": + return self + try: + result = RuntimeLifecycle().run(stage, self) + except RuntimeLifecycleError as exc: + self.stage_results[stage] = exc.result + self._write_metadata() + raise + self.stage_results[stage] = result + self.state = next_state self._write_metadata() return self + def deploy(self) -> RuntimePool: + return self._run_stage("deploy", "deployed") + def ensure_observability(self) -> RuntimePool: - self.state = "observability_ready" - self._write_metadata() - return self + return self._run_stage("observability", "observability_ready") def ensure_pingmesh(self) -> RuntimePool: - self.state = "pingmesh_ready" - self._write_metadata() - return self + return self._run_stage("pingmesh", "pingmesh_ready") def warm(self) -> RuntimePool: - self.state = "warm" - self._write_metadata() - return self + return self._run_stage("warm", "warm") def status(self) -> dict[str, object]: return {"id": self.id, "name": self.name, "scale": self.scale, "state": self.state} def teardown(self) -> RuntimePool: - if self.workers: - teardown_workers(self.workers, str(repo_root())) + if self.state == "torn_down": + return self + try: + RuntimeLifecycle().run("teardown", self) + except RuntimeLifecycleError as exc: + self.stage_results["teardown"] = exc.result + self._write_metadata() + raise self.state = "torn_down" metadata_path = self.root_dir / "runtime.json" if metadata_path.exists(): @@ -114,7 +110,7 @@ def teardown(self) -> RuntimePool: class RuntimeManager: def __init__(self, workspace: str = "."): - self.workspace = Path(workspace) + self.workspace = Path(workspace).expanduser().resolve() self.runtime_root_dir = self.workspace / ".netopsbench" / "runtimes" self.runtime_root_dir.mkdir(parents=True, exist_ok=True) @@ -122,119 +118,51 @@ def _build_runtime( self, *, scale: str, workers: int = 1, name: str | None = None, root_dir: Path | None = None ) -> RuntimePool: worker_count = max(1, int(workers)) - runtime_name = str(name or f"{scale}-{worker_count}").strip() + runtime_name = str(name or f"{scale}-{worker_count}-{uuid.uuid4().hex[:8]}").strip() runtime_root = Path(root_dir) if root_dir is not None else (self.runtime_root_dir / runtime_name) - subnet_base = _runtime_subnet_base(scale, runtime_name) runtime_root.mkdir(parents=True, exist_ok=True) logs_dir = runtime_root / "logs" logs_dir.mkdir(parents=True, exist_ok=True) - worker_items: list[WorkerSpec] = [] + worker_items: list[RuntimeIdentity] = [] for idx in range(1, worker_count + 1): worker_name = f"worker-{idx}" worker_dir = runtime_root / worker_name worker_dir.mkdir(exist_ok=True) - worker_items.append( - WorkerSpec( - id=worker_name, - name=worker_name, - root_dir=worker_dir, - index=idx, - lab_name=(runtime_name if worker_count == 1 else f"{runtime_name}-w{idx:02d}"), - topology_dir=str(worker_dir), - mgmt_subnet=f"172.31.{min(254, subnet_base + idx)}.0/24", - bucket=_worker_bucket(scale, idx), - shard_dir=str(worker_dir / "scenarios"), - report_path=str(logs_dir / f"worker_{idx:02d}.report.json"), - log_path=str(logs_dir / f"worker_{idx:02d}.log"), - deploy_log_path=str(logs_dir / f"worker_{idx:02d}.deploy.log"), - ) + lab_name = runtime_name if worker_count == 1 else f"{runtime_name}-w{idx:02d}" + mgmt_subnet = management_subnet(scale, idx) + identity = RuntimeIdentity.create( + runtime_id=runtime_name, + worker_id=worker_name, + worker_index=idx, + lab_name=lab_name, + topology_dir=worker_dir, + mgmt_subnet=mgmt_subnet, + mgmt_network=f"clab-mgmt-{lab_name}", ) + worker_items.append(identity) runtime = RuntimePool( - id=runtime_name, name=runtime_name, scale=scale, root_dir=runtime_root, workers=worker_items + id=runtime_name, + name=runtime_name, + scale=scale, + root_dir=runtime_root, + workers=worker_items, ) runtime._write_metadata() return runtime - def _used_management_subnets(self) -> set[str]: - try: - result = safe_run( - [*sudo_prefix(), "docker", "network", "ls", "--format", "{{.ID}}"], - capture_output=True, - text=True, - check=False, - timeout=30, - ) - except Exception: - logger.warning("docker network ls failed; assuming no networks in use", exc_info=True) - return set() - used: set[str] = set() - for network_id in [line.strip() for line in result.stdout.splitlines() if line.strip()]: - inspect = safe_run( - [ - *sudo_prefix(), - "docker", - "network", - "inspect", - network_id, - "--format", - "{{range .IPAM.Config}}{{println .Subnet}}{{end}}", - ], - capture_output=True, - text=True, - check=False, - timeout=30, - ) - for subnet in inspect.stdout.splitlines(): - subnet = subnet.strip() - if subnet: - used.add(subnet) - return used - - def _allocate_management_subnets(self, scale: str, worker_count: int) -> builtins.list[str]: - used = self._used_management_subnets() - start = int(_worker_mgmt_subnet(scale, 1).split(".")[2]) - selected: list[str] = [] - for octet in range(start, 255): - subnet = f"172.31.{octet}.0/24" - if subnet in used or subnet in selected: - continue - selected.append(subnet) - if len(selected) == worker_count: - return selected - raise RuntimeError(f"unable to allocate {worker_count} unique management subnets for scale {scale}") - def provision( self, *, scale: str, workers: int = 1, name: str | None = None, root_dir: Path | None = None ) -> RuntimePool: runtime = self._build_runtime(scale=scale, workers=workers, name=name, root_dir=root_dir) try: - allocated_subnets = self._allocate_management_subnets(scale, runtime.size) - runtime.workers = [ - WorkerSpec( - id=worker.id, - name=worker.name, - root_dir=worker.root_dir, - index=worker.index, - lab_name=worker.lab_name, - topology_dir=worker.topology_dir, - mgmt_subnet=allocated_subnets[idx], - bucket=worker.bucket, - shard_dir=worker.shard_dir, - report_path=worker.report_path, - log_path=worker.log_path, - deploy_log_path=worker.deploy_log_path, - metadata=dict(worker.metadata), - ) - for idx, worker in enumerate(runtime.workers) - ] - deploy_workers(runtime.workers, scale, str(repo_root())) + runtime.deploy().ensure_observability().ensure_pingmesh().warm() except Exception: logger.warning("Worker deployment failed; tearing down partial state", exc_info=True) # Best-effort teardown of partially-deployed workers (Docker # networks, containers, telegraf instances, etc.) before # cleaning up metadata on disk. try: - teardown_workers(runtime.workers, str(repo_root())) + teardown_workers(runtime.workers) except Exception: logger.warning("Best-effort teardown_workers failed during cleanup", exc_info=True) # Use sudo rm to handle root-owned files left by containerlab. @@ -251,7 +179,7 @@ def provision( shutil.rmtree(runtime.root_dir, ignore_errors=True) raise runtime.metadata["provisioning_mode"] = "worker_pool" - runtime.state = "deployed" + runtime.state = "warm" runtime._write_metadata() return runtime @@ -260,31 +188,31 @@ def create(self, *, scale: str, workers: int = 1, name: str | None = None) -> Ru def attach(self, root_dir: Path) -> RuntimePool: runtime_path = Path(root_dir) - payload = json.loads((runtime_path / "runtime.json").read_text(encoding="utf-8")) - required_keys = {"id", "name", "scale", "workers"} + metadata_path = runtime_path / "runtime.json" + try: + payload = json.loads(metadata_path.read_text(encoding="utf-8")) + except FileNotFoundError: + raise + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeMetadataError(f"Unable to read runtime metadata {metadata_path}: {exc}") from exc + if payload.get("schema_version") != "3": + raise RuntimeMetadataError( + "Unsupported runtime.json schema; recreate the runtime with the current RuntimeManager" + ) + required_keys = {"schema_version", "id", "name", "scale", "workers", "stage_results"} missing = sorted(required_keys - set(payload)) if missing: - raise ValueError(f"missing required runtime metadata: {', '.join(missing)}") - worker_items = [ - WorkerSpec( - id=str(item.get("id") or item.get("name") or f"worker-{index+1}"), - name=str(item.get("name") or item.get("id") or f"worker-{index+1}"), - root_dir=Path(item["root_dir"]) if item.get("root_dir") else None, - index=int(item.get("index", index + 1)), - lab_name=item.get("lab_name") or str(item.get("name") or item.get("id") or f"worker-{index+1}"), - topology_dir=item.get("topology_dir") - or str(item.get("root_dir") or runtime_path / f"worker-{index+1}"), - mgmt_subnet=item.get("mgmt_subnet") or _worker_mgmt_subnet(str(payload["scale"]), index + 1), - bucket=item.get("bucket") or _worker_bucket(str(payload["scale"]), index + 1), - shard_dir=item.get("shard_dir") or str(runtime_path / f"worker-{index+1}" / "scenarios"), - report_path=item.get("report_path") or str(runtime_path / "logs" / f"worker_{index+1:02d}.report.json"), - log_path=item.get("log_path") or str(runtime_path / "logs" / f"worker_{index+1:02d}.log"), - deploy_log_path=item.get("deploy_log_path") - or str(runtime_path / "logs" / f"worker_{index+1:02d}.deploy.log"), - metadata=dict(item.get("metadata", {})), - ) - for index, item in enumerate(payload["workers"]) - ] + raise RuntimeMetadataError(f"missing required runtime metadata: {', '.join(missing)}") + try: + worker_items = [RuntimeIdentity.model_validate(item) for item in payload["workers"]] + stage_results = { + stage: LifecycleStageResult.model_validate(result) + for stage, result in dict(payload.get("stage_results", {})).items() + } + except (KeyError, TypeError, ValidationError, ValueError) as exc: + raise RuntimeMetadataError( + f"Invalid schema-v3 runtime metadata in {metadata_path}; recreate the runtime: {exc}" + ) from exc return RuntimePool( id=str(payload["id"]), name=str(payload["name"]), @@ -293,6 +221,7 @@ def attach(self, root_dir: Path) -> RuntimePool: workers=worker_items, state=str(payload.get("state", "created")), metadata=dict(payload.get("metadata", {})), + stage_results=stage_results, ) def list(self) -> builtins.list[RuntimePool]: @@ -313,4 +242,4 @@ def get(self, name: str) -> RuntimePool | None: return None -__all__ = ["RuntimeManager", "RuntimePool"] +__all__ = ["RuntimeManager", "RuntimeMetadataError", "RuntimePool"] diff --git a/netopsbench/platform/scenario/__init__.py b/netopsbench/platform/scenario/__init__.py index cce9099..7bb28a1 100644 --- a/netopsbench/platform/scenario/__init__.py +++ b/netopsbench/platform/scenario/__init__.py @@ -1,9 +1 @@ -""" -Scenario Management for Automated Fault Injection -""" - -from .models import Episode, Scenario -from .parser import parse_scenario_file -from .validator import validate_scenario, validate_scenario_topology - -__all__ = ["Scenario", "Episode", "parse_scenario_file", "validate_scenario", "validate_scenario_topology"] +"""Internal scenario execution subsystem.""" diff --git a/netopsbench/platform/scenario/episode_runner.py b/netopsbench/platform/scenario/episode_runner.py index 49f5318..b9c1958 100644 --- a/netopsbench/platform/scenario/episode_runner.py +++ b/netopsbench/platform/scenario/episode_runner.py @@ -1,225 +1,138 @@ -"""Per-episode execution helper extracted from :class:`ScenarioExecutor`. - -Keeping the episode loop in its own class makes it easier to unit-test the -inject → observe → diagnose → recover state machine in isolation, and shrinks -``executor.py`` so it can focus on scenario-level orchestration. -""" +"""Per-episode execution for :class:`ScenarioExecutor`.""" from __future__ import annotations import time from collections.abc import Callable -from datetime import datetime -from typing import Any, Protocol, runtime_checkable +from datetime import UTC, datetime +from typing import Any -from netopsbench.platform.utils.events import emit as _emit +from netopsbench.logging_utils import get_logger from .models import Episode +logger = get_logger(__name__) -@runtime_checkable -class EpisodeExecutionPort(Protocol): - """Contract that :class:`EpisodeRunner` requires from its host executor. - - This Protocol exists to make the (historically implicit) coupling - between :class:`EpisodeRunner` and :class:`ScenarioExecutor` explicit, - so test stubs and future executors can satisfy a typed surface instead - of duck-typing five underscore methods. The underscore names are - preserved to keep the existing executor implementation source-compatible. - """ - - skip_none_episodes: bool - post_recovery_wait_seconds: float - - def _build_skipped_episode_result( - self, - episode: Episode, - *, - start_time: str, - end_time: str, - waited_seconds: int, - ) -> dict[str, Any]: ... - - def _inject_fault(self, episode: Episode) -> dict[str, Any]: ... - - def _wait_and_observe( - self, - seconds: int, - *, - baseline_end_time: datetime | None = None, - ) -> dict[str, Any]: ... - - def _merge_observation_windows( - self, - windows: list[dict[str, Any]], - *, - total_duration_seconds: int, - ) -> dict[str, Any]: ... - - def _recover_fault(self) -> dict[str, Any]: ... - - -class EpisodeRunner: - """Run a single :class:`Episode` against an :class:`EpisodeExecutionPort`. - - The runner does not own state of its own — it delegates fault injection, - observation, and recovery to the executor it was constructed with. This - keeps backward-compatible behaviour while letting tests substitute a - minimal stub executor that satisfies :class:`EpisodeExecutionPort`. - """ - - def __init__(self, executor: EpisodeExecutionPort): - self.executor = executor - - # ------------------------------------------------------------------ - # Public entry point - # ------------------------------------------------------------------ - - def run( - self, - episode: Episode, - diagnosis_callback: Callable[[dict], dict] | None = None, - diagnose_if_skipped: bool = False, - ) -> dict: - """Execute ``episode`` and return its result dict. - - Args: - diagnose_if_skipped: When True and the episode is skipped (fault_type=none), - still invoke ``diagnosis_callback`` after the observation wait. Used for - negative-sample (healthy-network) scenarios so the agent can produce a - verdict that is checked for false-positives. - """ - executor = self.executor - _emit(f"\n{'='*70}") - _emit(f"Episode: {episode.episode_id}") - _emit(f"Description: {episode.description}") - _emit(f"{'='*70}") - - episode_result: dict[str, Any] = { - "episode_id": episode.episode_id, - "description": episode.description, - "episode": _episode_payload(episode), - "start_time": datetime.now().isoformat(), - "success": False, - "error": None, - } - try: - if executor.skip_none_episodes and episode.fault_type == "none": - _emit("\n[Episode] Skipping baseline fault actions (fault_type=none)") - skipped_start = datetime.now().isoformat() - waited_seconds = max(0, int(episode.duration_seconds or 0)) - if waited_seconds > 0: - _emit(f"[Episode] Waiting {waited_seconds}s to reserve a clean pre-fault baseline window...") - if diagnose_if_skipped: - skipped_observations = executor._wait_and_observe(waited_seconds) - else: - _sleep(executor, waited_seconds) - skipped_observations = None +def run_episode( + executor: Any, + episode: Episode, + diagnosis_callback: Callable[[dict], dict] | None = None, + diagnose_if_skipped: bool = False, +) -> dict: + """Execute one episode using the owning scenario executor.""" + logger.info(f"\n{'=' * 70}") + logger.info(f"Episode: {episode.episode_id}") + logger.info(f"Description: {episode.description}") + logger.info(f"{'=' * 70}") + + episode_result: dict[str, Any] = { + "episode_id": episode.episode_id, + "description": episode.description, + "episode": _episode_payload(episode), + "start_time": datetime.now(UTC).isoformat(), + "success": False, + "error": None, + } + + try: + if executor.skip_none_episodes and episode.fault_type == "none": + logger.info("\n[Episode] Skipping baseline fault actions (fault_type=none)") + skipped_start = datetime.now(UTC).isoformat() + waited_seconds = max(0, int(episode.duration_seconds or 0)) + if waited_seconds > 0: + logger.info(f"[Episode] Waiting {waited_seconds}s to reserve a clean pre-fault baseline window...") + if diagnose_if_skipped: + skipped_observations = executor._wait_and_observe(waited_seconds) else: + _sleep(executor, waited_seconds) skipped_observations = None - skipped_end = datetime.now().isoformat() - skipped_result = executor._build_skipped_episode_result( - episode, - start_time=skipped_start, - end_time=skipped_end, - waited_seconds=waited_seconds, - ) - if skipped_observations is not None: - skipped_result["observations"] = skipped_observations - # For negative-sample (healthy-network) episodes, call the agent so - # we can measure false-positive rate. The skipped_result already - # contains the episode payload needed by the callback. - if diagnosis_callback and diagnose_if_skipped: - _emit("\n[Diagnosis] Calling agent on healthy-network episode (negative sample)...") - try: - skipped_result["diagnosis"] = diagnosis_callback(skipped_result) - except Exception as diagnosis_error: # noqa: BLE001 - skipped_result["diagnosis"] = { - "error": str(diagnosis_error), - "success": False, - } - return skipped_result - - pre_fault_reference = datetime.utcnow().replace(microsecond=0) - - # Step 1: Inject fault - injection_result = executor._inject_fault(episode) - episode_result["injection"] = injection_result - - if not injection_result.get("success"): - episode_result["error"] = "Fault injection failed" - return episode_result - - # Step 2: Dual-window observation (early + steady) for transient capture. - early_observation_seconds = int( - episode.metadata.get( - "early_observation_seconds", - min(20, max(10, episode.duration_seconds // 3)), - ) - ) - if early_observation_seconds >= episode.duration_seconds: - early_observation_seconds = max(0, episode.duration_seconds - 10) - steady_observation_seconds = max(1, episode.duration_seconds - early_observation_seconds) - - observation_windows: list[dict] = [] - if early_observation_seconds > 0: - _emit("\n[Early Observation] " f"Monitoring immediately for {early_observation_seconds} seconds...") - early_observation = executor._wait_and_observe( - early_observation_seconds, - baseline_end_time=pre_fault_reference, - ) - observation_windows.append(early_observation) - - # Stabilization wait before the steady window. - _emit(f"\n[Stabilization] Waiting {episode.stabilization_time}s...") - _sleep(executor, episode.stabilization_time) - - _emit("\n[Steady Observation] " f"Monitoring stabilized fault for {steady_observation_seconds} seconds...") - steady_observation = executor._wait_and_observe( - steady_observation_seconds, - baseline_end_time=pre_fault_reference, - ) - observation_windows.append(steady_observation) - - episode_result["observations"] = executor._merge_observation_windows( - observation_windows, - total_duration_seconds=episode.duration_seconds, + else: + skipped_observations = None + skipped_end = datetime.now(UTC).isoformat() + skipped_result = executor._build_skipped_episode_result( + episode, + start_time=skipped_start, + end_time=skipped_end, + waited_seconds=waited_seconds, ) - - # Step 3.5: Optional in-fault diagnosis (skip baseline episodes). - if diagnosis_callback and episode.fault_type != "none": + if skipped_observations is not None: + skipped_result["observations"] = skipped_observations + if diagnosis_callback and diagnose_if_skipped: + logger.info("\n[Diagnosis] Calling agent on healthy-network episode (negative sample)...") try: - episode_result["diagnosis"] = diagnosis_callback(episode_result) - except Exception as diagnosis_error: # noqa: BLE001 — surface via result - episode_result["diagnosis"] = { + skipped_result["diagnosis"] = diagnosis_callback(skipped_result) + except Exception as diagnosis_error: # noqa: BLE001 + skipped_result["diagnosis"] = { "error": str(diagnosis_error), "success": False, } + return skipped_result - # Step 4: Recovery + post-recovery settle. - recovery_results = executor._recover_fault() - episode_result["recovery"] = recovery_results - - _emit(f"\n[Post-Recovery] Waiting {executor.post_recovery_wait_seconds}s for recovery...") - _sleep(executor, executor.post_recovery_wait_seconds) - - episode_result["success"] = True - episode_result["end_time"] = datetime.now().isoformat() + pre_fault_reference = datetime.now(UTC).replace(microsecond=0) + injection_result = executor._inject_fault(episode) + episode_result["injection"] = injection_result + if not injection_result.get("success"): + episode_result["error"] = "Fault injection failed" return episode_result - except Exception as exc: # noqa: BLE001 — preserve historical behaviour - _emit(f"\n✗ Episode failed: {exc}") - episode_result["error"] = str(exc) - episode_result["end_time"] = datetime.now().isoformat() - - # Best-effort recovery even on failure. + early_observation_seconds = int( + episode.metadata.get( + "early_observation_seconds", + min(20, max(10, episode.duration_seconds // 3)), + ) + ) + if early_observation_seconds >= episode.duration_seconds: + early_observation_seconds = max(0, episode.duration_seconds - 10) + steady_observation_seconds = max(1, episode.duration_seconds - early_observation_seconds) + + observation_windows: list[dict] = [] + if early_observation_seconds > 0: + logger.info(f"\n[Early Observation] Monitoring immediately for {early_observation_seconds} seconds...") + observation_windows.append(executor._capture_observation_window(early_observation_seconds, "early")) + + logger.info(f"\n[Stabilization] Waiting {episode.stabilization_time}s...") + _sleep(executor, episode.stabilization_time) + + logger.info(f"\n[Steady Observation] Monitoring stabilized fault for {steady_observation_seconds} seconds...") + observation_windows.append(executor._capture_observation_window(steady_observation_seconds, "steady")) + + episode_result["observations"] = executor._merge_observation_windows( + observation_windows, + total_duration_seconds=episode.duration_seconds, + baseline_end_time=pre_fault_reference, + ) + coverage_audit = episode_result["observations"].pop("_coverage_audit", None) + + if diagnosis_callback and episode.fault_type != "none": try: - executor._recover_fault() - except Exception as recovery_error: # noqa: BLE001 - _emit(f" Warning: Recovery failed: {recovery_error}") - - return episode_result + episode_result["diagnosis"] = diagnosis_callback(episode_result) + except Exception as diagnosis_error: # noqa: BLE001 + episode_result["diagnosis"] = { + "error": str(diagnosis_error), + "success": False, + } + + if coverage_audit is not None: + episode_result["coverage_audit"] = coverage_audit + + episode_result["recovery"] = executor._recover_fault() + logger.info(f"\n[Post-Recovery] Waiting {executor.post_recovery_wait_seconds}s for recovery...") + _sleep(executor, executor.post_recovery_wait_seconds) + + episode_result["success"] = True + episode_result["end_time"] = datetime.now(UTC).isoformat() + return episode_result + + except Exception as exc: # noqa: BLE001 + logger.info(f"\nEpisode failed: {exc}") + episode_result["error"] = str(exc) + episode_result["end_time"] = datetime.now(UTC).isoformat() + try: + executor._recover_fault() + except Exception as recovery_error: # noqa: BLE001 + logger.info(f"Warning: Recovery failed: {recovery_error}") + return episode_result def _episode_payload(episode: Episode) -> dict[str, Any]: @@ -237,9 +150,8 @@ def _episode_payload(episode: Episode) -> dict[str, Any]: } -def _sleep(executor: EpisodeExecutionPort, seconds: float) -> None: - sleep_fn = getattr(executor, "sleep", time.sleep) - sleep_fn(seconds) +def _sleep(executor: Any, seconds: float) -> None: + getattr(executor, "sleep", time.sleep)(seconds) -__all__ = ["EpisodeExecutionPort", "EpisodeRunner"] +__all__ = ["run_episode"] diff --git a/netopsbench/platform/scenario/executor.py b/netopsbench/platform/scenario/executor.py index f168f59..c5c0dcd 100644 --- a/netopsbench/platform/scenario/executor.py +++ b/netopsbench/platform/scenario/executor.py @@ -7,21 +7,23 @@ import json import time from collections.abc import Callable -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from netopsbench.logging_utils import get_logger from netopsbench.platform.faults.injector import FaultInjector from netopsbench.platform.faults.scenario_execution import inject_fault as _inject_fault_impl from netopsbench.platform.faults.scenario_execution import recover_fault as _recover_fault_impl +from netopsbench.platform.faults.specs import FaultSpecRegistry, create_fault_registry +from netopsbench.platform.topology.topology_utils import coerce_topology_manifest, load_topology_manifest from netopsbench.platform.traffic.controller import TrafficController from netopsbench.platform.traffic.scenario_execution import setup_traffic as _setup_traffic_impl from netopsbench.platform.traffic.scenario_execution import stop_traffic as _stop_traffic_impl -from netopsbench.platform.utils.events import emit as _emit -from .episode_runner import EpisodeRunner +from .episode_runner import run_episode from .models import Episode, Scenario -from .observation import merge_observation_windows as _merge_observation_windows_impl +from .observation import analyze_observation_windows as _analyze_observation_windows_impl +from .observation import capture_observation_window as _capture_observation_window_impl from .observation import wait_and_observe as _wait_and_observe_impl logger = get_logger(__name__) @@ -46,6 +48,7 @@ def __init__( topology_id: str | None = None, sleep_fn: Callable[[float], None] | None = None, persist_results: bool = True, + fault_registry: FaultSpecRegistry | None = None, ): """ Initialize scenario runner. @@ -58,21 +61,20 @@ def __init__( if metadata is None: topology_file = Path(topology_dir) / "topology.json" if topology_file.exists(): - with open(topology_file, encoding="utf-8") as handle: - metadata = json.load(handle) - self.topology_metadata = metadata - try: - self.injector = FaultInjector( - clab_dir=topology_dir, - topology_metadata=metadata, - ) - except TypeError: - # Support injectors that still expose a no-argument constructor. - self.injector = FaultInjector() + metadata = load_topology_manifest(topology_file).model_dump(mode="json") + if metadata is None: + raise ValueError(f"Canonical topology metadata is required for ScenarioExecutor: {topology_dir}") + manifest = coerce_topology_manifest(metadata) + self.fault_registry = fault_registry or create_fault_registry() + self.topology_metadata = manifest.model_dump(mode="json") + self.injector = FaultInjector( + clab_dir=topology_dir, + topology_metadata=self.topology_metadata, + fault_registry=self.fault_registry, + ) self.traffic_controller: TrafficController | None = None self.results_dir = Path("scenario_results") - self.results_dir.mkdir(exist_ok=True) - self.topology_id = topology_id or Path(self.topology_dir).resolve().name + self.topology_id = topology_id or manifest.topology_id self.influxdb_url = influxdb_url self.influxdb_token = influxdb_token self.influxdb_org = influxdb_org @@ -98,16 +100,29 @@ def _inject_fault(self, episode: Episode) -> dict: def _wait_and_observe(self, duration: int, baseline_end_time: datetime | None = None) -> dict: return _wait_and_observe_impl(self, duration, baseline_end_time) + def _capture_observation_window(self, duration: int, name: str) -> dict: + return _capture_observation_window_impl(self, duration, name=name) + def _recover_fault(self): return _recover_fault_impl(self) - def _merge_observation_windows(self, windows: list[dict], total_duration_seconds: int) -> dict: - return _merge_observation_windows_impl(windows, total_duration_seconds) + def _merge_observation_windows( + self, + windows: list[dict], + total_duration_seconds: int, + baseline_end_time: datetime | None = None, + ) -> dict: + return _analyze_observation_windows_impl( + self, + windows, + total_duration_seconds, + baseline_end_time=baseline_end_time, + ) def _build_skipped_episode_result( self, episode: Episode, start_time: str | None = None, end_time: str | None = None, waited_seconds: int = 0 ) -> dict: - now = datetime.now().isoformat() + now = datetime.now(UTC).isoformat() return { "episode_id": episode.episode_id, "description": episode.description, @@ -132,8 +147,9 @@ def _build_skipped_episode_result( } def run_episode(self, episode: Episode, diagnosis_callback=None, diagnose_if_skipped: bool = False) -> dict: - """Run a single episode (delegates to :class:`EpisodeRunner`).""" - return EpisodeRunner(self).run( + """Run a single episode.""" + return run_episode( + self, episode, diagnosis_callback=diagnosis_callback, diagnose_if_skipped=diagnose_if_skipped, @@ -149,19 +165,19 @@ def run_scenario(self, scenario: Scenario, diagnosis_callback=None) -> dict: Returns: Scenario result dict """ - _emit(f"\n{'#'*70}") - _emit(f"# Scenario: {scenario.name}") - _emit(f"# ID: {scenario.scenario_id}") - _emit(f"# Description: {scenario.description}") - _emit(f"# Topology: {scenario.topology_scale}") - _emit(f"# Traffic Profile: {scenario.traffic_profile}") - _emit(f"# Episodes: {len(scenario.episodes)}") - _emit(f"{'#'*70}") + logger.info(f"\n{'#'*70}") + logger.info(f"# Scenario: {scenario.name}") + logger.info(f"# ID: {scenario.scenario_id}") + logger.info(f"# Description: {scenario.description}") + logger.info(f"# Topology: {scenario.topology_scale}") + logger.info(f"# Traffic Profile: {scenario.traffic_profile}") + logger.info(f"# Episodes: {len(scenario.episodes)}") + logger.info(f"{'#'*70}") scenario_result = { "scenario_id": scenario.scenario_id, "name": scenario.name, - "start_time": datetime.now().isoformat(), + "start_time": datetime.now(UTC).isoformat(), "topology_scale": scenario.topology_scale, "traffic_profile": scenario.traffic_profile, "episodes": [], @@ -174,14 +190,14 @@ def run_scenario(self, scenario: Scenario, diagnosis_callback=None) -> dict: scenario_result["traffic_config"] = traffic_config # Wait for traffic to stabilize - _emit(f"\n[Baseline] Waiting {self.baseline_wait_seconds}s for traffic baseline...") + logger.info(f"\n[Baseline] Waiting {self.baseline_wait_seconds}s for traffic baseline...") self.sleep(self.baseline_wait_seconds) # Run each episode is_negative_sample = bool((scenario.metadata or {}).get("negative_sample", False)) n_episodes = len(scenario.episodes) for i, episode in enumerate(scenario.episodes, 1): - _emit(f"\n[Episode {i}/{n_episodes}]") + logger.info(f"\n[Episode {i}/{n_episodes}]") # For negative-sample scenarios, diagnose the middle episode so the # agent observes a representative healthy window (false-positive check). diagnose_if_skipped = is_negative_sample and (i - 1) == n_episodes // 2 @@ -193,13 +209,13 @@ def run_scenario(self, scenario: Scenario, diagnosis_callback=None) -> dict: scenario_result["episodes"].append(episode_result) if not episode_result.get("success"): - _emit(f"\n✗ Episode {episode.episode_id} failed, continuing...") + logger.info(f"\n✗ Episode {episode.episode_id} failed, continuing...") # Mark scenario as successful if all episodes completed scenario_result["success"] = all(ep.get("success", False) for ep in scenario_result["episodes"]) except Exception as e: - _emit(f"\n✗ Scenario failed: {e}") + logger.info(f"\n✗ Scenario failed: {e}") scenario_result["error"] = str(e) finally: @@ -207,7 +223,7 @@ def run_scenario(self, scenario: Scenario, diagnosis_callback=None) -> dict: self._stop_traffic() self._recover_fault() - scenario_result["end_time"] = datetime.now().isoformat() + scenario_result["end_time"] = datetime.now(UTC).isoformat() if self.persist_results: result_file = self._persist_scenario_result(scenario, scenario_result) @@ -215,17 +231,18 @@ def run_scenario(self, scenario: Scenario, diagnosis_callback=None) -> dict: else: result_file = scenario_result.get("result_file") - _emit(f"\n{'#'*70}") - _emit("# Scenario Complete") - _emit(f"# Success: {scenario_result['success']}") + logger.info(f"\n{'#'*70}") + logger.info("# Scenario Complete") + logger.info(f"# Success: {scenario_result['success']}") if result_file: - _emit(f"# Results saved to: {result_file}") - _emit(f"{'#'*70}") + logger.info(f"# Results saved to: {result_file}") + logger.info(f"{'#'*70}") return scenario_result def _persist_scenario_result(self, scenario: Scenario, scenario_result: dict) -> Path: - result_file = self.results_dir / f"{scenario.scenario_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + self.results_dir.mkdir(parents=True, exist_ok=True) + result_file = self.results_dir / f"{scenario.scenario_id}_{datetime.now(UTC).strftime('%Y%m%d_%H%M%S')}.json" with open(result_file, "w", encoding="utf-8") as f: json.dump(scenario_result, f, indent=2) return result_file diff --git a/netopsbench/platform/scenario/generator.py b/netopsbench/platform/scenario/generator.py index 16925f8..6f33d79 100644 --- a/netopsbench/platform/scenario/generator.py +++ b/netopsbench/platform/scenario/generator.py @@ -1,30 +1,34 @@ -#!/usr/bin/env python3 """Generate standardized scenario YAML files from topology metadata.""" from __future__ import annotations -import argparse import ipaddress -import json import random import re +from collections.abc import Callable from dataclasses import dataclass, field +from importlib.resources import files from pathlib import Path from typing import Any import yaml -ROOT = Path(__file__).resolve().parents[3] +from netopsbench.models.topology import DeviceRole, TopologyManifest +from netopsbench.platform.topology.configdb_payload import interface_names_for_config +from netopsbench.platform.topology.topology_utils import load_topology_manifest + + +def default_campaign_spec() -> Path: + resource = files("netopsbench.platform.scenario").joinpath("specs", "fault_campaign.yaml") + if not resource.is_file(): + raise FileNotFoundError("Packaged default scenario campaign is missing") + return Path(str(resource)) @dataclass class TopologyContext: - scale: str topology_dir: Path - metadata: dict[str, Any] - spines: list[str] - leafs: list[str] - clients: list[dict[str, Any]] + manifest: TopologyManifest device_interfaces: dict[str, list[str]] leaf_interface_roles: dict[str, dict[str, list[str]]] client_interfaces: dict[str, list[str]] @@ -32,6 +36,41 @@ class TopologyContext: bgp_neighbors: dict[str, list[dict[str, Any]]] = field(default_factory=dict) bgp_networks: dict[str, list[dict[str, Any]]] = field(default_factory=dict) + @property + def scale(self) -> str: + return self.manifest.scale + + @property + def metadata(self) -> dict[str, Any]: + return self.manifest.to_agent_topology() + + def _device_names(self, role: DeviceRole) -> list[str]: + return [device.name for device in self.manifest.devices_by_role(role)] + + @property + def spines(self) -> list[str]: + return self._device_names(DeviceRole.SPINE) + + @property + def leafs(self) -> list[str]: + return self._device_names(DeviceRole.LEAF) + + @property + def cores(self) -> list[str]: + return self._device_names(DeviceRole.CORE) + + @property + def aggs(self) -> list[str]: + return self._device_names(DeviceRole.AGG) + + @property + def edges(self) -> list[str]: + return self._device_names(DeviceRole.EDGE) + + @property + def clients(self) -> list[dict[str, Any]]: + return self.metadata["devices"]["clients"] + def load_yaml(path: Path) -> dict[str, Any]: with path.open("r", encoding="utf-8") as f: @@ -42,56 +81,41 @@ def load_topology(scale: str, topology_dir: str | None) -> TopologyContext: if topology_dir: topo_dir = Path(topology_dir) else: - topo_dir = ROOT / "lab-topology" / f"generated_topology_{scale}" + topo_dir = Path.cwd() / "lab-topology" / f"generated_topology_{scale}" metadata_path = topo_dir / "topology.json" if not metadata_path.exists(): raise FileNotFoundError(f"Topology metadata not found: {metadata_path}") - with metadata_path.open("r", encoding="utf-8") as f: - metadata = json.load(f) - - devices = metadata.get("devices", {}) - spines = [d["name"] for d in devices.get("spines", [])] - leafs = [d["name"] for d in devices.get("leafs", [])] - clients = list(devices.get("clients", [])) - - device_interfaces: dict[str, list[str]] = {} - leaf_interface_roles: dict[str, dict[str, list[str]]] = {} - client_interfaces: dict[str, list[str]] = {} + manifest = load_topology_manifest(metadata_path) + device_interfaces, leaf_interface_roles, client_interfaces = _interface_facts_from_manifest(manifest) device_asns: dict[str, int] = {} bgp_neighbors: dict[str, list[dict[str, Any]]] = {} bgp_networks: dict[str, list[dict[str, Any]]] = {} - clab_path = topo_dir / "dcn.clab.yaml" - if clab_path.exists(): - device_interfaces, leaf_interface_roles, client_interfaces = parse_clab_topology(clab_path) - config_dir = topo_dir / "configs" - for device in spines + leafs: - cfg = config_dir / f"{device}.sh" - if device not in device_interfaces: - device_interfaces[device] = parse_network_interfaces(cfg) - bgp_info = parse_bgp_config(cfg) - if bgp_info.get("local_as") is not None: - device_asns[device] = int(bgp_info["local_as"]) + switch_devices = [device.name for device in manifest.switches()] + for device in switch_devices: + cfg = _device_config_path(config_dir, device) + frr_cfg = config_dir / "frr" / f"{device}.conf" + if not cfg.is_file(): + raise FileNotFoundError(f"SONiC ConfigDB artifact not found: {cfg}") + if not frr_cfg.is_file(): + raise FileNotFoundError(f"FRR artifact not found: {frr_cfg}") + parsed_interfaces = parse_network_interfaces(cfg) + if not parsed_interfaces: + raise ValueError(f"SONiC ConfigDB has no interfaces: {cfg}") + device_interfaces[device] = parsed_interfaces + bgp_info = parse_bgp_config(frr_cfg) + if bgp_info.get("local_as") is None: + raise ValueError(f"FRR artifact has no BGP router stanza: {frr_cfg}") + device_asns[device] = int(bgp_info["local_as"]) bgp_neighbors[device] = list(bgp_info.get("neighbors") or []) bgp_networks[device] = list(bgp_info.get("networks") or []) - if not leaf_interface_roles: - leaf_interface_roles = build_leaf_interface_roles(device_interfaces, metadata) - - if not client_interfaces: - for client in clients: - client_interfaces[client["name"]] = ["eth1"] - return TopologyContext( - scale=scale, topology_dir=topo_dir, - metadata=metadata, - spines=spines, - leafs=leafs, - clients=clients, + manifest=manifest, device_interfaces=device_interfaces, leaf_interface_roles=leaf_interface_roles, client_interfaces=client_interfaces, @@ -101,32 +125,16 @@ def load_topology(scale: str, topology_dir: str | None) -> TopologyContext: ) -def parse_network_interfaces(cfg_path: Path) -> list[str]: - if not cfg_path.exists(): - return ["Ethernet0"] +def _device_config_path(config_dir: Path, device: str) -> Path: + return config_dir / "sonic" / device / "config_db.json" - interfaces = set() - with cfg_path.open("r", encoding="utf-8") as f: - for raw in f: - line = raw.strip() - if line.startswith("config interface startup"): - parts = line.split() - if len(parts) >= 4: - normalized = normalize_sonic_interface(parts[3]) - if normalized: - interfaces.add(normalized) - elif line.startswith("config interface ip add"): - parts = line.split() - if len(parts) >= 5: - normalized = normalize_sonic_interface(parts[4]) - if normalized: - interfaces.add(normalized) - - return sorted(interfaces) if interfaces else ["Ethernet0"] + +def parse_network_interfaces(cfg_path: Path) -> list[str]: + return interface_names_for_config(cfg_path) def parse_bgp_config(cfg_path: Path) -> dict[str, Any]: - """Parse local ASN, neighbors, and advertised networks from one SONiC config script.""" + """Parse local ASN, neighbors, and advertised networks from one FRR config.""" info: dict[str, Any] = {"local_as": None, "neighbors": [], "networks": []} if not cfg_path.exists(): return info @@ -184,58 +192,56 @@ def parse_bgp_config(cfg_path: Path) -> dict[str, Any]: return info -def parse_clab_topology(clab_path: Path): - data = load_yaml(clab_path) - topo = data.get("topology", {}) - links = topo.get("links", []) or [] - - device_interfaces: dict[str, set] = {} - leaf_interface_roles: dict[str, dict[str, set]] = {} - client_interfaces: dict[str, set] = {} +def _interface_facts_from_manifest( + manifest: TopologyManifest, +) -> tuple[dict[str, list[str]], dict[str, dict[str, list[str]]], dict[str, list[str]]]: + """Derive interface ownership and fabric direction from canonical links.""" + devices = {device.name: device for device in manifest.devices} + switch_interfaces: dict[str, set[str]] = {} + client_interfaces: dict[str, set[str]] = {} + interface_roles: dict[str, dict[str, set[str]]] = {} - def add_iface(device: str, iface: str): - if device.startswith("client"): - client_interfaces.setdefault(device, set()).add(iface) - else: - normalized = normalize_sonic_interface(iface) - if normalized: - device_interfaces.setdefault(device, set()).add(normalized) - - def add_leaf_role(leaf: str, role: str, iface: str): - normalized = normalize_sonic_interface(iface) - if not normalized: + def add_role(device: str, interface: str, role: str) -> None: + normalized = normalize_sonic_interface(interface) + if normalized is None: return - leaf_interface_roles.setdefault(leaf, {"uplink": set(), "downlink": set(), "any": set()}) - leaf_interface_roles[leaf][role].add(normalized) - leaf_interface_roles[leaf]["any"].add(normalized) - - for link in links: - endpoints = link.get("endpoints", []) - if len(endpoints) != 2: - continue - left = endpoints[0] - right = endpoints[1] - if ":" not in left or ":" not in right: - continue - dev_a, if_a = left.split(":", 1) - dev_b, if_b = right.split(":", 1) - - add_iface(dev_a, if_a) - add_iface(dev_b, if_b) - - if dev_a.startswith("leaf") and dev_b.startswith("spine"): - add_leaf_role(dev_a, "uplink", if_a) - if dev_b.startswith("leaf") and dev_a.startswith("spine"): - add_leaf_role(dev_b, "uplink", if_b) - if dev_a.startswith("leaf") and dev_b.startswith("client"): - add_leaf_role(dev_a, "downlink", if_a) - if dev_b.startswith("leaf") and dev_a.startswith("client"): - add_leaf_role(dev_b, "downlink", if_b) + roles = interface_roles.setdefault(device, {"uplink": set(), "downlink": set(), "any": set()}) + roles[role].add(normalized) + roles["any"].add(normalized) + + uplink_peers = { + DeviceRole.LEAF: {DeviceRole.SPINE}, + DeviceRole.EDGE: {DeviceRole.AGG}, + DeviceRole.AGG: {DeviceRole.CORE}, + } + downlink_peers = { + DeviceRole.LEAF: {DeviceRole.CLIENT}, + DeviceRole.EDGE: {DeviceRole.CLIENT}, + DeviceRole.AGG: {DeviceRole.EDGE}, + } + for link in manifest.links: + left, right = link.endpoints + for endpoint, peer in ((left, right), (right, left)): + device = devices[endpoint.device] + peer_role = devices[peer.device].role + if device.role is DeviceRole.CLIENT: + client_interfaces.setdefault(device.name, set()).add(endpoint.interface) + continue + normalized = normalize_sonic_interface(endpoint.interface) + if normalized is not None: + switch_interfaces.setdefault(device.name, set()).add(normalized) + if peer_role in uplink_peers.get(device.role, set()): + add_role(device.name, endpoint.interface, "uplink") + elif peer_role in downlink_peers.get(device.role, set()): + add_role(device.name, endpoint.interface, "downlink") return ( - {k: sorted(v) for k, v in device_interfaces.items()}, - {k: {rk: sorted(rv) for rk, rv in roles.items()} for k, roles in leaf_interface_roles.items()}, - {k: sorted(v) for k, v in client_interfaces.items()}, + {name: sorted(interfaces) for name, interfaces in switch_interfaces.items()}, + { + name: {role: sorted(interfaces) for role, interfaces in roles.items()} + for name, roles in interface_roles.items() + }, + {name: sorted(interfaces) for name, interfaces in client_interfaces.items()}, ) @@ -265,35 +271,6 @@ def normalize_sonic_interface(name: str | None) -> str | None: return raw -def build_leaf_interface_roles( - device_interfaces: dict[str, list[str]], - metadata: dict[str, Any], -) -> dict[str, dict[str, list[str]]]: - roles: dict[str, dict[str, list[str]]] = {} - num_spines = int(metadata.get("scale", {}).get("num_spines", 0) or 0) - - for device, interfaces in device_interfaces.items(): - if not device.startswith("leaf"): - continue - roles[device] = {"uplink": [], "downlink": [], "any": []} - for iface in interfaces: - roles[device]["any"].append(iface) - if not (iface.startswith("Ethernet") and iface[8:].isdigit()): - continue - port_idx = int(iface[8:]) - if port_idx % 4 != 0: - continue - eth_idx = (port_idx // 4) + 1 - if num_spines and eth_idx <= num_spines: - roles[device]["uplink"].append(iface) - elif num_spines and eth_idx > num_spines: - roles[device]["downlink"].append(iface) - for role in ("uplink", "downlink", "any"): - roles[device][role] = sorted(set(roles[device][role])) or ["Ethernet0"] - - return roles - - def count_for_scale(item: dict[str, Any], scale: str, default_count: int) -> int: cps = item.get("count_per_scale", {}) if isinstance(cps, dict) and scale in cps: @@ -303,9 +280,17 @@ def count_for_scale(item: dict[str, Any], scale: str, default_count: int) -> int def pick_device(role: str, topo: TopologyContext, rng: random.Random) -> str: if role == "spine": - return rng.choice(topo.spines) + return rng.choice(topo.cores if topo.cores else topo.spines) + if role == "core": + return rng.choice(topo.cores if topo.cores else topo.spines) if role == "leaf": - return rng.choice(topo.leafs) + return rng.choice(topo.edges if topo.edges else topo.leafs) + if role == "edge": + return rng.choice(topo.edges if topo.edges else topo.leafs) + if role == "agg": + if not topo.aggs: + raise ValueError("device_role 'agg' requires a fat-tree topology") + return rng.choice(topo.aggs) if role == "client": return rng.choice([c["name"] for c in topo.clients]) raise ValueError(f"Unknown role: {role}") @@ -331,6 +316,10 @@ def pick_advertised_network(device: str, topo: TopologyContext, rng: random.Rand return dict(rng.choice(candidates)) +def _client_leaf(client: dict[str, Any]) -> str: + return str(client.get("leaf") or client.get("edge") or "").strip() + + def _pick_client_for_route_fault( topo: TopologyContext, rng: random.Random, @@ -340,19 +329,13 @@ def _pick_client_for_route_fault( # on the same leaf where the route fault is injected. candidates = list(topo.clients) if excluded_leaf: - remote = [c for c in candidates if c.get("leaf") != excluded_leaf] + remote = [c for c in candidates if _client_leaf(c) != excluded_leaf] if remote: candidates = remote return rng.choice(candidates) -def pick_client_subnet( - topo: TopologyContext, - rng: random.Random, - prefix_len: int = 30, - excluded_leaf: str | None = None, -) -> str: - client = _pick_client_for_route_fault(topo, rng, excluded_leaf=excluded_leaf) +def _client_subnet(client: dict[str, Any], prefix_len: int = 30) -> str: ip_str = str(client.get("data_ip") or "").strip() if not ip_str: raise ValueError(f"Client {client.get('name', 'unknown')} missing data_ip in topology metadata") @@ -361,12 +344,7 @@ def pick_client_subnet( return str(net) -def pick_client_host_route( - topo: TopologyContext, - rng: random.Random, - excluded_leaf: str | None = None, -) -> str: - client = _pick_client_for_route_fault(topo, rng, excluded_leaf=excluded_leaf) +def _client_host_route(client: dict[str, Any]) -> str: ip_str = str(client.get("data_ip") or "").strip() if not ip_str: raise ValueError(f"Client {client.get('name', 'unknown')} missing data_ip in topology metadata") @@ -378,7 +356,7 @@ def pick_link_down_interface(device: str, topo: TopologyContext, rng: random.Ran candidates = [normalize_sonic_interface(c) for c in candidates if normalize_sonic_interface(c)] if not candidates: candidates = ["Ethernet0"] - if device.startswith("leaf"): + if device.startswith(("leaf", "edge")): roles = topo.leaf_interface_roles.get(device) if roles and roles.get("downlink"): return rng.choice(roles["downlink"]) @@ -408,6 +386,10 @@ def pick_leaf_interface(device: str, topo: TopologyContext, rng: random.Random, return pick_network_interface(device, topo, rng) +def is_role_aware_switch(device: str) -> bool: + return device.startswith(("leaf", "edge", "agg")) + + def pick_client_interface(device: str, topo: TopologyContext, rng: random.Random) -> str: candidates = topo.client_interfaces.get(device) if candidates: @@ -415,6 +397,142 @@ def pick_client_interface(device: str, topo: TopologyContext, rng: random.Random return "eth1" +@dataclass +class FaultBuildTarget: + device: str + interface: str | None = None + prefix: str | None = None + mtu: int | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +def _generic_fault_target(topo: TopologyContext, rng: random.Random, template: dict[str, Any]) -> FaultBuildTarget: + return FaultBuildTarget(device=pick_device(template.get("device_role", "leaf"), topo, rng)) + + +def _link_down_target(topo: TopologyContext, rng: random.Random, template: dict[str, Any]) -> FaultBuildTarget: + target = _generic_fault_target(topo, rng, template) + target.interface = pick_link_down_interface(target.device, topo, rng) + return target + + +def _role_interface_target( + topo: TopologyContext, + rng: random.Random, + template: dict[str, Any], +) -> FaultBuildTarget: + target = _generic_fault_target(topo, rng, template) + role = template.get("interface_role") or "uplink" + target.interface = ( + pick_leaf_interface(target.device, topo, rng, role) + if is_role_aware_switch(target.device) + else pick_network_interface(target.device, topo, rng) + ) + return target + + +def _packet_fault_target( + topo: TopologyContext, + rng: random.Random, + template: dict[str, Any], +) -> FaultBuildTarget: + target = _generic_fault_target(topo, rng, template) + if target.device.startswith("client"): + target.interface = pick_client_interface(target.device, topo, rng) + elif is_role_aware_switch(target.device): + target.interface = pick_leaf_interface( + target.device, + topo, + rng, + template.get("interface_role") or "uplink", + ) + else: + target.interface = pick_network_interface(target.device, topo, rng) + return target + + +def _blackhole_target(topo: TopologyContext, rng: random.Random, template: dict[str, Any]) -> FaultBuildTarget: + target = _generic_fault_target(topo, rng, template) + client = _pick_client_for_route_fault(topo, rng, excluded_leaf=target.device) + target.prefix = _client_subnet(client, prefix_len=30) + return target + + +def _static_route_target(topo: TopologyContext, rng: random.Random, template: dict[str, Any]) -> FaultBuildTarget: + device = pick_device("leaf", topo, rng) + client = _pick_client_for_route_fault(topo, rng, excluded_leaf=device) + return FaultBuildTarget( + device=device, + metadata={"target_ip": _client_host_route(client), "wrong_nexthop": "auto"}, + ) + + +def _bgp_neighbor_target(topo: TopologyContext, rng: random.Random, template: dict[str, Any]) -> FaultBuildTarget: + target = _generic_fault_target(topo, rng, template) + neighbor = pick_bgp_neighbor(target.device, topo, rng) + target.metadata = { + "peer_ip": neighbor["peer_ip"], + "misconfig_kind": template.get("misconfig_kind", "peer_as_mismatch"), + } + if neighbor.get("remote_as") is not None: + target.metadata["original_remote_as"] = int(neighbor["remote_as"]) + return target + + +def _route_policy_target(topo: TopologyContext, rng: random.Random, template: dict[str, Any]) -> FaultBuildTarget: + target = _generic_fault_target(topo, rng, template) + network = pick_advertised_network(target.device, topo, rng) + target.prefix = network["prefix"] + target.metadata = {"misconfig_kind": template.get("misconfig_kind", "network_statement_missing")} + if network.get("route_map"): + target.metadata["route_map"] = network["route_map"] + return target + + +def _acl_target(topo: TopologyContext, rng: random.Random, template: dict[str, Any]) -> FaultBuildTarget: + target = _generic_fault_target(topo, rng, template) + target.prefix = pick_advertised_network(target.device, topo, rng)["prefix"] + target.interface = ( + pick_leaf_interface(target.device, topo, rng, "uplink") + if is_role_aware_switch(target.device) + else pick_network_interface(target.device, topo, rng) + ) + target.metadata = {"direction": template.get("direction", "in")} + return target + + +FaultScenarioBuilder = Callable[[TopologyContext, random.Random, dict[str, Any]], FaultBuildTarget] + +FAULT_SCENARIO_BUILDERS: dict[str, FaultScenarioBuilder] = { + "device_down": _generic_fault_target, + "link_down": _link_down_target, + "link_flapping": _role_interface_target, + "mtu_mismatch": _role_interface_target, + "high_latency": _role_interface_target, + "packet_loss": _packet_fault_target, + "packet_corruption": _packet_fault_target, + "blackhole_route": _blackhole_target, + "static_route_misconfig": _static_route_target, + "bgp_neighbor_misconfig": _bgp_neighbor_target, + "route_policy_misconfig": _route_policy_target, + "acl_misconfig": _acl_target, +} + + +def diagnostic_observation_duration(base_duration: int, manifest: TopologyManifest) -> int: + """Return one complete canonical Pingmesh epoch for a diagnostic window.""" + coverage_seconds = manifest.pingmesh.coverage_epoch_seconds(manifest.facts.total_clients) + return max(base_duration, coverage_seconds) + + +def _validate_traffic_profiles(spec: dict[str, Any]) -> None: + configured = [spec.get("defaults", {}).get("traffic_profile")] + configured.extend(template.get("traffic_profile") for template in spec.get("fault_templates", [])) + invalid = next((profile for profile in configured if profile not in (None, "standard")), None) + if invalid is not None: + raise ValueError(f"Only the standard traffic profile is supported, got: {invalid}") + + def build_fault_instance( fault_type: str, difficulty: str, @@ -426,7 +544,10 @@ def build_fault_instance( ) -> dict[str, Any]: # Negative sample: healthy network, no fault injected. if fault_type == "none": - baseline_duration = int(defaults.get("baseline_duration_seconds", 20)) + baseline_duration = diagnostic_observation_duration( + int(defaults.get("baseline_duration_seconds", 20)), + topo.manifest, + ) recovery_duration = int(defaults.get("recovery_duration_seconds", 20)) baseline_stabilization = int(defaults.get("baseline_stabilization_seconds", 5)) scenario_id = f"generated_healthy_network_{topo.scale}_{idx:03d}" @@ -435,7 +556,7 @@ def build_fault_instance( "name": f"Generated healthy network case #{idx} ({topo.scale})", "description": f"Auto-generated negative sample — no fault injected ({topo.scale} topology).", "topology_scale": topo.scale, - "traffic_profile": template.get("traffic_profile", defaults.get("traffic_profile", "standard")), + "traffic_profile": "standard", "metadata": { "difficulty": difficulty, "negative_sample": True, @@ -475,94 +596,47 @@ def build_fault_instance( # are visible to ``template.get("misconfig_kind")`` lookups below. if "parameters" in template: template = {**template, **template["parameters"]} - device_role = template.get("device_role", "leaf") - target_device = pick_device(device_role, topo, rng) - target_interface = None - extra_episode_metadata: dict[str, Any] = {} - target_prefix = None - mtu = None + try: + builder = FAULT_SCENARIO_BUILDERS[fault_type] + except KeyError as exc: + raise ValueError(f"No scenario builder registered for fault type: {fault_type}") from exc + target = builder(topo, rng, template) + extra_episode_metadata = dict(target.metadata) if template.get("variant"): extra_episode_metadata["variant"] = template["variant"] if template.get("recovery_mode"): extra_episode_metadata["recovery_mode"] = template["recovery_mode"] - if fault_type == "link_down": - target_interface = pick_link_down_interface(target_device, topo, rng) - elif fault_type in {"link_flapping", "mtu_mismatch", "high_latency"}: - if target_device.startswith("leaf"): - interface_role = template.get("interface_role") - if not interface_role and fault_type in {"high_latency", "mtu_mismatch"}: - interface_role = "uplink" - target_interface = pick_leaf_interface(target_device, topo, rng, interface_role or "any") - else: - target_interface = pick_network_interface(target_device, topo, rng) - elif fault_type in {"packet_loss", "packet_corruption"}: - if target_device.startswith("client"): - target_interface = pick_client_interface(target_device, topo, rng) - elif target_device.startswith("leaf"): - interface_role = template.get("interface_role") or "uplink" - target_interface = pick_leaf_interface(target_device, topo, rng, interface_role) - else: - target_interface = pick_network_interface(target_device, topo, rng) - elif fault_type == "blackhole_route": - target_prefix = pick_client_subnet(topo, rng, prefix_len=30, excluded_leaf=target_device) - elif fault_type == "static_route_misconfig": - target_device = pick_device("leaf", topo, rng) - extra_episode_metadata["target_ip"] = pick_client_host_route( - topo, - rng, - excluded_leaf=target_device, - ) - extra_episode_metadata["wrong_nexthop"] = "auto" - elif fault_type == "bgp_neighbor_misconfig": - target_device = pick_device(device_role, topo, rng) - neighbor = pick_bgp_neighbor(target_device, topo, rng) - extra_episode_metadata["peer_ip"] = neighbor["peer_ip"] - if neighbor.get("remote_as") is not None: - extra_episode_metadata["original_remote_as"] = int(neighbor["remote_as"]) - extra_episode_metadata["misconfig_kind"] = template.get("misconfig_kind", "peer_as_mismatch") - elif fault_type == "route_policy_misconfig": - target_device = pick_device(device_role, topo, rng) - network = pick_advertised_network(target_device, topo, rng) - target_prefix = network["prefix"] - if network.get("route_map"): - extra_episode_metadata["route_map"] = network["route_map"] - extra_episode_metadata["misconfig_kind"] = template.get("misconfig_kind", "network_statement_missing") - elif fault_type == "acl_misconfig": - target_device = pick_device(device_role, topo, rng) - network = pick_advertised_network(target_device, topo, rng) - target_prefix = network["prefix"] - if target_device.startswith("leaf"): - target_interface = pick_leaf_interface(target_device, topo, rng, "uplink") - else: - target_interface = pick_network_interface(target_device, topo, rng) - extra_episode_metadata["direction"] = template.get("direction", "in") - if fault_type == "link_flapping": - extra_episode_metadata["iterations"] = int(template.get("iterations", 6)) - extra_episode_metadata["down_time"] = int(template.get("down_time", 2)) - extra_episode_metadata["up_time"] = int(template.get("up_time", 3)) - elif fault_type == "packet_loss": - extra_episode_metadata["loss_pct"] = int(template.get("loss_pct", 20)) - elif fault_type == "packet_corruption": - extra_episode_metadata["corruption_pct"] = int(template.get("corruption_pct", 20)) - elif fault_type == "high_latency": - extra_episode_metadata["latency_ms"] = int(template.get("latency_ms", 100)) - elif fault_type == "mtu_mismatch": - mtu = int(template.get("mtu", 1400)) - - expected_location = {"device": target_device} - if target_interface: - if target_device.startswith("client"): - expected_location["interface"] = target_interface + parameter_defaults = { + "link_flapping": {"iterations": 6, "down_time": 2, "up_time": 3}, + "packet_loss": {"loss_pct": 20}, + "packet_corruption": {"corruption_pct": 20}, + "high_latency": {"latency_ms": 100}, + } + for key, default in parameter_defaults.get(fault_type, {}).items(): + extra_episode_metadata[key] = int(template.get(key, default)) + if fault_type == "mtu_mismatch": + target.mtu = int(template.get("mtu", 1400)) + + expected_location = {"device": target.device} + if target.interface: + if target.device.startswith("client"): + expected_location["interface"] = target.interface else: - expected_location["interface"] = normalize_sonic_interface(target_interface) + expected_location["interface"] = normalize_sonic_interface(target.interface) scenario_id = f"generated_{fault_type}_{topo.scale}_{idx:03d}" scenario_name = f"Generated {fault_type} case #{idx} ({topo.scale})" - baseline_duration = int(defaults.get("baseline_duration_seconds", 20)) - fault_duration = int(defaults.get("fault_duration_seconds", 30)) + baseline_duration = diagnostic_observation_duration( + int(defaults.get("baseline_duration_seconds", 20)), + topo.manifest, + ) + fault_duration = diagnostic_observation_duration( + int(defaults.get("fault_duration_seconds", 30)), + topo.manifest, + ) recovery_duration = int(defaults.get("recovery_duration_seconds", 20)) baseline_stabilization = int(defaults.get("baseline_stabilization_seconds", 5)) fault_stabilization = int( @@ -572,19 +646,19 @@ def build_fault_instance( episode_fault = { "episode_id": "ep002_fault", - "description": f"Inject {fault_type} on {target_device}", + "description": f"Inject {fault_type} on {target.device}", "fault_type": fault_type, - "target_device": target_device, + "target_device": target.device, "duration_seconds": fault_duration, "stabilization_time": fault_stabilization, "metadata": {"severity": template.get("severity", "medium")}, } - if target_interface: - episode_fault["target_interface"] = target_interface - if target_prefix: - episode_fault["target_prefix"] = target_prefix - if mtu: - episode_fault["mtu"] = mtu + if target.interface: + episode_fault["target_interface"] = target.interface + if target.prefix: + episode_fault["target_prefix"] = target.prefix + if target.mtu: + episode_fault["mtu"] = target.mtu episode_fault["metadata"].update(extra_episode_metadata) return { @@ -595,7 +669,7 @@ def build_fault_instance( f"Auto-generated {fault_type} localization scenario for {topo.scale} topology.", ), "topology_scale": topo.scale, - "traffic_profile": template.get("traffic_profile", defaults.get("traffic_profile", "standard")), + "traffic_profile": "standard", "metadata": { "difficulty": difficulty, "expected_diagnosis": fault_type, @@ -611,7 +685,7 @@ def build_fault_instance( "episode_id": "ep001_baseline", "description": "Establish baseline - no faults", "fault_type": "none", - "target_device": target_device, + "target_device": target.device, "duration_seconds": baseline_duration, "stabilization_time": baseline_stabilization, }, @@ -620,7 +694,7 @@ def build_fault_instance( "episode_id": "ep003_recovery_verify", "description": "Verify recovery after fault removal", "fault_type": "none", - "target_device": target_device, + "target_device": target.device, "duration_seconds": recovery_duration, "stabilization_time": recovery_stabilization, }, @@ -629,6 +703,7 @@ def build_fault_instance( def generate(spec: dict[str, Any], topo: TopologyContext, out_dir: Path, seed: int) -> list[Path]: + _validate_traffic_profiles(spec) rng = random.Random(seed) defaults = dict(spec.get("defaults", {})) defaults["seed"] = seed @@ -670,57 +745,3 @@ def cleanup_existing_outputs(out_dir: Path) -> int: existing.unlink() removed += 1 return removed - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Generate randomized NetOpsBench scenarios") - parser.add_argument( - "--spec", - default="scenarios/specs/fault_campaign.yaml", - help="Scenario generation spec file", - ) - parser.add_argument( - "--scale", - required=True, - choices=["xs", "small", "medium", "large"], - help="Topology scale to generate scenarios for", - ) - parser.add_argument( - "--topology-dir", - help="Override topology directory (default: lab-topology/generated_topology_)", - ) - parser.add_argument( - "--out", - help="Output directory (default: scenarios/generated/)", - ) - parser.add_argument( - "--seed", - type=int, - default=42, - help="Random seed for reproducible generation", - ) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - spec_path = Path(args.spec) - spec = load_yaml(spec_path) - topo = load_topology(args.scale, args.topology_dir) - - out_dir = Path(args.out) if args.out else ROOT / "scenarios" / "generated" / args.scale - removed = cleanup_existing_outputs(out_dir) - if removed: - print(f"Removed {removed} existing scenario file(s) under: {out_dir}") - generated = generate(spec, topo, out_dir, args.seed) - - print(f"Generated {len(generated)} scenario file(s) under: {out_dir}") - for path in generated[:8]: - print(f" - {path}") - if len(generated) > 8: - print(f" ... and {len(generated) - 8} more") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/netopsbench/platform/scenario/models.py b/netopsbench/platform/scenario/models.py index 162cdf3..50269e5 100644 --- a/netopsbench/platform/scenario/models.py +++ b/netopsbench/platform/scenario/models.py @@ -30,7 +30,7 @@ class Scenario: name: str description: str topology_scale: str # xs, small, medium, large - traffic_profile: str # light, standard, stress + traffic_profile: str # canonical value: standard episodes: list[Episode] metadata: dict = field(default_factory=dict) parameters: dict = field(default_factory=dict) diff --git a/netopsbench/platform/scenario/observation.py b/netopsbench/platform/scenario/observation.py index df2c921..979b98d 100644 --- a/netopsbench/platform/scenario/observation.py +++ b/netopsbench/platform/scenario/observation.py @@ -1,4 +1,4 @@ -"""Observation runtime helpers for scenario execution.""" +"""Observation timing and Pingmesh analysis for scenario execution.""" from __future__ import annotations @@ -7,209 +7,133 @@ from netopsbench.config import config from netopsbench.logging_utils import get_logger -from netopsbench.platform.utils.events import emit as _emit +from netopsbench.platform.topology.topology_utils import coerce_topology_manifest logger = get_logger(__name__) -# Minimum baseline window (seconds) to ensure enough samples for statistical detection. -# With a 5s ping cycle, 60s guarantees ~12 data points per path. _MIN_BASELINE_WINDOW_SECONDS = 60 def _utc_iso(dt: datetime) -> str: - """Format *dt* as an InfluxDB-compatible UTC ISO-8601 string ending in 'Z'. + value = dt.isoformat() + if value.endswith("+00:00"): + return value[:-6] + "Z" + return value if value.endswith("Z") else value + "Z" - ``datetime.isoformat()`` on an aware UTC datetime returns ``+00:00``; - appending ``Z`` on top of that produces the invalid ``+00:00Z`` which - InfluxDB's Flux ``time()`` rejects with HTTP 400. - """ - s = dt.isoformat() - if s.endswith("+00:00"): - return s[:-6] + "Z" - return s if s.endswith("Z") else s + "Z" +def _coverage_epoch_seconds(runner) -> int: + manifest = coerce_topology_manifest(runner.topology_metadata) + return manifest.pingmesh.coverage_epoch_seconds(manifest.facts.total_clients) -def wait_and_observe(runner, duration: int, baseline_end_time: datetime | None = None) -> dict: - """Collect a single observation window and run Pingmesh anomaly detection.""" - - _emit(f"\n[Observation] Monitoring for {duration} seconds...") +def capture_observation_window(runner, duration: int, *, name: str = "window") -> dict: + """Record one observation interval without querying observability backends.""" + safe_duration = max(0, int(duration)) + logger.info(f"\n[Observation] Monitoring for {safe_duration} seconds...") start_time = datetime.now(UTC).replace(microsecond=0) - observations = { + sleep_fn = getattr(runner, "sleep", time.sleep) + for index in range(safe_duration): + sleep_fn(1) + if (index + 1) % 10 == 0: + logger.info(f" Observed {index + 1}/{safe_duration}s...") + end_time = datetime.now(UTC).replace(microsecond=0) + return { + "name": name, "start_time": _utc_iso(start_time), - "duration_seconds": duration, - "pingmesh_metrics": [], - "anomalies_detected": False, - "data_source_status": "unavailable", + "end_time": _utc_iso(end_time), + "duration_seconds": safe_duration, } - for i in range(duration): - time.sleep(1) - if (i + 1) % 10 == 0: - _emit(f" Observed {i + 1}/{duration}s...") - end_time = datetime.now(UTC).replace(microsecond=0) - observations["end_time"] = _utc_iso(end_time) +def _summary_for_window(anomalies: list[dict], window_name: str) -> dict: + selected = [item for item in anomalies if window_name in (item.get("windows_observed") or [])] + return { + "total_anomalies": len(selected), + "latency_spikes": sum(item.get("type") == "latency_spike" for item in selected), + "packet_loss_events": sum(item.get("type") == "packet_loss" for item in selected), + "path_unreachable_events": sum(item.get("type") == "path_unreachable" for item in selected), + "mtu_or_fragmentation_events": sum(item.get("type") == "mtu_or_fragmentation_suspect" for item in selected), + "jitter_spikes": sum(item.get("type") == "jitter_spike" for item in selected), + } - baseline_end_dt = baseline_end_time or start_time - # Ensure minimum baseline window for reliable statistical detection - baseline_window = max(duration, _MIN_BASELINE_WINDOW_SECONDS) - baseline_start = _utc_iso(baseline_end_dt - timedelta(seconds=baseline_window)) - baseline_end = _utc_iso(baseline_end_dt) - current_start = _utc_iso(start_time) - current_end = _utc_iso(end_time) - - try: - from netopsbench.platform.pingmesh.detector import AnomalyDetector - - detector = AnomalyDetector( - influxdb_url=runner.influxdb_url or config.influxdb_url, - token=runner.influxdb_token or config.influxdb_token, - org=runner.influxdb_org or config.influxdb_org, - bucket=runner.influxdb_bucket or config.influxdb_bucket, - topology_id=runner.topology_id, - ) - report = detector.generate_anomaly_report( - baseline_start=baseline_start, - baseline_end=baseline_end, - current_start=current_start, - current_end=current_end, - ) - observations["pingmesh_metrics"] = report - observations["anomalies_detected"] = report.get("summary", {}).get("total_anomalies", 0) > 0 - query_status = report.get("query_status", {}) if isinstance(report, dict) else {} - if query_status.get("ok", False): - observations["data_source_status"] = "ok" - else: - error_msg = query_status.get("error") or "query_failed" - observations["data_source_status"] = f"error: {error_msg}" - except Exception as e: - observations["data_source_status"] = f"error: {e}" - - return observations - - -def merge_observation_windows(windows: list[dict], total_duration_seconds: int) -> dict: - """Merge early/steady observation windows into one agent-facing payload.""" - if not windows: + +def analyze_observation_windows( + runner, + windows: list[dict], + total_duration_seconds: int, + baseline_end_time: datetime | None = None, +) -> dict: + """Analyze captured intervals using one baseline and one current snapshot.""" + valid_windows = [window for window in windows if isinstance(window, dict) and window.get("start_time")] + if not valid_windows: now = _utc_iso(datetime.now(UTC).replace(microsecond=0)) return { "start_time": now, + "end_time": now, "duration_seconds": total_duration_seconds, - "pingmesh_metrics": { - "summary": { - "total_anomalies": 0, - "latency_spikes": 0, - "packet_loss_events": 0, - "path_unreachable_events": 0, - "mtu_or_fragmentation_events": 0, - "jitter_spikes": 0, - }, - "anomalies": [], - }, + "pingmesh_metrics": {"summary": {"total_anomalies": 0}, "anomalies": []}, "anomalies_detected": False, + "coverage_status": "incomplete", "data_source_status": "unavailable", - "end_time": now, "observation_windows": [], } - valid_windows = [w for w in windows if isinstance(w, dict) and w] - if not valid_windows: - return merge_observation_windows([], total_duration_seconds) - - combined_anomalies: list[dict] = [] - combined_src_leaf: dict[str, dict[str, int]] = {} - combined_dst_leaf: dict[str, dict[str, int]] = {} - combined_spine: dict[str, dict[str, int]] = {} - latency_spikes = 0 - packet_loss_events = 0 - path_unreachable_events = 0 - mtu_or_fragmentation_events = 0 - jitter_spikes = 0 - query_ok = True - query_errors = [] - observation_windows = [] - - for idx, observation in enumerate(valid_windows, start=1): - pm = observation.get("pingmesh_metrics", {}) - if not isinstance(pm, dict): - continue - - summary = pm.get("summary", {}) or {} - latency_spikes += int(summary.get("latency_spikes", 0) or 0) - packet_loss_events += int(summary.get("packet_loss_events", 0) or 0) - path_unreachable_events += int(summary.get("path_unreachable_events", 0) or 0) - mtu_or_fragmentation_events += int(summary.get("mtu_or_fragmentation_events", 0) or 0) - jitter_spikes += int(summary.get("jitter_spikes", 0) or 0) - - for anomaly in pm.get("anomalies", []) or []: - anomaly_with_window = dict(anomaly) - anomaly_with_window["observation_window"] = f"window_{idx}" - combined_anomalies.append(anomaly_with_window) - - agg = pm.get("aggregated_anomalies", {}) or {} - _agg_keys = ("drop_count", "latency_spikes", "jitter_spikes", "path_unreachable") - for leaf, values in (agg.get("by_src_leaf", {}) or {}).items(): - bucket = combined_src_leaf.setdefault(leaf, {k: 0 for k in _agg_keys}) - for k in _agg_keys: - bucket[k] += int(values.get(k, 0) or 0) - for leaf, values in (agg.get("by_dst_leaf", {}) or {}).items(): - bucket = combined_dst_leaf.setdefault(leaf, {k: 0 for k in _agg_keys}) - for k in _agg_keys: - bucket[k] += int(values.get(k, 0) or 0) - for spine, values in (agg.get("by_spine", {}) or {}).items(): - bucket = combined_spine.setdefault(spine, {k: 0 for k in _agg_keys}) - for k in _agg_keys: - bucket[k] += int(values.get(k, 0) or 0) - - query_status = pm.get("query_status", {}) or {} - if not query_status.get("ok", False): - query_ok = False - if query_status.get("error"): - query_errors.append(str(query_status["error"])) - - observation_windows.append( - { - "name": f"window_{idx}", - "start_time": observation.get("start_time"), - "end_time": observation.get("end_time"), - "duration_seconds": observation.get("duration_seconds"), - "summary": summary, - } - ) - - merged_query_error = "; ".join(dict.fromkeys(query_errors)) if query_errors else None + from netopsbench.platform.pingmesh.detector import AnomalyDetector + baseline_end_dt = baseline_end_time or datetime.fromisoformat( + str(valid_windows[0]["start_time"]).replace("Z", "+00:00") + ) + baseline_seconds = max(_MIN_BASELINE_WINDOW_SECONDS, _coverage_epoch_seconds(runner)) + baseline_start = _utc_iso(baseline_end_dt - timedelta(seconds=baseline_seconds)) + baseline_end = _utc_iso(baseline_end_dt) + current_start = str(valid_windows[0]["start_time"]) + current_end = str(valid_windows[-1]["end_time"]) + + detector = AnomalyDetector( + influxdb_url=runner.influxdb_url or config.influxdb_url, + token=runner.influxdb_token or config.influxdb_token, + org=runner.influxdb_org or config.influxdb_org, + bucket=runner.influxdb_bucket or config.influxdb_bucket, + topology_metadata=runner.topology_metadata, + topology_id=runner.topology_id, + ) + report = detector.generate_windowed_anomaly_report( + baseline_start=baseline_start, + baseline_end=baseline_end, + current_start=current_start, + current_end=current_end, + windows=valid_windows, + ) + query_status = report.get("query_status", {}) + query_ok = bool(query_status.get("ok")) + anomalies = report.get("anomalies", []) or [] + coverage = report.get("coverage", {}) or {} + observation_windows = [ + { + **window, + "summary": _summary_for_window(anomalies, str(window.get("name") or "window")), + } + for window in valid_windows + ] return { - "start_time": valid_windows[0].get("start_time"), + "start_time": current_start, + "end_time": current_end, "duration_seconds": total_duration_seconds, - "pingmesh_metrics": { - "timestamp": datetime.now(UTC).isoformat(), - "windows": { - "window_1": valid_windows[0].get("pingmesh_metrics", {}).get("windows", {}), - "window_2": valid_windows[-1].get("pingmesh_metrics", {}).get("windows", {}), - }, - "query_status": { - "ok": query_ok, - "error": merged_query_error, - }, - "summary": { - "total_anomalies": len(combined_anomalies), - "latency_spikes": latency_spikes, - "packet_loss_events": packet_loss_events, - "path_unreachable_events": path_unreachable_events, - "mtu_or_fragmentation_events": mtu_or_fragmentation_events, - "jitter_spikes": jitter_spikes, - }, - "anomalies": combined_anomalies, - "aggregated_anomalies": { - "by_src_leaf": combined_src_leaf, - "by_dst_leaf": combined_dst_leaf, - "by_spine": combined_spine, - }, - }, - "anomalies_detected": len(combined_anomalies) > 0, - "data_source_status": "ok" if query_ok else f"error: {merged_query_error or 'query_failed'}", - "end_time": valid_windows[-1].get("end_time"), + "pingmesh_metrics": report, + "anomalies_detected": bool(report.get("summary", {}).get("total_anomalies", 0)), + "coverage_status": coverage.get("coverage_status", "error"), + "data_source_status": "ok" if query_ok else f"error: {query_status.get('error') or 'query_failed'}", "observation_windows": observation_windows, + "_coverage_audit": coverage, } + + +def wait_and_observe(runner, duration: int, baseline_end_time: datetime | None = None) -> dict: + """Capture and analyze one observation window.""" + window = capture_observation_window(runner, duration, name="steady") + return analyze_observation_windows( + runner, + [window], + total_duration_seconds=duration, + baseline_end_time=baseline_end_time, + ) diff --git a/scenarios/specs/fault_campaign.yaml b/netopsbench/platform/scenario/specs/fault_campaign.yaml similarity index 68% rename from scenarios/specs/fault_campaign.yaml rename to netopsbench/platform/scenario/specs/fault_campaign.yaml index b70903a..a36b68a 100644 --- a/scenarios/specs/fault_campaign.yaml +++ b/netopsbench/platform/scenario/specs/fault_campaign.yaml @@ -20,6 +20,10 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: blackhole_route_leaf fault_type: blackhole_route @@ -29,6 +33,10 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: mtu_mismatch_core fault_type: mtu_mismatch @@ -39,17 +47,24 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: high_latency_leaf fault_type: high_latency difficulty: medium device_role: leaf latency_ms: 120 - traffic_profile: stress count_per_scale: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: static_route_misconfig fault_type: static_route_misconfig @@ -59,6 +74,10 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: packet_loss_edge fault_type: packet_loss @@ -66,11 +85,14 @@ fault_templates: device_role: leaf interface_role: uplink loss_pct: 30 - traffic_profile: stress count_per_scale: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: packet_corruption_edge fault_type: packet_corruption @@ -78,11 +100,14 @@ fault_templates: device_role: leaf interface_role: uplink corruption_pct: 25 - traffic_profile: stress count_per_scale: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: link_flapping_core fault_type: link_flapping @@ -93,6 +118,10 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: bgp_neighbor_misconfig_leaf fault_type: bgp_neighbor_misconfig @@ -104,6 +133,10 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: route_policy_misconfig_leaf fault_type: route_policy_misconfig @@ -115,6 +148,10 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: device_down_leaf fault_type: device_down @@ -124,6 +161,10 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: acl_misconfig_leaf fault_type: acl_misconfig @@ -133,6 +174,25 @@ fault_templates: xs: 1 small: 1 medium: 2 + large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 + + - name: bgp_neighbor_misconfig_agg + fault_type: bgp_neighbor_misconfig + difficulty: hard + device_role: agg + parameters: + misconfig_kind: peer_as_mismatch + count_per_scale: + xs: 0 + small: 0 + medium: 0 + large: 0 + xlarge: 0 + fat-tree-k8: 4 + fat-tree-k12: 4 - name: healthy_network fault_type: none @@ -142,3 +202,6 @@ fault_templates: small: 3 medium: 4 large: 4 + xlarge: 4 + fat-tree-k8: 4 + fat-tree-k12: 4 diff --git a/netopsbench/platform/scenario/validator.py b/netopsbench/platform/scenario/validator.py index 31838ac..2f3ffba 100644 --- a/netopsbench/platform/scenario/validator.py +++ b/netopsbench/platform/scenario/validator.py @@ -2,15 +2,15 @@ from __future__ import annotations -import json import os -import re +from netopsbench.models.profiles import supported_scales from netopsbench.platform.faults.specs import ( - canonicalize_fault_name, - get_fault_spec, - get_supported_scenario_faults, + FaultSpecRegistry, + create_fault_registry, ) +from netopsbench.platform.topology.configdb_payload import interface_names_for_config +from netopsbench.platform.topology.topology_utils import load_topology_manifest from netopsbench.platform.utils.interface_names import interface_aliases from .models import Scenario @@ -20,32 +20,28 @@ # Constants # --------------------------------------------------------------------------- -_SUPPORTED_TOPOLOGY_SCALES = ["xs", "small", "medium", "large"] -_SUPPORTED_TRAFFIC_PROFILES = ["light", "standard", "stress"] +_SUPPORTED_TOPOLOGY_SCALES = supported_scales() +_SUPPORTED_TRAFFIC_PROFILES = ("standard",) -CLIENT_COUNT_TO_SCALE = { - 2: "xs", - 4: "xs", - 8: "small", - 16: "medium", - 32: "large", - 64: "large", -} - -NETWORK_DEVICE_PREFIXES = ("spine", "leaf") +NETWORK_DEVICE_PREFIXES = ("spine", "leaf", "core", "agg", "edge") # --------------------------------------------------------------------------- # Schema validation # --------------------------------------------------------------------------- -def supported_scenario_faults() -> list[str]: - faults = sorted(set(get_supported_scenario_faults())) +def supported_scenario_faults(fault_registry: FaultSpecRegistry | None = None) -> list[str]: + registry = fault_registry or create_fault_registry() + faults = sorted(set(registry.supported_scenario_faults())) return ["none", *faults] -def validate_scenario(scenario: Scenario) -> list[str]: +def validate_scenario( + scenario: Scenario, + fault_registry: FaultSpecRegistry | None = None, +) -> list[str]: """Validate a scenario's schema and fault-specific constraints.""" + registry = fault_registry or create_fault_registry() errors: list[str] = [] if not scenario.scenario_id: @@ -57,22 +53,22 @@ def validate_scenario(scenario: Scenario) -> list[str]: if scenario.topology_scale not in _SUPPORTED_TOPOLOGY_SCALES: errors.append(f"Invalid topology_scale: {scenario.topology_scale}") if scenario.traffic_profile not in _SUPPORTED_TRAFFIC_PROFILES: - errors.append(f"Invalid traffic_profile: {scenario.traffic_profile}") + errors.append(f"Invalid traffic_profile: {scenario.traffic_profile}; only 'standard' is supported") - canonical_fault_types = [canonicalize_fault_name(ep.fault_type) for ep in scenario.episodes] + canonical_fault_types = [registry.canonicalize(ep.fault_type) for ep in scenario.episodes] has_fault_episode = any(fault_type != "none" for fault_type in canonical_fault_types) if has_fault_episode: difficulty = (scenario.metadata or {}).get("difficulty") if difficulty not in ["easy", "medium", "hard"]: errors.append("Scenario metadata requires difficulty in [easy, medium, hard] for benchmark scoring") - expected_diagnosis = canonicalize_fault_name((scenario.metadata or {}).get("expected_diagnosis")) + expected_diagnosis = registry.canonicalize((scenario.metadata or {}).get("expected_diagnosis")) if not expected_diagnosis: errors.append("Scenario metadata missing expected_diagnosis for benchmark scoring") - supported_faults = supported_scenario_faults() + supported_faults = supported_scenario_faults(registry) for i, episode in enumerate(scenario.episodes): - canonical_fault_type = canonicalize_fault_name(episode.fault_type) + canonical_fault_type = registry.canonicalize(episode.fault_type) if not episode.episode_id: errors.append(f"Episode {i}: Missing episode_id") if not canonical_fault_type: @@ -85,7 +81,7 @@ def validate_scenario(scenario: Scenario) -> list[str]: if canonical_fault_type != "none" and not episode.target_device: errors.append(f"Episode {i}: Missing target_device") - spec = get_fault_spec(canonical_fault_type) + spec = registry.get(canonical_fault_type) if spec is not None: episode_view = episode_from_dict({**episode_to_dict(episode), "fault_type": canonical_fault_type}) errors.extend(spec.validate_episode(episode_view, episode_index=i)) @@ -98,77 +94,8 @@ def validate_scenario(scenario: Scenario) -> list[str]: # --------------------------------------------------------------------------- -def load_topology_metadata(topology_dir: str) -> dict: - """Load topology metadata from /topology.json.""" - metadata_path = os.path.join(topology_dir, "topology.json") - if not os.path.exists(metadata_path): - raise FileNotFoundError( - f"Topology metadata not found: {metadata_path}. " - "Run deploy/generation first or provide --topology-dir with topology.json." - ) - - with open(metadata_path, encoding="utf-8") as handle: - return json.load(handle) - - -def infer_topology_scale(metadata: dict) -> str: - """Infer topology scale from metadata explicit fields or client count.""" - if not isinstance(metadata, dict): - return "unknown" - - explicit = metadata.get("topology_scale") or metadata.get("scale_name") - if isinstance(explicit, str): - return explicit - - scale_block = metadata.get("scale", {}) - if isinstance(scale_block, dict): - explicit_from_scale = scale_block.get("name") - if isinstance(explicit_from_scale, str): - return explicit_from_scale - - total_clients = scale_block.get("total_clients") - if isinstance(total_clients, int): - return CLIENT_COUNT_TO_SCALE.get(total_clients, "unknown") - - clients = metadata.get("devices", {}).get("clients", []) - if isinstance(clients, list): - return CLIENT_COUNT_TO_SCALE.get(len(clients), "unknown") - - return "unknown" - - -def _all_topology_device_names(metadata: dict) -> set[str]: - names: set[str] = set() - devices = metadata.get("devices", {}) if isinstance(metadata, dict) else {} - for role in ("spines", "leafs", "clients"): - for device in devices.get(role, []) or []: - name = device.get("name") - if name: - names.add(name) - return names - - def _parse_config_interfaces(config_path: str) -> set[str]: - interfaces: set[str] = set() - if not os.path.exists(config_path): - return interfaces - - try: - with open(config_path, encoding="utf-8") as handle: - for raw_line in handle: - line = raw_line.strip() - if line.startswith("config interface startup"): - parts = line.split() - if len(parts) >= 4: - interfaces.add(parts[3]) - elif line.startswith("config interface ip add"): - parts = line.split() - if len(parts) >= 5: - interfaces.add(parts[4]) - except (OSError, ValueError): - return set() - - return interfaces + return set(interface_names_for_config(config_path)) def _validate_episode_target_interface( @@ -183,17 +110,16 @@ def _validate_episode_target_interface( if not target_device.startswith(NETWORK_DEVICE_PREFIXES): return [] - config_path = os.path.join(topology_dir, "configs", f"{target_device}.sh") + config_path = os.path.join(topology_dir, "configs", "sonic", target_device, "config_db.json") + if not os.path.isfile(config_path): + return [f"[scenario={scenario_id} episode={episode_id}] Required ConfigDB artifact is missing: {config_path}"] config_interfaces = _parse_config_interfaces(config_path) if not config_interfaces: - if re.match(r"^(Ethernet\d+|eth\d+|e\d+-\d+|ethernet-\d+/\d+)$", target_interface): - return [] return [ ( - f"[scenario={scenario_id} episode={episode_id}] Invalid target_interface " - f"'{target_interface}' for device '{target_device}'. " - "Expected format EthernetX or ethX for network devices." + f"[scenario={scenario_id} episode={episode_id}] ConfigDB artifact has no interfaces " + f"for device '{target_device}': {config_path}" ) ] @@ -215,8 +141,8 @@ def _validate_episode_target_interface( def validate_scenario_topology(scenario, topology_dir: str) -> dict: """Validate scenario topology compatibility and episode target consistency.""" - metadata = load_topology_metadata(topology_dir) - actual_scale = infer_topology_scale(metadata) + manifest = load_topology_manifest(os.path.join(topology_dir, "topology.json")) + actual_scale = manifest.scale declared_scale = scenario.topology_scale errors: list[str] = [] @@ -232,7 +158,7 @@ def validate_scenario_topology(scenario, topology_dir: str) -> dict: f"actual='{actual_scale}'. Use matching scenario files or redeploy the topology." ) - device_names = _all_topology_device_names(metadata) + device_names = {device.name for device in manifest.devices} for episode in scenario.episodes: if episode.fault_type != "none" and episode.target_device not in device_names: errors.append( @@ -264,7 +190,5 @@ def validate_scenario_topology(scenario, topology_dir: str) -> dict: __all__ = [ "supported_scenario_faults", "validate_scenario", - "load_topology_metadata", - "infer_topology_scale", "validate_scenario_topology", ] diff --git a/netopsbench/platform/session/__init__.py b/netopsbench/platform/session/__init__.py index 8747a65..8c9e86c 100644 --- a/netopsbench/platform/session/__init__.py +++ b/netopsbench/platform/session/__init__.py @@ -1,13 +1 @@ -"""Internal helpers for session execution.""" - -from .types import ScenarioExecutionRef - -__all__ = ["SessionOrchestrator", "ScenarioExecutionRef"] - - -def __getattr__(name: str): - if name == "SessionOrchestrator": - from .orchestrator import SessionOrchestrator - - return SessionOrchestrator - raise AttributeError(name) +"""Internal benchmark session orchestration package.""" diff --git a/netopsbench/platform/session/atif.py b/netopsbench/platform/session/atif.py new file mode 100644 index 0000000..b7598f5 --- /dev/null +++ b/netopsbench/platform/session/atif.py @@ -0,0 +1,490 @@ +"""ATIF trajectory construction and serialization.""" + +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any + +from netopsbench.agents._trace_utils import jsonable as _jsonable +from netopsbench.agents.tracing import AgentTraceRecorder + +from .trace_utils import isoformat as _isoformat + +ATIF_SCHEMA_VERSION = "ATIF-v1.7" + + +def build_trace_payload( + *, + trace_id: str, + run_id: str, + case_id: str, + scenario_id: str, + episode_result: dict[str, Any], + worker: str, + topology_id: str | None, + runtime_id: str, + agent: Any, + diagnosis: Any, + diagnosis_payload: dict[str, Any], + started_at: datetime, + ended_at: datetime, + pingmesh_window: dict[str, Any] | None = None, + error: str | None = None, + diagnostic_context: Any = None, + trace_recorder: AgentTraceRecorder | None = None, + topology_scale: str | None = None, +) -> dict[str, Any]: + metadata = dict(getattr(diagnosis, "metadata", None) or {}) + model = _agent_model_payload(agent) + model = {**model, **_model_payload(metadata)} + if trace_recorder is not None: + model = {**model, **trace_recorder.model_metadata()} + steps = _context_steps(diagnostic_context) + runtime_steps = trace_recorder.to_steps() if trace_recorder is not None else [] + steps.extend(_normalise_steps(runtime_steps, model_metadata=model)) + if not runtime_steps: + steps.extend(_steps_from_tool_calls(diagnosis_payload.get("tool_calls") or metadata.get("tool_calls") or [])) + steps.append(_final_diagnosis_step(diagnosis_payload, ended_at=ended_at)) + return { + "trace_id": trace_id, + "run_id": run_id, + "case_id": case_id, + "scenario_id": scenario_id, + "episode_id": (episode_result.get("episode") or {}).get("episode_id"), + "worker": worker, + "topology_id": topology_id, + "topology_scale": topology_scale, + "runtime_id": runtime_id, + "agent": _agent_payload(agent, diagnosis), + "model": model, + "pingmesh_window": _jsonable(pingmesh_window or {}), + "started_at": _isoformat(started_at), + "ended_at": _isoformat(ended_at), + "steps": _jsonable(steps), + "final_diagnosis": _diagnosis_payload(diagnosis_payload), + "metrics": _metrics_payload( + metadata, + diagnosis_payload, + trace_recorder=trace_recorder, + started_at=started_at, + ended_at=ended_at, + ), + "error": error or _diagnosis_error(diagnosis, diagnosis_payload), + } + + +def build_atif_payload(trace: dict[str, Any]) -> dict[str, Any]: + return { + "schema_version": ATIF_SCHEMA_VERSION, + "session_id": trace.get("run_id"), + "trajectory_id": trace.get("trace_id"), + "agent": _atif_agent(trace), + "steps": _atif_steps(trace), + "notes": None, + "final_metrics": _atif_final_metrics(trace), + "extra": { + "framework": "netopsbench", + "case_id": trace.get("case_id"), + "scenario_id": trace.get("scenario_id"), + "episode_id": trace.get("episode_id"), + "runtime_id": trace.get("runtime_id"), + "worker": trace.get("worker"), + "pingmesh_window": trace.get("pingmesh_window") or {}, + "topology_id": trace.get("topology_id"), + "topology_scale": trace.get("topology_scale"), + "started_at": trace.get("started_at"), + "ended_at": trace.get("ended_at"), + "final_diagnosis": trace.get("final_diagnosis") or {}, + "error": trace.get("error"), + }, + } + + +def _normalise_steps(steps: list[Any], *, model_metadata: dict[str, Any]) -> list[dict[str, Any]]: + normalised: list[dict[str, Any]] = [] + saw_runtime_event = False + seen_copied_context: set[str] = set() + for index, item in enumerate(steps, 1): + step = dict(item) if isinstance(item, dict) else {"content": str(item)} + step.setdefault("index", index) + step["type"] = str(step.get("type") or step.get("role") or step.get("source") or "message") + if step.get("is_copied_context"): + fingerprint = _message_fingerprint(step) + if saw_runtime_event or fingerprint in seen_copied_context: + continue + seen_copied_context.add(fingerprint) + step.pop("is_copied_context", None) + step["is_initial_context"] = True + else: + saw_runtime_event = True + if model_metadata and step["type"] == "llm": + step.setdefault("model", model_metadata.get("model")) + step.setdefault("provider", model_metadata.get("provider")) + normalised.append(_jsonable(step)) + return normalised + + +def _atif_steps(trace: dict[str, Any]) -> list[dict[str, Any]]: + atif_steps: list[dict[str, Any]] = [] + for step_id, native in enumerate(_collapse_copied_context_steps(trace.get("steps") or []), 1): + step = native if isinstance(native, dict) else {"content": str(native)} + atif_step: dict[str, Any] = { + "step_id": step_id, + "timestamp": step.get("started_at") or step.get("ended_at") or trace.get("started_at"), + "source": _atif_source(step), + "message": _atif_message_content(_step_message(step) or ""), + "extra": _compact_extra(step), + } + if atif_step["source"] == "agent" and (step.get("model") or (trace.get("model") or {}).get("model")): + atif_step["model_name"] = step.get("model") or (trace.get("model") or {}).get("model") + if step.get("reasoning_content"): + atif_step["reasoning_content"] = step.get("reasoning_content") + metrics = _atif_step_metrics(step) + if metrics: + atif_step["metrics"] = metrics + if _is_tool_step(step): + tool_call_id = str(step.get("tool_call_id") or step.get("run_id") or f"tool-{step_id}") + function_name = step.get("name") or step.get("tool_name") or step.get("tool") or "tool" + atif_step["source"] = "agent" + atif_step["message"] = atif_step["message"] or f"Tool call: {function_name}" + atif_step["tool_calls"] = [ + { + "tool_call_id": tool_call_id, + "function_name": function_name, + "arguments": step.get("args") or step.get("input") or {}, + "extra": {"run_id": step.get("run_id"), "parent_run_id": step.get("parent_run_id")}, + } + ] + observation = step.get("observation") + if observation is None and step.get("error") is not None: + observation = {"error": step.get("error")} + if observation is not None: + atif_step["observation"] = { + "results": [ + { + "source_call_id": tool_call_id, + "content": _atif_observation_content(observation), + "extra": {"status": "error" if step.get("error") else "success"}, + } + ] + } + atif_steps.append(_jsonable(atif_step)) + return atif_steps + + +def _atif_agent(trace: dict[str, Any]) -> dict[str, Any]: + agent = trace.get("agent") or {} + model = trace.get("model") or {} + return _jsonable( + { + "name": agent.get("name") or "unknown", + "version": agent.get("class") or "unknown", + "model_name": model.get("model") or "unknown", + "extra": {"class": agent.get("class"), "provider": model.get("provider"), "runtime": model.get("runtime")}, + } + ) + + +def _atif_final_metrics(trace: dict[str, Any]) -> dict[str, Any]: + metrics = trace.get("metrics") or {} + return _jsonable( + { + "total_prompt_tokens": metrics.get("input_tokens"), + "total_completion_tokens": metrics.get("output_tokens"), + "total_steps": len(trace.get("steps") or []), + "extra": { + "total_tokens": metrics.get("total_tokens"), + "llm_call_count": metrics.get("llm_call_count"), + "tool_calls_count": metrics.get("tool_calls_count"), + "time_taken_seconds": metrics.get("time_taken_seconds"), + }, + } + ) + + +def _agent_payload(agent: Any, diagnosis: Any) -> dict[str, Any]: + return { + "name": str(getattr(diagnosis, "agent_name", None) or getattr(agent, "name", None) or "unknown"), + "class": getattr(getattr(agent, "__class__", None), "__name__", type(agent).__name__), + } + + +def _agent_model_payload(agent: Any) -> dict[str, Any]: + payload: dict[str, Any] = {} + provider = _first_agent_attr(agent, ("vendor", "provider")) + model = _first_agent_attr(agent, ("model", "model_name")) + runtime = _first_agent_attr(agent, ("runtime", "runtime_name")) + if provider: + payload["provider"] = provider + if model: + payload["model"] = model + if runtime: + payload["runtime"] = runtime + return _jsonable(payload) + + +def _first_agent_attr(agent: Any, names: tuple[str, ...]) -> str | None: + for candidate in _agent_candidates(agent): + for name in names: + value = getattr(candidate, name, None) + if value is None or callable(value): + continue + text = str(value).strip() + if text: + return text + return None + + +def _agent_candidates(agent: Any): + seen: set[int] = set() + stack = [agent] + while stack: + candidate = stack.pop(0) + if candidate is None or id(candidate) in seen: + continue + seen.add(id(candidate)) + yield candidate + for attr in ("agent", "_agent", "wrapped_agent", "inner"): + inner = getattr(candidate, attr, None) + if inner is not None and id(inner) not in seen: + stack.append(inner) + + +def _model_payload(metadata: dict[str, Any]) -> dict[str, Any]: + return { + key: _jsonable(metadata[key]) + for key in ("provider", "model", "runtime") + if key in metadata and metadata.get(key) is not None + } + + +def _metrics_payload( + metadata: dict[str, Any], + diagnosis_payload: dict[str, Any], + *, + trace_recorder: AgentTraceRecorder | None = None, + started_at: datetime, + ended_at: datetime, +) -> dict[str, Any]: + recorder_metrics = trace_recorder.metrics() if trace_recorder is not None else {} + payload = { + "time_taken_seconds": float( + diagnosis_payload.get("time_taken_seconds") or max(0.0, (ended_at - started_at).total_seconds()) + ), + "tool_calls_count": len( + (trace_recorder.tool_calls() if trace_recorder is not None else []) + or diagnosis_payload.get("tool_calls") + or metadata.get("tool_calls") + or [] + ), + } + for key in ("input_tokens", "output_tokens", "total_tokens", "llm_call_count"): + if recorder_metrics.get(key): + payload[key] = _jsonable(recorder_metrics[key]) + elif key in metadata: + payload[key] = _jsonable(metadata[key]) + return payload + + +def _diagnosis_payload(diagnosis_payload: dict[str, Any]) -> dict[str, Any]: + final = dict(diagnosis_payload) + metadata = dict(final.get("metadata") or {}) + metadata.pop("trace", None) + metadata.pop("trajectory", None) + final["metadata"] = metadata + return _jsonable(final) + + +def _diagnosis_error(diagnosis: Any, diagnosis_payload: dict[str, Any]) -> str | None: + if diagnosis_payload.get("error"): + return str(diagnosis_payload["error"]) + if getattr(diagnosis, "success", True) is False: + findings = getattr(diagnosis, "findings", None) or {} + if isinstance(findings, dict) and findings.get("error"): + return str(findings["error"]) + return None + + +def _context_steps(diagnostic_context: Any) -> list[dict[str, Any]]: + if diagnostic_context is None: + return [] + payload = { + "scenario_id": getattr(diagnostic_context, "scenario_id", None), + "topology": getattr(diagnostic_context, "topology", None), + "symptoms": getattr(diagnostic_context, "symptoms", None), + } + return [ + { + "type": "message", + "source": "user", + "message": _atif_message_content(_jsonable(payload)), + "is_copied_context": True, + } + ] + + +def _safe_int(value: Any) -> int: + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +def _steps_from_tool_calls(tool_calls: Any) -> list[dict[str, Any]]: + if not isinstance(tool_calls, list): + return [] + return [ + { + "index": index, + "type": "tool_call", + "name": (item if isinstance(item, dict) else {"tool": str(item)}).get("tool") + or (item if isinstance(item, dict) else {}).get("name") + or "tool", + "args": (item if isinstance(item, dict) else {}).get("args") + or (item if isinstance(item, dict) else {}).get("input") + or {}, + } + for index, item in enumerate(tool_calls, 1) + ] + + +def _final_diagnosis_step(diagnosis_payload: dict[str, Any], *, ended_at: datetime) -> dict[str, Any]: + final = _diagnosis_payload(diagnosis_payload) + verdict = final.get("verdict") or ("error" if final.get("error") else "unknown") + fault_type = final.get("fault_type") + location = final.get("location") if isinstance(final.get("location"), dict) else {} + location_text = ", ".join(str(value) for value in (location or {}).values() if value not in (None, "")) + parts = [f"Final diagnosis: {verdict}"] + if fault_type: + parts.append(f"fault_type={fault_type}") + if location_text: + parts.append(f"location={location_text}") + return { + "type": "final_diagnosis", + "source": "agent", + "message": "; ".join(parts), + "ended_at": _isoformat(ended_at), + "extra": {"final": True, "diagnosis": _final_diagnosis_summary(final)}, + } + + +def _final_diagnosis_summary(final: dict[str, Any]) -> dict[str, Any]: + return _jsonable( + { + key: final[key] + for key in ("verdict", "fault_type", "location", "evidence", "confidence", "reasoning", "error") + if key in final and final[key] not in (None, {}, []) + } + ) + + +def _atif_source(step: dict[str, Any]) -> str: + raw = str(step.get("source") or step.get("role") or step.get("type") or "agent").lower() + if raw == "human": + return "user" + if raw in {"system", "user"}: + return raw + return "agent" + + +def _is_tool_step(step: dict[str, Any]) -> bool: + if str(step.get("type") or "").lower() in {"tool", "tool_call"}: + return True + if step.get("tool_call_id"): + return True + return bool(step.get("name") and ("args" in step or "input" in step)) + + +def _step_message(step: dict[str, Any]) -> Any: + for key in ("message", "content", "completion", "prompt"): + if step.get(key) is not None: + return step[key] + return None + + +def _atif_step_metrics(step: dict[str, Any]) -> dict[str, Any]: + usage = step.get("usage") if isinstance(step.get("usage"), dict) else {} + metrics: dict[str, Any] = {} + if usage: + metrics["prompt_tokens"] = usage.get("input_tokens") or usage.get("prompt_tokens") + metrics["completion_tokens"] = usage.get("output_tokens") or usage.get("completion_tokens") + if usage.get("total_tokens") is not None: + metrics.setdefault("extra", {})["total_tokens"] = usage["total_tokens"] + if step.get("duration_seconds") is not None: + metrics.setdefault("extra", {})["duration_seconds"] = step["duration_seconds"] + return {key: value for key, value in metrics.items() if value not in (None, {})} + + +def _atif_observation_content(value: Any) -> str | None: + if value is None: + return None + return value if isinstance(value, str) else json.dumps(_jsonable(value), sort_keys=True, default=str) + + +def _atif_message_content(value: Any) -> str: + return value if isinstance(value, str) else json.dumps(_jsonable(value), sort_keys=True, default=str) + + +def _compact_extra(step: dict[str, Any]) -> dict[str, Any]: + excluded = { + "message", + "content", + "prompt", + "completion", + "reasoning_content", + "tool_calls", + "observation", + "args", + "input", + "output", + "usage", + "extra", + } + extra = dict(step.get("extra") or {}) if isinstance(step.get("extra"), dict) else {} + for key, value in step.items(): + if key not in excluded: + extra[key] = value + return extra + + +def _collapse_copied_context_steps(steps: list[Any]) -> list[Any]: + collapsed: list[Any] = [] + saw_runtime_event = False + seen_context: set[str] = set() + for item in steps: + if not isinstance(item, dict): + collapsed.append(item) + saw_runtime_event = True + continue + if item.get("is_copied_context"): + fingerprint = _message_fingerprint(item) + if saw_runtime_event or fingerprint in seen_context: + continue + copied = dict(item) + copied.pop("is_copied_context", None) + copied["is_initial_context"] = True + collapsed.append(copied) + seen_context.add(fingerprint) + continue + collapsed.append(item) + saw_runtime_event = True + return collapsed + + +def _message_fingerprint(step: dict[str, Any]) -> str: + return json.dumps( + _jsonable( + { + "type": step.get("type"), + "source": step.get("source") or step.get("role"), + "message": step.get("message") if "message" in step else step.get("content"), + "tool_call_id": step.get("tool_call_id"), + } + ), + sort_keys=True, + default=str, + ) + + +__all__ = ["ATIF_SCHEMA_VERSION", "build_atif_payload", "build_trace_payload"] diff --git a/netopsbench/platform/session/context.py b/netopsbench/platform/session/context.py index 3e4c6ec..d8936de 100644 --- a/netopsbench/platform/session/context.py +++ b/netopsbench/platform/session/context.py @@ -2,18 +2,28 @@ from __future__ import annotations -import glob -import json -import os +import copy +import hashlib +from pathlib import Path from typing import Any from netopsbench.logging_utils import get_logger +from netopsbench.models.runtime import RuntimeIdentity # Internal platform code can call AgentToolkit directly; external integrations should use SDK MCP helpers. +from netopsbench.platform.session.types import WorkerExecutionContext from netopsbench.platform.toolkit.toolkit import AgentToolkit +from netopsbench.platform.topology.topology_utils import load_topology_manifest logger = get_logger(__name__) +_EPISODE_ALLOWED_KEYS = { + "episode_id", + "duration_seconds", + "stabilization_time", +} +_MAX_AGENT_ANOMALIES = 100 + def build_topology_snapshot(toolkit: AgentToolkit) -> dict: topology_result = toolkit.get_topology() @@ -23,26 +33,9 @@ def build_topology_snapshot(toolkit: AgentToolkit) -> dict: def _build_toolkit_for_topology(topology_dir: str) -> AgentToolkit: - topology_file = os.path.join(topology_dir, "dcn.clab.yaml") - if not os.path.exists(topology_file): - candidates = sorted(glob.glob(os.path.join(topology_dir, "*.clab.y*ml"))) - if candidates: - topology_file = candidates[0] - topology_metadata = None - topology_json = os.path.join(topology_dir, "topology.json") - if os.path.exists(topology_json): - try: - with open(topology_json, encoding="utf-8") as f: - topology_metadata = json.load(f) - except Exception: - logger.debug("failed to parse topology.json at %s", topology_json, exc_info=True) - topology_metadata = None - toolkit_kwargs: dict[str, Any] = {} - if os.path.exists(topology_file): - toolkit_kwargs["topology_file"] = topology_file - if isinstance(topology_metadata, dict): - toolkit_kwargs["topology_metadata"] = topology_metadata - return AgentToolkit(**toolkit_kwargs) + topology_path = Path(topology_dir).resolve() + manifest = load_topology_manifest(topology_path) + return AgentToolkit(topology_dir=topology_path, topology_metadata=manifest.model_dump(mode="json")) def _extract_episode_pingmesh_query_window(episode_result: dict[str, Any]) -> dict[str, str | None]: @@ -73,3 +66,102 @@ def _extract_episode_pingmesh_query_window(episode_result: dict[str, Any]) -> di if current_starts and current_ends: return {"start_time": min(current_starts), "end_time": max(current_ends)} return {"start_time": None, "end_time": None} + + +def build_worker_execution_context(worker: RuntimeIdentity, topology_dir: Path) -> WorkerExecutionContext: + """Build an execution context from the worker's canonical runtime identity.""" + resolved_topology_dir = Path(topology_dir) + if resolved_topology_dir.resolve() != worker.topology_dir.resolve(): + raise ValueError( + f"Worker topology directory {resolved_topology_dir} does not match runtime identity " + f"{worker.topology_dir}" + ) + return WorkerExecutionContext( + topology_dir=resolved_topology_dir, + topology_id=worker.topology_id, + influxdb_bucket=worker.bucket, + ) + + +def _bounded_observations(observations: dict[str, Any]) -> dict[str, Any]: + bounded = copy.deepcopy(observations) + metrics = bounded.get("pingmesh_metrics") + if not isinstance(metrics, dict): + return bounded + anomalies = metrics.get("anomalies") + if not isinstance(anomalies, list) or len(anomalies) <= _MAX_AGENT_ANOMALIES: + if isinstance(anomalies, list): + metrics["returned_anomalies"] = len(anomalies) + metrics["truncated"] = False + return bounded + + severity_rank = {"high": 2, "medium": 1, "low": 0} + persistence_rank = {"persistent": 3, "steady_only": 2, "early_only": 1, "full_window": 0} + + def rank(item: dict[str, Any]) -> tuple[Any, ...]: + return ( + -severity_rank.get(str(item.get("severity")), 0), + -persistence_rank.get(str(item.get("persistence")), 0), + -float(item.get("value", 0.0) or 0.0), + str(item.get("type", "")), + str(item.get("src_ip", "")), + str(item.get("dst_ip", "")), + ) + + ordered = sorted((item for item in anomalies if isinstance(item, dict)), key=rank) + selected: list[dict[str, Any]] = [] + selected_ids: set[int] = set() + represented: set[tuple[str, str, str]] = set() + for item in ordered: + identity = (str(item.get("type")), str(item.get("src_leaf")), str(item.get("dst_leaf"))) + if identity in represented: + continue + represented.add(identity) + selected.append(item) + selected_ids.add(id(item)) + if len(selected) == _MAX_AGENT_ANOMALIES: + break + if len(selected) < _MAX_AGENT_ANOMALIES: + for item in ordered: + if id(item) in selected_ids: + continue + selected.append(item) + if len(selected) == _MAX_AGENT_ANOMALIES: + break + + metrics["anomalies"] = selected + metrics["returned_anomalies"] = len(selected) + metrics["truncated"] = len(selected) < len(anomalies) + return bounded + + +def build_public_case_id(*, scenario_id: str, episode_result: dict[str, Any]) -> str: + """Return a stable, non-semantic case id for agent context.""" + episode = episode_result.get("episode", {}) if isinstance(episode_result, dict) else {} + episode_id = episode.get("episode_id") if isinstance(episode, dict) else None + source = f"{scenario_id}:{episode_id or 'unknown'}" + digest = hashlib.sha1(source.encode("utf-8")).hexdigest()[:12] + return f"case-{digest}" + + +def build_public_symptoms(*, episode_result: dict[str, Any], pingmesh_query_window: dict[str, Any]) -> dict[str, Any]: + """Build the bounded symptom payload exposed to a diagnosis agent.""" + episode = episode_result.get("episode", {}) if isinstance(episode_result, dict) else {} + observations = episode_result.get("observations", {}) if isinstance(episode_result, dict) else {} + safe_episode = ( + {key: episode.get(key) for key in _EPISODE_ALLOWED_KEYS if key in episode} if isinstance(episode, dict) else {} + ) + return { + "episode": safe_episode, + "observations": _bounded_observations(observations) if isinstance(observations, dict) else {}, + "pingmesh_query_window": pingmesh_query_window if isinstance(pingmesh_query_window, dict) else {}, + "observation_type": "scenario_episode", + } + + +__all__ = [ + "build_public_case_id", + "build_public_symptoms", + "build_topology_snapshot", + "build_worker_execution_context", +] diff --git a/netopsbench/platform/session/diagnosis.py b/netopsbench/platform/session/diagnosis.py index 7aacf69..148018f 100644 --- a/netopsbench/platform/session/diagnosis.py +++ b/netopsbench/platform/session/diagnosis.py @@ -4,11 +4,28 @@ import asyncio import inspect +import json from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace from typing import Any from netopsbench.agents.base import DiagnosticContext +from netopsbench.agents.tracing import AgentTraceRecorder +from netopsbench.logging_utils import get_logger +from netopsbench.platform.session.context import ( + _build_toolkit_for_topology, + _extract_episode_pingmesh_query_window, + build_public_case_id, + build_public_symptoms, + build_topology_snapshot, +) +from netopsbench.platform.session.trace_store import TraceWriter +from netopsbench.platform.session.types import WorkerExecutionContext + +logger = get_logger(__name__) async def _maybe_await(value: Any) -> Any: @@ -40,6 +57,7 @@ async def diagnose(self, context: DiagnosticContext): def run_agent_diagnose(handle: Any, context: DiagnosticContext): + """Execute a sync or async diagnosis handle from a synchronous session.""" try: asyncio.get_running_loop() except RuntimeError: @@ -47,3 +65,172 @@ def run_agent_diagnose(handle: Any, context: DiagnosticContext): with ThreadPoolExecutor(max_workers=1) as executor: future = executor.submit(lambda: asyncio.run(handle.diagnose(context))) return future.result() + + +def _strip_runtime_trace_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + cleaned = dict(metadata or {}) + for key in ("messages", "raw_messages", "trace_events", "trace", "trajectory", "conversation"): + cleaned.pop(key, None) + return cleaned + + +def build_runtime_diagnosis_callback( + agent: Any, + topology_dir: str, + scenario_id: str, + worker_context: WorkerExecutionContext | None = None, + trace_writer: TraceWriter | None = None, + worker_name: str | None = None, + runtime_id: str | None = None, + scenario_scale: str | None = None, +): + """Build the episode callback that presents observations to one agent.""" + toolkit = _build_toolkit_for_topology(topology_dir) + if worker_context is not None: + toolkit.influxdb_bucket = worker_context.influxdb_bucket + toolkit.topology_id = worker_context.topology_id + handle = agent if isinstance(agent, AgentHandleAdapter) else AgentHandleAdapter(agent) + context_dir = Path(topology_dir) / ".netopsbench" + context_file = context_dir / "pingmesh_context.json" + worker_env = worker_context.as_env() if worker_context is not None else {} + worker_env["NETOPSBENCH_PINGMESH_CONTEXT_FILE"] = str(context_file) + + def callback(episode_result: dict) -> dict: + start_time = datetime.now(UTC) + trace_recorder = AgentTraceRecorder(enabled=trace_writer is not None) + pingmesh_query_window = _extract_episode_pingmesh_query_window(episode_result) + window_start = pingmesh_query_window.get("start_time") + window_end = pingmesh_query_window.get("end_time") + toolkit.set_pingmesh_time_window(window_start, window_end) + if window_start and window_end: + try: + context_dir.mkdir(parents=True, exist_ok=True) + context_file.write_text( + json.dumps({"start_time": window_start, "end_time": window_end}), + encoding="utf-8", + ) + except OSError: + logger.debug("failed to write pingmesh context file", exc_info=True) + + context = DiagnosticContext( + scenario_id=build_public_case_id(scenario_id=scenario_id, episode_result=episode_result), + topology=build_topology_snapshot(toolkit), + symptoms=build_public_symptoms( + episode_result=episode_result, + pingmesh_query_window=pingmesh_query_window, + ), + ground_truth=None, + tools=toolkit, + trace=trace_recorder, + metadata={"worker_env": worker_env} if worker_env else {}, + ) + try: + diagnosis = run_agent_diagnose(handle, context) + except Exception as exc: + trace_recorder.record_error(stage="agent", error=exc) + ended_at = datetime.now(UTC) + diagnosis_payload: dict[str, Any] = { + "error": str(exc), + "success": False, + "time_taken_seconds": max(0.0, (ended_at - start_time).total_seconds()), + "metadata": {"agent_failure_stage": "diagnose", "error_type": type(exc).__name__}, + } + if trace_writer is not None: + try: + trace_result = trace_writer.write_case_trace( + case_id=context.scenario_id, + scenario_id=scenario_id, + episode_result=episode_result, + worker=worker_name or "worker", + topology_id=(worker_context.topology_id if worker_context is not None else None), + topology_scale=scenario_scale, + runtime_id=runtime_id or "", + agent=agent, + diagnostic_context=context, + diagnosis=SimpleNamespace( + agent_name=getattr(handle, "name", "agent"), + success=False, + findings={"error": str(exc)}, + metadata=diagnosis_payload["metadata"], + ), + diagnosis_payload=diagnosis_payload, + started_at=start_time, + ended_at=ended_at, + pingmesh_window=pingmesh_query_window, + error=str(exc), + trace_recorder=trace_recorder, + ) + diagnosis_payload["trace"] = { + "trace_id": trace_result.trace_id, + "case_id": trace_result.case_id, + "worker": trace_result.worker, + "atif_path": trace_result.atif_path, + } + except Exception: + logger.debug("failed to persist failed agent runtime trace", exc_info=True) + diagnosis_payload["metadata"] = _strip_runtime_trace_metadata(diagnosis_payload["metadata"]) + return diagnosis_payload + + findings = dict(diagnosis.findings or {}) + location = findings.get("location") or {} + if not isinstance(location, dict): + location = {} + ended_at = datetime.now(UTC) + metadata = dict(diagnosis.metadata or {}) + recorder_metrics = trace_recorder.metrics() + for key in ("input_tokens", "output_tokens", "total_tokens", "llm_call_count"): + if recorder_metrics.get(key): + metadata[key] = recorder_metrics[key] + recorded_tool_calls = trace_recorder.tool_calls() + diagnosis_payload = { + "verdict": diagnosis.verdict, + "fault_type": findings.get("fault_type") or metadata.get("fault_type"), + "location": { + key: value + for key, value in { + "device": location.get("device") or findings.get("device"), + "interface": location.get("interface") or findings.get("interface"), + }.items() + if value is not None + }, + "evidence": list(findings.get("evidence") or []), + "confidence": float(diagnosis.confidence or 0.0), + "reasoning": diagnosis.reasoning, + "tool_calls": recorded_tool_calls or list(metadata.get("tool_calls") or []), + "time_taken_seconds": max(0.0, (ended_at - start_time).total_seconds()), + "metadata": metadata, + } + if trace_writer is not None: + try: + trace_result = trace_writer.write_case_trace( + case_id=context.scenario_id, + scenario_id=scenario_id, + episode_result=episode_result, + worker=worker_name or "worker", + topology_id=(worker_context.topology_id if worker_context is not None else None), + topology_scale=scenario_scale, + runtime_id=runtime_id or "", + agent=agent, + diagnostic_context=context, + diagnosis=diagnosis, + diagnosis_payload=diagnosis_payload, + started_at=start_time, + ended_at=ended_at, + pingmesh_window=pingmesh_query_window, + trace_recorder=trace_recorder, + ) + diagnosis_payload["trace"] = { + "trace_id": trace_result.trace_id, + "case_id": trace_result.case_id, + "worker": trace_result.worker, + "atif_path": trace_result.atif_path, + } + except Exception: + logger.debug("failed to persist agent runtime trace", exc_info=True) + diagnosis_payload["metadata"] = _strip_runtime_trace_metadata(diagnosis_payload["metadata"]) + return diagnosis_payload + + return callback + + +__all__ = ["AgentHandleAdapter", "build_runtime_diagnosis_callback", "run_agent_diagnose"] diff --git a/netopsbench/platform/session/dispatch.py b/netopsbench/platform/session/dispatch.py index 89ef79d..2eb87da 100644 --- a/netopsbench/platform/session/dispatch.py +++ b/netopsbench/platform/session/dispatch.py @@ -1,4 +1,4 @@ -"""Worker-pool execution helpers for SDK sessions.""" +"""Worker-pool execution for SDK sessions.""" from __future__ import annotations @@ -7,221 +7,165 @@ from collections.abc import Callable, Sequence from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Protocol +from typing import Any +from netopsbench.evaluator.fault_type_judge import create_judge_from_env +from netopsbench.evaluator.scorer import Evaluator +from netopsbench.models.runtime import RuntimeIdentity from netopsbench.platform.runtime.manager import RuntimePool -from netopsbench.platform.session.tracing import TraceWriter +from netopsbench.platform.scenario.executor import ScenarioExecutor +from netopsbench.platform.session.context import build_worker_execution_context +from netopsbench.platform.session.diagnosis import build_runtime_diagnosis_callback +from netopsbench.platform.session.reporting import load_topology_metadata +from netopsbench.platform.session.scoring import score_scenario_fault_episodes +from netopsbench.platform.session.trace_store import TraceWriter from netopsbench.platform.session.types import ScenarioExecutionRef, WorkerExecutionContext -from netopsbench.platform.worker.pool import WorkerSpec logger = logging.getLogger(__name__) +WorkerRunResult = tuple[list[Any], list[dict[str, Any]], dict[str, Any]] -class AgentLike(Protocol): - name: str - def diagnose(self, context: Any) -> Any: ... - - -class EvaluatorLike(Protocol): - def generate_report( - self, - results: list[Any], - agent_name: str = "unknown", - topology_scale: str = "unknown", - ) -> dict[str, Any]: ... - - -class ScenarioRunnerLike(Protocol): - results_dir: Path - - def run_scenario( - self, - scenario: Any, - diagnosis_callback: Callable[[dict[str, Any]], dict[str, Any]] | None = None, - ) -> dict[str, Any]: ... - - -class ScenarioRunnerFactory(Protocol): - def __call__(self, *args: Any, **kwargs: Any) -> ScenarioRunnerLike: ... - - -class ReportSaver(Protocol): - def __call__(self, report_payload: dict[str, Any], report_path: Path) -> None: ... - - -class RunHandleBuilder(Protocol): - def __call__(self, **kwargs: Any) -> dict[str, Any]: ... - - -# --------------------------------------------------------------------------- -# Configuration object for execute_on_runtime_pool -# --------------------------------------------------------------------------- - - -@dataclass -class RuntimePoolConfig: - """Groups the parameters for :func:`execute_on_runtime_pool`.""" - - # -- core execution context --- - run_id: str - mode: str - scenarios: list[ScenarioExecutionRef] - runtime: RuntimePool - agent: Any - - # -- artifact paths --- - artifact_dir: Path - raw_dir: Path - report_path: Path - metadata_path: Path - - # -- lifecycle --- - runtime_owner: str - teardown: str - started_at: Any - completed_at_factory: Callable[[], Any] - - # -- factory callbacks --- - scenario_runner_cls: Callable[..., ScenarioRunnerLike] - evaluator_factory: Callable[[], EvaluatorLike] - score_fault_episodes: Callable[..., list[Any]] - diagnosis_callback_builder: Callable[ - [Any, str, str, WorkerExecutionContext, TraceWriter | None, str, str, str, str | None], - Callable[[dict[str, Any]], dict[str, Any]], - ] - worker_context_builder: Callable[[WorkerSpec, Path], WorkerExecutionContext] - topology_metadata_loader: Callable[[Path], dict[str, Any] | None] - - # -- reporting callbacks --- - create_run_report: Callable[..., dict[str, Any]] - save_run_report: Callable[[dict[str, Any], Path], None] - save_run_metadata: Callable[..., None] - build_run_handle: Callable[..., dict[str, Any]] - run_handle_adapter: Callable[[dict[str, Any]], Any] - artifact_manager: Any - traces_dir: Path | None = None - trace_writer: TraceWriter | None = None - - # -- scenario executor knobs (defaults preserve historical behaviour) --- - baseline_wait_seconds: float = 60 - post_recovery_wait_seconds: float = 2 - skip_none_episodes: bool = True +@dataclass(frozen=True) +class PoolDispatchResult: + evaluations: list[Any] + scenarios: list[dict[str, Any]] + workers: list[dict[str, Any]] def assign_scenarios_to_workers( scenarios: list[ScenarioExecutionRef], - workers: Sequence[WorkerSpec], + workers: Sequence[RuntimeIdentity], ) -> dict[str, list[ScenarioExecutionRef]]: if not workers: raise ValueError("runtime pool must contain at least one worker") - assignments: dict[str, list[ScenarioExecutionRef]] = { - str(worker.id or f"worker-{worker.index}"): [] for worker in workers - } + assignments: dict[str, list[ScenarioExecutionRef]] = {worker.worker_id: [] for worker in workers} for index, scenario in enumerate(scenarios): - worker = workers[index % len(workers)] - worker_id = str(worker.id or f"worker-{worker.index}") - assignments[worker_id].append(scenario) + assignments[workers[index % len(workers)].worker_id].append(scenario) return assignments -def build_scenario_executor( - pool_config: RuntimePoolConfig, +def _dispatch_workers( + workers: Sequence[RuntimeIdentity], + scenarios: list[ScenarioExecutionRef], + run_worker: Callable[[RuntimeIdentity, list[ScenarioExecutionRef]], WorkerRunResult], +) -> PoolDispatchResult: + if not workers: + raise ValueError("runtime must contain at least one worker") + ordered_workers = sorted(workers, key=lambda item: item.worker_index) + assignments = assign_scenarios_to_workers(scenarios, ordered_workers) + + if len(ordered_workers) == 1: + worker = ordered_workers[0] + results = {worker.worker_index: run_worker(worker, assignments[worker.worker_id])} + else: + logger.info("Executing %d workers in parallel", len(ordered_workers)) + results = {} + with ThreadPoolExecutor(max_workers=len(ordered_workers)) as pool: + futures = { + pool.submit(run_worker, worker, assignments[worker.worker_id]): worker for worker in ordered_workers + } + for future in as_completed(futures): + worker = futures[future] + results[worker.worker_index] = future.result() + + evaluations: list[Any] = [] + scenario_summaries: list[dict[str, Any]] = [] + worker_summaries: list[dict[str, Any]] = [] + for worker in ordered_workers: + worker_evaluations, worker_scenarios, worker_summary = results[worker.worker_index] + evaluations.extend(worker_evaluations) + scenario_summaries.extend(worker_scenarios) + worker_summaries.append(worker_summary) + return PoolDispatchResult(evaluations, scenario_summaries, worker_summaries) + + +def _build_scenario_executor( worker_context: WorkerExecutionContext, worker_raw_dir: Path, -) -> ScenarioRunnerLike: - """Construct a scenario executor for a single worker. - - Centralising the construction lets callers (and tests) override - ``pool_config.scenario_runner_cls`` with a stub while keeping the kwargs - used by the dispatcher in one place. The ``pool_config`` parameter is - named to avoid shadowing the global :data:`netopsbench.config` singleton. - """ - kwargs = { - "topology_dir": str(worker_context.topology_dir), - "topology_metadata": pool_config.topology_metadata_loader(worker_context.topology_dir), - "baseline_wait_seconds": pool_config.baseline_wait_seconds, - "post_recovery_wait_seconds": pool_config.post_recovery_wait_seconds, - "skip_none_episodes": pool_config.skip_none_episodes, - "influxdb_bucket": worker_context.influxdb_bucket, - "topology_id": worker_context.topology_id, - "persist_results": False, - } - try: - runner = pool_config.scenario_runner_cls(**kwargs) - except TypeError: - # Compatibility for test doubles or older runner classes that do not - # expose the session-owned persistence switch yet. - kwargs.pop("persist_results", None) - runner = pool_config.scenario_runner_cls(**kwargs) + *, + fault_registry: Any, + baseline_wait_seconds: int, + post_recovery_wait_seconds: int, + skip_none_episodes: bool, +) -> ScenarioExecutor: + runner = ScenarioExecutor( + topology_dir=str(worker_context.topology_dir), + topology_metadata=load_topology_metadata(worker_context.topology_dir), + baseline_wait_seconds=baseline_wait_seconds, + post_recovery_wait_seconds=post_recovery_wait_seconds, + skip_none_episodes=skip_none_episodes, + influxdb_bucket=worker_context.influxdb_bucket, + topology_id=worker_context.topology_id, + persist_results=False, + fault_registry=fault_registry, + ) runner.results_dir = worker_raw_dir return runner -def _run_worker( - config: RuntimePoolConfig, - worker: WorkerSpec, - worker_scenarios: list[ScenarioExecutionRef], -) -> tuple[list[Any], list[dict[str, Any]], dict[str, Any]]: - """Execute all scenarios assigned to a single worker. +def _create_evaluator() -> Evaluator: + judge = create_judge_from_env() + return Evaluator(fault_type_judge=judge) if judge is not None else Evaluator() - Returns ``(eval_results, scenario_summaries, worker_summary)`` without - touching any shared mutable state so this function is safe to call from - a thread-pool. - """ - worker_id = str(worker.id or f"worker-{worker.index}") - worker_name = str(worker.name or worker_id) - topology_root = worker.topology_dir or (str(worker.root_dir) if worker.root_dir is not None else None) - if topology_root is None: - raise ValueError(f"worker '{worker_id}' is missing topology_dir/root_dir") - topology_dir = Path(topology_root) - worker_context = config.worker_context_builder(worker, topology_dir) - worker_raw_dir = config.raw_dir / worker_name +def _run_worker( + *, + runtime: RuntimePool, + agent: Any, + raw_dir: Path, + trace_writer: TraceWriter | None, + fault_registry: Any, + baseline_wait_seconds: int, + post_recovery_wait_seconds: int, + skip_none_episodes: bool, + worker: RuntimeIdentity, + scenarios: list[ScenarioExecutionRef], +) -> WorkerRunResult: + worker_context = build_worker_execution_context(worker, worker.topology_dir) + worker_raw_dir = raw_dir / worker.worker_id worker_raw_dir.mkdir(parents=True, exist_ok=True) + runner = _build_scenario_executor( + worker_context, + worker_raw_dir, + fault_registry=fault_registry, + baseline_wait_seconds=baseline_wait_seconds, + post_recovery_wait_seconds=post_recovery_wait_seconds, + skip_none_episodes=skip_none_episodes, + ) + evaluator = _create_evaluator() - runner = build_scenario_executor(config, worker_context, worker_raw_dir) - evaluator = config.evaluator_factory() - - eval_results: list[Any] = [] + evaluations: list[Any] = [] scenario_summaries: list[dict[str, Any]] = [] worker_success = True - executed_count = 0 - for scenario in worker_scenarios: - diagnosis_callback = config.diagnosis_callback_builder( - config.agent, + for scenario in scenarios: + callback = build_runtime_diagnosis_callback( + agent, str(worker_context.topology_dir), scenario.id, worker_context, - config.trace_writer, - worker_name, - config.run_id, - config.runtime.id, + trace_writer, + worker.worker_id, + runtime.id, scenario.scale, ) - scenario_result = runner.run_scenario( - scenario.to_scenario(), - diagnosis_callback=diagnosis_callback, - ) - raw_result_path = _persist_raw_scenario_result( - worker_raw_dir=worker_raw_dir, - scenario_id=scenario.id, - scenario_result=scenario_result, - ) + parsed_scenario = scenario.to_scenario() + scenario_result = runner.run_scenario(parsed_scenario, diagnosis_callback=callback) + raw_result_path = _persist_raw_scenario_result(worker_raw_dir, scenario.id, scenario_result) try: - scored = config.score_fault_episodes( - scenario.to_scenario(), + scored = score_scenario_fault_episodes( + parsed_scenario, scenario_result, evaluator, topology_dir=str(worker_context.topology_dir), ) except Exception as exc: - if config.trace_writer is not None: + if trace_writer is not None: try: - config.trace_writer.write_failure_result( + trace_writer.write_failure_result( scenario_id=scenario.id, scenario_result=scenario_result, stage="evaluator", @@ -230,193 +174,80 @@ def _run_worker( except Exception: logger.debug("failed to persist evaluator failure trace result", exc_info=True) raise - if config.trace_writer is not None: + if trace_writer is not None: try: - config.trace_writer.write_evaluation_results( + trace_writer.write_evaluation_results( evaluation_results=scored, scenario_result=scenario_result, ) except Exception: logger.debug("failed to persist trace evaluation results", exc_info=True) - executed_count += 1 - eval_results.extend(scored) + evaluations.extend(scored) + success = bool(scenario_result.get("success")) scenario_summaries.append( { "scenario_id": scenario.id, - "status": "completed" if scenario_result.get("success") else "failed", + "status": "completed" if success else "failed", "scale": scenario.scale, - "worker": worker_name, + "worker": worker.worker_id, "raw_result_path": raw_result_path, } ) - if not scenario_result.get("success"): - worker_success = False + worker_success &= success worker_summary = { - "worker_id": worker_id, - "worker_name": worker_name, + "worker_id": worker.worker_id, + "worker_name": worker.worker_id, "lab_name": worker.lab_name, - "scenario_count": len(worker_scenarios), - "executed_count": executed_count, + "scenario_count": len(scenarios), + "executed_count": len(scenarios), "success": worker_success, } - return eval_results, scenario_summaries, worker_summary + return evaluations, scenario_summaries, worker_summary def _persist_raw_scenario_result( - *, worker_raw_dir: Path, scenario_id: str, scenario_result: dict[str, Any], ) -> str: - existing = scenario_result.get("result_file") - if existing: - result_path = Path(str(existing)) - if result_path.exists(): - return str(result_path) - result_path.parent.mkdir(parents=True, exist_ok=True) - else: - safe_id = "".join(ch if ch.isalnum() or ch in {"-", "_", "."} else "_" for ch in str(scenario_id)) - timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f") - result_path = worker_raw_dir / f"{safe_id}_{timestamp}.json" - + safe_id = "".join(ch if ch.isalnum() or ch in {"-", "_", "."} else "_" for ch in str(scenario_id)) + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") + result_path = worker_raw_dir / f"{safe_id}_{timestamp}.json" result_path.write_text(json.dumps(scenario_result, indent=2, default=str), encoding="utf-8") scenario_result["result_file"] = str(result_path) return str(result_path) def execute_on_runtime_pool( - config: RuntimePoolConfig | None = None, - **kwargs, -): - """Execute scenarios across a runtime worker pool. - - Accepts either a :class:`RuntimePoolConfig` instance or keyword arguments - (for backward compatibility). - - When the pool contains more than one worker the workers execute **in - parallel** using threads so that each Containerlab topology can run its - assigned scenarios concurrently. - """ - if config is None: - config = RuntimePoolConfig(**kwargs) - - if not config.runtime.workers: - raise ValueError("runtime must contain at least one worker") - - assignments = assign_scenarios_to_workers(config.scenarios, config.runtime.workers) - # -- parallel worker execution ----------------------------------------- - num_workers = len(config.runtime.workers) - all_eval_results: list[Any] = [] - scenario_summaries: list[dict[str, Any]] = [] - worker_summaries: list[dict[str, Any]] = [] - - if num_workers == 1: - # Fast path: single worker, skip thread-pool overhead. - worker = config.runtime.workers[0] - wid = str(worker.id or f"worker-{worker.index}") - evals, sc_sums, w_sum = _run_worker( - config, - worker, - assignments.get(wid, []), - ) - all_eval_results.extend(evals) - scenario_summaries.extend(sc_sums) - worker_summaries.append(w_sum) - else: - logger.info("Executing %d workers in parallel", num_workers) - # Map futures back to workers so results can be merged in - # deterministic (worker-index) order after all complete. - futures_map: dict[Any, WorkerSpec] = {} - with ThreadPoolExecutor(max_workers=num_workers) as pool: - for worker in config.runtime.workers: - wid = str(worker.id or f"worker-{worker.index}") - fut = pool.submit( - _run_worker, - config, - worker, - assignments.get(wid, []), - ) - futures_map[fut] = worker - - # Wait for all to complete; propagate first failure. - results_by_worker: dict[int, tuple[list[Any], list[dict[str, Any]], dict[str, Any]]] = {} - for fut in as_completed(futures_map): - worker = futures_map[fut] - results_by_worker[worker.index] = fut.result() # raises on error - - # Merge in stable worker-index order. - for worker in sorted(config.runtime.workers, key=lambda w: w.index): - evals, sc_sums, w_sum = results_by_worker[worker.index] - all_eval_results.extend(evals) - scenario_summaries.extend(sc_sums) - worker_summaries.append(w_sum) + *, + runtime: RuntimePool, + scenarios: list[ScenarioExecutionRef], + agent: Any, + raw_dir: Path, + trace_writer: TraceWriter | None = None, + fault_registry: Any = None, + baseline_wait_seconds: int = 60, + post_recovery_wait_seconds: int = 2, + skip_none_episodes: bool = True, +) -> PoolDispatchResult: + """Run scenarios on an existing runtime and return ordered execution data.""" + return _dispatch_workers( + runtime.workers, + scenarios, + lambda worker, assigned: _run_worker( + runtime=runtime, + agent=agent, + raw_dir=raw_dir, + trace_writer=trace_writer, + fault_registry=fault_registry, + baseline_wait_seconds=baseline_wait_seconds, + post_recovery_wait_seconds=post_recovery_wait_seconds, + skip_none_episodes=skip_none_episodes, + worker=worker, + scenarios=assigned, + ), + ) - if all_eval_results: - report_evaluator = config.evaluator_factory() - aggregate_report = report_evaluator.generate_report( - all_eval_results, - agent_name=getattr(config.agent, "name", config.agent.__class__.__name__), - topology_scale=config.runtime.scale, - ) - else: - aggregate_report = {"summary": {"total_cases": 0, "average_score": 0.0}, "detailed_results": []} - completed_at = config.completed_at_factory() - report_payload = config.create_run_report( - run_id=config.run_id, - mode=config.mode, - started_at=config.started_at, - completed_at=completed_at, - runtime=config.runtime, - runtime_owner=config.runtime_owner, - teardown=config.teardown, - scenarios=config.scenarios, - agent=config.agent, - worker_summaries=worker_summaries, - scenario_summaries=scenario_summaries, - aggregate_report=aggregate_report, - artifact_dir=config.artifact_dir, - raw_dir=config.raw_dir, - traces_dir=config.traces_dir, - trace_index_path=(config.trace_writer.index_path if config.trace_writer is not None else None), - trace_results_path=(config.trace_writer.results_path if config.trace_writer is not None else None), - report_path=config.report_path, - metadata_path=config.metadata_path, - ) - config.save_run_report(report_payload, config.report_path) - report_status = str( - (report_payload.get("raw") or {}).get("status") - or (report_payload.get("summary") or {}).get("status") - or report_payload.get("status") - or "unknown" - ) - config.save_run_metadata( - config.artifact_manager, - config.artifact_dir, - run_id=config.run_id, - mode=config.mode, - status=report_status, - runtime_id=config.runtime.id, - runtime_owner=config.runtime_owner, - teardown=config.teardown, - started_at=config.started_at, - completed_at=completed_at, - scenarios=config.scenarios, - worker_summaries=worker_summaries, - traces_dir=config.traces_dir, - trace_index_path=(config.trace_writer.index_path if config.trace_writer is not None else None), - trace_results_path=(config.trace_writer.results_path if config.trace_writer is not None else None), - ) - handle_payload = config.build_run_handle( - run_id=config.run_id, - mode=config.mode, - status=report_status, - started_at=config.started_at, - completed_at=completed_at, - artifact_dir=config.artifact_dir, - scenarios=config.scenarios, - runtime_id=config.runtime.id, - report_path=config.report_path, - ) - return config.run_handle_adapter(handle_payload) +__all__ = ["PoolDispatchResult", "assign_scenarios_to_workers", "execute_on_runtime_pool"] diff --git a/netopsbench/platform/session/env.py b/netopsbench/platform/session/env.py deleted file mode 100644 index d5b47d2..0000000 --- a/netopsbench/platform/session/env.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Explicit worker execution context helpers for session execution.""" - -from __future__ import annotations - -from pathlib import Path - -from netopsbench.platform.session.types import WorkerExecutionContext -from netopsbench.platform.worker.pool import WorkerSpec - - -def build_worker_execution_context(worker: WorkerSpec, topology_dir: Path) -> WorkerExecutionContext: - resolved_topology_dir = Path(topology_dir) - return WorkerExecutionContext( - topology_dir=resolved_topology_dir, - topology_id=resolved_topology_dir.name, - influxdb_bucket=str(worker.bucket or ""), - ) diff --git a/netopsbench/platform/session/harbor_export.py b/netopsbench/platform/session/harbor_export.py new file mode 100644 index 0000000..639f22d --- /dev/null +++ b/netopsbench/platform/session/harbor_export.py @@ -0,0 +1,307 @@ +"""Harbor viewer export for persisted NetOpsBench traces.""" + +from __future__ import annotations + +import json +import shutil +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from harbor.models.agent.context import AgentContext +from harbor.models.job.config import DatasetConfig, JobConfig +from harbor.models.job.result import JobResult, JobStats +from harbor.models.task.id import LocalTaskId +from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, TrialConfig, VerifierConfig +from harbor.models.trial.result import AgentInfo, ModelInfo, TimingInfo, TrialResult +from harbor.models.verifier.result import VerifierResult + +from .trace_utils import isoformat as _isoformat +from .trace_utils import load_jsonl as _load_jsonl +from .trace_utils import safe_path_part as _safe_path_part +from .trace_utils import to_json as _to_json + + +def export_traces(run_dir: str | Path, *, output: str | Path) -> Path: + """Export a NetOpsBench run into a local Harbor viewer jobs directory.""" + run_path = Path(run_dir) + traces_dir = run_path / "traces" + if not traces_dir.is_dir(): + raise FileNotFoundError(f"trace directory not found: {traces_dir}") + + index_rows = load_trace_index(run_path) + if not index_rows: + raise FileNotFoundError(f"trace index is empty or missing: {traces_dir / 'index.jsonl'}") + + output_root = Path(output) + output_root.mkdir(parents=True, exist_ok=True) + job_name = f"netopsbench-{run_path.name}" + job_dir = output_root / job_name + if job_dir.exists(): + shutil.rmtree(job_dir) + job_dir.mkdir(parents=True) + + job_id = str(uuid.uuid5(uuid.NAMESPACE_URL, f"netopsbench:{run_path.name}")) + result_rows = load_trace_results(run_path) + trial_results = [ + _write_harbor_trial( + job_dir, + job_id=job_id, + index_row=row, + atif=json.loads(_resolve_atif_path(run_path, row).read_text(encoding="utf-8")), + result_row=_matching_result_row(result_rows, row), + ) + for row in index_rows + ] + job_config = _harbor_job_config(job_name, output_root, index_rows) + (job_dir / "config.json").write_text(_to_json(job_config.model_dump(mode="json")), encoding="utf-8") + (job_dir / "result.json").write_text( + _to_json(_harbor_job_result(job_id, _run_times(run_path, index_rows), trial_results).model_dump(mode="json")), + encoding="utf-8", + ) + return output_root + + +def load_trace_index(run_dir: str | Path) -> list[dict[str, Any]]: + return _load_jsonl(Path(run_dir) / "traces" / "index.jsonl") + + +def load_trace_results(run_dir: str | Path) -> list[dict[str, Any]]: + return _load_jsonl(Path(run_dir) / "traces" / "results.jsonl") + + +def _write_harbor_trial( + job_dir: Path, + *, + job_id: str, + index_row: dict[str, Any], + atif: dict[str, Any], + result_row: dict[str, Any] | None, +) -> TrialResult: + case_id = str(index_row.get("case_id") or (atif.get("extra") or {}).get("case_id") or "case") + scenario_id = str(index_row.get("scenario_id") or (atif.get("extra") or {}).get("scenario_id") or "scenario") + trial_name = _safe_path_part(f"{scenario_id}__{case_id}") + trial_dir = job_dir / trial_name + agent_dir = trial_dir / "agent" + verifier_dir = trial_dir / "verifier" + task_dir = trial_dir / "task" + agent_dir.mkdir(parents=True) + verifier_dir.mkdir(parents=True) + task_dir.mkdir(parents=True) + + score = float((result_row or {}).get("score") or 0.0) + (agent_dir / "trajectory.json").write_text(_to_json(atif), encoding="utf-8") + (verifier_dir / "reward.txt").write_text(str(score), encoding="utf-8") + (verifier_dir / "result.json").write_text( + _to_json({"reward": score, "netopsbench_result": result_row or {}}), + encoding="utf-8", + ) + (verifier_dir / "test-stdout.txt").write_text(_to_json(result_row or {}), encoding="utf-8") + (verifier_dir / "test-stderr.txt").write_text("", encoding="utf-8") + (task_dir / "instruction.md").write_text( + f"NetOpsBench scenario {scenario_id} episode {index_row.get('episode_id') or ''}\n", + encoding="utf-8", + ) + + config = _harbor_trial_config(job_id, job_dir, task_dir, trial_name, index_row) + trial_result = _harbor_trial_result(trial_dir, task_dir, trial_name, scenario_id, case_id, config, index_row, score) + (trial_dir / "config.json").write_text(_to_json(config.model_dump(mode="json")), encoding="utf-8") + (trial_dir / "result.json").write_text(_to_json(trial_result.model_dump(mode="json")), encoding="utf-8") + return trial_result + + +def _harbor_trial_config( + job_id: str, + job_dir: Path, + task_dir: Path, + trial_name: str, + index_row: dict[str, Any], +) -> TrialConfig: + return TrialConfig( + task=TaskConfig(path=task_dir, source=_dataset_source(index_row)), + trial_name=trial_name, + trials_dir=job_dir, + agent=AgentConfig(name=index_row.get("agent") or "unknown", model_name=index_row.get("model")), + environment=EnvironmentConfig(type="docker"), + verifier=VerifierConfig(disable=False), + artifacts=[], + extra_instruction_paths=[], + job_id=job_id, + ) + + +def _harbor_trial_result( + trial_dir: Path, + task_dir: Path, + trial_name: str, + scenario_id: str, + case_id: str, + config: TrialConfig, + index_row: dict[str, Any], + score: float, +) -> TrialResult: + started_at = index_row.get("started_at") + ended_at = index_row.get("ended_at") or started_at + return TrialResult( + id=uuid.uuid5(uuid.NAMESPACE_URL, f"{index_row.get('trace_id')}:{trial_name}"), + task_name=scenario_id, + trial_name=trial_name, + trial_uri=str(trial_dir), + task_id=LocalTaskId(path=task_dir), + source=_dataset_source(index_row), + task_checksum=case_id, + config=config, + agent_info=AgentInfo( + name=index_row.get("agent") or "unknown", + version=index_row.get("agent_class") or "unknown", + model_info=ModelInfo(name=index_row.get("model") or "unknown", provider=index_row.get("provider")), + ), + agent_result=AgentContext( + n_input_tokens=index_row.get("input_tokens"), + n_output_tokens=index_row.get("output_tokens"), + metadata={ + "trace_id": index_row.get("trace_id"), + "step_count": index_row.get("step_count"), + "topology_scale": index_row.get("topology_scale"), + "provider": index_row.get("provider"), + "model": index_row.get("model"), + }, + ), + verifier_result=VerifierResult(rewards={"reward": score, "score": score}), + started_at=started_at, + finished_at=ended_at, + agent_execution=TimingInfo(started_at=started_at, finished_at=ended_at), + verifier=TimingInfo(started_at=ended_at, finished_at=ended_at), + ) + + +def _harbor_job_config(job_name: str, output_root: Path, index_rows: list[dict[str, Any]]) -> JobConfig: + agent_configs: dict[tuple[str, str | None], AgentConfig] = {} + dataset_names = sorted({_dataset_source(row) for row in index_rows}) + for row in index_rows: + agent_name = str(row.get("agent") or "unknown") + model_name = _job_config_model_name(row) + agent_configs.setdefault((agent_name, model_name), AgentConfig(name=agent_name, model_name=model_name)) + return JobConfig( + job_name=job_name, + jobs_dir=output_root, + agents=list(agent_configs.values()), + datasets=[DatasetConfig(name=name) for name in dataset_names], + environment=EnvironmentConfig(type="docker"), + ) + + +def _harbor_job_result(job_id: str, run_times: dict[str, str], trial_results: list[TrialResult]) -> JobResult: + stats = JobStats.from_trial_results(trial_results, n_total_trials=len(trial_results)) + _attach_mean_reward_metrics(stats, trial_results) + return JobResult( + id=job_id, + started_at=run_times["started_at"], + updated_at=run_times["finished_at"], + finished_at=run_times["finished_at"], + n_total_trials=len(trial_results), + stats=stats, + trial_results=trial_results, + ) + + +def _resolve_atif_path(run_path: Path, row: dict[str, Any]) -> Path: + raw_path = row.get("atif_path") + if raw_path and Path(raw_path).exists(): + return Path(raw_path) + case_id = _safe_path_part(row.get("case_id") or "case") + matches = sorted((run_path / "traces").glob(f"*/{case_id}/trajectory.atif.json")) + if matches: + return matches[0] + worker = _safe_path_part(row.get("worker") or "worker") + candidate = run_path / "traces" / worker / case_id / "trajectory.atif.json" + if candidate.exists(): + return candidate + raise FileNotFoundError(f"ATIF trajectory not found for trace {row.get('trace_id')}") + + +def _matching_result_row(result_rows: list[dict[str, Any]], index_row: dict[str, Any]) -> dict[str, Any] | None: + trace_id = index_row.get("trace_id") + for row in result_rows: + if row.get("trace_id") == trace_id: + return row + for row in result_rows: + if row.get("scenario_id") == index_row.get("scenario_id") and row.get("episode_id") == index_row.get( + "episode_id" + ): + return row + return None + + +def _dataset_source(index_row: dict[str, Any]) -> str: + scale = str(index_row.get("topology_scale") or "").strip() + if scale and scale.lower() != "unknown": + return f"netopsbench-{scale}" + return "netopsbench" + + +def _job_config_model_name(index_row: dict[str, Any]) -> str | None: + model = index_row.get("model") + if not model: + return None + model_text = str(model) + provider = str(index_row.get("provider") or "").strip() + if provider and not model_text.startswith(f"{provider}/"): + return f"{provider}/{model_text}" + return model_text + + +def _attach_mean_reward_metrics(stats: JobStats, trial_results: list[TrialResult]) -> None: + rewards_by_eval: dict[str, list[dict[str, float | int] | None]] = {} + for trial_result in trial_results: + agent_name = trial_result.agent_info.name + model_name = trial_result.agent_info.model_info.name if trial_result.agent_info.model_info else None + dataset_name = trial_result.source or "adhoc" + evals_key = JobStats.format_agent_evals_key(agent_name, model_name, dataset_name) + rewards = trial_result.verifier_result.rewards if trial_result.verifier_result else None + rewards_by_eval.setdefault(evals_key, []).append(rewards) + for evals_key, rewards in rewards_by_eval.items(): + eval_stats = stats.evals.get(evals_key) + if eval_stats is not None: + eval_stats.metrics = [_mean_reward_metric(rewards)] + + +def _mean_reward_metric(rewards: list[dict[str, float | int] | None]) -> dict[str, float | int]: + reward_keys = sorted({key for reward in rewards if reward is not None for key in reward}) + if len(reward_keys) <= 1: + values = [0 if reward is None else next(iter(reward.values()), 0) for reward in rewards] + return {"mean": sum(values) / len(values) if values else 0.0} + return { + key: sum(0 if reward is None else reward.get(key, 0) for reward in rewards) / len(rewards) + for key in reward_keys + } + + +def _run_times(run_path: Path, index_rows: list[dict[str, Any]]) -> dict[str, str]: + report_path = run_path / "report.json" + if report_path.exists(): + try: + summary = json.loads(report_path.read_text(encoding="utf-8")).get("summary") or {} + if summary.get("started_at") and summary.get("completed_at"): + return { + "started_at": _normalise_iso_z(summary["started_at"]), + "finished_at": _normalise_iso_z(summary["completed_at"]), + } + except Exception: + pass + starts = [str(value) for row in index_rows if (value := row.get("started_at"))] + ends = [str(value) for row in index_rows if (value := row.get("ended_at"))] + now = _isoformat(datetime.now(UTC)) + return { + "started_at": _normalise_iso_z(min(starts) if starts else now), + "finished_at": _normalise_iso_z(max(ends) if ends else now), + } + + +def _normalise_iso_z(value: Any) -> str: + text = str(value) + return text[:-6] + "Z" if text.endswith("+00:00") else text + + +__all__ = ["export_traces", "load_trace_index", "load_trace_results"] diff --git a/netopsbench/platform/session/orchestrator.py b/netopsbench/platform/session/orchestrator.py index 407ae63..fc07671 100644 --- a/netopsbench/platform/session/orchestrator.py +++ b/netopsbench/platform/session/orchestrator.py @@ -2,47 +2,27 @@ from __future__ import annotations -import json from collections.abc import Iterable, Sequence from datetime import UTC, datetime from pathlib import Path -from types import SimpleNamespace from typing import Any -from netopsbench.agents.base import DiagnosticContext -from netopsbench.agents.tracing import AgentTraceRecorder -from netopsbench.evaluator.fault_type_judge import create_judge_from_env from netopsbench.evaluator.scorer import Evaluator from netopsbench.logging_utils import get_logger from netopsbench.platform.runtime.manager import RuntimeManager, RuntimePool -from netopsbench.platform.scenario.executor import ScenarioExecutor -from netopsbench.platform.session.context import ( - _build_toolkit_for_topology, - _extract_episode_pingmesh_query_window, - build_topology_snapshot, -) -from netopsbench.platform.session.diagnosis import AgentHandleAdapter, run_agent_diagnose from netopsbench.platform.session.dispatch import execute_on_runtime_pool -from netopsbench.platform.session.env import build_worker_execution_context from netopsbench.platform.session.reporting import ( LocalArtifactStore, artifacts_root, build_run_handle, create_run_report, - load_topology_metadata, next_run_id, resolve_scale, save_run_metadata, save_run_report, ) -from netopsbench.platform.session.scoring import score_scenario_fault_episodes -from netopsbench.platform.session.tracing import TraceWriter -from netopsbench.platform.session.types import ScenarioExecutionRef, WorkerExecutionContext -from netopsbench.platform.worker.pool import WorkerSpec -from netopsbench.platform.worker.runtime_agent_input import ( - build_public_case_id, - build_public_symptoms, -) +from netopsbench.platform.session.trace_store import TraceWriter +from netopsbench.platform.session.types import ScenarioExecutionRef logger = get_logger(__name__) @@ -232,45 +212,87 @@ def _execute_on_runtime_pool( ) -> Any: artifact_dir = self._artifacts_root(artifacts_dir) / run_id raw_dir = artifact_dir / "raw" - trace_writer = TraceWriter(artifact_dir / "traces", run_id=run_id) if self._trace_enabled(trace) else None + trace_writer = TraceWriter(artifact_dir / "traces", run_id=run_id) if trace else None artifact_dir.mkdir(parents=True, exist_ok=True) raw_dir.mkdir(parents=True, exist_ok=True) report_path = artifact_dir / "report.json" metadata_path = artifact_dir / "metadata.json" - return execute_on_runtime_pool( + dispatched = execute_on_runtime_pool( + scenarios=scenarios, + runtime=runtime, + agent=agent, + raw_dir=raw_dir, + trace_writer=trace_writer, + fault_registry=getattr(getattr(self.platform, "faults", None), "spec_registry", None), + ) + aggregate_report = ( + Evaluator().generate_report( + dispatched.evaluations, + agent_name=getattr(agent, "name", agent.__class__.__name__), + topology_scale=runtime.scale, + ) + if dispatched.evaluations + else {"summary": {"total_cases": 0, "average_score": 0.0}, "detailed_results": []} + ) + completed_at = self._timestamp() + traces_dir = trace_writer.root_dir if trace_writer is not None else None + report_payload = create_run_report( run_id=run_id, mode=mode, - scenarios=scenarios, + started_at=started_at, + completed_at=completed_at, runtime=runtime, + runtime_owner=runtime_owner, + teardown=teardown, + scenarios=scenarios, agent=agent, + worker_summaries=dispatched.workers, + scenario_summaries=dispatched.scenarios, + aggregate_report=aggregate_report, artifact_dir=artifact_dir, raw_dir=raw_dir, - traces_dir=(trace_writer.root_dir if trace_writer is not None else None), - trace_writer=trace_writer, + traces_dir=traces_dir, + trace_index_path=(trace_writer.index_path if trace_writer is not None else None), + trace_results_path=(trace_writer.results_path if trace_writer is not None else None), report_path=report_path, metadata_path=metadata_path, + ) + self._save_report_adapter(report_payload, report_path) + report_status = str( + (report_payload.get("raw") or {}).get("status") + or (report_payload.get("summary") or {}).get("status") + or report_payload.get("status") + or "unknown" + ) + save_run_metadata( + self.artifacts, + artifact_dir, + run_id=run_id, + mode=mode, + status=report_status, + runtime_id=runtime.id, runtime_owner=runtime_owner, teardown=teardown, started_at=started_at, - completed_at_factory=self._timestamp, - scenario_runner_cls=ScenarioExecutor, - evaluator_factory=lambda: ( - Evaluator(fault_type_judge=_j) if (_j := create_judge_from_env()) is not None else Evaluator() - ), - score_fault_episodes=score_scenario_fault_episodes, - diagnosis_callback_builder=self._build_runtime_diagnosis_callback, - worker_context_builder=self._build_worker_execution_context, - topology_metadata_loader=self._load_topology_metadata, - create_run_report=create_run_report, - save_run_report=self._save_report_adapter, - save_run_metadata=save_run_metadata, - build_run_handle=build_run_handle, - run_handle_adapter=self._run_handle_adapter, - artifact_manager=self.artifacts, + completed_at=completed_at, + scenarios=scenarios, + worker_summaries=dispatched.workers, + traces_dir=traces_dir, + trace_index_path=(trace_writer.index_path if trace_writer is not None else None), + trace_results_path=(trace_writer.results_path if trace_writer is not None else None), ) - - def _build_worker_execution_context(self, worker: WorkerSpec, topology_dir: Path) -> WorkerExecutionContext: - return build_worker_execution_context(worker, topology_dir) + handle_payload = build_run_handle( + run_id=run_id, + mode=mode, + status=report_status, + started_at=started_at, + completed_at=completed_at, + artifact_dir=artifact_dir, + scenarios=scenarios, + runtime_id=runtime.id, + report_path=report_path, + ) + return self._run_handle_adapter(handle_payload) def _coerce_scenario(self, scenario: Any) -> ScenarioExecutionRef: return ScenarioExecutionRef.coerce(scenario) @@ -287,180 +309,6 @@ def _provision_runtime(self, *, scale: str, workers: int, name: str, root_dir: s runtime_root = (Path(root_dir) / name) if root_dir is not None else None return self.runtimes.provision(scale=scale, workers=workers, name=name, root_dir=runtime_root) - def _build_runtime_diagnosis_callback( - self, - agent: Any, - topology_dir: str, - scenario_id: str, - worker_context: WorkerExecutionContext | None = None, - trace_writer: TraceWriter | None = None, - worker_name: str | None = None, - run_id: str | None = None, - runtime_id: str | None = None, - scenario_scale: str | None = None, - ): - toolkit = _build_toolkit_for_topology(topology_dir) - if worker_context is not None: - try: - toolkit.influxdb_bucket = worker_context.influxdb_bucket - toolkit.topology_id = worker_context.topology_id - except Exception: - logger.debug("failed to attach worker context to toolkit", exc_info=True) - handle = agent if isinstance(agent, AgentHandleAdapter) else AgentHandleAdapter(agent) - context_dir = Path(topology_dir) / ".netopsbench" - context_file = context_dir / "pingmesh_context.json" - - # Worker env vars (e.g. NETOPSBENCH_TOPOLOGY_DIR) for agent-spawned - # subprocesses (MCP servers). Stored in context.metadata["worker_env"] - # so agents can forward them without mutating process env. - _worker_env = worker_context.as_env() if worker_context is not None else {} - _worker_env["NETOPSBENCH_PINGMESH_CONTEXT_FILE"] = str(context_file) - - def callback(episode_result: dict) -> dict: - start_time = self._timestamp() - trace_recorder = AgentTraceRecorder(enabled=trace_writer is not None) - pingmesh_query_window = _extract_episode_pingmesh_query_window(episode_result) - window_start = pingmesh_query_window.get("start_time") - window_end = pingmesh_query_window.get("end_time") - try: - toolkit.set_pingmesh_time_window(window_start, window_end) - except Exception: - logger.debug("failed to set toolkit pingmesh time window", exc_info=True) - if window_start and window_end: - try: - context_dir.mkdir(parents=True, exist_ok=True) - context_file.write_text( - json.dumps({"start_time": window_start, "end_time": window_end}), - encoding="utf-8", - ) - except Exception: - logger.debug("failed to write pingmesh context file", exc_info=True) - - context = DiagnosticContext( - scenario_id=build_public_case_id(scenario_id=scenario_id, episode_result=episode_result), - topology=build_topology_snapshot(toolkit), - symptoms=build_public_symptoms( - episode_result=episode_result, - pingmesh_query_window=pingmesh_query_window, - ), - ground_truth=None, - tools=toolkit, - trace=trace_recorder, - metadata={"worker_env": _worker_env} if _worker_env else {}, - ) - try: - diagnosis = self._run_agent_diagnose(handle, context) - except Exception as exc: - trace_recorder.record_error(stage="agent", error=exc) - ended_at = self._timestamp() - diagnosis_payload: dict[str, Any] = { - "error": str(exc), - "success": False, - "time_taken_seconds": max(0.0, (ended_at - start_time).total_seconds()), - "metadata": {"agent_failure_stage": "diagnose", "error_type": type(exc).__name__}, - } - if trace_writer is not None: - try: - trace_result = trace_writer.write_case_trace( - case_id=context.scenario_id, - scenario_id=scenario_id, - episode_result=episode_result, - worker=worker_name or "worker", - topology_id=(worker_context.topology_id if worker_context is not None else None), - topology_scale=scenario_scale, - runtime_id=runtime_id or "", - agent=agent, - diagnostic_context=context, - diagnosis=SimpleNamespace( - agent_name=getattr(handle, "name", "agent"), - success=False, - findings={"error": str(exc)}, - metadata=diagnosis_payload["metadata"], - ), - diagnosis_payload=diagnosis_payload, - started_at=start_time, - ended_at=ended_at, - pingmesh_window=pingmesh_query_window, - error=str(exc), - trace_recorder=trace_recorder, - ) - diagnosis_payload["trace"] = { - "trace_id": trace_result.trace_id, - "case_id": trace_result.case_id, - "worker": trace_result.worker, - "atif_path": trace_result.atif_path, - } - except Exception: - logger.debug("failed to persist failed agent runtime trace", exc_info=True) - diagnosis_payload["metadata"] = _strip_runtime_trace_metadata(diagnosis_payload["metadata"]) - return diagnosis_payload - findings = dict(diagnosis.findings or {}) - location = findings.get("location") or {} - if not isinstance(location, dict): - location = {} - ended_at = self._timestamp() - metadata = dict(diagnosis.metadata or {}) - recorder_metrics = trace_recorder.metrics() - for key in ("input_tokens", "output_tokens", "total_tokens", "llm_call_count"): - if recorder_metrics.get(key): - metadata[key] = recorder_metrics[key] - recorded_tool_calls = trace_recorder.tool_calls() - diagnosis_payload = { - "verdict": diagnosis.verdict, - "fault_type": findings.get("fault_type") or metadata.get("fault_type"), - "location": { - key: value - for key, value in { - "device": location.get("device") or findings.get("device"), - "interface": location.get("interface") or findings.get("interface"), - }.items() - if value is not None - }, - "evidence": list(findings.get("evidence") or []), - "confidence": float(diagnosis.confidence or 0.0), - "reasoning": diagnosis.reasoning, - "tool_calls": recorded_tool_calls or list(metadata.get("tool_calls") or []), - "time_taken_seconds": max(0.0, (ended_at - start_time).total_seconds()), - "metadata": metadata, - } - if trace_writer is not None: - try: - trace_result = trace_writer.write_case_trace( - case_id=context.scenario_id, - scenario_id=scenario_id, - episode_result=episode_result, - worker=worker_name or "worker", - topology_id=(worker_context.topology_id if worker_context is not None else None), - topology_scale=scenario_scale, - runtime_id=runtime_id or "", - agent=agent, - diagnostic_context=context, - diagnosis=diagnosis, - diagnosis_payload=diagnosis_payload, - started_at=start_time, - ended_at=ended_at, - pingmesh_window=pingmesh_query_window, - trace_recorder=trace_recorder, - ) - diagnosis_payload["trace"] = { - "trace_id": trace_result.trace_id, - "case_id": trace_result.case_id, - "worker": trace_result.worker, - "atif_path": trace_result.atif_path, - } - except Exception: - logger.debug("failed to persist agent runtime trace", exc_info=True) - diagnosis_payload["metadata"] = _strip_runtime_trace_metadata(diagnosis_payload["metadata"]) - return diagnosis_payload - - return callback - - def _run_agent_diagnose(self, handle: Any, context): - return run_agent_diagnose(handle, context) - - def _load_topology_metadata(self, topology_dir: Path): - return load_topology_metadata(topology_dir) - def _artifacts_root(self, artifacts_dir: str | Path | None) -> Path: return artifacts_root(self.artifacts, artifacts_dir) @@ -468,31 +316,10 @@ def _next_run_id(self, artifacts_root_dir: Path, *, started_at: datetime | None return next_run_id(artifacts_root_dir, started_at=started_at) def _resolve_scale(self, scenarios: Iterable[ScenarioExecutionRef]) -> str: - return resolve_scale(self.platform, scenarios) + return resolve_scale(scenarios) def _timestamp(self) -> datetime: return datetime.now(UTC) - def _trace_enabled(self, trace: bool) -> bool: - if not trace: - return False - raw = str(self.env_value("NETOPSBENCH_TRACE", "1")).strip().lower() - return raw not in {"0", "false", "no", "off"} - - def env_value(self, key: str, default: str = "") -> str: - env = getattr(self.platform, "env", None) - if isinstance(env, dict) and key in env: - return str(env[key]) - import os - - return os.environ.get(key, default) - - -def _strip_runtime_trace_metadata(metadata: dict[str, Any]) -> dict[str, Any]: - public_metadata = dict(metadata or {}) - public_metadata.pop("trace", None) - public_metadata.pop("trajectory", None) - return public_metadata - __all__ = ["SessionOrchestrator"] diff --git a/netopsbench/platform/session/reporting.py b/netopsbench/platform/session/reporting.py index c9f5f1f..f64c7a5 100644 --- a/netopsbench/platform/session/reporting.py +++ b/netopsbench/platform/session/reporting.py @@ -10,6 +10,7 @@ from netopsbench.logging_utils import get_logger from netopsbench.platform.session.types import ScenarioExecutionRef +from netopsbench.platform.topology.topology_utils import load_topology_manifest logger = get_logger(__name__) @@ -27,15 +28,8 @@ def save_metadata(self, artifact_dir: Path, payload: dict[str, Any]) -> None: metadata_path.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") -def load_topology_metadata(topology_dir: Path) -> dict[str, Any] | None: - topology_json = topology_dir / "topology.json" - if not topology_json.exists(): - return None - try: - return json.loads(topology_json.read_text(encoding="utf-8")) - except Exception: - logger.debug("failed to parse topology.json at %s", topology_json, exc_info=True) - return None +def load_topology_metadata(topology_dir: Path) -> dict[str, Any]: + return load_topology_manifest(topology_dir).model_dump(mode="json") def artifacts_root(artifact_manager: Any, artifacts_dir: str | Path | None) -> Path: @@ -53,12 +47,9 @@ def next_run_id(artifact_root: Path, *, started_at: datetime | None = None) -> s return f"{base}-{suffix:02d}" -def resolve_scale(platform: Any, scenarios: Iterable[ScenarioExecutionRef]) -> str: +def resolve_scale(scenarios: Iterable[ScenarioExecutionRef]) -> str: scenario_list = list(scenarios) - if scenario_list: - return scenario_list[0].scale - defaults = getattr(platform, "defaults", {}) or {} - return str(defaults.get("scale") or "xs") + return scenario_list[0].scale if scenario_list else "xs" def create_run_report( diff --git a/netopsbench/platform/session/scoring.py b/netopsbench/platform/session/scoring.py index 84fca5e..0a107f3 100644 --- a/netopsbench/platform/session/scoring.py +++ b/netopsbench/platform/session/scoring.py @@ -1,10 +1,10 @@ """Scenario scoring helpers.""" -import ipaddress from pathlib import Path from typing import Any from netopsbench.evaluator.scorer import AgentOutput, EvaluationResult, Evaluator +from netopsbench.platform.topology.configdb_payload import interface_networks_for_config from netopsbench.platform.utils.interface_names import are_interfaces_equivalent # Fault types where the injected interface and its link-peer are both valid answers. @@ -24,23 +24,7 @@ def _parse_device_interface_networks(config_path: Path) -> dict[str, str]: - interface_networks: dict[str, str] = {} - if not config_path.exists(): - return interface_networks - for raw_line in config_path.read_text(encoding="utf-8").splitlines(): - line = raw_line.strip() - if not line.startswith("config interface ip add "): - continue - parts = line.split() - if len(parts) < 6: - continue - interface_name = parts[4] - cidr = parts[5] - try: - interface_networks[interface_name] = str(ipaddress.ip_interface(cidr).network) - except ValueError: - continue - return interface_networks + return interface_networks_for_config(config_path) def _resolve_config_interface(target_interface: str | None, interface_names: list[str]) -> str | None: @@ -52,6 +36,23 @@ def _resolve_config_interface(target_interface: str | None, interface_names: lis return None +def _device_config_path(configs_dir: Path, device: str) -> Path: + return configs_dir / "sonic" / device / "config_db.json" + + +def _iter_device_config_paths(configs_dir: Path) -> list[Path]: + preseed_root = configs_dir / "sonic" + if not preseed_root.exists(): + return [] + return sorted(preseed_root.glob("*/config_db.json")) + + +def _device_name_for_config_path(config_path: Path) -> str: + if config_path.name == "config_db.json": + return config_path.parent.name + return config_path.stem + + def _find_link_peer_locations( topology_dir: str | None, target_device: str | None, @@ -62,7 +63,7 @@ def _find_link_peer_locations( configs_dir = Path(topology_dir) / "configs" if not configs_dir.exists(): return [] - target_config = configs_dir / f"{target_device}.sh" + target_config = _device_config_path(configs_dir, target_device) target_networks = _parse_device_interface_networks(target_config) target_config_interface = _resolve_config_interface(target_interface, list(target_networks.keys())) if not target_config_interface: @@ -72,8 +73,8 @@ def _find_link_peer_locations( return [] peers: list[dict[str, str]] = [] seen = set() - for config_path in sorted(configs_dir.glob("*.sh")): - peer_device = config_path.stem + for config_path in _iter_device_config_paths(configs_dir): + peer_device = _device_name_for_config_path(config_path) if peer_device == target_device: continue for peer_interface, peer_network in _parse_device_interface_networks(config_path).items(): @@ -173,17 +174,3 @@ def score_scenario_fault_episodes( eval_result.details["episode_id"] = episode_info.get("episode_id") scored_results.append(eval_result) return scored_results - - -def resolve_scenario_files(path: str) -> list[str]: - target = Path(path) - if not target.exists(): - raise FileNotFoundError(f"Scenario path not found: {path}") - if target.is_file(): - if target.suffix not in {".yaml", ".yml"}: - raise ValueError(f"Scenario file must be .yaml/.yml: {path}") - return [str(target)] - files = sorted([str(p) for p in target.iterdir() if p.is_file() and p.suffix in {".yaml", ".yml"}]) - if not files: - raise ValueError(f"No scenario YAML files found directly under {path}") - return files diff --git a/netopsbench/platform/session/trace_store.py b/netopsbench/platform/session/trace_store.py new file mode 100644 index 0000000..4825a03 --- /dev/null +++ b/netopsbench/platform/session/trace_store.py @@ -0,0 +1,262 @@ +"""Thread-safe trace persistence and evaluation indexes.""" + +from __future__ import annotations + +import json +import threading +import uuid +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from netopsbench.agents._trace_utils import jsonable as _jsonable +from netopsbench.agents.tracing import AgentTraceRecorder + +from .atif import build_atif_payload, build_trace_payload +from .trace_utils import ( + duration_seconds as _duration_seconds, +) +from .trace_utils import ( + load_jsonl as _load_jsonl, +) +from .trace_utils import ( + safe_path_part as _safe_path_part, +) +from .trace_utils import ( + to_json as _to_json, +) + + +@dataclass(frozen=True) +class TraceWriteResult: + """Trajectory path written for one diagnosis.""" + + trace_id: str + case_id: str + worker: str + atif_path: str + + +class TraceWriter: + """Thread-safe writer for per-case ATIF trajectories plus run indexes.""" + + def __init__(self, root_dir: str | Path, *, run_id: str): + self.root_dir = Path(root_dir) + self.run_id = str(run_id) + self.index_path = self.root_dir / "index.jsonl" + self.results_path = self.root_dir / "results.jsonl" + self._lock = threading.Lock() + self.root_dir.mkdir(parents=True, exist_ok=True) + + def write_case_trace( + self, + *, + case_id: str, + scenario_id: str, + episode_result: dict[str, Any], + worker: str, + topology_id: str | None, + runtime_id: str, + agent: Any, + diagnostic_context: Any, + diagnosis: Any, + diagnosis_payload: dict[str, Any], + started_at: datetime, + ended_at: datetime, + pingmesh_window: dict[str, Any] | None = None, + error: str | None = None, + trace_recorder: AgentTraceRecorder | None = None, + topology_scale: str | None = None, + ) -> TraceWriteResult: + safe_worker = _safe_path_part(worker) + safe_case_id = _safe_path_part(case_id) + trace_id = f"{self.run_id}:{safe_worker}:{safe_case_id}:{uuid.uuid4().hex[:12]}" + case_dir = self.root_dir / safe_worker / safe_case_id + case_dir.mkdir(parents=True, exist_ok=True) + + trace = build_trace_payload( + trace_id=trace_id, + run_id=self.run_id, + case_id=str(case_id), + scenario_id=str(scenario_id), + episode_result=episode_result, + worker=str(worker), + topology_id=topology_id, + runtime_id=str(runtime_id), + topology_scale=topology_scale, + agent=agent, + diagnosis=diagnosis, + diagnosis_payload=diagnosis_payload, + started_at=started_at, + ended_at=ended_at, + pingmesh_window=pingmesh_window, + error=error, + diagnostic_context=diagnostic_context, + trace_recorder=trace_recorder, + ) + atif_path = case_dir / "trajectory.atif.json" + atif_path.write_text(_to_json(build_atif_payload(trace)), encoding="utf-8") + + result = TraceWriteResult(trace_id=trace_id, case_id=str(case_id), worker=str(worker), atif_path=str(atif_path)) + self._append_index(result, trace) + return result + + def write_evaluation_results(self, *, evaluation_results: list[Any], scenario_result: dict[str, Any]) -> None: + trace_by_episode = _trace_refs_by_episode(scenario_result) + trace_by_scenario = _trace_refs_by_scenario(self.index_path) + rows: list[dict[str, Any]] = [] + for result in evaluation_results: + payload = result.to_dict() if hasattr(result, "to_dict") else dict(result) + details = payload.get("details") if isinstance(payload, dict) else {} + episode_id = (details or {}).get("episode_id") or _episode_id_from_testcase(payload.get("testcase_id")) + scenario_id = (details or {}).get("scenario_id") + trace_ref = trace_by_episode.get(str(episode_id)) or trace_by_scenario.get(str(scenario_id)) or {} + rows.append( + _jsonable( + { + "trace_id": trace_ref.get("trace_id"), + "atif_path": trace_ref.get("atif_path"), + "run_id": self.run_id, + "case_id": trace_ref.get("case_id"), + "scenario_id": scenario_id, + "episode_id": episode_id, + "testcase_id": payload.get("testcase_id"), + "score": payload.get("score"), + "correct_verdict": payload.get("correct_verdict"), + "correct_device": payload.get("correct_device"), + "correct_interface": payload.get("correct_interface"), + "correct_fault_type": payload.get("correct_fault_type"), + "details": details or {}, + } + ) + ) + if rows: + self._append_jsonl(self.results_path, rows) + + def write_failure_result( + self, + *, + scenario_id: str, + scenario_result: dict[str, Any], + stage: str, + error: BaseException | str, + ) -> None: + trace_by_episode = _trace_refs_by_episode(scenario_result) + trace_by_scenario = _trace_refs_by_scenario(self.index_path) + error_type = type(error).__name__ if isinstance(error, BaseException) else "Error" + rows = [ + { + "trace_id": trace_ref.get("trace_id"), + "atif_path": trace_ref.get("atif_path"), + "run_id": self.run_id, + "case_id": trace_ref.get("case_id"), + "scenario_id": scenario_id, + "episode_id": episode_id, + "status": "error", + "error_stage": stage, + "error_type": error_type, + "error": str(error), + } + for episode_id, trace_ref in trace_by_episode.items() + ] + if not rows: + trace_ref = trace_by_scenario.get(str(scenario_id)) or {} + rows.append( + { + "trace_id": trace_ref.get("trace_id"), + "atif_path": trace_ref.get("atif_path"), + "run_id": self.run_id, + "case_id": trace_ref.get("case_id"), + "scenario_id": scenario_id, + "status": "error", + "error_stage": stage, + "error_type": error_type, + "error": str(error), + } + ) + self._append_jsonl(self.results_path, [_jsonable(row) for row in rows]) + + def _append_index(self, result: TraceWriteResult, trace: dict[str, Any]) -> None: + metrics = trace.get("metrics") or {} + agent = trace.get("agent") or {} + model = trace.get("model") or {} + started_at = trace.get("started_at") + ended_at = trace.get("ended_at") + self._append_jsonl( + self.index_path, + [ + { + "trace_id": result.trace_id, + "run_id": self.run_id, + "case_id": result.case_id, + "scenario_id": trace.get("scenario_id"), + "episode_id": trace.get("episode_id"), + "worker": result.worker, + "topology_scale": trace.get("topology_scale"), + "status": "error" if trace.get("error") else "completed", + "started_at": started_at, + "ended_at": ended_at, + "duration_seconds": _duration_seconds(started_at, ended_at), + "agent": agent.get("name"), + "agent_class": agent.get("class"), + "model": model.get("model"), + "provider": model.get("provider"), + "runtime": model.get("runtime"), + "step_count": len(trace.get("steps") or []), + "tool_call_count": metrics.get("tool_calls_count"), + "llm_call_count": metrics.get("llm_call_count"), + "input_tokens": metrics.get("input_tokens"), + "output_tokens": metrics.get("output_tokens"), + "total_tokens": metrics.get("total_tokens"), + "error_stage": _error_stage(trace), + "atif_path": result.atif_path, + } + ], + ) + + def _append_jsonl(self, path: Path, rows: list[dict[str, Any]]) -> None: + with self._lock: + with path.open("a", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, sort_keys=True, default=str) + "\n") + + +def _trace_refs_by_episode(scenario_result: dict[str, Any]) -> dict[str, dict[str, Any]]: + refs: dict[str, dict[str, Any]] = {} + for episode_result in scenario_result.get("episodes") or []: + if not isinstance(episode_result, dict): + continue + episode_id = ((episode_result.get("episode") or {}).get("episode_id")) or "unknown" + diagnosis = episode_result.get("diagnosis") or {} + trace = dict(diagnosis.get("trace") or {}) + if trace: + trace.setdefault("case_id", diagnosis.get("case_id") or diagnosis.get("scenario_id")) + refs[str(episode_id)] = trace + return refs + + +def _trace_refs_by_scenario(index_path: Path) -> dict[str, dict[str, Any]]: + refs: dict[str, dict[str, Any]] = {} + for row in _load_jsonl(index_path): + scenario_id = row.get("scenario_id") + if scenario_id: + refs.setdefault(str(scenario_id), row) + return refs + + +def _episode_id_from_testcase(testcase_id: Any) -> str | None: + if not testcase_id: + return None + text = str(testcase_id) + return text.rsplit(":", 1)[1] if ":" in text else None + + +def _error_stage(trace: dict[str, Any]) -> str | None: + if not trace.get("error"): + return None + metadata = (trace.get("final_diagnosis") or {}).get("metadata") or {} + return metadata.get("agent_failure_stage") or metadata.get("failure_stage") or "diagnose" + + +__all__ = ["TraceWriter", "TraceWriteResult"] diff --git a/netopsbench/platform/session/trace_utils.py b/netopsbench/platform/session/trace_utils.py new file mode 100644 index 0000000..14b8927 --- /dev/null +++ b/netopsbench/platform/session/trace_utils.py @@ -0,0 +1,39 @@ +"""Shared serialization helpers for session traces.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +def load_jsonl(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def duration_seconds(started_at: Any, ended_at: Any) -> float | None: + if not started_at or not ended_at: + return None + try: + start = datetime.fromisoformat(str(started_at).replace("Z", "+00:00")) + end = datetime.fromisoformat(str(ended_at).replace("Z", "+00:00")) + except (TypeError, ValueError): + return None + return max(0.0, (end - start).total_seconds()) + + +def safe_path_part(value: Any) -> str: + safe = "".join(ch if ch.isalnum() or ch in {"-", "_", "."} else "_" for ch in str(value)) + return safe[:120] or "unknown" + + +def isoformat(value: datetime) -> str: + timestamp = value if value.tzinfo is not None else value.replace(tzinfo=UTC) + return timestamp.astimezone(UTC).isoformat() + + +def to_json(payload: Any) -> str: + return json.dumps(payload, indent=2, sort_keys=True, default=str) diff --git a/netopsbench/platform/session/tracing.py b/netopsbench/platform/session/tracing.py deleted file mode 100644 index 4e51118..0000000 --- a/netopsbench/platform/session/tracing.py +++ /dev/null @@ -1,1054 +0,0 @@ -"""Harbor-compatible trajectory persistence for session execution.""" - -from __future__ import annotations - -import json -import shutil -import threading -import uuid -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from harbor.models.agent.context import AgentContext -from harbor.models.job.config import DatasetConfig, JobConfig -from harbor.models.job.result import JobResult, JobStats -from harbor.models.task.id import LocalTaskId -from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, TrialConfig, VerifierConfig -from harbor.models.trial.result import AgentInfo, ModelInfo, TimingInfo, TrialResult -from harbor.models.verifier.result import VerifierResult - -from netopsbench.agents._trace_utils import jsonable as _jsonable -from netopsbench.agents.tracing import AgentTraceRecorder - -ATIF_SCHEMA_VERSION = "ATIF-v1.7" - - -@dataclass(frozen=True) -class TraceWriteResult: - """Trajectory path written for one diagnosis.""" - - trace_id: str - case_id: str - worker: str - atif_path: str - - -class TraceWriter: - """Thread-safe writer for per-case ATIF trajectories plus run indexes.""" - - def __init__(self, root_dir: str | Path, *, run_id: str): - self.root_dir = Path(root_dir) - self.run_id = str(run_id) - self.index_path = self.root_dir / "index.jsonl" - self.results_path = self.root_dir / "results.jsonl" - self._lock = threading.Lock() - self.root_dir.mkdir(parents=True, exist_ok=True) - - def write_case_trace( - self, - *, - case_id: str, - scenario_id: str, - episode_result: dict[str, Any], - worker: str, - topology_id: str | None, - runtime_id: str, - agent: Any, - diagnostic_context: Any, - diagnosis: Any, - diagnosis_payload: dict[str, Any], - started_at: datetime, - ended_at: datetime, - pingmesh_window: dict[str, Any] | None = None, - error: str | None = None, - trace_recorder: AgentTraceRecorder | None = None, - topology_scale: str | None = None, - ) -> TraceWriteResult: - safe_worker = _safe_path_part(worker) - safe_case_id = _safe_path_part(case_id) - trace_id = f"{self.run_id}:{safe_worker}:{safe_case_id}:{uuid.uuid4().hex[:12]}" - case_dir = self.root_dir / safe_worker / safe_case_id - case_dir.mkdir(parents=True, exist_ok=True) - - trace = build_trace_payload( - trace_id=trace_id, - run_id=self.run_id, - case_id=str(case_id), - scenario_id=str(scenario_id), - episode_result=episode_result, - worker=str(worker), - topology_id=topology_id, - runtime_id=str(runtime_id), - topology_scale=topology_scale, - agent=agent, - diagnosis=diagnosis, - diagnosis_payload=diagnosis_payload, - started_at=started_at, - ended_at=ended_at, - pingmesh_window=pingmesh_window, - error=error, - diagnostic_context=diagnostic_context, - trace_recorder=trace_recorder, - ) - atif_path = case_dir / "trajectory.atif.json" - atif_path.write_text(_to_json(build_atif_payload(trace)), encoding="utf-8") - - result = TraceWriteResult(trace_id=trace_id, case_id=str(case_id), worker=str(worker), atif_path=str(atif_path)) - self._append_index(result, trace) - return result - - def write_evaluation_results(self, *, evaluation_results: list[Any], scenario_result: dict[str, Any]) -> None: - trace_by_episode = _trace_refs_by_episode(scenario_result) - trace_by_scenario = _trace_refs_by_scenario(self.index_path) - rows: list[dict[str, Any]] = [] - for result in evaluation_results: - payload = result.to_dict() if hasattr(result, "to_dict") else dict(result) - details = payload.get("details") if isinstance(payload, dict) else {} - episode_id = (details or {}).get("episode_id") or _episode_id_from_testcase(payload.get("testcase_id")) - scenario_id = (details or {}).get("scenario_id") - trace_ref = trace_by_episode.get(str(episode_id)) or trace_by_scenario.get(str(scenario_id)) or {} - rows.append( - _jsonable( - { - "trace_id": trace_ref.get("trace_id"), - "atif_path": trace_ref.get("atif_path"), - "run_id": self.run_id, - "case_id": trace_ref.get("case_id"), - "scenario_id": scenario_id, - "episode_id": episode_id, - "testcase_id": payload.get("testcase_id"), - "score": payload.get("score"), - "correct_verdict": payload.get("correct_verdict"), - "correct_device": payload.get("correct_device"), - "correct_interface": payload.get("correct_interface"), - "correct_fault_type": payload.get("correct_fault_type"), - "details": details or {}, - } - ) - ) - if rows: - self._append_jsonl(self.results_path, rows) - - def write_failure_result( - self, - *, - scenario_id: str, - scenario_result: dict[str, Any], - stage: str, - error: BaseException | str, - ) -> None: - trace_by_episode = _trace_refs_by_episode(scenario_result) - trace_by_scenario = _trace_refs_by_scenario(self.index_path) - error_type = type(error).__name__ if isinstance(error, BaseException) else "Error" - rows = [ - { - "trace_id": trace_ref.get("trace_id"), - "atif_path": trace_ref.get("atif_path"), - "run_id": self.run_id, - "case_id": trace_ref.get("case_id"), - "scenario_id": scenario_id, - "episode_id": episode_id, - "status": "error", - "error_stage": stage, - "error_type": error_type, - "error": str(error), - } - for episode_id, trace_ref in trace_by_episode.items() - ] - if not rows: - trace_ref = trace_by_scenario.get(str(scenario_id)) or {} - rows.append( - { - "trace_id": trace_ref.get("trace_id"), - "atif_path": trace_ref.get("atif_path"), - "run_id": self.run_id, - "case_id": trace_ref.get("case_id"), - "scenario_id": scenario_id, - "status": "error", - "error_stage": stage, - "error_type": error_type, - "error": str(error), - } - ) - self._append_jsonl(self.results_path, [_jsonable(row) for row in rows]) - - def _append_index(self, result: TraceWriteResult, trace: dict[str, Any]) -> None: - metrics = trace.get("metrics") or {} - agent = trace.get("agent") or {} - model = trace.get("model") or {} - started_at = trace.get("started_at") - ended_at = trace.get("ended_at") - self._append_jsonl( - self.index_path, - [ - { - "trace_id": result.trace_id, - "run_id": self.run_id, - "case_id": result.case_id, - "scenario_id": trace.get("scenario_id"), - "episode_id": trace.get("episode_id"), - "worker": result.worker, - "topology_scale": trace.get("topology_scale"), - "status": "error" if trace.get("error") else "completed", - "started_at": started_at, - "ended_at": ended_at, - "duration_seconds": _duration_seconds(started_at, ended_at), - "agent": agent.get("name"), - "agent_class": agent.get("class"), - "model": model.get("model"), - "provider": model.get("provider"), - "runtime": model.get("runtime"), - "step_count": len(trace.get("steps") or []), - "tool_call_count": metrics.get("tool_calls_count"), - "llm_call_count": metrics.get("llm_call_count"), - "input_tokens": metrics.get("input_tokens"), - "output_tokens": metrics.get("output_tokens"), - "total_tokens": metrics.get("total_tokens"), - "error_stage": _error_stage(trace), - "atif_path": result.atif_path, - } - ], - ) - - def _append_jsonl(self, path: Path, rows: list[dict[str, Any]]) -> None: - with self._lock: - with path.open("a", encoding="utf-8") as handle: - for row in rows: - handle.write(json.dumps(row, sort_keys=True, default=str) + "\n") - - -def build_trace_payload( - *, - trace_id: str, - run_id: str, - case_id: str, - scenario_id: str, - episode_result: dict[str, Any], - worker: str, - topology_id: str | None, - runtime_id: str, - agent: Any, - diagnosis: Any, - diagnosis_payload: dict[str, Any], - started_at: datetime, - ended_at: datetime, - pingmesh_window: dict[str, Any] | None = None, - error: str | None = None, - diagnostic_context: Any = None, - trace_recorder: AgentTraceRecorder | None = None, - topology_scale: str | None = None, -) -> dict[str, Any]: - metadata = dict(getattr(diagnosis, "metadata", None) or {}) - model = _agent_model_payload(agent) - model = {**model, **_model_payload(metadata)} - if trace_recorder is not None: - model = {**model, **trace_recorder.model_metadata()} - steps = _context_steps(diagnostic_context) - runtime_steps = trace_recorder.to_steps() if trace_recorder is not None else [] - steps.extend(_normalise_steps(runtime_steps, model_metadata=model)) - if not runtime_steps: - steps.extend(_steps_from_tool_calls(diagnosis_payload.get("tool_calls") or metadata.get("tool_calls") or [])) - steps.append(_final_diagnosis_step(diagnosis_payload, ended_at=ended_at)) - return { - "trace_id": trace_id, - "run_id": run_id, - "case_id": case_id, - "scenario_id": scenario_id, - "episode_id": (episode_result.get("episode") or {}).get("episode_id"), - "worker": worker, - "topology_id": topology_id, - "topology_scale": topology_scale, - "runtime_id": runtime_id, - "agent": _agent_payload(agent, diagnosis), - "model": model, - "pingmesh_window": _jsonable(pingmesh_window or {}), - "started_at": _isoformat(started_at), - "ended_at": _isoformat(ended_at), - "steps": _jsonable(steps), - "final_diagnosis": _diagnosis_payload(diagnosis_payload), - "metrics": _metrics_payload( - metadata, - diagnosis_payload, - trace_recorder=trace_recorder, - started_at=started_at, - ended_at=ended_at, - ), - "error": error or _diagnosis_error(diagnosis, diagnosis_payload), - } - - -def build_atif_payload(trace: dict[str, Any]) -> dict[str, Any]: - return { - "schema_version": ATIF_SCHEMA_VERSION, - "session_id": trace.get("run_id"), - "trajectory_id": trace.get("trace_id"), - "agent": _atif_agent(trace), - "steps": _atif_steps(trace), - "notes": None, - "final_metrics": _atif_final_metrics(trace), - "extra": { - "framework": "netopsbench", - "case_id": trace.get("case_id"), - "scenario_id": trace.get("scenario_id"), - "episode_id": trace.get("episode_id"), - "runtime_id": trace.get("runtime_id"), - "worker": trace.get("worker"), - "pingmesh_window": trace.get("pingmesh_window") or {}, - "topology_id": trace.get("topology_id"), - "topology_scale": trace.get("topology_scale"), - "started_at": trace.get("started_at"), - "ended_at": trace.get("ended_at"), - "final_diagnosis": trace.get("final_diagnosis") or {}, - "error": trace.get("error"), - }, - } - - -def export_traces(run_dir: str | Path, *, output: str | Path) -> Path: - """Export a NetOpsBench run into a local Harbor viewer jobs directory.""" - run_path = Path(run_dir) - traces_dir = run_path / "traces" - if not traces_dir.is_dir(): - raise FileNotFoundError(f"trace directory not found: {traces_dir}") - - index_rows = load_trace_index(run_path) - if not index_rows: - raise FileNotFoundError(f"trace index is empty or missing: {traces_dir / 'index.jsonl'}") - - output_root = Path(output) - output_root.mkdir(parents=True, exist_ok=True) - job_name = f"netopsbench-{run_path.name}" - job_dir = output_root / job_name - if job_dir.exists(): - shutil.rmtree(job_dir) - job_dir.mkdir(parents=True) - - job_id = str(uuid.uuid5(uuid.NAMESPACE_URL, f"netopsbench:{run_path.name}")) - result_rows = load_trace_results(run_path) - trial_results = [ - _write_harbor_trial( - job_dir, - job_id=job_id, - index_row=row, - atif=json.loads(_resolve_atif_path(run_path, row).read_text(encoding="utf-8")), - result_row=_matching_result_row(result_rows, row), - ) - for row in index_rows - ] - job_config = _harbor_job_config(job_name, output_root, index_rows) - (job_dir / "config.json").write_text(_to_json(job_config.model_dump(mode="json")), encoding="utf-8") - (job_dir / "result.json").write_text( - _to_json(_harbor_job_result(job_id, _run_times(run_path, index_rows), trial_results).model_dump(mode="json")), - encoding="utf-8", - ) - return output_root - - -def load_trace_index(run_dir: str | Path) -> list[dict[str, Any]]: - return _load_jsonl(Path(run_dir) / "traces" / "index.jsonl") - - -def load_trace_results(run_dir: str | Path) -> list[dict[str, Any]]: - return _load_jsonl(Path(run_dir) / "traces" / "results.jsonl") - - -def _write_harbor_trial( - job_dir: Path, - *, - job_id: str, - index_row: dict[str, Any], - atif: dict[str, Any], - result_row: dict[str, Any] | None, -) -> TrialResult: - case_id = str(index_row.get("case_id") or (atif.get("extra") or {}).get("case_id") or "case") - scenario_id = str(index_row.get("scenario_id") or (atif.get("extra") or {}).get("scenario_id") or "scenario") - trial_name = _safe_path_part(f"{scenario_id}__{case_id}") - trial_dir = job_dir / trial_name - agent_dir = trial_dir / "agent" - verifier_dir = trial_dir / "verifier" - task_dir = trial_dir / "task" - agent_dir.mkdir(parents=True) - verifier_dir.mkdir(parents=True) - task_dir.mkdir(parents=True) - - score = float((result_row or {}).get("score") or 0.0) - (agent_dir / "trajectory.json").write_text(_to_json(atif), encoding="utf-8") - (verifier_dir / "reward.txt").write_text(str(score), encoding="utf-8") - (verifier_dir / "result.json").write_text( - _to_json({"reward": score, "netopsbench_result": result_row or {}}), - encoding="utf-8", - ) - (verifier_dir / "test-stdout.txt").write_text(_to_json(result_row or {}), encoding="utf-8") - (verifier_dir / "test-stderr.txt").write_text("", encoding="utf-8") - (task_dir / "instruction.md").write_text( - f"NetOpsBench scenario {scenario_id} episode {index_row.get('episode_id') or ''}\n", - encoding="utf-8", - ) - - config = _harbor_trial_config(job_id, job_dir, task_dir, trial_name, index_row) - trial_result = _harbor_trial_result(trial_dir, task_dir, trial_name, scenario_id, case_id, config, index_row, score) - (trial_dir / "config.json").write_text(_to_json(config.model_dump(mode="json")), encoding="utf-8") - (trial_dir / "result.json").write_text(_to_json(trial_result.model_dump(mode="json")), encoding="utf-8") - return trial_result - - -def _harbor_trial_config( - job_id: str, - job_dir: Path, - task_dir: Path, - trial_name: str, - index_row: dict[str, Any], -) -> TrialConfig: - return TrialConfig( - task=TaskConfig(path=task_dir, source=_dataset_source(index_row)), - trial_name=trial_name, - trials_dir=job_dir, - agent=AgentConfig(name=index_row.get("agent") or "unknown", model_name=index_row.get("model")), - environment=EnvironmentConfig(type="docker"), - verifier=VerifierConfig(disable=False), - artifacts=[], - extra_instruction_paths=[], - job_id=job_id, - ) - - -def _harbor_trial_result( - trial_dir: Path, - task_dir: Path, - trial_name: str, - scenario_id: str, - case_id: str, - config: TrialConfig, - index_row: dict[str, Any], - score: float, -) -> TrialResult: - started_at = index_row.get("started_at") - ended_at = index_row.get("ended_at") or started_at - return TrialResult( - id=uuid.uuid5(uuid.NAMESPACE_URL, f"{index_row.get('trace_id')}:{trial_name}"), - task_name=scenario_id, - trial_name=trial_name, - trial_uri=str(trial_dir), - task_id=LocalTaskId(path=task_dir), - source=_dataset_source(index_row), - task_checksum=case_id, - config=config, - agent_info=AgentInfo( - name=index_row.get("agent") or "unknown", - version=index_row.get("agent_class") or "unknown", - model_info=ModelInfo(name=index_row.get("model") or "unknown", provider=index_row.get("provider")), - ), - agent_result=AgentContext( - n_input_tokens=index_row.get("input_tokens"), - n_output_tokens=index_row.get("output_tokens"), - metadata={ - "trace_id": index_row.get("trace_id"), - "step_count": index_row.get("step_count"), - "topology_scale": index_row.get("topology_scale"), - "provider": index_row.get("provider"), - "model": index_row.get("model"), - }, - ), - verifier_result=VerifierResult(rewards={"reward": score, "score": score}), - started_at=started_at, - finished_at=ended_at, - agent_execution=TimingInfo(started_at=started_at, finished_at=ended_at), - verifier=TimingInfo(started_at=ended_at, finished_at=ended_at), - ) - - -def _harbor_job_config(job_name: str, output_root: Path, index_rows: list[dict[str, Any]]) -> JobConfig: - agent_configs: dict[tuple[str, str | None], AgentConfig] = {} - dataset_names = sorted({_dataset_source(row) for row in index_rows}) - for row in index_rows: - agent_name = str(row.get("agent") or "unknown") - model_name = _job_config_model_name(row) - agent_configs.setdefault((agent_name, model_name), AgentConfig(name=agent_name, model_name=model_name)) - return JobConfig( - job_name=job_name, - jobs_dir=output_root, - agents=list(agent_configs.values()), - datasets=[DatasetConfig(name=name) for name in dataset_names], - environment=EnvironmentConfig(type="docker"), - ) - - -def _harbor_job_result(job_id: str, run_times: dict[str, str], trial_results: list[TrialResult]) -> JobResult: - stats = JobStats.from_trial_results(trial_results, n_total_trials=len(trial_results)) - _attach_mean_reward_metrics(stats, trial_results) - return JobResult( - id=job_id, - started_at=run_times["started_at"], - updated_at=run_times["finished_at"], - finished_at=run_times["finished_at"], - n_total_trials=len(trial_results), - stats=stats, - trial_results=trial_results, - ) - - -def _normalise_steps(steps: list[Any], *, model_metadata: dict[str, Any]) -> list[dict[str, Any]]: - normalised: list[dict[str, Any]] = [] - saw_runtime_event = False - seen_copied_context: set[str] = set() - for index, item in enumerate(steps, 1): - step = dict(item) if isinstance(item, dict) else {"content": str(item)} - step.setdefault("index", index) - step["type"] = str(step.get("type") or step.get("role") or step.get("source") or "message") - if step.get("is_copied_context"): - fingerprint = _message_fingerprint(step) - if saw_runtime_event or fingerprint in seen_copied_context: - continue - seen_copied_context.add(fingerprint) - step.pop("is_copied_context", None) - step["is_initial_context"] = True - else: - saw_runtime_event = True - if model_metadata and step["type"] == "llm": - step.setdefault("model", model_metadata.get("model")) - step.setdefault("provider", model_metadata.get("provider")) - normalised.append(_jsonable(step)) - return normalised - - -def _atif_steps(trace: dict[str, Any]) -> list[dict[str, Any]]: - atif_steps: list[dict[str, Any]] = [] - for step_id, native in enumerate(_collapse_copied_context_steps(trace.get("steps") or []), 1): - step = native if isinstance(native, dict) else {"content": str(native)} - atif_step: dict[str, Any] = { - "step_id": step_id, - "timestamp": step.get("started_at") or step.get("ended_at") or trace.get("started_at"), - "source": _atif_source(step), - "message": _atif_message_content(_step_message(step) or ""), - "extra": _compact_extra(step), - } - if atif_step["source"] == "agent" and (step.get("model") or (trace.get("model") or {}).get("model")): - atif_step["model_name"] = step.get("model") or (trace.get("model") or {}).get("model") - if step.get("reasoning_content"): - atif_step["reasoning_content"] = step.get("reasoning_content") - metrics = _atif_step_metrics(step) - if metrics: - atif_step["metrics"] = metrics - if _is_tool_step(step): - tool_call_id = str(step.get("tool_call_id") or step.get("run_id") or f"tool-{step_id}") - function_name = step.get("name") or step.get("tool_name") or step.get("tool") or "tool" - atif_step["source"] = "agent" - atif_step["message"] = atif_step["message"] or f"Tool call: {function_name}" - atif_step["tool_calls"] = [ - { - "tool_call_id": tool_call_id, - "function_name": function_name, - "arguments": step.get("args") or step.get("input") or {}, - "extra": {"run_id": step.get("run_id"), "parent_run_id": step.get("parent_run_id")}, - } - ] - observation = step.get("observation") - if observation is None and step.get("error") is not None: - observation = {"error": step.get("error")} - if observation is not None: - atif_step["observation"] = { - "results": [ - { - "source_call_id": tool_call_id, - "content": _atif_observation_content(observation), - "extra": {"status": "error" if step.get("error") else "success"}, - } - ] - } - atif_steps.append(_jsonable(atif_step)) - return atif_steps - - -def _atif_agent(trace: dict[str, Any]) -> dict[str, Any]: - agent = trace.get("agent") or {} - model = trace.get("model") or {} - return _jsonable( - { - "name": agent.get("name") or "unknown", - "version": agent.get("class") or "unknown", - "model_name": model.get("model") or "unknown", - "extra": {"class": agent.get("class"), "provider": model.get("provider"), "runtime": model.get("runtime")}, - } - ) - - -def _atif_final_metrics(trace: dict[str, Any]) -> dict[str, Any]: - metrics = trace.get("metrics") or {} - return _jsonable( - { - "total_prompt_tokens": metrics.get("input_tokens"), - "total_completion_tokens": metrics.get("output_tokens"), - "total_steps": len(trace.get("steps") or []), - "extra": { - "total_tokens": metrics.get("total_tokens"), - "llm_call_count": metrics.get("llm_call_count"), - "tool_calls_count": metrics.get("tool_calls_count"), - "time_taken_seconds": metrics.get("time_taken_seconds"), - }, - } - ) - - -def _trace_refs_by_episode(scenario_result: dict[str, Any]) -> dict[str, dict[str, Any]]: - refs: dict[str, dict[str, Any]] = {} - for episode_result in scenario_result.get("episodes") or []: - if not isinstance(episode_result, dict): - continue - episode_id = ((episode_result.get("episode") or {}).get("episode_id")) or "unknown" - diagnosis = episode_result.get("diagnosis") or {} - trace = dict(diagnosis.get("trace") or {}) - if trace: - trace.setdefault("case_id", diagnosis.get("case_id") or diagnosis.get("scenario_id")) - refs[str(episode_id)] = trace - return refs - - -def _trace_refs_by_scenario(index_path: Path) -> dict[str, dict[str, Any]]: - refs: dict[str, dict[str, Any]] = {} - for row in _load_jsonl(index_path): - scenario_id = row.get("scenario_id") - if scenario_id: - refs.setdefault(str(scenario_id), row) - return refs - - -def _agent_payload(agent: Any, diagnosis: Any) -> dict[str, Any]: - return { - "name": str(getattr(diagnosis, "agent_name", None) or getattr(agent, "name", None) or "unknown"), - "class": getattr(getattr(agent, "__class__", None), "__name__", type(agent).__name__), - } - - -def _agent_model_payload(agent: Any) -> dict[str, Any]: - payload: dict[str, Any] = {} - provider = _first_agent_attr(agent, ("vendor", "provider")) - model = _first_agent_attr(agent, ("model", "model_name")) - runtime = _first_agent_attr(agent, ("runtime", "runtime_name")) - if provider: - payload["provider"] = provider - if model: - payload["model"] = model - if runtime: - payload["runtime"] = runtime - return _jsonable(payload) - - -def _first_agent_attr(agent: Any, names: tuple[str, ...]) -> str | None: - for candidate in _agent_candidates(agent): - for name in names: - value = getattr(candidate, name, None) - if value is None or callable(value): - continue - text = str(value).strip() - if text: - return text - return None - - -def _agent_candidates(agent: Any): - seen: set[int] = set() - stack = [agent] - while stack: - candidate = stack.pop(0) - if candidate is None or id(candidate) in seen: - continue - seen.add(id(candidate)) - yield candidate - for attr in ("agent", "_agent", "wrapped_agent", "inner"): - inner = getattr(candidate, attr, None) - if inner is not None and id(inner) not in seen: - stack.append(inner) - - -def _model_payload(metadata: dict[str, Any]) -> dict[str, Any]: - return { - key: _jsonable(metadata[key]) - for key in ("provider", "model", "runtime") - if key in metadata and metadata.get(key) is not None - } - - -def _metrics_payload( - metadata: dict[str, Any], - diagnosis_payload: dict[str, Any], - *, - trace_recorder: AgentTraceRecorder | None = None, - started_at: datetime, - ended_at: datetime, -) -> dict[str, Any]: - recorder_metrics = trace_recorder.metrics() if trace_recorder is not None else {} - payload = { - "time_taken_seconds": float( - diagnosis_payload.get("time_taken_seconds") or max(0.0, (ended_at - started_at).total_seconds()) - ), - "tool_calls_count": len( - (trace_recorder.tool_calls() if trace_recorder is not None else []) - or diagnosis_payload.get("tool_calls") - or metadata.get("tool_calls") - or [] - ), - } - for key in ("input_tokens", "output_tokens", "total_tokens", "llm_call_count"): - if recorder_metrics.get(key): - payload[key] = _jsonable(recorder_metrics[key]) - elif key in metadata: - payload[key] = _jsonable(metadata[key]) - return payload - - -def _diagnosis_payload(diagnosis_payload: dict[str, Any]) -> dict[str, Any]: - final = dict(diagnosis_payload) - metadata = dict(final.get("metadata") or {}) - metadata.pop("trace", None) - metadata.pop("trajectory", None) - final["metadata"] = metadata - return _jsonable(final) - - -def _diagnosis_error(diagnosis: Any, diagnosis_payload: dict[str, Any]) -> str | None: - if diagnosis_payload.get("error"): - return str(diagnosis_payload["error"]) - if getattr(diagnosis, "success", True) is False: - findings = getattr(diagnosis, "findings", None) or {} - if isinstance(findings, dict) and findings.get("error"): - return str(findings["error"]) - return None - - -def _context_steps(diagnostic_context: Any) -> list[dict[str, Any]]: - if diagnostic_context is None: - return [] - payload = { - "scenario_id": getattr(diagnostic_context, "scenario_id", None), - "topology": getattr(diagnostic_context, "topology", None), - "symptoms": getattr(diagnostic_context, "symptoms", None), - } - return [ - { - "type": "message", - "source": "user", - "message": _atif_message_content(_jsonable(payload)), - "is_copied_context": True, - } - ] - - -def _safe_int(value: Any) -> int: - try: - return int(value or 0) - except (TypeError, ValueError): - return 0 - - -def _steps_from_tool_calls(tool_calls: Any) -> list[dict[str, Any]]: - if not isinstance(tool_calls, list): - return [] - return [ - { - "index": index, - "type": "tool_call", - "name": (item if isinstance(item, dict) else {"tool": str(item)}).get("tool") - or (item if isinstance(item, dict) else {}).get("name") - or "tool", - "args": (item if isinstance(item, dict) else {}).get("args") - or (item if isinstance(item, dict) else {}).get("input") - or {}, - } - for index, item in enumerate(tool_calls, 1) - ] - - -def _final_diagnosis_step(diagnosis_payload: dict[str, Any], *, ended_at: datetime) -> dict[str, Any]: - final = _diagnosis_payload(diagnosis_payload) - verdict = final.get("verdict") or ("error" if final.get("error") else "unknown") - fault_type = final.get("fault_type") - location = final.get("location") if isinstance(final.get("location"), dict) else {} - location_text = ", ".join(str(value) for value in (location or {}).values() if value not in (None, "")) - parts = [f"Final diagnosis: {verdict}"] - if fault_type: - parts.append(f"fault_type={fault_type}") - if location_text: - parts.append(f"location={location_text}") - return { - "type": "final_diagnosis", - "source": "agent", - "message": "; ".join(parts), - "ended_at": _isoformat(ended_at), - "extra": {"final": True, "diagnosis": _final_diagnosis_summary(final)}, - } - - -def _final_diagnosis_summary(final: dict[str, Any]) -> dict[str, Any]: - return _jsonable( - { - key: final[key] - for key in ("verdict", "fault_type", "location", "evidence", "confidence", "reasoning", "error") - if key in final and final[key] not in (None, {}, []) - } - ) - - -def _load_jsonl(path: Path) -> list[dict[str, Any]]: - if not path.exists(): - return [] - return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] - - -def _resolve_atif_path(run_path: Path, row: dict[str, Any]) -> Path: - raw_path = row.get("atif_path") - if raw_path and Path(raw_path).exists(): - return Path(raw_path) - case_id = _safe_path_part(row.get("case_id") or "case") - matches = sorted((run_path / "traces").glob(f"*/{case_id}/trajectory.atif.json")) - if matches: - return matches[0] - worker = _safe_path_part(row.get("worker") or "worker") - candidate = run_path / "traces" / worker / case_id / "trajectory.atif.json" - if candidate.exists(): - return candidate - raise FileNotFoundError(f"ATIF trajectory not found for trace {row.get('trace_id')}") - - -def _matching_result_row(result_rows: list[dict[str, Any]], index_row: dict[str, Any]) -> dict[str, Any] | None: - trace_id = index_row.get("trace_id") - for row in result_rows: - if row.get("trace_id") == trace_id: - return row - for row in result_rows: - if row.get("scenario_id") == index_row.get("scenario_id") and row.get("episode_id") == index_row.get( - "episode_id" - ): - return row - return None - - -def _dataset_source(index_row: dict[str, Any]) -> str: - scale = str(index_row.get("topology_scale") or "").strip() - if scale and scale.lower() != "unknown": - return f"netopsbench-{scale}" - return "netopsbench" - - -def _job_config_model_name(index_row: dict[str, Any]) -> str | None: - model = index_row.get("model") - if not model: - return None - model_text = str(model) - provider = str(index_row.get("provider") or "").strip() - if provider and not model_text.startswith(f"{provider}/"): - return f"{provider}/{model_text}" - return model_text - - -def _attach_mean_reward_metrics(stats: JobStats, trial_results: list[TrialResult]) -> None: - rewards_by_eval: dict[str, list[dict[str, float | int] | None]] = {} - for trial_result in trial_results: - agent_name = trial_result.agent_info.name - model_name = trial_result.agent_info.model_info.name if trial_result.agent_info.model_info else None - dataset_name = trial_result.source or "adhoc" - evals_key = JobStats.format_agent_evals_key(agent_name, model_name, dataset_name) - rewards = trial_result.verifier_result.rewards if trial_result.verifier_result else None - rewards_by_eval.setdefault(evals_key, []).append(rewards) - for evals_key, rewards in rewards_by_eval.items(): - eval_stats = stats.evals.get(evals_key) - if eval_stats is not None: - eval_stats.metrics = [_mean_reward_metric(rewards)] - - -def _mean_reward_metric(rewards: list[dict[str, float | int] | None]) -> dict[str, float | int]: - reward_keys = sorted({key for reward in rewards if reward is not None for key in reward}) - if len(reward_keys) <= 1: - values = [0 if reward is None else next(iter(reward.values()), 0) for reward in rewards] - return {"mean": sum(values) / len(values) if values else 0.0} - return { - key: sum(0 if reward is None else reward.get(key, 0) for reward in rewards) / len(rewards) - for key in reward_keys - } - - -def _run_times(run_path: Path, index_rows: list[dict[str, Any]]) -> dict[str, str]: - report_path = run_path / "report.json" - if report_path.exists(): - try: - summary = json.loads(report_path.read_text(encoding="utf-8")).get("summary") or {} - if summary.get("started_at") and summary.get("completed_at"): - return { - "started_at": _normalise_iso_z(summary["started_at"]), - "finished_at": _normalise_iso_z(summary["completed_at"]), - } - except Exception: - pass - starts = [str(value) for row in index_rows if (value := row.get("started_at"))] - ends = [str(value) for row in index_rows if (value := row.get("ended_at"))] - now = _isoformat(datetime.now(UTC)) - return { - "started_at": _normalise_iso_z(min(starts) if starts else now), - "finished_at": _normalise_iso_z(max(ends) if ends else now), - } - - -def _normalise_iso_z(value: Any) -> str: - text = str(value) - return text[:-6] + "Z" if text.endswith("+00:00") else text - - -def _atif_source(step: dict[str, Any]) -> str: - raw = str(step.get("source") or step.get("role") or step.get("type") or "agent").lower() - if raw == "human": - return "user" - if raw in {"system", "user"}: - return raw - return "agent" - - -def _is_tool_step(step: dict[str, Any]) -> bool: - if str(step.get("type") or "").lower() in {"tool", "tool_call"}: - return True - if step.get("tool_call_id"): - return True - return bool(step.get("name") and ("args" in step or "input" in step)) - - -def _step_message(step: dict[str, Any]) -> Any: - for key in ("message", "content", "completion", "prompt"): - if step.get(key) is not None: - return step[key] - return None - - -def _atif_step_metrics(step: dict[str, Any]) -> dict[str, Any]: - usage = step.get("usage") if isinstance(step.get("usage"), dict) else {} - metrics: dict[str, Any] = {} - if usage: - metrics["prompt_tokens"] = usage.get("input_tokens") or usage.get("prompt_tokens") - metrics["completion_tokens"] = usage.get("output_tokens") or usage.get("completion_tokens") - if usage.get("total_tokens") is not None: - metrics.setdefault("extra", {})["total_tokens"] = usage["total_tokens"] - if step.get("duration_seconds") is not None: - metrics.setdefault("extra", {})["duration_seconds"] = step["duration_seconds"] - return {key: value for key, value in metrics.items() if value not in (None, {})} - - -def _atif_observation_content(value: Any) -> str | None: - if value is None: - return None - return value if isinstance(value, str) else json.dumps(_jsonable(value), sort_keys=True, default=str) - - -def _atif_message_content(value: Any) -> str: - return value if isinstance(value, str) else json.dumps(_jsonable(value), sort_keys=True, default=str) - - -def _compact_extra(step: dict[str, Any]) -> dict[str, Any]: - excluded = { - "message", - "content", - "prompt", - "completion", - "reasoning_content", - "tool_calls", - "observation", - "args", - "input", - "output", - "usage", - "extra", - } - extra = dict(step.get("extra") or {}) if isinstance(step.get("extra"), dict) else {} - for key, value in step.items(): - if key not in excluded: - extra[key] = value - return extra - - -def _collapse_copied_context_steps(steps: list[Any]) -> list[Any]: - collapsed: list[Any] = [] - saw_runtime_event = False - seen_context: set[str] = set() - for item in steps: - if not isinstance(item, dict): - collapsed.append(item) - saw_runtime_event = True - continue - if item.get("is_copied_context"): - fingerprint = _message_fingerprint(item) - if saw_runtime_event or fingerprint in seen_context: - continue - copied = dict(item) - copied.pop("is_copied_context", None) - copied["is_initial_context"] = True - collapsed.append(copied) - seen_context.add(fingerprint) - continue - collapsed.append(item) - saw_runtime_event = True - return collapsed - - -def _message_fingerprint(step: dict[str, Any]) -> str: - return json.dumps( - _jsonable( - { - "type": step.get("type"), - "source": step.get("source") or step.get("role"), - "message": step.get("message") if "message" in step else step.get("content"), - "tool_call_id": step.get("tool_call_id"), - } - ), - sort_keys=True, - default=str, - ) - - -def _episode_id_from_testcase(testcase_id: Any) -> str | None: - if not testcase_id: - return None - text = str(testcase_id) - return text.rsplit(":", 1)[1] if ":" in text else None - - -def _duration_seconds(started_at: Any, ended_at: Any) -> float | None: - try: - if not started_at or not ended_at: - return None - start = datetime.fromisoformat(str(started_at).replace("Z", "+00:00")) - end = datetime.fromisoformat(str(ended_at).replace("Z", "+00:00")) - return max(0.0, (end - start).total_seconds()) - except Exception: - return None - - -def _error_stage(trace: dict[str, Any]) -> str | None: - if not trace.get("error"): - return None - metadata = (trace.get("final_diagnosis") or {}).get("metadata") or {} - return metadata.get("agent_failure_stage") or metadata.get("failure_stage") or "diagnose" - - -def _safe_path_part(value: Any) -> str: - safe = "".join(ch if ch.isalnum() or ch in {"-", "_", "."} else "_" for ch in str(value)) - return safe[:120] or "unknown" - - -def _isoformat(value: datetime) -> str: - if value.tzinfo is None: - value = value.replace(tzinfo=UTC) - return value.isoformat() - - -def _to_json(payload: Any) -> str: - return json.dumps(payload, indent=2, sort_keys=True, default=str) - - -__all__ = [ - "ATIF_SCHEMA_VERSION", - "TraceWriter", - "TraceWriteResult", - "build_atif_payload", - "build_trace_payload", - "export_traces", - "load_trace_index", - "load_trace_results", -] diff --git a/netopsbench/platform/toolkit/__init__.py b/netopsbench/platform/toolkit/__init__.py index 9052d76..694ed67 100644 --- a/netopsbench/platform/toolkit/__init__.py +++ b/netopsbench/platform/toolkit/__init__.py @@ -1,10 +1 @@ -"""Toolkit interfaces for NetOpsBench platform internals. - -External integrations should use SDK MCP helpers from ``netopsbench.sdk.mcp``. -This module remains importable for internal platform code and tests. -""" - -from ._core.common import ToolResult -from .toolkit import AgentToolkit - -__all__ = ["AgentToolkit", "ToolResult"] +"""Internal troubleshooting toolkit subsystem.""" diff --git a/netopsbench/platform/toolkit/_core/device/bgp_parsers.py b/netopsbench/platform/toolkit/_core/device/bgp_parsers.py deleted file mode 100644 index 8d61a19..0000000 --- a/netopsbench/platform/toolkit/_core/device/bgp_parsers.py +++ /dev/null @@ -1,57 +0,0 @@ -"""BGP parsing helpers for device toolkit internals.""" - -from __future__ import annotations - -import ipaddress -from typing import Any - -from .text_parsers import coerce_value, normalize_key - - -def parse_bgp_summary(text: str) -> list[dict[str, Any]]: - if not text: - return [] - lines = [line.rstrip() for line in text.splitlines() if line.strip()] - header_idx = None - for idx, line in enumerate(lines): - if line.startswith("Neighbor") and "State" in line: - header_idx = idx - break - if header_idx is None: - return [] - headers = lines[header_idx].split() - rows: list[dict[str, Any]] = [] - for line in lines[header_idx + 1 :]: - parts = line.split() - if len(parts) < 2: - continue - neighbor = parts[0].strip() - try: - ipaddress.ip_address(neighbor) - except ValueError: - continue - if len(parts) > len(headers): - parts = parts[: len(headers) - 1] + [" ".join(parts[len(headers) - 1 :])] - row = {normalize_key(k): v for k, v in zip(headers, parts, strict=False)} - state_value = row.get("state_pfxrcd") or row.get("state_pfxrcd".lower()) - prefixes = None - state = None - if state_value is not None and str(state_value).isdigit(): - prefixes = int(state_value) - state = "Established" - else: - state = state_value - rows.append( - { - "neighbor": row.get("neighbor"), - "asn": coerce_value(row.get("as")), - "state": state, - "prefixes_received": prefixes, - "up_down": row.get("up_down"), - "msg_rcvd": coerce_value(row.get("msgrcvd")), - "msg_sent": coerce_value(row.get("msgsent")), - "in_q": coerce_value(row.get("inq")), - "out_q": coerce_value(row.get("outq")), - } - ) - return rows diff --git a/netopsbench/platform/toolkit/_core/device/device_ops.py b/netopsbench/platform/toolkit/_core/device/device_ops.py deleted file mode 100644 index 7bac8f0..0000000 --- a/netopsbench/platform/toolkit/_core/device/device_ops.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Compatibility aggregate mixin for device toolkit operations.""" - -from __future__ import annotations - -from .connectivity_ops import ConnectivityOpsMixin -from .interface_ops import InterfaceOpsMixin -from .log_ops import LogOpsMixin -from .routing_ops import RoutingOpsMixin - - -class DeviceOpsMixin(InterfaceOpsMixin, RoutingOpsMixin, LogOpsMixin, ConnectivityOpsMixin): - """Aggregate device toolkit operations mixin.""" - - -__all__ = [ - "ConnectivityOpsMixin", - "DeviceOpsMixin", - "InterfaceOpsMixin", - "LogOpsMixin", - "RoutingOpsMixin", -] diff --git a/netopsbench/platform/toolkit/_core/device/interface_ops.py b/netopsbench/platform/toolkit/_core/device/interface_ops.py index b3593cd..5d4f213 100644 --- a/netopsbench/platform/toolkit/_core/device/interface_ops.py +++ b/netopsbench/platform/toolkit/_core/device/interface_ops.py @@ -4,7 +4,11 @@ import subprocess +from netopsbench.platform.topology.topology_utils import is_network_device_name + from ..common import ToolResult +from .interface_parsers import merge_interface_tables, parse_ip_link_stats +from .text_parsers import parse_table class InterfaceOpsMixin: @@ -15,7 +19,7 @@ def get_device_interfaces(self, device: str, format: str = "structured") -> Tool if safe_format not in {"raw", "structured", "summary", "both"}: return ToolResult(success=False, data=None, error=f"Invalid format: {format}") container = self._resolve_container(device) - if device.startswith(("spine", "leaf")): + if is_network_device_name(device): brief_result = self._docker_exec(container, ["bash", "-lc", "show interfaces status"], timeout=30) detail_result = self._docker_exec(container, ["bash", "-lc", "show interfaces counters"], timeout=30) if brief_result.returncode != 0 and detail_result.returncode != 0: @@ -27,7 +31,7 @@ def get_device_interfaces(self, device: str, format: str = "structured") -> Tool "brief": "show interfaces status unavailable; using ip link", "detail": fallback.stdout, } - structured = self._parse_ip_link_stats(fallback.stdout) + structured = parse_ip_link_stats(fallback.stdout) if safe_format == "summary": return ToolResult(success=True, data=self._summarize_interface_data(device, structured)) if safe_format == "structured": @@ -43,9 +47,9 @@ def get_device_interfaces(self, device: str, format: str = "structured") -> Tool "detail": detail_result.stdout if detail_result.returncode == 0 else "Unable to get details", } if safe_format in {"structured", "summary", "both"}: - status_rows = self._parse_table(brief_result.stdout if brief_result.returncode == 0 else "") - counter_rows = self._parse_table(detail_result.stdout if detail_result.returncode == 0 else "") - merged = self._merge_interface_tables(status_rows, counter_rows) + status_rows = parse_table(brief_result.stdout if brief_result.returncode == 0 else "") + counter_rows = parse_table(detail_result.stdout if detail_result.returncode == 0 else "") + merged = merge_interface_tables(status_rows, counter_rows) structured = {"device": device, "interfaces": merged} if not merged: structured["warning"] = "Unable to parse interface tables; check raw output." @@ -60,7 +64,7 @@ def get_device_interfaces(self, device: str, format: str = "structured") -> Tool return ToolResult(success=False, data=None, error=result.stderr) raw_data = {"device": device, "interfaces": result.stdout} if safe_format in {"structured", "summary", "both"}: - structured = self._parse_ip_link_stats(result.stdout) + structured = parse_ip_link_stats(result.stdout) if safe_format == "summary": return ToolResult(success=True, data=self._summarize_interface_data(device, structured)) if safe_format == "structured": diff --git a/netopsbench/platform/toolkit/_core/device/interface_parsers.py b/netopsbench/platform/toolkit/_core/device/interface_parsers.py index 3da9ace..eefcbe2 100644 --- a/netopsbench/platform/toolkit/_core/device/interface_parsers.py +++ b/netopsbench/platform/toolkit/_core/device/interface_parsers.py @@ -5,6 +5,8 @@ import re from typing import Any +from netopsbench.platform.utils.interface_names import resolve_interface_metric_identities + from .text_parsers import coerce_value, extract_interface_name, normalize_key @@ -78,7 +80,7 @@ def get_live_interface_snapshot(toolkit, device: str, interface: str) -> dict[st interfaces = data.get("interfaces", []) if not isinstance(interfaces, list): return None - candidate_names = set(toolkit._resolve_interface_metric_identities(interface)["names"]) + candidate_names = set(resolve_interface_metric_identities(interface)["names"]) for entry in interfaces: if not isinstance(entry, dict): continue diff --git a/netopsbench/platform/toolkit/_core/device/log_ops.py b/netopsbench/platform/toolkit/_core/device/log_ops.py index 6b2a724..1a43984 100644 --- a/netopsbench/platform/toolkit/_core/device/log_ops.py +++ b/netopsbench/platform/toolkit/_core/device/log_ops.py @@ -2,9 +2,10 @@ from __future__ import annotations -import requests +from netopsbench.platform.observability.influxdb import query_flux from ..common import ToolResult +from .log_parsers import get_device_logs_fallback, parse_influx_syslog_rows class LogOpsMixin: @@ -23,21 +24,10 @@ def get_device_logs( else: safe_severity = None query = f"""\nfrom(bucket: "{self.influxdb_bucket}")\n |> range(start: -{safe_minutes}m)\n |> filter(fn: (r) => r._measurement == "syslog")\n |> filter(fn: (r) => r.source == "{safe_device}")\n |> filter(fn: (r) => r._field == "message")\n {severity_filter}\n |> sort(columns: ["_time"], desc: true)\n |> limit(n: 100)\n""" - headers = { - "Authorization": f"Token {self.influxdb_token}", - "Content-Type": "application/vnd.flux", - "Accept": "application/csv", - } - response = requests.post( - f"{self.influxdb_url}/api/v2/query?org={self.influxdb_org}", - headers=headers, - data=query, - timeout=30, - proxies={"http": "", "https": ""}, - ) - if response.status_code != 200: - return ToolResult(success=False, data=None, error=f"InfluxDB query failed: {response.text}") - structured_logs = self._parse_influx_syslog_rows(response.text) + result = query_flux(self.influxdb_url, self.influxdb_token, self.influxdb_org, query) + if result.status != "ok": + return ToolResult(success=False, data=None, error=f"InfluxDB query failed: {result.error}") + structured_logs = parse_influx_syslog_rows(result.text) if structured_logs: data = { "device": safe_device, @@ -47,10 +37,10 @@ def get_device_logs( "logs": structured_logs, } if include_raw: - data["raw_csv"] = response.text + data["raw_csv"] = result.text return ToolResult(success=True, data=data) - fallback_logs = self._get_device_logs_fallback( - safe_device, time_range_minutes=safe_minutes, severity=safe_severity + fallback_logs = get_device_logs_fallback( + self, safe_device, time_range_minutes=safe_minutes, severity=safe_severity ) warning = None source = "influxdb" @@ -66,9 +56,7 @@ def get_device_logs( "warning": warning, } if include_raw: - data["raw_csv"] = response.text + data["raw_csv"] = result.text return ToolResult(success=True, data=data) - except requests.exceptions.RequestException as e: - return ToolResult(success=False, data=None, error=f"Request failed: {str(e)}") except Exception as e: return ToolResult(success=False, data=None, error=str(e)) diff --git a/netopsbench/platform/toolkit/_core/device/log_parsers.py b/netopsbench/platform/toolkit/_core/device/log_parsers.py index e9a5502..c95ad00 100644 --- a/netopsbench/platform/toolkit/_core/device/log_parsers.py +++ b/netopsbench/platform/toolkit/_core/device/log_parsers.py @@ -6,6 +6,8 @@ from datetime import UTC, datetime, timedelta from typing import Any +from netopsbench.platform.topology.topology_utils import is_network_device_name + def parse_influx_syslog_rows(csv_text: str) -> list[dict[str, Any]]: import csv @@ -100,7 +102,7 @@ def get_device_logs_fallback( safe_minutes = max(1, min(int(time_range_minutes), 24 * 60)) cutoff = datetime.now(UTC) - timedelta(minutes=safe_minutes) candidate_files = ["/var/log/syslog"] - if str(device).startswith(("leaf", "spine")): + if is_network_device_name(str(device)): candidate_files.append("/var/log/frr/frr.log") quoted_files = " ".join(candidate_files) command = ( diff --git a/netopsbench/platform/toolkit/_core/device/parsers.py b/netopsbench/platform/toolkit/_core/device/parsers.py deleted file mode 100644 index c3f65d8..0000000 --- a/netopsbench/platform/toolkit/_core/device/parsers.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Compatibility re-export layer for device parser helpers.""" - -from __future__ import annotations - -from .bgp_parsers import parse_bgp_summary -from .interface_parsers import ( - get_active_interface_names, - get_live_interface_snapshot, - merge_interface_tables, - parse_ip_link_stats, -) -from .log_parsers import ( - get_device_logs_fallback, - parse_influx_syslog_rows, - parse_local_syslog_lines, -) -from .route_parsers import parse_route_table -from .telemetry_parsers import ( - get_recent_influx_interface_identities, - parse_influx_interface_identity_rows, - parse_influx_metric_rows, - parse_influx_timestamp, - query_influx_rows, - summarize_counter_points, -) -from .text_parsers import coerce_value, extract_interface_name, normalize_key, parse_table, preview_items - -__all__ = [ - "coerce_value", - "extract_interface_name", - "get_active_interface_names", - "get_device_logs_fallback", - "get_live_interface_snapshot", - "get_recent_influx_interface_identities", - "merge_interface_tables", - "normalize_key", - "parse_bgp_summary", - "parse_influx_interface_identity_rows", - "parse_influx_metric_rows", - "parse_influx_syslog_rows", - "parse_influx_timestamp", - "parse_ip_link_stats", - "parse_local_syslog_lines", - "parse_route_table", - "parse_table", - "preview_items", - "query_influx_rows", - "summarize_counter_points", -] diff --git a/netopsbench/platform/toolkit/_core/device/route_parsers.py b/netopsbench/platform/toolkit/_core/device/route_parsers.py index e4b0444..a8918c3 100644 --- a/netopsbench/platform/toolkit/_core/device/route_parsers.py +++ b/netopsbench/platform/toolkit/_core/device/route_parsers.py @@ -31,6 +31,21 @@ def parse_nexthops(rest: str) -> list[dict[str, str | None]]: hops.append({"via": via, "interface": iface}) return hops + def add_route_state(route: dict[str, Any], raw_text: str) -> None: + code = str(route.get("code") or "") + route["selected"] = ">" in code or "best" in raw_text.lower() + discard_match = re.search(r"\b(Null0|blackhole|reject)\b", raw_text, re.IGNORECASE) + discard_hop = next( + ( + hop.get("interface") + for hop in route.get("nexthops", []) + if str(hop.get("interface") or "").lower() == "null0" + ), + None, + ) + route["is_discard"] = bool(discard_match or discard_hop) + route["discard_interface"] = discard_hop or (discard_match.group(1) if discard_match else None) + lines = [line.rstrip() for line in text.splitlines() if line.strip()] if lines and lines[0].startswith("Routing entry for "): prefix = lines[0].split("Routing entry for ", 1)[1].strip() @@ -55,6 +70,7 @@ def parse_nexthops(rest: str) -> list[dict[str, str | None]]: if nh_match: via, iface = nh_match.groups() route["nexthops"].append({"via": via, "interface": iface}) + add_route_state(route, text) return [route] for raw in text.splitlines(): if not raw.strip(): @@ -62,12 +78,19 @@ def parse_nexthops(rest: str) -> list[dict[str, str | None]]: if raw.startswith(" "): if current: current["nexthops"].extend(parse_nexthops(raw)) + current["_raw"] = f"{current.get('_raw', '')} {raw.strip()}" continue match = re.match(r"^([A-Z*>]+)\s+([0-9.]+/\d+)\s*(.*)$", raw.strip()) if not match: continue code, prefix, rest = match.groups() - route = {"prefix": prefix, "code": code, "protocol": protocol_from_code(code), "nexthops": []} + route = { + "prefix": prefix, + "code": code, + "protocol": protocol_from_code(code), + "nexthops": [], + "_raw": rest, + } metric_match = re.search(r"\[(\d+)/(\d+)\]", rest) if metric_match: route["admin_distance"] = int(metric_match.group(1)) @@ -79,4 +102,6 @@ def parse_nexthops(rest: str) -> list[dict[str, str | None]]: route["nexthops"].extend(parse_nexthops(rest)) routes.append(route) current = route + for route in routes: + add_route_state(route, str(route.pop("_raw", ""))) return routes diff --git a/netopsbench/platform/toolkit/_core/device/routing_ops.py b/netopsbench/platform/toolkit/_core/device/routing_ops.py index e26421b..a1d042a 100644 --- a/netopsbench/platform/toolkit/_core/device/routing_ops.py +++ b/netopsbench/platform/toolkit/_core/device/routing_ops.py @@ -5,8 +5,11 @@ import subprocess from netopsbench.logging_utils import get_logger +from netopsbench.platform.observability.bgp_parser import parse_bgp_summary +from netopsbench.platform.topology.topology_utils import is_network_device_name from ..common import ToolResult, truncate_text_lines +from .route_parsers import parse_route_table logger = get_logger(__name__) @@ -18,7 +21,7 @@ def get_bgp_neighbors(self, device: str, format: str = "structured") -> ToolResu safe_format = (format or "raw").lower() if safe_format not in {"raw", "structured", "both"}: return ToolResult(success=False, data=None, error=f"Invalid format: {format}") - if not device.startswith(("spine", "leaf")): + if not is_network_device_name(device): return ToolResult(success=False, data=None, error=f"Device {device} does not run BGP") container = self._resolve_container(device) result = self._docker_exec(container, ["vtysh", "-c", "show ip bgp summary"], timeout=30) @@ -26,7 +29,7 @@ def get_bgp_neighbors(self, device: str, format: str = "structured") -> ToolResu return ToolResult(success=False, data=None, error=result.stderr) raw_data = {"device": device, "bgp_neighbors": result.stdout} if safe_format in {"structured", "both"}: - neighbors = self._parse_bgp_summary(result.stdout) + neighbors = parse_bgp_summary(result.stdout) # Enrich non-Established peers with detail (last error, reset reason) for nbr in neighbors: if nbr.get("state") and nbr["state"] != "Established" and nbr.get("neighbor"): @@ -45,6 +48,59 @@ def get_bgp_neighbors(self, device: str, format: str = "structured") -> ToolResu except Exception as e: return ToolResult(success=False, data=None, error=str(e)) + def get_bgp_neighbor(self, device: str, peer: str) -> ToolResult: + """Return live detail for one BGP session on one device.""" + try: + self._validate_device_name(device) + safe_peer = self._validate_ip_address(peer, "peer") + if not is_network_device_name(device): + return ToolResult(success=False, data=None, error=f"Device {device} does not run BGP") + container = self._resolve_container(device) + detail_result = self._docker_exec( + container, + ["vtysh", "-c", f"show ip bgp neighbors {safe_peer}"], + timeout=15, + ) + if detail_result.returncode != 0: + return ToolResult(success=False, data=None, error=detail_result.stderr) + detail = self._parse_bgp_neighbor_detail(detail_result.stdout) + summary_result = self._docker_exec(container, ["vtysh", "-c", "show ip bgp summary"], timeout=15) + summary = None + if summary_result.returncode == 0: + summary = next( + (row for row in parse_bgp_summary(summary_result.stdout) if row.get("neighbor") == safe_peer), + None, + ) + if not detail and summary is None: + return ToolResult(success=False, data=None, error=f"BGP neighbor {safe_peer} not found on {device}") + summary = summary or {} + return ToolResult( + success=True, + data={ + "device": device, + "peer": safe_peer, + "state": summary.get("state") or detail.get("bgp_state"), + "peer_as": summary.get("asn") or detail.get("remote_as"), + "configured_as": detail.get("remote_as"), + "local_address": detail.get("local_address"), + "foreign_address": detail.get("foreign_address"), + "uptime": summary.get("up_down") or detail.get("uptime"), + "last_reset": detail.get("last_reset"), + "last_error": detail.get("last_error"), + "notifications": detail.get("notifications", []), + "update_source": detail.get("update_source"), + "message_counters": { + "received": summary.get("msg_rcvd"), + "sent": summary.get("msg_sent"), + }, + "prefixes_received": summary.get("prefixes_received"), + }, + ) + except subprocess.TimeoutExpired: + return ToolResult(success=False, data=None, error="Command timed out") + except Exception as exc: + return ToolResult(success=False, data=None, error=str(exc)) + def _get_bgp_neighbor_detail(self, container: str, neighbor_ip: str) -> dict | None: """Fetch per-neighbor detail for a non-Established peer.""" try: @@ -76,14 +132,14 @@ def _parse_bgp_neighbor_detail(text: str) -> dict: detail["last_reset"] = line_s elif "Notification" in line_s and ("sent" in line_s or "rcvd" in line_s): detail.setdefault("notifications", []).append(line_s) - elif "remote AS" in line_s.lower(): - m = re.search(r"remote AS (\d+)", line_s) + elif "remote as" in line_s.lower(): + m = re.search(r"remote AS (\d+)", line_s, re.IGNORECASE) if m: detail["remote_as"] = int(m.group(1)) elif "Local host:" in line_s: - detail["local_host"] = line_s + detail["local_address"] = line_s.split("Local host:", 1)[1].split(",", 1)[0].strip() elif "Foreign host:" in line_s: - detail["foreign_host"] = line_s + detail["foreign_address"] = line_s.split("Foreign host:", 1)[1].split(",", 1)[0].strip() elif "update-source" in line_s.lower(): detail["update_source"] = line_s elif "password" in line_s.lower() and "configured" in line_s.lower(): @@ -105,7 +161,7 @@ def get_route_table( safe_format = (format or "raw").lower() if safe_format not in {"raw", "structured", "both"}: return ToolResult(success=False, data=None, error=f"Invalid format: {format}") - if not device.startswith(("spine", "leaf")): + if not is_network_device_name(device): return ToolResult(success=False, data=None, error=f"Device {device} is not a router") container = self._resolve_container(device) if prefix: @@ -120,7 +176,7 @@ def get_route_table( route_text, route_meta = truncate_text_lines(result.stdout, max_lines) raw_data = {"device": device, "prefix": safe_prefix, "route_table": route_text, **route_meta} if safe_format in {"structured", "both"}: - routes = self._parse_route_table(result.stdout) + routes = parse_route_table(result.stdout) safe_max_routes = max(1, int(max_routes)) structured = { "device": device, @@ -141,26 +197,11 @@ def get_route_table( except Exception as e: return ToolResult(success=False, data=None, error=str(e)) - def get_all_bgp_status(self) -> ToolResult: - results = {} - devices = [] - if self.topology_metadata: - for spine in self.topology_metadata.get("devices", {}).get("spines", []): - devices.append(spine["name"]) - for leaf in self.topology_metadata.get("devices", {}).get("leafs", []): - devices.append(leaf["name"]) - else: - devices = [name for name in self.container_names.keys() if name.startswith(("spine", "leaf"))] - for device in devices: - result = self.get_bgp_neighbors(device) - results[device] = result.data if result.success else {"error": result.error} - return ToolResult(success=True, data=results) - def get_device_config(self, device: str, section: str = "", max_lines: int = 500) -> ToolResult: """Retrieve running configuration from a SONiC/FRR device via vtysh.""" try: self._validate_device_name(device) - if not device.startswith(("spine", "leaf")): + if not is_network_device_name(device): return ToolResult(success=False, data=None, error=f"Device {device} does not support running-config") container = self._resolve_container(device) if section: @@ -192,7 +233,7 @@ def get_bgp_rib(self, device: str, prefix: str | None = None, max_lines: int = 5 """Retrieve BGP RIB entries with AS path, origin, next-hop, local-pref details.""" try: self._validate_device_name(device) - if not device.startswith(("spine", "leaf")): + if not is_network_device_name(device): return ToolResult(success=False, data=None, error=f"Device {device} does not run BGP") container = self._resolve_container(device) if prefix: @@ -230,7 +271,7 @@ def get_device_acl(self, device: str, view: str = "summary", max_lines: int = 30 safe_view = (view or "summary").lower() if safe_view not in {"summary", "frr", "iptables", "all"}: return ToolResult(success=False, data=None, error=f"Invalid view: {view}") - if not device.startswith(("spine", "leaf")): + if not is_network_device_name(device): return ToolResult(success=False, data=None, error=f"Device {device} does not support ACLs") container = self._resolve_container(device) diff --git a/netopsbench/platform/toolkit/_core/device/telemetry_parsers.py b/netopsbench/platform/toolkit/_core/device/telemetry_parsers.py index 44f697f..0f5c5b4 100644 --- a/netopsbench/platform/toolkit/_core/device/telemetry_parsers.py +++ b/netopsbench/platform/toolkit/_core/device/telemetry_parsers.py @@ -4,11 +4,12 @@ import csv import re +from dataclasses import dataclass, field from datetime import datetime from io import StringIO -from typing import Any +from typing import Any, Literal -import requests +from netopsbench.platform.observability.influxdb import query_flux def parse_influx_timestamp(value: str | None) -> datetime | None: @@ -78,26 +79,20 @@ def parse_influx_interface_identity_rows(csv_text: str) -> list[dict[str, str | def get_recent_influx_interface_identities( - toolkit, device: str, time_range_minutes: int, headers: dict[str, str] | None = None + toolkit, device: str, time_range_minutes: int ) -> list[dict[str, str | None]]: safe_device = toolkit._validate_device_name(device) safe_minutes = max(1, min(int(time_range_minutes), 24 * 60)) - request_headers = headers or { - "Authorization": f"Token {toolkit.influxdb_token}", - "Content-Type": "application/vnd.flux", - "Accept": "application/csv", - } query = f"""\nfrom(bucket: "{toolkit.influxdb_bucket}")\n |> range(start: -{safe_minutes}m)\n |> filter(fn: (r) => r._measurement == "interfaces")\n |> filter(fn: (r) => r.source == "{safe_device}")\n |> keep(columns: ["_time", "name", "path", "source"])\n |> limit(n: 2000)\n""" - response = requests.post( - f"{toolkit.influxdb_url}/api/v2/query?org={toolkit.influxdb_org}", - headers=request_headers, - data=query, - timeout=30, - proxies={"http": "", "https": ""}, + result = query_flux( + toolkit.influxdb_url, + toolkit.influxdb_token, + toolkit.influxdb_org, + query, ) - if response.status_code != 200: + if result.status != "ok": return [] - return parse_influx_interface_identity_rows(response.text) + return parse_influx_interface_identity_rows(result.text) def summarize_counter_points(field: str, points: list[dict[str, Any]]) -> dict[str, Any]: @@ -141,22 +136,23 @@ def summarize_counter_points(field: str, points: list[dict[str, Any]]) -> dict[s return summary -def query_influx_rows(toolkit, query: str, require_value: bool = True) -> list[dict[str, Any]]: - headers = { - "Authorization": f"Token {toolkit.influxdb_token}", - "Content-Type": "application/vnd.flux", - "Accept": "application/csv", - } - response = requests.post( - f"{toolkit.influxdb_url}/api/v2/query?org={toolkit.influxdb_org}", - headers=headers, - data=query, - timeout=30, - proxies={"http": "", "https": ""}, - ) - response.raise_for_status() +@dataclass(frozen=True) +class InfluxQueryResult: + status: Literal["ok", "error"] + rows: list[dict[str, Any]] = field(default_factory=list) + error: str | None = None + + +class InfluxQueryError(RuntimeError): + """Raised when a structured InfluxDB query result reports failure.""" + + +def query_influx(toolkit, query: str, require_value: bool = True) -> InfluxQueryResult: + result = query_flux(toolkit.influxdb_url, toolkit.influxdb_token, toolkit.influxdb_org, query) + if result.status != "ok": + return InfluxQueryResult(status="error", error=result.error) rows: list[dict[str, Any]] = [] - reader = csv.DictReader(StringIO(response.text)) + reader = csv.DictReader(StringIO(result.text)) for row in reader: if row.get("result", "") == "result": continue @@ -176,4 +172,12 @@ def query_influx_rows(toolkit, query: str, require_value: bool = True) -> list[d if not has_payload: continue rows.append(parsed) - return rows + return InfluxQueryResult(status="ok", rows=rows) + + +def query_influx_rows(toolkit, query: str, require_value: bool = True) -> list[dict[str, Any]]: + """Return parsed rows, preserving query failure as an exception.""" + result = query_influx(toolkit, query, require_value=require_value) + if result.status == "error": + raise InfluxQueryError(result.error or "InfluxDB query failed") + return result.rows diff --git a/netopsbench/platform/toolkit/_core/device/validators.py b/netopsbench/platform/toolkit/_core/device/validators.py index f5e4233..d54b2a2 100644 --- a/netopsbench/platform/toolkit/_core/device/validators.py +++ b/netopsbench/platform/toolkit/_core/device/validators.py @@ -6,7 +6,7 @@ import re import subprocess -from netopsbench.platform.utils.proc import safe_run, sudo_prefix +from netopsbench.platform.utils.proc import docker_prefix, safe_run DEVICE_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{1,64}$") INTERFACE_PATTERN = re.compile(r"^[a-zA-Z0-9./-]{1,64}$") @@ -46,30 +46,6 @@ def resolve_container(toolkit, device: str, field_name: str = "device") -> str: return container -def get_client_device_names(toolkit) -> list[str]: - if getattr(toolkit, "topology_metadata", None): - clients = [ - str(client.get("name")) - for client in toolkit.topology_metadata.get("devices", {}).get("clients", []) - if client.get("name") - ] - if clients: - return sorted(clients) - return sorted(name for name in toolkit.container_names if str(name).lower().startswith("client")) - - -def require_client_source(toolkit, src: str, tool_name: str) -> str: - safe_src = validate_device_name(src, field_name="source") - client_names = get_client_device_names(toolkit) - if safe_src not in client_names: - available = ", ".join(client_names) if client_names else "no clients discovered" - raise ValueError( - f"{tool_name} only supports client sources to avoid source-address ambiguity on infrastructure " - f"devices. Received source '{safe_src}'. Choose one of the discovered clients: {available}." - ) - return safe_src - - def docker_exec(container: str, cmd_args: list[str], timeout: int) -> subprocess.CompletedProcess: - args = [*sudo_prefix(), "docker", "exec", container] + [str(arg) for arg in cmd_args] + args = [*docker_prefix(), "docker", "exec", container] + [str(arg) for arg in cmd_args] return safe_run(args, capture_output=True, text=True, timeout=timeout, check=False) diff --git a/netopsbench/platform/toolkit/_core/observability/bgp_ops.py b/netopsbench/platform/toolkit/_core/observability/bgp_ops.py new file mode 100644 index 0000000..cc04616 --- /dev/null +++ b/netopsbench/platform/toolkit/_core/observability/bgp_ops.py @@ -0,0 +1,265 @@ +"""Historical BGP event discovery from centralized telemetry.""" + +from __future__ import annotations + +from collections import defaultdict +from datetime import UTC, datetime +from typing import Any + +from ..common import ToolResult +from .pingmesh_scope import parse_iso8601_timestamp + +_VALID_STATES = {"non_established", "all", "established"} + + +def _timestamp(value: object) -> datetime | None: + if not value: + return None + try: + return parse_iso8601_timestamp(str(value), "_time") + except ValueError: + return None + + +def _bool(value: object) -> bool: + return value is True or str(value).lower() == "true" + + +class BgpOpsMixin: + def query_bgp_events( + self, + start_time: str | None = None, + end_time: str | None = None, + time_range_minutes: int = 10, + device: str | None = None, + peer: str | None = None, + role: str | None = None, + state: str = "non_established", + limit: int = 100, + ) -> ToolResult: + """Find BGP session transitions without logging in to every router.""" + try: + if state not in _VALID_STATES: + raise ValueError(f"Invalid state: {state}. Expected one of {sorted(_VALID_STATES)}") + safe_device = self._validate_device_name(device) if device else None + safe_peer = self._validate_ip_address(peer, "peer") if peer else None + safe_limit = max(1, min(int(limit), 500)) + roles = self._bgp_device_roles() + if role and role not in set(roles.values()): + raise ValueError(f"Invalid or unavailable role: {role}") + if safe_device and safe_device not in roles: + raise ValueError(f"Unknown routing device: {safe_device}") + + scope = self._resolve_pingmesh_time_scope(time_range_minutes, start_time, end_time) + filters = self._bgp_flux_filters(safe_device, safe_peer) + device_filter = self._bgp_flux_filters(safe_device, None) + neighbor_query = f""" +from(bucket: "{self.influxdb_bucket}") +{scope["range_clause"]} |> filter(fn: (r) => r._measurement == "bgp_neighbors") + |> filter(fn: (r) => r.topology_id == "{self._flux_string(self.topology_id)}") +{filters} |> pivot(rowKey: ["_time", "_measurement", "source", "neighbor_address"], columnKey: ["_field"], valueColumn: "_value") + |> sort(columns: ["_time"]) +""" + + collection_query = f""" +from(bucket: "{self.influxdb_bucket}") +{scope["range_clause"]} |> filter(fn: (r) => r._measurement == "bgp_collection") + |> filter(fn: (r) => r.topology_id == "{self._flux_string(self.topology_id)}") +{device_filter} |> pivot(rowKey: ["_time", "_measurement", "source"], columnKey: ["_field"], valueColumn: "_value") + |> sort(columns: ["_time"]) +""" + window_rows = self._query_influx_rows(neighbor_query, require_value=False) + window_rows.extend(self._query_influx_rows(collection_query, require_value=False)) + prior_rows: list[dict[str, Any]] = [] + if scope["mode"] == "absolute": + prior_query = f""" +from(bucket: "{self.influxdb_bucket}") + |> range(start: -30d, stop: time(v: "{scope["start_time"]}")) + |> filter(fn: (r) => r._measurement == "bgp_neighbors") + |> filter(fn: (r) => r.topology_id == "{self._flux_string(self.topology_id)}") +{filters} |> pivot(rowKey: ["_time", "_measurement", "source", "neighbor_address"], columnKey: ["_field"], valueColumn: "_value") + |> group(columns: ["source", "neighbor_address"]) + |> last(column: "_time") +""" + prior_rows = self._query_influx_rows(prior_query, require_value=False) + rows = [dict(row, _scope_prior=True) for row in prior_rows] + window_rows + events = self._build_bgp_events(rows, scope, roles, safe_device, safe_peer, role, state) + truncated = len(events) > safe_limit + returned = events[:safe_limit] + freshness = self._bgp_freshness_seconds(rows, scope) + return ToolResult( + success=True, + data={ + "status": "ok", + "time_scope": { + key: "episode_context" if key == "source" and value == "toolkit_default" else value + for key, value in scope.items() + if key != "range_clause" + }, + "events": returned, + "returned_events": len(returned), + "truncated": truncated, + "data_freshness_seconds": freshness, + }, + ) + except Exception as exc: + return ToolResult(success=False, data=None, error=str(exc)) + + @staticmethod + def _flux_string(value: object) -> str: + return str(value or "").replace("\\", "\\\\").replace('"', '\\"') + + def _bgp_flux_filters(self, device: str | None, peer: str | None) -> str: + rendered = "" + if device: + rendered += f' |> filter(fn: (r) => r.source == "{self._flux_string(device)}")\n' + if peer: + rendered += f' |> filter(fn: (r) => r.neighbor_address == "{self._flux_string(peer)}")\n' + return rendered + + def _bgp_device_roles(self) -> dict[str, str]: + devices = self.topology_metadata.get("devices", []) if self.topology_metadata else [] + if isinstance(devices, dict): + return { + str(item["name"]): str(role).removesuffix("s") + for role, entries in devices.items() + for item in entries + if role != "clients" and item.get("name") + } + return { + str(item["name"]): str(item.get("role", "unknown")) + for item in devices + if item.get("name") and item.get("role") != "client" + } + + def _build_bgp_events( + self, + rows: list[dict[str, Any]], + scope: dict[str, Any], + roles: dict[str, str], + device: str | None, + peer: str | None, + role: str | None, + state_filter: str, + ) -> list[dict[str, Any]]: + sessions: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + collections: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + source = str(row.get("source") or "") + if not source or source not in roles or (role and roles[source] != role): + continue + measurement = row.get("_measurement") + if measurement == "bgp_collection": + collections[source].append(row) + elif measurement == "bgp_neighbors" and row.get("neighbor_address"): + sessions[(source, str(row["neighbor_address"]))].append(row) + + events: list[dict[str, Any]] = [] + for (source, neighbor), samples in sessions.items(): + if device and source != device or peer and neighbor != peer: + continue + samples.sort(key=lambda row: _timestamp(row.get("_time")) or datetime.min.replace(tzinfo=UTC)) + prior = [row for row in samples if row.get("_scope_prior")] + window = [row for row in samples if not row.get("_scope_prior")] + if not window: + continue + states = [str(row.get("session_state") or "UNKNOWN").upper() for row in window] + previous = str(prior[-1].get("session_state") or "UNKNOWN").upper() if prior else None + latest = states[-1] + saw_down = any(value != "ESTABLISHED" for value in states) + transition_states = ([previous] if previous else []) + states + down_transition = any( + before == "ESTABLISHED" and after != "ESTABLISHED" + for before, after in zip(transition_states, transition_states[1:], strict=False) + ) + recovery_transition = any( + before != "ESTABLISHED" and after == "ESTABLISHED" + for before, after in zip(transition_states, transition_states[1:], strict=False) + ) + if down_transition and recovery_transition: + event_type = "session_flap" + elif down_transition: + event_type = "session_down" + elif recovery_transition: + event_type = "session_recovered" + elif saw_down: + event_type = "non_established_observed" + elif state_filter == "all": + event_type = "established_observed" + else: + continue + if state_filter == "non_established" and not saw_down: + continue + if state_filter == "established" and latest != "ESTABLISHED": + continue + unique_states = list(dict.fromkeys(states)) + events.append( + { + "device": source, + "role": roles[source], + "peer": neighbor, + "peer_as": window[-1].get("asn"), + "event_type": event_type, + "previous_state": previous, + "latest_state": latest, + "states_observed": unique_states, + "first_seen": window[0].get("_time"), + "last_seen": window[-1].get("_time"), + "sample_count": len(window), + "prefixes_before": prior[-1].get("prefixes_received") if prior else None, + "prefixes_after": window[-1].get("prefixes_received"), + } + ) + + for source, samples in collections.items(): + latest = samples[-1] + if not _bool(latest.get("collection_ok")): + events.append( + { + "device": source, + "role": roles[source], + "peer": None, + "event_type": "collection_gap", + "previous_state": None, + "latest_state": None, + "states_observed": [], + "first_seen": samples[0].get("_time"), + "last_seen": latest.get("_time"), + "sample_count": len(samples), + "error_type": latest.get("error_type"), + } + ) + selected_devices = { + name + for name, device_role in roles.items() + if (not device or name == device) and (not role or device_role == role) + } + missing_devices = selected_devices - collections.keys() if not peer or device else set() + for source in missing_devices: + events.append( + { + "device": source, + "role": roles[source], + "peer": None, + "event_type": "collection_gap", + "previous_state": None, + "latest_state": None, + "states_observed": [], + "first_seen": None, + "last_seen": None, + "sample_count": 0, + "error_type": "no_collection_samples", + } + ) + return sorted(events, key=lambda event: (str(event.get("last_seen") or ""), event["device"]), reverse=True) + + @staticmethod + def _bgp_freshness_seconds(rows: list[dict[str, Any]], scope: dict[str, Any]) -> int | None: + timestamps = [_timestamp(row.get("_time")) for row in rows] + latest = max((value for value in timestamps if value), default=None) + if latest is None: + return None + reference = ( + parse_iso8601_timestamp(scope["end_time"], "end_time") if scope["mode"] == "absolute" else datetime.now(UTC) + ) + return max(0, round((reference - latest).total_seconds())) diff --git a/netopsbench/platform/toolkit/_core/observability/grafana_ops.py b/netopsbench/platform/toolkit/_core/observability/grafana_ops.py deleted file mode 100644 index 11a70a4..0000000 --- a/netopsbench/platform/toolkit/_core/observability/grafana_ops.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Grafana screenshot and panel helpers for AgentToolkit.""" - -from __future__ import annotations - -import base64 -import os -from datetime import datetime - -import requests - -from ..common import ToolResult - - -class GrafanaOpsMixin: - def get_grafana_screenshot( - self, - panel_name: str, - time_range: str = "1h", - width: int = 1000, - height: int = 500, - save_to_file: bool = True, - include_base64: bool = False, - ) -> ToolResult: - panel_id = self.panel_mapping.get(panel_name.lower().replace(" ", "_").replace("-", "_")) - - if panel_id is None: - return ToolResult( - success=False, - data=None, - error=f"Unknown panel: {panel_name}. Available panels: {list(self.panel_mapping.keys())}", - ) - - dashboard_uid = "dcn-overview" - - try: - render_url = ( - f"{self.grafana_url}/render/d-solo/{dashboard_uid}/" - f"?orgId=1&panelId={panel_id}" - f"&from=now-{time_range}&to=now" - f"&width={width}&height={height}" - f"&tz=UTC" - ) - - response = requests.get( - render_url, - auth=self.grafana_auth, - timeout=30, - proxies={"http": "", "https": ""}, - ) - - if response.status_code == 200: - image_data = response.content - - result_data = { - "panel_name": panel_name, - "panel_id": panel_id, - "time_range": time_range, - "width": width, - "height": height, - "content_type": response.headers.get("Content-Type", "image/png"), - } - - if include_base64: - result_data["image_base64"] = base64.b64encode(image_data).decode("utf-8") - - if save_to_file: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = f"{panel_name}_{timestamp}.png" - filepath = os.path.join(self.screenshot_dir, filename) - with open(filepath, "wb") as f: - f.write(image_data) - result_data["file_path"] = filepath - - return ToolResult(success=True, data=result_data) - - return ToolResult( - success=False, - data=None, - error=( - f"Grafana render failed (status {response.status_code}). " - "Note: Grafana Image Renderer plugin may need to be installed. " - "Use get_grafana_panel_data() for text-based data instead." - ), - ) - - except requests.exceptions.RequestException as e: - return ToolResult(success=False, data=None, error=f"Request failed: {str(e)}") - except Exception as e: - return ToolResult(success=False, data=None, error=str(e)) - - def get_dashboard_screenshot( - self, - time_range: str = "1h", - width: int = 1920, - height: int = 1080, - save_to_file: bool = True, - include_base64: bool = False, - ) -> ToolResult: - dashboard_uid = "dcn-overview" - - try: - render_url = ( - f"{self.grafana_url}/render/d/{dashboard_uid}/dcn-overview" - f"?orgId=1" - f"&from=now-{time_range}&to=now" - f"&width={width}&height={height}" - f"&tz=UTC" - ) - - response = requests.get( - render_url, - auth=self.grafana_auth, - timeout=60, - proxies={"http": "", "https": ""}, - ) - - if response.status_code == 200: - image_data = response.content - - result_data = { - "dashboard": "DCN Overview", - "time_range": time_range, - "width": width, - "height": height, - "content_type": response.headers.get("Content-Type", "image/png"), - } - - if include_base64: - result_data["image_base64"] = base64.b64encode(image_data).decode("utf-8") - - if save_to_file: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = f"dashboard_{timestamp}.png" - filepath = os.path.join(self.screenshot_dir, filename) - with open(filepath, "wb") as f: - f.write(image_data) - result_data["file_path"] = filepath - - return ToolResult(success=True, data=result_data) - - return ToolResult( - success=False, - data=None, - error=( - f"Dashboard render failed (status {response.status_code}). " - "Note: Grafana Image Renderer plugin may need to be installed." - ), - ) - - except requests.exceptions.RequestException as e: - return ToolResult(success=False, data=None, error=f"Request failed: {str(e)}") - except Exception as e: - return ToolResult(success=False, data=None, error=str(e)) - - def get_multiple_panel_screenshots( - self, - panel_names: list[str], - time_range: str = "1h", - include_base64: bool = False, - ) -> ToolResult: - results = [] - errors = [] - - for panel_name in panel_names: - result = self.get_grafana_screenshot(panel_name, time_range, include_base64=include_base64) - if result.success: - panel_result = { - "panel_name": panel_name, - "file_path": result.data.get("file_path"), - } - if include_base64 and "image_base64" in result.data: - panel_result["image_base64"] = result.data["image_base64"] - results.append(panel_result) - else: - errors.append({"panel_name": panel_name, "error": result.error}) - - return ToolResult( - success=len(results) > 0, - data={ - "captured": results, - "failed": errors, - "total_requested": len(panel_names), - "total_captured": len(results), - }, - error=f"Failed to capture {len(errors)} panels" if errors else None, - ) - - def get_troubleshooting_screenshots(self, time_range: str = "30m", include_base64: bool = False) -> ToolResult: - troubleshooting_panels = [ - "bgp_timeline", - "interface_in_throughput", - "interface_out_throughput", - "physical_errors", - "logical_discards", - "queue_drops", - "syslog_events", - ] - - return self.get_multiple_panel_screenshots( - troubleshooting_panels, - time_range=time_range, - include_base64=include_base64, - ) - - def get_grafana_panel_data(self, panel_name: str, time_range_minutes: int = 60) -> ToolResult: - panel_queries = { - "bgp_session": f""" -from(bucket: "{self.influxdb_bucket}") - |> range(start: -{time_range_minutes}m) - |> filter(fn: (r) => r._measurement == "bgp_neighbors") - |> filter(fn: (r) => r._field == "session_state") - |> last() -""", - "interface_throughput": f""" -from(bucket: "{self.influxdb_bucket}") - |> range(start: -{time_range_minutes}m) - |> filter(fn: (r) => r._measurement == "interfaces") - |> filter(fn: (r) => r._field == "in_octets" or r._field == "out_octets") - |> derivative(unit: 1s, nonNegative: true) - |> map(fn: (r) => ({{ r with _value: r._value * 8.0 }})) - |> aggregateWindow(every: 1m, fn: mean, createEmpty: false) -""", - "interface_errors": f""" -from(bucket: "{self.influxdb_bucket}") - |> range(start: -{time_range_minutes}m) - |> filter(fn: (r) => r._measurement == "interfaces") - |> filter(fn: (r) => r._field == "in_error_packets" or r._field == "out_error_packets" or r._field == "in_errors" or r._field == "out_errors" or r._field == "in_discarded_packets" or r._field == "out_discarded_packets") - |> derivative(unit: 1s, nonNegative: true) - |> aggregateWindow(every: 1m, fn: mean, createEmpty: false) -""", - "queue_drops": f""" -from(bucket: "{self.influxdb_bucket}") - |> range(start: -{time_range_minutes}m) - |> filter(fn: (r) => r._measurement == "interfaces") - |> filter(fn: (r) => r._field == "out_discarded_packets" or r._field == "in_discarded_packets") - |> derivative(unit: 1s, nonNegative: true) - |> aggregateWindow(every: 1m, fn: mean, createEmpty: false) -""", - "cpu_usage": f""" -from(bucket: "{self.influxdb_bucket}") - |> range(start: -{time_range_minutes}m) - |> filter(fn: (r) => r._measurement == "cpu_usage") - |> filter(fn: (r) => r._field == "instant") - |> aggregateWindow(every: 1m, fn: mean, createEmpty: false) -""", - "syslog_events": f""" -from(bucket: "{self.influxdb_bucket}") - |> range(start: -{time_range_minutes}m) - |> filter(fn: (r) => r._measurement == "syslog") - |> filter(fn: (r) => r._field == "message") - |> sort(columns: ["_time"], desc: true) - |> limit(n: 50) -""", - } - - panel_key = panel_name.lower().replace(" ", "_").replace("-", "_") - query = None - for key, q in panel_queries.items(): - if key in panel_key or panel_key in key: - query = q - break - - if not query: - return ToolResult( - success=False, - data=None, - error=f"Unknown panel: {panel_name}. Available panels: {list(panel_queries.keys())}", - ) - - try: - headers = { - "Authorization": f"Token {self.influxdb_token}", - "Content-Type": "application/vnd.flux", - "Accept": "application/csv", - } - - response = requests.post( - f"{self.influxdb_url}/api/v2/query?org={self.influxdb_org}", - headers=headers, - data=query, - timeout=30, - proxies={"http": "", "https": ""}, - ) - - if response.status_code != 200: - return ToolResult(success=False, data=None, error=f"InfluxDB query failed: {response.text}") - - return ToolResult( - success=True, - data={ - "panel_name": panel_name, - "time_range_minutes": time_range_minutes, - "data": response.text, - }, - ) - except requests.exceptions.RequestException as e: - return ToolResult(success=False, data=None, error=f"Request failed: {str(e)}") - except Exception as e: - return ToolResult(success=False, data=None, error=str(e)) diff --git a/netopsbench/platform/toolkit/_core/observability/metrics_ops.py b/netopsbench/platform/toolkit/_core/observability/metrics_ops.py index fa113c1..18f871b 100644 --- a/netopsbench/platform/toolkit/_core/observability/metrics_ops.py +++ b/netopsbench/platform/toolkit/_core/observability/metrics_ops.py @@ -4,9 +4,17 @@ from typing import Any -import requests +from netopsbench.platform.observability.influxdb import query_flux +from netopsbench.platform.utils.interface_names import resolve_interface_metric_identities from ..common import ToolResult +from ..device.interface_parsers import get_active_interface_names, get_live_interface_snapshot +from ..device.telemetry_parsers import ( + get_recent_influx_interface_identities, + parse_influx_metric_rows, + summarize_counter_points, +) +from ..device.text_parsers import preview_items class MetricsOpsMixin: @@ -35,7 +43,7 @@ def get_interface_metrics( if safe_view not in allowed_views: return ToolResult(success=False, data=None, error=f"Invalid view: {view}") - identities = self._resolve_interface_metric_identities(safe_interface) + identities = resolve_interface_metric_identities(safe_interface) type_fields = { "throughput": ["in_octets", "out_octets"], @@ -63,22 +71,9 @@ def get_interface_metrics( |> yield(name: "metrics") """ - headers = { - "Authorization": f"Token {self.influxdb_token}", - "Content-Type": "application/vnd.flux", - "Accept": "application/csv", - } - - response = requests.post( - f"{self.influxdb_url}/api/v2/query?org={self.influxdb_org}", - headers=headers, - data=query, - timeout=30, - proxies={"http": "", "https": ""}, - ) - - if response.status_code != 200: - return ToolResult(success=False, data=None, error=f"InfluxDB query failed: {response.text}") + result = query_flux(self.influxdb_url, self.influxdb_token, self.influxdb_org, query) + if result.status != "ok": + return ToolResult(success=False, data=None, error=f"InfluxDB query failed: {result.error}") if safe_view == "raw": return ToolResult( @@ -89,39 +84,34 @@ def get_interface_metrics( "time_range_minutes": safe_minutes, "metric_type": safe_metric_type, "view": safe_view, - "metrics": response.text, + "metrics": result.text, }, ) - metric_rows = self._parse_influx_metric_rows(response.text) + metric_rows = parse_influx_metric_rows(result.text) if not metric_rows: - observed_identities = self._get_recent_influx_interface_identities( - safe_device, - safe_minutes, - headers=headers, - ) + observed_identities = get_recent_influx_interface_identities(self, safe_device, safe_minutes) observed_interfaces = sorted( {str(item.get("name")).strip() for item in observed_identities if item.get("name")} ) - active_interfaces = self._get_active_interface_names(safe_device) + active_interfaces = get_active_interface_names(self, safe_device) missing_active_interfaces = [name for name in active_interfaces if name not in observed_interfaces] - live_snapshot = self._get_live_interface_snapshot(safe_device, safe_interface) + live_snapshot = get_live_interface_snapshot(self, safe_device, safe_interface) warning_parts: list[str] = [] if observed_interfaces: warning_parts.append( "Interface metrics exist for the device, but per-interface samples are missing for " f"{safe_interface}. Recent Influx interfaces for {safe_device}: " - f"{self._preview_items(observed_interfaces)}." + f"{preview_items(observed_interfaces)}." ) else: warning_parts.append( - "No interface time-series samples were available in InfluxDB for this device/interface " - "window." + "No interface time-series samples were available in InfluxDB for this device/interface window." ) if missing_active_interfaces: warning_parts.append( "Active interfaces missing from recent Influx data: " - f"{self._preview_items(missing_active_interfaces)}." + f"{preview_items(missing_active_interfaces)}." ) if live_snapshot: warning_parts.append("Returning a live CLI snapshot as fallback context.") @@ -155,7 +145,7 @@ def get_interface_metrics( summary: dict[str, dict[str, Any]] = {} for field, points in series.items(): - field_summary = self._summarize_counter_points(field, points) + field_summary = summarize_counter_points(field, points) if not field_summary: continue summary[field] = field_summary @@ -189,7 +179,5 @@ def get_interface_metrics( "series": series, }, ) - except requests.exceptions.RequestException as e: - return ToolResult(success=False, data=None, error=f"Request failed: {str(e)}") except Exception as e: return ToolResult(success=False, data=None, error=str(e)) diff --git a/netopsbench/platform/toolkit/_core/observability/observability_facade.py b/netopsbench/platform/toolkit/_core/observability/observability_facade.py deleted file mode 100644 index 0f51f30..0000000 --- a/netopsbench/platform/toolkit/_core/observability/observability_facade.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Observability-focused AgentToolkit facade.""" - -from __future__ import annotations - -from .grafana_ops import GrafanaOpsMixin -from .metrics_ops import MetricsOpsMixin -from .pingmesh_ops import PingmeshOpsMixin - - -class ObservabilityFacadeMixin(GrafanaOpsMixin, MetricsOpsMixin, PingmeshOpsMixin): - """Thin facade that groups observability-related toolkit capabilities.""" - - pass diff --git a/netopsbench/platform/toolkit/_core/observability/pingmesh_ops.py b/netopsbench/platform/toolkit/_core/observability/pingmesh_ops.py index e4b3305..a5fce93 100644 --- a/netopsbench/platform/toolkit/_core/observability/pingmesh_ops.py +++ b/netopsbench/platform/toolkit/_core/observability/pingmesh_ops.py @@ -84,7 +84,8 @@ def get_pingmesh_hotspots( |> aggregateWindow(every: 30s, fn: mean, createEmpty: false) |> last() |> pivot(rowKey: ["src_leaf", "dst_leaf"], columnKey: ["_field"], valueColumn: "_value") - |> sort(columns: ["rtt_p99"], desc: true) + |> group() + |> sort(columns: ["packet_loss", "rtt_p99"], desc: true) |> limit(n: {safe_limit}) """ diff --git a/netopsbench/platform/toolkit/_core/topology/__init__.py b/netopsbench/platform/toolkit/_core/topology/__init__.py deleted file mode 100644 index 9104824..0000000 --- a/netopsbench/platform/toolkit/_core/topology/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Topology-related toolkit internals.""" diff --git a/netopsbench/platform/toolkit/_core/topology/topology_ops.py b/netopsbench/platform/toolkit/_core/topology/topology_ops.py deleted file mode 100644 index f319d33..0000000 --- a/netopsbench/platform/toolkit/_core/topology/topology_ops.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Topology-specific AgentToolkit methods.""" - -from __future__ import annotations - -import json - -from ..common import ToolResult - - -class TopologyOpsMixin: - def reload_topology(self, topology_metadata: dict = None, metadata_file: str = None) -> ToolResult: - """ - Reload topology configuration at runtime. - - Args: - topology_metadata: Direct metadata dictionary - metadata_file: Path to metadata JSON file - - Returns: - ToolResult indicating success or failure - """ - try: - if topology_metadata: - self._load_topology_metadata(topology_metadata) - elif metadata_file: - with open(metadata_file) as f: - self._load_topology_metadata(json.load(f)) - else: - return ToolResult( - success=False, data=None, error="Either topology_metadata or metadata_file must be provided" - ) - - return ToolResult( - success=True, - data={ - "topology_name": self.topology_name, - "devices": list(self.container_names.keys()), - "total_devices": len(self.container_names), - }, - ) - except Exception as e: - return ToolResult(success=False, data=None, error=str(e)) - - def get_topology(self) -> ToolResult: - """ - Get the network topology information. - - Returns topology structure including devices, links, and IP assignments. - Requires runtime topology metadata to be loaded during toolkit initialization. - """ - try: - if self.topology_metadata: - return ToolResult(success=True, data=self._enrich_topology_metadata(self.topology_metadata)) - return ToolResult( - success=False, - data=None, - error="Topology metadata is not loaded; initialize toolkit with generated topology metadata.", - ) - except Exception as e: - return ToolResult(success=False, data=None, error=str(e)) diff --git a/netopsbench/platform/toolkit/_toolkit_bootstrap.py b/netopsbench/platform/toolkit/_toolkit_bootstrap.py deleted file mode 100644 index b48a859..0000000 --- a/netopsbench/platform/toolkit/_toolkit_bootstrap.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Bootstrap mixin for AgentToolkit runtime state.""" - -from __future__ import annotations - -import json -import os - -from netopsbench.config import config -from netopsbench.platform.topology.topology_utils import build_topology_state_from_metadata - - -class AgentToolkitBootstrapMixin: - def __init__( - self, - grafana_url: str = "http://localhost:3000", - grafana_user: str = "admin", - grafana_password: str = "admin", - influxdb_url: str = config.influxdb_url, - influxdb_token: str = config.influxdb_token, - influxdb_org: str = config.influxdb_org, - influxdb_bucket: str = config.influxdb_bucket, - topology_file: str = None, - topology_metadata: dict = None, - ): - self.grafana_url = config.grafana_url or grafana_url - self.grafana_auth = (grafana_user, grafana_password) - self.influxdb_url = config.influxdb_url or influxdb_url - self.influxdb_token = config.influxdb_token or influxdb_token - self.influxdb_org = config.influxdb_org or influxdb_org - self.influxdb_bucket = config.influxdb_bucket or influxdb_bucket - - base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - auto_topology_dir = self._discover_topology_dir(base_dir) - self.topology_file = topology_file or os.path.join(auto_topology_dir, "dcn.clab.yaml") - - self.device_mgmt_ips = {} - self.container_names = {} - self.topology_metadata = None - self.topology_name = "dcn" - self.topology_id = config.topology_id - self._pingmesh_default_start_time = None - self._pingmesh_default_end_time = None - - if topology_metadata: - build_topology_state_from_metadata(topology_metadata).apply_to(self) - else: - metadata_file = os.path.join(auto_topology_dir, "topology.json") - if os.path.exists(metadata_file): - with open(metadata_file, encoding="utf-8") as handle: - build_topology_state_from_metadata(json.load(handle)).apply_to(self) - else: - raise FileNotFoundError( - f"Topology metadata not found: {metadata_file}. " - "Set NETOPSBENCH_TOPOLOGY_DIR to a generated topology directory." - ) - if not self.topology_id and self.topology_file: - self.topology_id = os.path.basename(os.path.dirname(self.topology_file)) - - self.screenshot_dir = os.path.join(base_dir, "screenshots") - os.makedirs(self.screenshot_dir, exist_ok=True) - self.panel_mapping = { - "bgp_established": 20, - "bgp_not_established": 21, - "syslog_warnings": 22, - "syslog_total": 23, - "bgp_timeline": 24, - "interface_in_throughput": 1, - "interface_out_throughput": 3, - "syslog_events": 2, - "syslog_by_severity": 25, - "bgp_neighbors_table": 5, - "interface_packets": 4, - "queue_drops": 9, - "physical_errors": 10, - "logical_discards": 26, - "pingmesh_heatmap": 100, - "pingmesh_drops": 101, - "rack_latency": 102, - "path_comparison": 103, - } diff --git a/netopsbench/platform/toolkit/_toolkit_helpers.py b/netopsbench/platform/toolkit/_toolkit_helpers.py deleted file mode 100644 index 36cc76a..0000000 --- a/netopsbench/platform/toolkit/_toolkit_helpers.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Shared helper mixin for AgentToolkit wiring.""" - -from __future__ import annotations - -import subprocess -from typing import Any - -from netopsbench.platform.topology.topology_utils import ( - build_topology_state_from_metadata, - discover_topology_dir, - enrich_topology_metadata, - resolve_interface_metric_identities, -) - -from ._core.device.parsers import ( - get_active_interface_names, - get_device_logs_fallback, - get_live_interface_snapshot, - get_recent_influx_interface_identities, - merge_interface_tables, - parse_bgp_summary, - parse_influx_metric_rows, - parse_influx_syslog_rows, - parse_influx_timestamp, - parse_ip_link_stats, - parse_route_table, - parse_table, - preview_items, - query_influx_rows, - summarize_counter_points, -) -from ._core.device.validators import ( - docker_exec, - require_client_source, - resolve_container, - validate_device_name, - validate_interface_name, - validate_ip_address, - validate_prefix, -) -from ._core.observability.pingmesh_scope import resolve_pingmesh_time_scope - - -class AgentToolkitHelperMixin: - _SEVERITY_OPTIONS = { - "emergency", - "alert", - "critical", - "error", - "warning", - "notice", - "info", - "debug", - } - _PINGMESH_RANGE_ENV_START = "NETOPSBENCH_PINGMESH_START_TIME" - _PINGMESH_RANGE_ENV_END = "NETOPSBENCH_PINGMESH_END_TIME" - _DEFAULT_SONIC_PORT_MTU = 9100 - - def _validate_device_name(self, device: str, field_name: str = "device") -> str: - return validate_device_name(device, field_name) - - def _validate_interface_name(self, interface: str) -> str: - return validate_interface_name(interface) - - def _validate_ip_address(self, ip_value: str, field_name: str = "ip") -> str: - return validate_ip_address(ip_value, field_name) - - def _validate_prefix(self, prefix: str) -> str: - return validate_prefix(prefix) - - def _resolve_container(self, device: str, field_name: str = "device") -> str: - return resolve_container(self, device, field_name) - - def _require_client_source(self, src: str, tool_name: str) -> str: - return require_client_source(self, src, tool_name) - - def _docker_exec(self, container: str, cmd_args: list[str], timeout: int) -> subprocess.CompletedProcess: - return docker_exec(container, cmd_args, timeout) - - def _discover_topology_dir(self, base_dir: str) -> str: - return discover_topology_dir(base_dir) - - def _load_topology_metadata(self, metadata: dict): - build_topology_state_from_metadata(metadata).apply_to(self) - - def _enrich_topology_metadata(self, topology: dict[str, Any]) -> dict[str, Any]: - return enrich_topology_metadata(topology, self._DEFAULT_SONIC_PORT_MTU) - - def _resolve_interface_metric_identities(self, interface: str) -> dict[str, list[str]]: - return resolve_interface_metric_identities(interface) - - @staticmethod - def _parse_influx_timestamp(value: str | None): - return parse_influx_timestamp(value) - - def _parse_influx_metric_rows(self, csv_text: str) -> list[dict[str, Any]]: - return parse_influx_metric_rows(csv_text) - - def _get_recent_influx_interface_identities( - self, device: str, time_range_minutes: int, headers: dict[str, str] | None = None - ) -> list[dict[str, str | None]]: - return get_recent_influx_interface_identities(self, device, time_range_minutes, headers) - - @staticmethod - def _preview_items(items: list[str], limit: int = 8) -> str: - return preview_items(items, limit) - - def _summarize_counter_points(self, field: str, points: list[dict[str, Any]]) -> dict[str, Any]: - return summarize_counter_points(field, points) - - def _parse_table(self, text: str) -> list[dict[str, str]]: - return parse_table(text) - - def _merge_interface_tables( - self, status_rows: list[dict[str, str]], counter_rows: list[dict[str, str]] - ) -> list[dict[str, Any]]: - return merge_interface_tables(status_rows, counter_rows) - - def _parse_ip_link_stats(self, text: str) -> list[dict[str, Any]]: - return parse_ip_link_stats(text) - - def _parse_bgp_summary(self, text: str) -> list[dict[str, Any]]: - return parse_bgp_summary(text) - - def _parse_influx_syslog_rows(self, csv_text: str) -> list[dict[str, Any]]: - return parse_influx_syslog_rows(csv_text) - - def _get_device_logs_fallback( - self, device: str, time_range_minutes: int, severity: str | None = None - ) -> list[dict[str, Any]]: - return get_device_logs_fallback(self, device, time_range_minutes, severity=severity) - - def _get_live_interface_snapshot(self, device: str, interface: str) -> dict[str, Any] | None: - return get_live_interface_snapshot(self, device, interface) - - def _get_active_interface_names(self, device: str) -> list[str]: - return get_active_interface_names(self, device) - - def _parse_route_table(self, text: str) -> list[dict[str, Any]]: - return parse_route_table(text) - - def _resolve_pingmesh_time_scope( - self, time_range_minutes: int, start_time: str | None = None, end_time: str | None = None - ) -> dict[str, Any]: - return resolve_pingmesh_time_scope(self, time_range_minutes, start_time=start_time, end_time=end_time) - - def set_pingmesh_time_window(self, start_time: str | None, end_time: str | None) -> None: - self._pingmesh_default_start_time = start_time - self._pingmesh_default_end_time = end_time - - def _query_influx_rows(self, query: str, require_value: bool = True) -> list[dict[str, Any]]: - return query_influx_rows(self, query, require_value=require_value) diff --git a/netopsbench/platform/toolkit/fastmcp_server.py b/netopsbench/platform/toolkit/fastmcp_server.py index f2acc1d..03c9194 100644 --- a/netopsbench/platform/toolkit/fastmcp_server.py +++ b/netopsbench/platform/toolkit/fastmcp_server.py @@ -1,41 +1,38 @@ -#!/usr/bin/env python3 """FastMCP server exposing NetOpsBench toolkit tools.""" -import importlib - -try: - FastMCP = importlib.import_module("fastmcp").FastMCP -except Exception: - - class FastMCP: # type: ignore[override] - def __init__(self, *args, **kwargs): - pass - - def tool(self, *args, **kwargs): - def decorator(func): - return func - - return decorator - - def run(self, *args, **kwargs): - raise RuntimeError( - "fastmcp is not installed. Install with: pip install -e '.[agent]' or pip install -e '.[dev,agent]'" - ) - - from .mcp.registry import group_tool_names, load_tool_specs -mcp = FastMCP("netopsbench") - _TOOL_SPECS = load_tool_specs() for _spec in _TOOL_SPECS: globals()[_spec.name] = _spec.handler - mcp.tool(name=_spec.name)(_spec.handler) EXPOSED_TOOLS = [_spec.name for _spec in _TOOL_SPECS] EXPOSED_TOOLS_BY_GROUP = group_tool_names(_TOOL_SPECS) +def create_server(): + """Create the optional FastMCP server only when it is actually used.""" + try: + from fastmcp import FastMCP + except ImportError as exc: + raise RuntimeError( + "fastmcp is not installed. Install with: pip install -e '.[agent]' or pip install -e '.[dev,agent]'" + ) from exc + server = FastMCP("netopsbench") + for spec in _TOOL_SPECS: + server.tool(name=spec.name)(spec.handler) + return server + + def run_server(): """Entry point used by CLI command.""" - mcp.run(transport="stdio") + create_server().run(transport="stdio") + + +def main() -> int: + run_server() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/netopsbench/platform/toolkit/mcp/inventory.py b/netopsbench/platform/toolkit/mcp/inventory.py index 20c13ff..76a8d49 100644 --- a/netopsbench/platform/toolkit/mcp/inventory.py +++ b/netopsbench/platform/toolkit/mcp/inventory.py @@ -17,6 +17,11 @@ def get_bgp_neighbors(device: str, format: str = "structured"): return as_payload(get_toolkit().get_bgp_neighbors(device=device, format=format)) +def get_bgp_neighbor(device: str, peer: str): + """Get live detail for one BGP neighbor on one network device.""" + return as_payload(get_toolkit().get_bgp_neighbor(device=device, peer=peer)) + + def get_route_table( device: str, prefix: str | None = None, @@ -55,6 +60,7 @@ def get_device_acl(device: str, view: str = "summary", max_lines: int = 300): ToolSpec(name="get_topology", group="inventory", handler=get_topology), ToolSpec(name="get_device_interfaces", group="inventory", handler=get_device_interfaces), ToolSpec(name="get_bgp_neighbors", group="inventory", handler=get_bgp_neighbors), + ToolSpec(name="get_bgp_neighbor", group="inventory", handler=get_bgp_neighbor), ToolSpec(name="get_route_table", group="inventory", handler=get_route_table), ToolSpec(name="get_device_config", group="inventory", handler=get_device_config), ToolSpec(name="get_bgp_rib", group="inventory", handler=get_bgp_rib), diff --git a/netopsbench/platform/toolkit/mcp/observability.py b/netopsbench/platform/toolkit/mcp/observability.py index b0387b6..906f579 100644 --- a/netopsbench/platform/toolkit/mcp/observability.py +++ b/netopsbench/platform/toolkit/mcp/observability.py @@ -40,9 +40,29 @@ def get_interface_metrics( ) -def get_all_bgp_status(): - """Get BGP status summary across all devices.""" - return as_payload(get_toolkit().get_all_bgp_status()) +def query_bgp_events( + start_time: str = "", + end_time: str = "", + time_range_minutes: int = 10, + device: str = "", + peer: str = "", + role: str = "", + state: str = "non_established", + limit: int = 100, +): + """Find BGP session events from centralized historical telemetry.""" + return as_payload( + get_toolkit().query_bgp_events( + start_time=start_time or None, + end_time=end_time or None, + time_range_minutes=time_range_minutes, + device=device or None, + peer=peer or None, + role=role or None, + state=state, + limit=limit, + ) + ) def get_pingmesh_summary( @@ -80,7 +100,7 @@ def get_pingmesh_hotspots( TOOL_SPECS = [ ToolSpec(name="get_device_logs", group="observability", handler=get_device_logs), ToolSpec(name="get_interface_metrics", group="observability", handler=get_interface_metrics), - ToolSpec(name="get_all_bgp_status", group="observability", handler=get_all_bgp_status), + ToolSpec(name="query_bgp_events", group="observability", handler=query_bgp_events), ToolSpec(name="get_pingmesh_summary", group="observability", handler=get_pingmesh_summary), ToolSpec(name="get_pingmesh_hotspots", group="observability", handler=get_pingmesh_hotspots), ] diff --git a/netopsbench/platform/toolkit/mcp/screenshot.py b/netopsbench/platform/toolkit/mcp/screenshot.py deleted file mode 100644 index 190c1b4..0000000 --- a/netopsbench/platform/toolkit/mcp/screenshot.py +++ /dev/null @@ -1,55 +0,0 @@ -from .context import as_payload, get_toolkit -from .contracts import ToolSpec - - -def get_grafana_screenshot( - panel_name: str, - time_range: str = "1h", - width: int = 1000, - height: int = 500, - include_base64: bool = False, -): - """Capture screenshot for one Grafana panel.""" - return as_payload( - get_toolkit().get_grafana_screenshot( - panel_name=panel_name, - time_range=time_range, - width=width, - height=height, - include_base64=include_base64, - ) - ) - - -def get_dashboard_screenshot( - time_range: str = "1h", - width: int = 1920, - height: int = 1080, - include_base64: bool = False, -): - """Capture screenshot for full dashboard.""" - return as_payload( - get_toolkit().get_dashboard_screenshot( - time_range=time_range, - width=width, - height=height, - include_base64=include_base64, - ) - ) - - -def get_troubleshooting_screenshots(time_range: str = "30m", include_base64: bool = False): - """Capture the standard troubleshooting screenshot set.""" - return as_payload( - get_toolkit().get_troubleshooting_screenshots( - time_range=time_range, - include_base64=include_base64, - ) - ) - - -TOOL_SPECS = [ - ToolSpec(name="get_grafana_screenshot", group="screenshot", handler=get_grafana_screenshot), - ToolSpec(name="get_dashboard_screenshot", group="screenshot", handler=get_dashboard_screenshot), - ToolSpec(name="get_troubleshooting_screenshots", group="screenshot", handler=get_troubleshooting_screenshots), -] diff --git a/netopsbench/platform/toolkit/toolkit.py b/netopsbench/platform/toolkit/toolkit.py index a6f2929..0ae1eb9 100644 --- a/netopsbench/platform/toolkit/toolkit.py +++ b/netopsbench/platform/toolkit/toolkit.py @@ -1,19 +1,139 @@ -"""Agent toolkit public entrypoint.""" +"""Agent-facing network troubleshooting toolkit.""" from __future__ import annotations +import subprocess +from pathlib import Path +from typing import Any + +from netopsbench.config import config +from netopsbench.models.topology import TopologyManifest +from netopsbench.platform.topology.topology_utils import ( + clab_container_name, + coerce_topology_manifest, + load_topology_manifest, +) + from ._core.common import ToolResult -from ._core.device.device_ops import DeviceOpsMixin -from ._core.observability.observability_facade import ObservabilityFacadeMixin -from ._core.topology.topology_ops import TopologyOpsMixin -from ._toolkit_bootstrap import AgentToolkitBootstrapMixin -from ._toolkit_helpers import AgentToolkitHelperMixin +from ._core.device.connectivity_ops import ConnectivityOpsMixin +from ._core.device.interface_ops import InterfaceOpsMixin +from ._core.device.log_ops import LogOpsMixin +from ._core.device.routing_ops import RoutingOpsMixin +from ._core.device.telemetry_parsers import query_influx_rows +from ._core.device.validators import ( + docker_exec, + resolve_container, + validate_device_name, + validate_interface_name, + validate_ip_address, + validate_prefix, +) +from ._core.observability.bgp_ops import BgpOpsMixin +from ._core.observability.metrics_ops import MetricsOpsMixin +from ._core.observability.pingmesh_ops import PingmeshOpsMixin +from ._core.observability.pingmesh_scope import resolve_pingmesh_time_scope class AgentToolkit( - AgentToolkitBootstrapMixin, AgentToolkitHelperMixin, TopologyOpsMixin, DeviceOpsMixin, ObservabilityFacadeMixin + InterfaceOpsMixin, + RoutingOpsMixin, + LogOpsMixin, + ConnectivityOpsMixin, + BgpOpsMixin, + MetricsOpsMixin, + PingmeshOpsMixin, ): - """Toolkit providing network troubleshooting capabilities to the AI Agent.""" + """Toolkit providing network troubleshooting capabilities to an agent.""" + + _SEVERITY_OPTIONS = { + "emergency", + "alert", + "critical", + "error", + "warning", + "notice", + "info", + "debug", + } + _PINGMESH_RANGE_ENV_START = "NETOPSBENCH_PINGMESH_START_TIME" + _PINGMESH_RANGE_ENV_END = "NETOPSBENCH_PINGMESH_END_TIME" + + def __init__( + self, + influxdb_url: str | None = None, + influxdb_token: str | None = None, + influxdb_org: str | None = None, + influxdb_bucket: str | None = None, + topology_dir: str | Path | None = None, + topology_metadata: dict[str, Any] | None = None, + ) -> None: + self.influxdb_url = influxdb_url or config.influxdb_url + self.influxdb_token = influxdb_token or config.influxdb_token + self.influxdb_org = influxdb_org or config.influxdb_org + self.influxdb_bucket = influxdb_bucket or config.influxdb_bucket + + topology_dir_value = topology_dir or config.topology_dir + resolved_topology_dir = ( + Path(topology_dir_value).expanduser().resolve() if topology_dir_value is not None else None + ) + if topology_metadata is not None: + manifest = coerce_topology_manifest(topology_metadata) + else: + if resolved_topology_dir is None: + raise ValueError("Pass topology_metadata or set an explicit topology_dir") + metadata_file = resolved_topology_dir / "topology.json" + if not metadata_file.is_file(): + raise FileNotFoundError( + f"Topology metadata not found: {metadata_file}. " + "Pass topology_metadata or set NETOPSBENCH_TOPOLOGY_DIR." + ) + manifest = load_topology_manifest(metadata_file) + self.manifest: TopologyManifest = manifest + self.topology_dir: Path | None = resolved_topology_dir + self.topology_metadata = manifest.model_dump(mode="json") + self.topology_name = manifest.name + self.topology_id = manifest.topology_id + self.container_names = { + device.name: clab_container_name(manifest.name, device.name) for device in manifest.devices + } + self._pingmesh_default_start_time: str | None = None + self._pingmesh_default_end_time: str | None = None + + def get_topology(self) -> ToolResult: + return ToolResult(success=True, data=self.manifest.to_agent_topology()) + + def _validate_device_name(self, device: str, field_name: str = "device") -> str: + return validate_device_name(device, field_name) + + def _validate_interface_name(self, interface: str) -> str: + return validate_interface_name(interface) + + def _validate_ip_address(self, ip_value: str, field_name: str = "ip") -> str: + return validate_ip_address(ip_value, field_name) + + def _validate_prefix(self, prefix: str) -> str: + return validate_prefix(prefix) + + def _resolve_container(self, device: str, field_name: str = "device") -> str: + return resolve_container(self, device, field_name) + + def _docker_exec(self, container: str, cmd_args: list[str], timeout: int) -> subprocess.CompletedProcess: + return docker_exec(container, cmd_args, timeout) + + def _resolve_pingmesh_time_scope( + self, + time_range_minutes: int, + start_time: str | None = None, + end_time: str | None = None, + ) -> dict[str, Any]: + return resolve_pingmesh_time_scope(self, time_range_minutes, start_time=start_time, end_time=end_time) + + def set_pingmesh_time_window(self, start_time: str | None, end_time: str | None) -> None: + self._pingmesh_default_start_time = start_time + self._pingmesh_default_end_time = end_time + + def _query_influx_rows(self, query: str, require_value: bool = True) -> list[dict[str, Any]]: + return query_influx_rows(self, query, require_value=require_value) __all__ = ["AgentToolkit", "ToolResult"] diff --git a/netopsbench/platform/topology/__init__.py b/netopsbench/platform/topology/__init__.py index a780fb7..de31ce2 100644 --- a/netopsbench/platform/topology/__init__.py +++ b/netopsbench/platform/topology/__init__.py @@ -1,21 +1 @@ -"""Platform topology subsystem.""" - -from .generator import TOPOLOGY_SCALES, TopologyConfig, TopologyGenerator, generate_topology -from .metadata_generator import generate_metadata_file, parse_clab_yaml -from .topology_utils import ( - TopologyState, - build_topology_state_from_metadata, - discover_topology_dir, -) - -__all__ = [ - "TOPOLOGY_SCALES", - "TopologyConfig", - "TopologyGenerator", - "generate_topology", - "generate_metadata_file", - "parse_clab_yaml", - "TopologyState", - "build_topology_state_from_metadata", - "discover_topology_dir", -] +"""Internal topology planning and rendering package.""" diff --git a/netopsbench/platform/topology/clos_builder.py b/netopsbench/platform/topology/clos_builder.py new file mode 100644 index 0000000..d1eedf3 --- /dev/null +++ b/netopsbench/platform/topology/clos_builder.py @@ -0,0 +1,226 @@ +"""Build complete artifact-independent plans for two-tier CLOS fabrics.""" + +from __future__ import annotations + +from netopsbench.models.topology import ( + Collector, + Device, + DeviceRole, + Link, + LinkEndpoint, + Management, + RoutingMetadata, + TopologyDefaults, + TopologyFacts, + TopologyManifest, +) + +from .config import TopologyConfig +from .plan import BGPNeighborPlan, DevicePlan, FabricPlan +from .planning import ( + ManagementAddressing, + build_render_settings, + client_commands, + pingmesh_policy_for_scale, + sonic_port_name, +) + + +def _validate_config(config: TopologyConfig) -> None: + if not 1 <= config.num_spines <= 255: + raise ValueError(f"num_spines must fit in one IPv4 octet (1-255), got {config.num_spines}") + if not 1 <= config.num_leafs <= 155: + raise ValueError( + "num_leafs must fit the client subnet scheme 192.168.(100+leaf).0/24 " f"(1-155), got {config.num_leafs}" + ) + if not 1 <= config.clients_per_leaf <= 64: + raise ValueError( + "clients_per_leaf must fit the per-leaf /30 allocation (1-64), " f"got {config.clients_per_leaf}" + ) + + +def build_clos_plan(config: TopologyConfig) -> FabricPlan: + """Return a complete CLOS plan without writing any artifacts.""" + _validate_config(config) + spine_mgmt_offset = 10 + leaf_mgmt_offset = spine_mgmt_offset + config.num_spines + client_mgmt_offset = leaf_mgmt_offset + config.num_leafs + total_clients = config.num_leafs * config.clients_per_leaf + last_device_offset = client_mgmt_offset + total_clients + allocated_offsets = { + *(spine_mgmt_offset + idx for idx in range(1, config.num_spines + 1)), + *(leaf_mgmt_offset + idx for idx in range(1, config.num_leafs + 1)), + *(client_mgmt_offset + idx for idx in range(1, total_clients + 1)), + } + addressing = ManagementAddressing.create(config.mgmt_ipv4_subnet, allocated_offsets) + collector_ip = config.collector_ip or addressing.default_collector(last_device_offset, prefer_offset_200=True) + addressing.validate_collector(collector_ip) + render_settings = build_render_settings(addressing, collector_ip) + + devices: list[Device] = [] + device_plans: list[DevicePlan] = [] + links: list[Link] = [] + + for spine_idx in range(1, config.num_spines + 1): + name = f"spine{spine_idx}" + router_id = f"10.0.0.{spine_idx}" + device = Device( + name=name, + role=DeviceRole.SPINE, + mgmt_ip=addressing.host_ip(spine_mgmt_offset + spine_idx), + asn=config.spine_asn, + router_id=router_id, + ) + interfaces = { + sonic_port_name(leaf_idx): (f"10.{spine_idx}.{leaf_idx}.1/30",) + for leaf_idx in range(1, config.num_leafs + 1) + } + neighbors = tuple( + BGPNeighborPlan( + peer_ip=f"10.{spine_idx}.{leaf_idx}.2", + remote_as=config.leaf_asn_start + leaf_idx - 1, + ) + for leaf_idx in range(1, config.num_leafs + 1) + ) + devices.append(device) + device_plans.append( + DevicePlan( + device=device, + required_ports=config.num_leafs, + configdb_interface_cidrs=interfaces, + bgp_asn=config.spine_asn, + bgp_router_id=router_id, + bgp_neighbors=neighbors, + ) + ) + + for leaf_idx in range(1, config.num_leafs + 1): + leaf_name = f"leaf{leaf_idx}" + router_id = f"10.0.0.{10 + leaf_idx}" + leaf_asn = config.leaf_asn_start + leaf_idx - 1 + subnet_octet = 100 + leaf_idx + leaf = Device( + name=leaf_name, + role=DeviceRole.LEAF, + mgmt_ip=addressing.host_ip(leaf_mgmt_offset + leaf_idx), + asn=leaf_asn, + router_id=router_id, + metadata={"client_subnet": f"192.168.{subnet_octet}.0/24"}, + ) + interfaces = { + sonic_port_name(spine_idx): (f"10.{spine_idx}.{leaf_idx}.2/30",) + for spine_idx in range(1, config.num_spines + 1) + } + neighbors = tuple( + BGPNeighborPlan(peer_ip=f"10.{spine_idx}.{leaf_idx}.1", remote_as=config.spine_asn) + for spine_idx in range(1, config.num_spines + 1) + ) + networks: list[str] = [] + for client_position in range(1, config.clients_per_leaf + 1): + subnet_base = (client_position - 1) * 4 + interfaces[sonic_port_name(config.num_spines + client_position)] = ( + f"192.168.{subnet_octet}.{subnet_base + 1}/30", + ) + networks.append(f"192.168.{subnet_octet}.{subnet_base}/30") + devices.append(leaf) + device_plans.append( + DevicePlan( + device=leaf, + required_ports=config.num_spines + config.clients_per_leaf, + configdb_interface_cidrs=interfaces, + bgp_asn=leaf_asn, + bgp_router_id=router_id, + bgp_neighbors=neighbors, + bgp_networks=tuple(networks), + ) + ) + + for client_position in range(1, config.clients_per_leaf + 1): + client_idx = (leaf_idx - 1) * config.clients_per_leaf + client_position + subnet_base = (client_position - 1) * 4 + client_ip = f"192.168.{subnet_octet}.{subnet_base + 2}" + gateway = f"192.168.{subnet_octet}.{subnet_base + 1}" + client = Device( + name=f"client{client_idx}", + role=DeviceRole.CLIENT, + mgmt_ip=addressing.host_ip(client_mgmt_offset + client_idx), + data_ip=client_ip, + attached_switch=leaf_name, + metadata={"rack": f"rack{leaf_idx}"}, + ) + devices.append(client) + device_plans.append(DevicePlan(device=client, client_commands=client_commands(client_ip, gateway))) + + for spine_idx in range(1, config.num_spines + 1): + for leaf_idx in range(1, config.num_leafs + 1): + links.append( + Link( + kind="spine-leaf", + endpoints=( + LinkEndpoint(device=f"spine{spine_idx}", interface=f"eth{leaf_idx}"), + LinkEndpoint(device=f"leaf{leaf_idx}", interface=f"eth{spine_idx}"), + ), + ) + ) + for leaf_idx in range(1, config.num_leafs + 1): + for client_position in range(1, config.clients_per_leaf + 1): + client_idx = (leaf_idx - 1) * config.clients_per_leaf + client_position + links.append( + Link( + kind="leaf-client", + endpoints=( + LinkEndpoint(device=f"leaf{leaf_idx}", interface=f"eth{config.num_spines + client_position}"), + LinkEndpoint(device=f"client{client_idx}", interface="eth1"), + ), + ) + ) + + scale_name = config.scale_name or "custom" + manifest = TopologyManifest( + topology_id=config.name, + name=config.name, + scale=scale_name, + family="clos", + management=Management( + network=config.mgmt_network_name or f"clab-mgmt-{config.name}", + ipv4_subnet=str(addressing.network), + ), + collector=Collector(ipv4=render_settings.syslog_collector), + defaults=TopologyDefaults(), + facts=TopologyFacts( + num_spines=config.num_spines, + num_leafs=config.num_leafs, + clients_per_attached_switch=config.clients_per_leaf, + total_clients=total_clients, + total_switches=config.num_spines + config.num_leafs, + ), + devices=devices, + links=links, + routing=RoutingMetadata( + spine_asn=config.spine_asn, + leaf_asn_range=f"{config.leaf_asn_start}-{config.leaf_asn_start + config.num_leafs - 1}", + ecmp_hash_policy_by_role={DeviceRole.SPINE: 1, DeviceRole.LEAF: 1}, + ), + pingmesh=pingmesh_policy_for_scale(scale_name), + ) + header = f"""# {config.name.upper()} Topology Configuration +# Generated by NetOpsBench Topology Generator +# Scale: {config.num_spines} spines, {config.num_leafs} leafs, {config.clients_per_leaf} clients/leaf +# +# Features: +# - BGP with ECMP +# - Observability: Syslog and gNMI + +""" + return FabricPlan( + manifest=manifest, + device_plans=tuple(device_plans), + nos_kind=config.nos_kind, + nos_image=config.nos_image, + client_image=config.client_image, + render_settings=render_settings, + yaml_header=header, + ) + + +__all__ = ["build_clos_plan"] diff --git a/netopsbench/platform/topology/config.py b/netopsbench/platform/topology/config.py new file mode 100644 index 0000000..43c92b4 --- /dev/null +++ b/netopsbench/platform/topology/config.py @@ -0,0 +1,146 @@ +"""Neutral topology configuration and artifact constants.""" + +from __future__ import annotations + +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path + +from netopsbench.models.profiles import ScaleProfile, get_scale_profile + +DEFAULT_SONIC_VS_IMAGE = "yyyyyt123/netopsbench-sonic-vs-202505-telemetry:202505-telemetry" +DEFAULT_CLIENT_IMAGE = "yyyyyt123/netopsbench-client:python3" +SONIC_PLATFORM = "x86_64-kvm_x86_64-r0" +SONIC_HWSKU = "Force10-S6000" +SONIC_HWSKU_PATH = f"/usr/share/sonic/device/{SONIC_PLATFORM}/{SONIC_HWSKU}" +SONIC_PORT_CONFIG_PATH = f"{SONIC_HWSKU_PATH}/port_config.ini" +SONIC_LANEMAP_PATH = f"{SONIC_HWSKU_PATH}/lanemap.ini" +SONIC_PORT_COUNTER_INTERVAL_MS = 10_000 +_TOPOLOGY_RESOURCES = files("netopsbench.platform.topology") +SONIC_BASE_CONFIG_DB = _TOPOLOGY_RESOURCES.joinpath("sonic_vs_base_config_db.json") +SONIC_START_WRAPPER_SOURCE = _TOPOLOGY_RESOURCES.joinpath("sonic_start.sh") + + +def default_output_dir() -> str: + return str(Path.cwd() / "generated_topology") + + +@dataclass +class TopologyConfig: + """Configuration for a two-tier CLOS topology.""" + + name: str = "dcn" + num_spines: int = 2 + num_leafs: int = 2 + clients_per_leaf: int = 1 + nos_kind: str = "sonic-vs" + nos_image: str = DEFAULT_SONIC_VS_IMAGE + client_image: str = DEFAULT_CLIENT_IMAGE + mgmt_ipv4_subnet: str = "172.20.20.0/24" + mgmt_network_name: str | None = None + collector_ip: str | None = None + spine_asn: int = 65001 + leaf_asn_start: int = 65011 + scale_name: str | None = None + + +@dataclass +class FatTreeConfig: + """Configuration for a k-ary fat-tree fabric.""" + + k: int + name: str = "dcn" + nos_kind: str = "sonic-vs" + nos_image: str = DEFAULT_SONIC_VS_IMAGE + client_image: str = DEFAULT_CLIENT_IMAGE + mgmt_ipv4_subnet: str = "172.20.20.0/24" + mgmt_network_name: str | None = None + collector_ip: str | None = None + core_asn_start: int = 65001 + agg_asn_start: int = 65101 + edge_asn_start: int = 65201 + clients_per_edge: int | None = None + scale_name: str | None = None + + def __post_init__(self) -> None: + if self.k < 2 or self.k % 2 != 0: + raise ValueError(f"fat-tree k must be a positive even integer, got {self.k}") + if self.clients_per_edge is None: + self.clients_per_edge = self.half + if not 1 <= int(self.clients_per_edge) <= self.half: + raise ValueError(f"clients_per_edge must be between 1 and k/2 ({self.half}), got {self.clients_per_edge}") + + @property + def half(self) -> int: + return self.k // 2 + + @property + def num_core(self) -> int: + return self.half * self.half + + @property + def num_pods(self) -> int: + return self.k + + @property + def num_total_agg(self) -> int: + return self.k * self.half + + @property + def num_total_edge(self) -> int: + return self.k * self.half + + @property + def num_total_clients(self) -> int: + return self.num_total_edge * int(self.clients_per_edge or 0) + + @property + def host_density(self) -> str: + return "standard" if self.clients_per_edge == self.half else "sparse" + + +def _topology_mgmt_subnet(profile: ScaleProfile) -> str: + return f"172.20.20.0/{profile.management_prefix}" + + +def _clos_config_from_profile(profile: ScaleProfile) -> TopologyConfig: + return TopologyConfig( + num_spines=int(profile.num_spines or 0), + num_leafs=int(profile.num_leafs or 0), + clients_per_leaf=profile.clients_per_attached_switch, + mgmt_ipv4_subnet=_topology_mgmt_subnet(profile), + scale_name=profile.name, + ) + + +def _fat_tree_config_from_profile(profile: ScaleProfile) -> FatTreeConfig: + return FatTreeConfig( + k=int(profile.fat_tree_k or 0), + clients_per_edge=profile.clients_per_attached_switch, + mgmt_ipv4_subnet=_topology_mgmt_subnet(profile), + scale_name=profile.name, + ) + + +def config_for_scale(scale: str) -> TopologyConfig | FatTreeConfig: + profile = get_scale_profile(scale) + if profile.family == "clos": + return _clos_config_from_profile(profile) + return _fat_tree_config_from_profile(profile) + + +__all__ = [ + "DEFAULT_CLIENT_IMAGE", + "DEFAULT_SONIC_VS_IMAGE", + "SONIC_BASE_CONFIG_DB", + "SONIC_HWSKU", + "SONIC_LANEMAP_PATH", + "SONIC_PLATFORM", + "SONIC_PORT_COUNTER_INTERVAL_MS", + "SONIC_PORT_CONFIG_PATH", + "SONIC_START_WRAPPER_SOURCE", + "FatTreeConfig", + "TopologyConfig", + "config_for_scale", + "default_output_dir", +] diff --git a/netopsbench/platform/topology/configdb_payload.py b/netopsbench/platform/topology/configdb_payload.py new file mode 100644 index 0000000..3f5fb0f --- /dev/null +++ b/netopsbench/platform/topology/configdb_payload.py @@ -0,0 +1,64 @@ +"""Helpers for generated SONiC ``config_db.json`` startup artifacts.""" + +from __future__ import annotations + +import ipaddress +import json +from pathlib import Path +from typing import Any + + +def _sort_key(interface_name: str) -> tuple[int, int | str]: + if interface_name.startswith("Ethernet"): + suffix = interface_name.removeprefix("Ethernet") + if suffix.isdigit(): + return (0, int(suffix)) + return (1, interface_name) + + +def load_configdb_payload(path: str | Path) -> dict[str, Any]: + payload_path = Path(path) + if not payload_path.exists(): + return {} + try: + with payload_path.open(encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, ValueError, TypeError): + return {} + return payload if isinstance(payload, dict) else {} + + +def interface_names_from_payload(payload: dict[str, Any]) -> list[str]: + configdb_interfaces = payload.get("INTERFACE") + if not isinstance(configdb_interfaces, dict): + return [] + names = {str(name).split("|", 1)[0] for name in configdb_interfaces.keys()} + return sorted(names, key=_sort_key) + + +def interface_names_from_configdb(path: str | Path) -> list[str]: + return interface_names_from_payload(load_configdb_payload(path)) + + +def interface_names_for_config(config_path: str | Path) -> list[str]: + return interface_names_from_configdb(config_path) + + +def interface_networks_from_payload(payload: dict[str, Any]) -> dict[str, str]: + configdb_interfaces = payload.get("INTERFACE") + networks: dict[str, str] = {} + if not isinstance(configdb_interfaces, dict): + return networks + for key in configdb_interfaces.keys(): + if "|" not in str(key): + continue + interface_name, cidr = str(key).split("|", 1) + try: + networks[interface_name] = str(ipaddress.ip_interface(cidr).network) + except ValueError: + continue + return networks + + +def interface_networks_for_config(config_path: str | Path) -> dict[str, str]: + return interface_networks_from_payload(load_configdb_payload(config_path)) diff --git a/netopsbench/platform/topology/fat_tree_builder.py b/netopsbench/platform/topology/fat_tree_builder.py new file mode 100644 index 0000000..7ab051d --- /dev/null +++ b/netopsbench/platform/topology/fat_tree_builder.py @@ -0,0 +1,348 @@ +"""Build complete artifact-independent plans for k-ary fat-tree fabrics.""" + +from __future__ import annotations + +import ipaddress + +from netopsbench.models.topology import ( + Collector, + Device, + DeviceRole, + Link, + LinkEndpoint, + Management, + RoutingMetadata, + TopologyDefaults, + TopologyFacts, + TopologyManifest, +) + +from .config import FatTreeConfig +from .plan import BGPNeighborPlan, DevicePlan, FabricPlan +from .planning import ( + ManagementAddressing, + build_render_settings, + client_commands, + pingmesh_policy_for_scale, + sonic_port_name, +) + + +def _validate_config(config: FatTreeConfig) -> None: + if config.num_total_edge > 155: + raise ValueError( + "num_total_edge must fit the client subnet scheme 192.168.(100+edge).0/24 " + f"(1-155), got {config.num_total_edge}" + ) + if int(config.clients_per_edge or 0) > 64: + raise ValueError("clients_per_edge must fit the per-edge /30 allocation") + + +def build_fat_tree_plan(config: FatTreeConfig) -> FabricPlan: + """Return a complete fat-tree plan without writing any artifacts.""" + _validate_config(config) + half = config.half + clients_per_edge = int(config.clients_per_edge or 0) + + core_agg_links: list[tuple[int, int]] = [] + agg_edge_links: list[tuple[int, int]] = [] + core_neighbors: dict[int, list[int]] = {} + agg_core_neighbors: dict[int, list[int]] = {} + agg_edge_neighbors: dict[int, list[int]] = {} + edge_agg_neighbors: dict[int, list[int]] = {} + agg_pod: dict[int, int] = {} + edge_pod: dict[int, int] = {} + + for pod in range(1, config.k + 1): + for local_agg in range(1, half + 1): + global_agg = (pod - 1) * half + local_agg + agg_pod[global_agg] = pod + agg_core_neighbors[global_agg] = [] + agg_edge_neighbors[global_agg] = [] + for core_offset in range(1, half + 1): + core_idx = (local_agg - 1) * half + core_offset + core_agg_links.append((core_idx, global_agg)) + core_neighbors.setdefault(core_idx, []).append(global_agg) + agg_core_neighbors[global_agg].append(core_idx) + for local_edge in range(1, half + 1): + global_edge = (pod - 1) * half + local_edge + agg_edge_links.append((global_agg, global_edge)) + agg_edge_neighbors[global_agg].append(global_edge) + edge_agg_neighbors.setdefault(global_edge, []).append(global_agg) + edge_pod.setdefault(global_edge, pod) + + core_agg_ips: dict[tuple[int, int], tuple[str, str]] = {} + core_agg_subnets = ipaddress.ip_network("10.1.0.0/16").subnets(new_prefix=30) + for link in sorted(core_agg_links): + hosts = tuple(next(core_agg_subnets).hosts()) + core_agg_ips[link] = (str(hosts[0]), str(hosts[1])) + + agg_edge_ips: dict[tuple[int, int], tuple[str, str]] = {} + agg_edge_subnets = ipaddress.ip_network("10.2.0.0/16").subnets(new_prefix=30) + for link in sorted(agg_edge_links): + hosts = tuple(next(agg_edge_subnets).hosts()) + agg_edge_ips[link] = (str(hosts[0]), str(hosts[1])) + + core_mgmt_offset = 10 + agg_mgmt_offset = core_mgmt_offset + config.num_core + edge_mgmt_offset = agg_mgmt_offset + config.num_total_agg + client_mgmt_offset = edge_mgmt_offset + config.num_total_edge + last_device_offset = client_mgmt_offset + config.num_total_clients + allocated_offsets = { + *(core_mgmt_offset + idx for idx in range(1, config.num_core + 1)), + *(agg_mgmt_offset + idx for idx in range(1, config.num_total_agg + 1)), + *(edge_mgmt_offset + idx for idx in range(1, config.num_total_edge + 1)), + *(client_mgmt_offset + idx for idx in range(1, config.num_total_clients + 1)), + } + addressing = ManagementAddressing.create(config.mgmt_ipv4_subnet, allocated_offsets) + collector_ip = config.collector_ip or addressing.default_collector(last_device_offset, prefer_offset_200=False) + addressing.validate_collector(collector_ip) + render_settings = build_render_settings(addressing, collector_ip) + + devices: list[Device] = [] + device_plans: list[DevicePlan] = [] + + for core_idx in range(1, config.num_core + 1): + name = f"core{core_idx}" + router_id = f"10.0.1.{core_idx}" + asn = config.core_asn_start + core_idx - 1 + device = Device( + name=name, + role=DeviceRole.CORE, + mgmt_ip=addressing.host_ip(core_mgmt_offset + core_idx), + asn=asn, + router_id=router_id, + ) + neighbors_for_device = sorted(core_neighbors[core_idx]) + interfaces = { + sonic_port_name(port_idx): (f"{core_agg_ips[(core_idx, agg_idx)][0]}/30",) + for port_idx, agg_idx in enumerate(neighbors_for_device, start=1) + } + neighbors = tuple( + BGPNeighborPlan( + peer_ip=core_agg_ips[(core_idx, agg_idx)][1], + remote_as=config.agg_asn_start + agg_idx - 1, + ) + for agg_idx in neighbors_for_device + ) + devices.append(device) + device_plans.append( + DevicePlan( + device=device, + required_ports=config.k, + configdb_interface_cidrs=interfaces, + bgp_asn=asn, + bgp_router_id=router_id, + bgp_neighbors=neighbors, + ) + ) + + for agg_idx in range(1, config.num_total_agg + 1): + name = f"agg{agg_idx}" + router_id = f"10.0.2.{agg_idx}" + asn = config.agg_asn_start + agg_idx - 1 + device = Device( + name=name, + role=DeviceRole.AGG, + mgmt_ip=addressing.host_ip(agg_mgmt_offset + agg_idx), + asn=asn, + router_id=router_id, + metadata={"pod": agg_pod[agg_idx]}, + ) + core_peers = sorted(agg_core_neighbors[agg_idx]) + edge_peers = sorted(agg_edge_neighbors[agg_idx]) + interfaces = { + sonic_port_name(port_idx): (f"{core_agg_ips[(core_idx, agg_idx)][1]}/30",) + for port_idx, core_idx in enumerate(core_peers, start=1) + } + interfaces.update( + { + sonic_port_name(half + port_offset): (f"{agg_edge_ips[(agg_idx, edge_idx)][0]}/30",) + for port_offset, edge_idx in enumerate(edge_peers, start=1) + } + ) + neighbors = tuple( + [ + BGPNeighborPlan( + peer_ip=core_agg_ips[(core_idx, agg_idx)][0], + remote_as=config.core_asn_start + core_idx - 1, + ) + for core_idx in core_peers + ] + + [ + BGPNeighborPlan( + peer_ip=agg_edge_ips[(agg_idx, edge_idx)][1], + remote_as=config.edge_asn_start + edge_idx - 1, + ) + for edge_idx in edge_peers + ] + ) + devices.append(device) + device_plans.append( + DevicePlan( + device=device, + required_ports=config.k, + configdb_interface_cidrs=interfaces, + bgp_asn=asn, + bgp_router_id=router_id, + bgp_neighbors=neighbors, + ) + ) + + for edge_idx in range(1, config.num_total_edge + 1): + name = f"edge{edge_idx}" + router_id = f"10.0.3.{edge_idx}" + asn = config.edge_asn_start + edge_idx - 1 + device = Device( + name=name, + role=DeviceRole.EDGE, + mgmt_ip=addressing.host_ip(edge_mgmt_offset + edge_idx), + asn=asn, + router_id=router_id, + metadata={ + "pod": edge_pod[edge_idx], + "client_subnet": f"192.168.{100 + edge_idx}.0/24", + }, + ) + agg_peers = sorted(edge_agg_neighbors[edge_idx]) + interfaces = { + sonic_port_name(port_idx): (f"{agg_edge_ips[(agg_idx, edge_idx)][1]}/30",) + for port_idx, agg_idx in enumerate(agg_peers, start=1) + } + networks: list[str] = [] + for client_position in range(1, clients_per_edge + 1): + subnet_base = (client_position - 1) * 4 + interfaces[sonic_port_name(half + client_position)] = (f"192.168.{100 + edge_idx}.{subnet_base + 1}/30",) + networks.append(f"192.168.{100 + edge_idx}.{subnet_base}/30") + neighbors = tuple( + BGPNeighborPlan( + peer_ip=agg_edge_ips[(agg_idx, edge_idx)][0], + remote_as=config.agg_asn_start + agg_idx - 1, + ) + for agg_idx in agg_peers + ) + devices.append(device) + device_plans.append( + DevicePlan( + device=device, + required_ports=half + clients_per_edge, + configdb_interface_cidrs=interfaces, + bgp_asn=asn, + bgp_router_id=router_id, + bgp_neighbors=neighbors, + bgp_networks=tuple(networks), + ) + ) + + client_idx = 0 + for edge_idx in range(1, config.num_total_edge + 1): + for client_position in range(1, clients_per_edge + 1): + client_idx += 1 + subnet_base = (client_position - 1) * 4 + client_ip = f"192.168.{100 + edge_idx}.{subnet_base + 2}" + gateway = f"192.168.{100 + edge_idx}.{subnet_base + 1}" + client = Device( + name=f"client{client_idx}", + role=DeviceRole.CLIENT, + mgmt_ip=addressing.host_ip(client_mgmt_offset + client_idx), + data_ip=client_ip, + attached_switch=f"edge{edge_idx}", + metadata={"rack": f"pod{edge_pod[edge_idx]}-edge{edge_idx}"}, + ) + devices.append(client) + device_plans.append(DevicePlan(device=client, client_commands=client_commands(client_ip, gateway))) + + links: list[Link] = [] + for core_idx, agg_idx in sorted(core_agg_links): + core_port = sorted(core_neighbors[core_idx]).index(agg_idx) + 1 + agg_port = sorted(agg_core_neighbors[agg_idx]).index(core_idx) + 1 + links.append( + Link( + kind="core-agg", + endpoints=( + LinkEndpoint(device=f"core{core_idx}", interface=f"eth{core_port}"), + LinkEndpoint(device=f"agg{agg_idx}", interface=f"eth{agg_port}"), + ), + ) + ) + for agg_idx, edge_idx in sorted(agg_edge_links): + agg_port = half + sorted(agg_edge_neighbors[agg_idx]).index(edge_idx) + 1 + edge_port = sorted(edge_agg_neighbors[edge_idx]).index(agg_idx) + 1 + links.append( + Link( + kind="agg-edge", + endpoints=( + LinkEndpoint(device=f"agg{agg_idx}", interface=f"eth{agg_port}"), + LinkEndpoint(device=f"edge{edge_idx}", interface=f"eth{edge_port}"), + ), + ) + ) + client_idx = 0 + for edge_idx in range(1, config.num_total_edge + 1): + for client_position in range(1, clients_per_edge + 1): + client_idx += 1 + links.append( + Link( + kind="edge-client", + endpoints=( + LinkEndpoint(device=f"edge{edge_idx}", interface=f"eth{half + client_position}"), + LinkEndpoint(device=f"client{client_idx}", interface="eth1"), + ), + ) + ) + + scale_name = config.scale_name or f"fat-tree-k{config.k}" + manifest = TopologyManifest( + topology_id=config.name, + name=config.name, + scale=scale_name, + family="fat-tree", + management=Management( + network=config.mgmt_network_name or f"clab-mgmt-{config.name}", + ipv4_subnet=str(addressing.network), + ), + collector=Collector(ipv4=render_settings.syslog_collector), + defaults=TopologyDefaults(), + facts=TopologyFacts( + num_cores=config.num_core, + num_aggs=config.num_total_agg, + num_edges=config.num_total_edge, + num_pods=config.num_pods, + clients_per_attached_switch=clients_per_edge, + total_clients=config.num_total_clients, + total_switches=config.num_core + config.num_total_agg + config.num_total_edge, + fat_tree_k=config.k, + full_density_clients_per_attached_switch=half, + host_density=config.host_density, + ), + devices=devices, + links=links, + routing=RoutingMetadata( + core_asn_range=f"{config.core_asn_start}-{config.core_asn_start + config.num_core - 1}", + agg_asn_range=f"{config.agg_asn_start}-{config.agg_asn_start + config.num_total_agg - 1}", + edge_asn_range=f"{config.edge_asn_start}-{config.edge_asn_start + config.num_total_edge - 1}", + ecmp_hash_policy_by_role={DeviceRole.CORE: 1, DeviceRole.AGG: 0, DeviceRole.EDGE: 1}, + ), + pingmesh=pingmesh_policy_for_scale(scale_name), + ) + header = f"""# {config.name.upper()} Fat-tree Topology Configuration +# Generated by NetOpsBench Topology Generator +# Scale: k={config.k}, {config.num_core} core, {config.num_total_agg} agg, {config.num_total_edge} edge, {clients_per_edge} clients/edge +# +# Features: +# - BGP with ECMP +# - Preseeded SONiC startup artifacts + +""" + return FabricPlan( + manifest=manifest, + device_plans=tuple(device_plans), + nos_kind=config.nos_kind, + nos_image=config.nos_image, + client_image=config.client_image, + render_settings=render_settings, + yaml_header=header, + ) + + +__all__ = ["build_fat_tree_plan"] diff --git a/netopsbench/platform/topology/generator.py b/netopsbench/platform/topology/generator.py index b05bedf..730aa5e 100644 --- a/netopsbench/platform/topology/generator.py +++ b/netopsbench/platform/topology/generator.py @@ -1,552 +1,59 @@ -""" -Topology Generator - Generate DCN network topologies for Containerlab. - -Supports generating 2-tier CLOS topologies with configurable scale. -""" +"""Generate canonical network topology artifacts.""" from __future__ import annotations -import ipaddress -import json -import os -from dataclasses import dataclass - -import yaml - -DEFAULT_SONIC_VS_IMAGE = "yyyyyt123/netopsbench-sonic-vs-202505-telemetry:202505-telemetry" - - -@dataclass -class TopologyConfig: - """Configuration for topology generation.""" - - name: str = "dcn" - num_spines: int = 2 - num_leafs: int = 2 - clients_per_leaf: int = 1 - nos_kind: str = "sonic-vs" - nos_image: str = DEFAULT_SONIC_VS_IMAGE - client_image: str = "yyyyyt123/netopsbench-client:python3" - mgmt_ipv4_subnet: str = "172.20.20.0/24" - mgmt_network_name: str | None = None - collector_ip: str | None = None - spine_asn: int = 65001 - leaf_asn_start: int = 65011 - - -# Predefined topology scales -TOPOLOGY_SCALES = { - "xs": TopologyConfig(num_spines=2, num_leafs=2, clients_per_leaf=1), - "small": TopologyConfig(num_spines=2, num_leafs=4, clients_per_leaf=2), - "medium": TopologyConfig(num_spines=4, num_leafs=8, clients_per_leaf=2), - "large": TopologyConfig(num_spines=4, num_leafs=16, clients_per_leaf=4), -} - - -class TopologyGenerator: - """ - Generates DCN network topologies for Containerlab. - - Creates CLOS topology YAML files and device configurations. - """ - - def __init__(self, config: TopologyConfig | None = None, output_dir: str | None = None): - self.config = config or TopologyConfig() - self.output_dir = output_dir or os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), - "generated_topology", - ) - self.mgmt_network = ipaddress.ip_network(self.config.mgmt_ipv4_subnet, strict=False) - if self.mgmt_network.version != 4: - raise ValueError(f"Only IPv4 management subnets are supported: {self.config.mgmt_ipv4_subnet}") - self.mgmt_network_name = self.config.mgmt_network_name or f"clab-mgmt-{self.config.name}" - - # Calculate IP offsets to avoid collisions - self.spine_mgmt_offset = 10 - self.leaf_mgmt_offset = self.spine_mgmt_offset + self.config.num_spines - self.client_mgmt_offset = self.leaf_mgmt_offset + self.config.num_leafs - self.collector_ip = self.config.collector_ip or self._default_collector_ip() - - self.syslog_collector = os.getenv("NETOPSBENCH_SYSLOG_COLLECTOR", self.collector_ip) - self.sflow_collector = os.getenv("NETOPSBENCH_SFLOW_COLLECTOR", self.syslog_collector) - self.sflow_port = int(os.getenv("NETOPSBENCH_SFLOW_PORT", "6343")) - self.sflow_polling_interval = int(os.getenv("NETOPSBENCH_SFLOW_POLLING_INTERVAL", "20")) - self.sflow_sample_rate = int(os.getenv("NETOPSBENCH_SFLOW_SAMPLE_RATE", "1000")) - self.sflow_sample_direction = os.getenv("NETOPSBENCH_SFLOW_SAMPLE_DIRECTION", "ingress") - self.snmp_community = os.getenv("NETOPSBENCH_SNMP_COMMUNITY", "public") - - def _mgmt_host_ip(self, host_offset: int) -> str: - """Return a deterministic management IPv4 address from the configured subnet.""" - if host_offset <= 0: - raise ValueError(f"Management host offset must be positive, got {host_offset}") - - candidate = ipaddress.ip_address(int(self.mgmt_network.network_address) + host_offset) - if candidate not in self.mgmt_network: - raise ValueError(f"Management host offset {host_offset} falls outside subnet {self.mgmt_network}") - if candidate in {self.mgmt_network.network_address, self.mgmt_network.broadcast_address}: - raise ValueError(f"Management host offset {host_offset} resolves to reserved address {candidate}") - return str(candidate) - - def _default_collector_ip(self) -> str: - """Pick a collector IP inside the management subnet, preferring .200 when possible.""" - preferred = int(self.mgmt_network.network_address) + 200 - preferred_ip = ipaddress.ip_address(preferred) - if preferred_ip in self.mgmt_network and preferred_ip not in { - self.mgmt_network.network_address, - self.mgmt_network.broadcast_address, - }: - return str(preferred_ip) - - fallback = ipaddress.ip_address(int(self.mgmt_network.broadcast_address) - 1) - if fallback in {self.mgmt_network.network_address, self.mgmt_network.broadcast_address}: - raise ValueError(f"Management subnet too small for collector IP: {self.mgmt_network}") - return str(fallback) - - def generate(self) -> dict: - """ - Generate complete topology including YAML and metadata. - - Returns: - Dictionary containing topology metadata and file paths - """ - os.makedirs(self.output_dir, exist_ok=True) - os.makedirs(os.path.join(self.output_dir, "configs"), exist_ok=True) - - # Generate topology structure - topology = self._generate_topology_structure() - - # Generate YAML file - yaml_path = self._write_yaml(topology) - - # Generate device configs - config_paths = self._generate_device_configs() - - # Generate metadata - metadata = self._generate_metadata() - metadata_path = os.path.join(self.output_dir, "topology.json") - with open(metadata_path, "w") as f: - json.dump(metadata, f, indent=2) - - return { - "yaml_file": yaml_path, - "metadata_file": metadata_path, - "config_files": config_paths, - "metadata": metadata, - } - - def _generate_topology_structure(self) -> dict: - """Generate Containerlab topology YAML structure.""" - topology = { - "name": self.config.name, - "mgmt": { - "network": self.mgmt_network_name, - "ipv4-subnet": str(self.mgmt_network), - }, - "topology": { - "kinds": { - self.config.nos_kind: {"image": self.config.nos_image}, - "linux": {"image": self.config.client_image}, - }, - "nodes": {}, - "links": [], - }, - } - - nodes = topology["topology"]["nodes"] - links = topology["topology"]["links"] - - # Generate spine nodes - for i in range(1, self.config.num_spines + 1): - spine_name = f"spine{i}" - nodes[spine_name] = { - "kind": self.config.nos_kind, - "group": "spine", - "mgmt-ipv4": self._mgmt_host_ip(self.spine_mgmt_offset + i), - } - - # Generate leaf nodes and clients - for i in range(1, self.config.num_leafs + 1): - leaf_name = f"leaf{i}" - nodes[leaf_name] = { - "kind": self.config.nos_kind, - "group": "leaf", - "mgmt-ipv4": self._mgmt_host_ip(self.leaf_mgmt_offset + i), - } - - # Generate clients for this leaf - for j in range(1, self.config.clients_per_leaf + 1): - client_idx = (i - 1) * self.config.clients_per_leaf + j - client_name = f"client{client_idx}" - # Point-to-point /30 subnet addressing - # Each client gets: 192.168.{octet}.{(j-1)*4 + 2}/30 - # Gateway is: 192.168.{octet}.{(j-1)*4 + 1} - octet = self._client_subnet_octet(i) - subnet_base = (j - 1) * 4 - client_ip = f"192.168.{octet}.{subnet_base + 2}" - gateway = f"192.168.{octet}.{subnet_base + 1}" - - nodes[client_name] = { - "kind": "linux", - "group": "client", - "mgmt-ipv4": self._mgmt_host_ip(self.client_mgmt_offset + client_idx), - "exec": [ - "ip link set dev eth1 mtu 9232", - f"ip addr add {client_ip}/30 dev eth1", - f"ip route add 192.168.0.0/16 via {gateway}", - "mkdir -p /var/log/pingmesh", - "iperf3 -s -D", - "ethtool -K eth1 rx off tx off tso off gso off gro off sg off tx-udp-segmentation off", - ], - } - - # Generate spine-leaf links - interface_counter = {} - for spine_idx in range(1, self.config.num_spines + 1): - spine_name = f"spine{spine_idx}" - for leaf_idx in range(1, self.config.num_leafs + 1): - leaf_name = f"leaf{leaf_idx}" - - # Get next available interface for each device - spine_if = interface_counter.get(spine_name, 0) + 1 - leaf_if = interface_counter.get(leaf_name, 0) + 1 - interface_counter[spine_name] = spine_if - interface_counter[leaf_name] = leaf_if - - links.append({"endpoints": [f"{spine_name}:eth{spine_if}", f"{leaf_name}:eth{leaf_if}"], "mtu": 9232}) - - # Generate leaf-client links - for i in range(1, self.config.num_leafs + 1): - leaf_name = f"leaf{i}" - for j in range(1, self.config.clients_per_leaf + 1): - client_idx = (i - 1) * self.config.clients_per_leaf + j - client_name = f"client{client_idx}" - - leaf_if = interface_counter.get(leaf_name, 0) + 1 - interface_counter[leaf_name] = leaf_if - - links.append({"endpoints": [f"{leaf_name}:eth{leaf_if}", f"{client_name}:eth1"], "mtu": 9232}) - - return topology - - def _write_yaml(self, topology: dict) -> str: - """Write topology YAML file.""" - yaml_path = os.path.join(self.output_dir, f"{self.config.name}.clab.yaml") - - # Add header comment - header = f"""# {self.config.name.upper()} Topology Configuration -# Generated by NetOpsBench Topology Generator -# Scale: {self.config.num_spines} spines, {self.config.num_leafs} leafs, {self.config.clients_per_leaf} clients/leaf -# -# Features: -# - BGP with ECMP -# - Observability: Syslog, gNMI, sFlow - -""" - with open(yaml_path, "w") as f: - f.write(header) - yaml.dump(topology, f, default_flow_style=False, sort_keys=False) - - return yaml_path - - def _generate_device_configs(self) -> list[str]: - """Generate device configuration files.""" - config_paths = [] - - # Generate spine configs - for i in range(1, self.config.num_spines + 1): - config_path = self._generate_spine_config(i) - config_paths.append(config_path) - - # Generate leaf configs - for i in range(1, self.config.num_leafs + 1): - config_path = self._generate_leaf_config(i) - config_paths.append(config_path) - - return config_paths - - def _sonic_port_name(self, eth_index: int) -> str: - """Map Linux eth index (eth1, eth2) to SONiC port name (Ethernet0, Ethernet4, ...).""" - return f"Ethernet{(eth_index - 1) * 4}" - - def _client_subnet_octet(self, leaf_idx: int) -> int: - """Return client subnet /24 octet for a leaf, avoiding spine-link collisions.""" - return 100 + leaf_idx - - def _telemetry_config_lines(self) -> list[str]: - return [ - f"config syslog add {self.syslog_collector} || true", - "sonic-db-cli CONFIG_DB hset 'FLEX_COUNTER_TABLE|PORT' FLEX_COUNTER_STATUS enable || true", - "sonic-db-cli CONFIG_DB hset 'FLEX_COUNTER_TABLE|PORT' POLL_INTERVAL 1000 || true", - "sonic-db-cli CONFIG_DB hmset 'GNMI|gnmi' port 50051 client_auth false log_level 2 || true", - ( - "sonic-db-cli CONFIG_DB hmset 'GNMI|certs' " - "ca_crt /etc/sonic/telemetry/dsmsroot.cer " - "server_crt /etc/sonic/telemetry/streamingtelemetryserver.cer " - "server_key /etc/sonic/telemetry/streamingtelemetryserver.key || true" - ), - "mkdir -p /var/run/telemetry /var/log/telemetry || true", - ( - "pgrep -x telemetry >/dev/null 2>&1 || " - "nohup /usr/sbin/telemetry -port 50051 -noTLS -client_auth none " - ">/var/log/telemetry/telemetry.log 2>&1 &" - ), - "config sflow agent-id add mgmt0 || true", - f"config sflow collector add telegraf {self.sflow_collector} --port {self.sflow_port} || true", - f"config sflow polling-interval {self.sflow_polling_interval} || true", - f"config sflow sample-direction {self.sflow_sample_direction} || true", - "config sflow enable || true", - ] - - def _generate_spine_config(self, spine_idx: int) -> str: - """Generate configuration for a spine switch.""" - spine_name = f"spine{spine_idx}" - router_id = f"10.0.0.{spine_idx}" - - script = [ - f"# Spine {spine_idx} Configuration", - "# Generated by NetOpsBench", - "set -e", - "", - ] - - vtysh_cmds = [ - "configure terminal", - "route-map RM-ALLOW permit 10", - "exit", - f"router bgp {self.config.spine_asn}", - f"bgp router-id {router_id}", - "no bgp ebgp-requires-policy", - ] - - # Add interfaces and BGP neighbors for each leaf - for leaf_idx in range(1, self.config.num_leafs + 1): - interface = self._sonic_port_name(leaf_idx) - subnet_idx = spine_idx * 10 + leaf_idx - spine_ip = f"192.168.{subnet_idx}.1" - leaf_ip = f"192.168.{subnet_idx}.2" - leaf_asn = self.config.leaf_asn_start + leaf_idx - 1 - - script.append(f"config interface startup {interface}") - script.append(f"config interface ip add {interface} {spine_ip}/30") - script.append(f"config sflow interface enable {interface} || true") - script.append(f"config sflow interface sample-rate {interface} {self.sflow_sample_rate} || true") - vtysh_cmds.append(f"neighbor {leaf_ip} remote-as {leaf_asn}") - - # Telemetry services (best-effort) - script.extend(self._telemetry_config_lines()) - - vtysh_cmds.extend( - [ - "address-family ipv4 unicast", - ] - ) - for leaf_idx in range(1, self.config.num_leafs + 1): - subnet_idx = spine_idx * 10 + leaf_idx - leaf_ip = f"192.168.{subnet_idx}.2" - vtysh_cmds.append(f"neighbor {leaf_ip} activate") - vtysh_cmds.append(f"neighbor {leaf_ip} route-map RM-ALLOW out") - vtysh_cmds.extend( - [ - "exit-address-family", - "end", - "write memory", - ] - ) - - script.append("supervisorctl start bgpd >/dev/null 2>&1 || true") - script.append("") - script.append("vtysh <<'VTY'") - script.extend(vtysh_cmds) - script.append("VTY") - - # Write config - config_path = os.path.join(self.output_dir, "configs", f"{spine_name}.sh") - with open(config_path, "w") as f: - f.write("\n".join(script) + "\n") - - return config_path - - def _generate_leaf_config(self, leaf_idx: int) -> str: - """Generate configuration for a leaf switch.""" - leaf_name = f"leaf{leaf_idx}" - router_id = f"10.0.0.{10 + leaf_idx}" - leaf_asn = self.config.leaf_asn_start + leaf_idx - 1 - - script = [ - f"# Leaf {leaf_idx} Configuration", - "# Generated by NetOpsBench", - "set -e", - "", - ] - - vtysh_cmds = [ - "configure terminal", - "route-map RM-ALLOW permit 10", - "exit", - f"router bgp {leaf_asn}", - f"bgp router-id {router_id}", - "no bgp ebgp-requires-policy", - ] - - # Add interfaces to spines - for spine_idx in range(1, self.config.num_spines + 1): - interface = self._sonic_port_name(spine_idx) - subnet_idx = spine_idx * 10 + leaf_idx - spine_ip = f"192.168.{subnet_idx}.1" - leaf_ip = f"192.168.{subnet_idx}.2" - - script.append(f"config interface startup {interface}") - script.append(f"config interface ip add {interface} {leaf_ip}/30") - script.append(f"config sflow interface enable {interface} || true") - script.append(f"config sflow interface sample-rate {interface} {self.sflow_sample_rate} || true") - vtysh_cmds.append(f"neighbor {spine_ip} remote-as {self.config.spine_asn}") - - # Add client-facing interfaces (one per client) - # Use point-to-point /30 subnets for each client to avoid subnet overlap - # Each client gets a dedicated /30 subnet from the leaf's /24 address space - for client_idx in range(1, self.config.clients_per_leaf + 1): - client_interface_idx = self.config.num_spines + client_idx - # Calculate /30 subnet: 192.168.{octet}.{(client_idx-1)*4}/30 - # Gateway IP is first usable IP in /30: 192.168.{octet}.{(client_idx-1)*4 + 1} - octet = self._client_subnet_octet(leaf_idx) - subnet_base = (client_idx - 1) * 4 - gateway_ip = subnet_base + 1 - interface = self._sonic_port_name(client_interface_idx) - script.append(f"config interface startup {interface}") - script.append(f"config interface ip add {interface} 192.168.{octet}.{gateway_ip}/30") - script.append(f"config sflow interface enable {interface} || true") - script.append(f"config sflow interface sample-rate {interface} {self.sflow_sample_rate} || true") - - # Telemetry services (best-effort) - script.extend(self._telemetry_config_lines()) - - vtysh_cmds.extend( - [ - "address-family ipv4 unicast", - ] - ) - for spine_idx in range(1, self.config.num_spines + 1): - subnet_idx = spine_idx * 10 + leaf_idx - spine_ip = f"192.168.{subnet_idx}.1" - vtysh_cmds.append(f"neighbor {spine_ip} activate") - vtysh_cmds.append(f"neighbor {spine_ip} route-map RM-ALLOW out") - octet = self._client_subnet_octet(leaf_idx) - for client_idx in range(1, self.config.clients_per_leaf + 1): - subnet_base = (client_idx - 1) * 4 - vtysh_cmds.append(f"network 192.168.{octet}.{subnet_base}/30 route-map RM-ALLOW") - vtysh_cmds.extend( - [ - "exit-address-family", - "end", - "write memory", - ] - ) - - script.append("supervisorctl start bgpd >/dev/null 2>&1 || true") - script.append("") - script.append("vtysh <<'VTY'") - script.extend(vtysh_cmds) - script.append("VTY") - - # Write config - config_path = os.path.join(self.output_dir, "configs", f"{leaf_name}.sh") - with open(config_path, "w") as f: - f.write("\n".join(script) + "\n") - - return config_path - - def _generate_metadata(self) -> dict: - """Generate topology metadata JSON.""" - devices = {"spines": [], "leafs": [], "clients": []} - - # Spines - for i in range(1, self.config.num_spines + 1): - devices["spines"].append( - { - "name": f"spine{i}", - "mgmt_ip": self._mgmt_host_ip(self.spine_mgmt_offset + i), - "router_id": f"10.0.0.{i}", - "asn": self.config.spine_asn, - } - ) - - # Leafs - for i in range(1, self.config.num_leafs + 1): - devices["leafs"].append( - { - "name": f"leaf{i}", - "mgmt_ip": self._mgmt_host_ip(self.leaf_mgmt_offset + i), - "router_id": f"10.0.0.{10 + i}", - "asn": self.config.leaf_asn_start + i - 1, - "client_subnet": f"192.168.{self._client_subnet_octet(i)}.0/24", - } - ) - - # Clients - for i in range(1, self.config.num_leafs + 1): - for j in range(1, self.config.clients_per_leaf + 1): - client_idx = (i - 1) * self.config.clients_per_leaf + j - # Point-to-point /30 subnet addressing - octet = self._client_subnet_octet(i) - subnet_base = (j - 1) * 4 - client_ip = f"192.168.{octet}.{subnet_base + 2}" - devices["clients"].append( - { - "name": f"client{client_idx}", - "mgmt_ip": self._mgmt_host_ip(self.client_mgmt_offset + client_idx), - "data_ip": client_ip, - "leaf": f"leaf{i}", - "rack": f"rack{i}", - } - ) - - # Links - links = [] - for spine_idx in range(1, self.config.num_spines + 1): - for leaf_idx in range(1, self.config.num_leafs + 1): - links.append({"endpoints": [f"spine{spine_idx}", f"leaf{leaf_idx}"], "type": "spine-leaf"}) - - for i in range(1, self.config.num_leafs + 1): - for j in range(1, self.config.clients_per_leaf + 1): - client_idx = (i - 1) * self.config.clients_per_leaf + j - links.append({"endpoints": [f"leaf{i}", f"client{client_idx}"], "type": "leaf-client"}) - - return { - "name": self.config.name, - "management": { - "network": self.mgmt_network_name, - "ipv4_subnet": str(self.mgmt_network), - }, - "collector": { - "ipv4": self.syslog_collector, - "sflow_port": self.sflow_port, - }, - "defaults": { - "link_mtu": 9232, - "sonic_port_mtu": 9100, - }, - "mtu_semantics": { - "link_mtu_scope": "containerlab/client link MTU", - "sonic_port_mtu_scope": "SONiC front-panel interface MTU", - "note": "Do not compare link_mtu 9232 directly against healthy SONiC port MTU 9100 when diagnosing faults.", - }, - "scale": { - "num_spines": self.config.num_spines, - "num_leafs": self.config.num_leafs, - "clients_per_leaf": self.config.clients_per_leaf, - "total_clients": self.config.num_leafs * self.config.clients_per_leaf, - "total_devices": self.config.num_spines + self.config.num_leafs, - }, - "devices": devices, - "links": links, - "routing": { - "protocol": "BGP", - "spine_asn": self.config.spine_asn, - "leaf_asn_range": f"{self.config.leaf_asn_start}-{self.config.leaf_asn_start + self.config.num_leafs - 1}", - "ecmp": True, - "bfd": True, - }, - } +from netopsbench.models.profiles import get_scale_profile + +from .clos_builder import build_clos_plan +from .config import ( + DEFAULT_SONIC_VS_IMAGE, + SONIC_LANEMAP_PATH, + SONIC_PORT_CONFIG_PATH, + FatTreeConfig, + TopologyConfig, + config_for_scale, + default_output_dir, +) +from .fat_tree_builder import build_fat_tree_plan +from .renderer import render_fabric_plan + + +def _clos_config( + scale: str, + *, + name: str | None, + mgmt_subnet: str | None, + mgmt_network: str | None, + collector_ip: str | None, +) -> TopologyConfig: + config = config_for_scale(scale) + if not isinstance(config, TopologyConfig): + raise ValueError(f"Scale {scale!r} is not a CLOS topology") + config.name = name or config.name + config.mgmt_ipv4_subnet = mgmt_subnet or config.mgmt_ipv4_subnet + config.mgmt_network_name = mgmt_network or config.mgmt_network_name + config.collector_ip = collector_ip or config.collector_ip + config.scale_name = scale + return config + + +def _fat_tree_config( + scale: str, + *, + name: str | None, + mgmt_subnet: str | None, + mgmt_network: str | None, + collector_ip: str | None, +) -> FatTreeConfig: + config = config_for_scale(scale) + if not isinstance(config, FatTreeConfig): + raise ValueError(f"Scale {scale!r} is not a fat-tree topology") + config.name = name or config.name + config.mgmt_ipv4_subnet = mgmt_subnet or config.mgmt_ipv4_subnet + config.mgmt_network_name = mgmt_network or config.mgmt_network_name + config.collector_ip = collector_ip or config.collector_ip + config.scale_name = scale + return config def generate_topology( @@ -557,33 +64,35 @@ def generate_topology( mgmt_network: str | None = None, collector_ip: str | None = None, ) -> dict: - """ - Convenience function to generate a topology of specified scale. - - Args: - scale: Topology scale ('xs', 'small', 'medium', 'large') - output_dir: Output directory path + """Generate one supported CLOS or fat-tree scale.""" + profile = get_scale_profile(scale) + if profile.family == "clos": + plan = build_clos_plan( + _clos_config( + scale, + name=name, + mgmt_subnet=mgmt_subnet, + mgmt_network=mgmt_network, + collector_ip=collector_ip, + ) + ) + else: + plan = build_fat_tree_plan( + _fat_tree_config( + scale, + name=name, + mgmt_subnet=mgmt_subnet, + mgmt_network=mgmt_network, + collector_ip=collector_ip, + ) + ) + return render_fabric_plan(plan, output_dir or default_output_dir()) - Returns: - Dictionary with generated file paths and metadata - """ - if scale not in TOPOLOGY_SCALES: - raise ValueError(f"Unknown scale: {scale}. Available: {list(TOPOLOGY_SCALES.keys())}") - base_config = TOPOLOGY_SCALES[scale] - config = TopologyConfig( - name=name or base_config.name, - num_spines=base_config.num_spines, - num_leafs=base_config.num_leafs, - clients_per_leaf=base_config.clients_per_leaf, - nos_kind=base_config.nos_kind, - nos_image=base_config.nos_image, - client_image=base_config.client_image, - mgmt_ipv4_subnet=mgmt_subnet or base_config.mgmt_ipv4_subnet, - mgmt_network_name=mgmt_network or base_config.mgmt_network_name, - collector_ip=collector_ip or base_config.collector_ip, - spine_asn=base_config.spine_asn, - leaf_asn_start=base_config.leaf_asn_start, - ) - generator = TopologyGenerator(config, output_dir) - return generator.generate() +__all__ = [ + "DEFAULT_SONIC_VS_IMAGE", + "SONIC_LANEMAP_PATH", + "SONIC_PORT_CONFIG_PATH", + "TopologyConfig", + "generate_topology", +] diff --git a/netopsbench/platform/topology/metadata_generator.py b/netopsbench/platform/topology/metadata_generator.py deleted file mode 100644 index d06e83c..0000000 --- a/netopsbench/platform/topology/metadata_generator.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Topology Metadata Generator - Parse existing Containerlab YAML and generate metadata. - -This utility creates topology.json metadata files from existing Containerlab YAML files. -Useful for pre-existing topologies that don't have metadata. -""" - -import json -from pathlib import Path - -import yaml - -from netopsbench.logging_utils import get_logger - -logger = get_logger(__name__) - - -def parse_clab_yaml(yaml_path: str) -> dict: - """ - Parse a Containerlab YAML file and extract topology metadata. - - Args: - yaml_path: Path to the .clab.yaml file - - Returns: - Dictionary containing topology metadata - """ - yaml_file = Path(yaml_path) - if not yaml_file.exists(): - raise FileNotFoundError(f"YAML file not found: {yaml_path}") - - with open(yaml_file) as f: - clab_config = yaml.safe_load(f) - - topology = clab_config.get("topology", {}) - nodes = topology.get("nodes", {}) - links = topology.get("links", []) - - # Extract devices by type - devices = {"spines": [], "leafs": [], "clients": []} - - # Parse nodes - for node_name, node_config in nodes.items(): - group = node_config.get("group", "") - mgmt_ip = node_config.get("mgmt-ipv4", "") - - if group == "spine": - # Extract spine info from configs or use defaults - devices["spines"].append( - { - "name": node_name, - "mgmt_ip": mgmt_ip, - "router_id": f"10.0.0.{len(devices['spines']) + 1}", - "asn": 65001, # Default spine ASN - } - ) - elif group == "leaf": - leaf_idx = len(devices["leafs"]) + 1 - devices["leafs"].append( - { - "name": node_name, - "mgmt_ip": mgmt_ip, - "router_id": f"10.0.0.{10 + leaf_idx}", - "asn": 65010 + leaf_idx, - "client_subnet": f"192.168.{leaf_idx}.0/24", - } - ) - elif group == "client": - # Parse client IP from exec commands - data_ip = None - leaf_name = None - - exec_commands = node_config.get("exec", []) - for cmd in exec_commands: - if "ip addr add" in cmd: - # Extract IP from command like "ip addr add 192.168.1.2/24 dev eth1" - parts = cmd.split() - if "add" in parts: - ip_idx = parts.index("add") + 1 - if ip_idx < len(parts): - data_ip = parts[ip_idx].split("/")[0] - - # Determine which leaf this client connects to from links - for link in links: - endpoints = link.get("endpoints", []) - for endpoint in endpoints: - if node_name in endpoint: - # Find the other endpoint (should be a leaf) - other_endpoint = [e for e in endpoints if node_name not in e][0] - leaf_name = other_endpoint.split(":")[0] - break - - # Determine rack from leaf - rack = None - if leaf_name: - # Extract leaf number - leaf_num = "".join(filter(str.isdigit, leaf_name)) - rack = f"rack{leaf_num}" - - devices["clients"].append( - { - "name": node_name, - "mgmt_ip": mgmt_ip, - "data_ip": data_ip or "unknown", - "leaf": leaf_name or "unknown", - "rack": rack or "unknown", - } - ) - - # Parse links - parsed_links = [] - for link in links: - endpoints = link.get("endpoints", []) - if len(endpoints) >= 2: - endpoint_names = [e.split(":")[0] for e in endpoints] - - # Determine link type - link_type = "unknown" - if any("spine" in e for e in endpoint_names) and any("leaf" in e for e in endpoint_names): - link_type = "spine-leaf" - elif any("leaf" in e for e in endpoint_names) and any("client" in e for e in endpoint_names): - link_type = "leaf-client" - - parsed_links.append({"endpoints": endpoint_names, "type": link_type}) - - # Calculate scale - num_spines = len(devices["spines"]) - num_leafs = len(devices["leafs"]) - num_clients = len(devices["clients"]) - clients_per_leaf = num_clients // num_leafs if num_leafs > 0 else 0 - - # Generate metadata - metadata = { - "name": clab_config.get("name", "dcn"), - "scale": { - "num_spines": num_spines, - "num_leafs": num_leafs, - "clients_per_leaf": clients_per_leaf, - "total_clients": num_clients, - "total_devices": num_spines + num_leafs, - }, - "devices": devices, - "links": parsed_links, - "routing": { - "protocol": "BGP", - "spine_asn": 65001, - "leaf_asn_range": f"65011-{65010 + num_leafs}", - "ecmp": True, - "bfd": True, - }, - } - - return metadata - - -def generate_metadata_file(yaml_path: str, output_path: str = None) -> str: - """ - Generate topology.json metadata file from Containerlab YAML. - - Args: - yaml_path: Path to the .clab.yaml file - output_path: Optional output path for topology.json - (defaults to same directory as YAML) - - Returns: - Path to the generated topology.json file - """ - yaml_file = Path(yaml_path) - - if output_path is None: - output_path = yaml_file.parent / "topology.json" - else: - output_path = Path(output_path) - - # Parse YAML and generate metadata - metadata = parse_clab_yaml(yaml_path) - - # Write metadata file - with open(output_path, "w") as f: - json.dump(metadata, f, indent=2) - - logger.info("Generated topology metadata: %s", output_path) - logger.info(" - Spines: %s", metadata["scale"]["num_spines"]) - logger.info(" - Leafs: %s", metadata["scale"]["num_leafs"]) - logger.info(" - Clients: %s", metadata["scale"]["total_clients"]) - - return str(output_path) diff --git a/netopsbench/platform/topology/plan.py b/netopsbench/platform/topology/plan.py new file mode 100644 index 0000000..979c6f6 --- /dev/null +++ b/netopsbench/platform/topology/plan.py @@ -0,0 +1,72 @@ +"""Topology-family-neutral fabric planning models.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from netopsbench.models.topology import Device, DeviceRole, TopologyManifest + + +@dataclass(frozen=True, slots=True) +class BGPNeighborPlan: + peer_ip: str + remote_as: int + + +@dataclass(frozen=True, slots=True) +class RenderSettings: + syslog_collector: str + + +@dataclass(frozen=True, slots=True) +class DevicePlan: + """Canonical identity and every input needed to render one node.""" + + device: Device + required_ports: int = 0 + configdb_interface_cidrs: dict[str, tuple[str, ...]] = field(default_factory=dict) + bgp_asn: int | None = None + bgp_router_id: str | None = None + bgp_neighbors: tuple[BGPNeighborPlan, ...] = () + bgp_networks: tuple[str, ...] = () + client_commands: tuple[str, ...] = () + + @property + def name(self) -> str: + return self.device.name + + @property + def is_client(self) -> bool: + return self.device.role is DeviceRole.CLIENT + + +@dataclass(frozen=True, slots=True) +class FabricPlan: + """A complete, artifact-independent plan for one generated fabric.""" + + manifest: TopologyManifest + device_plans: tuple[DevicePlan, ...] + nos_kind: str + nos_image: str + client_image: str + render_settings: RenderSettings + yaml_header: str + + def __post_init__(self) -> None: + manifest_names = [device.name for device in self.manifest.devices] + plan_names = [device_plan.name for device_plan in self.device_plans] + if manifest_names != plan_names: + raise ValueError("FabricPlan device plans must match canonical manifest device order") + for device_plan in self.device_plans: + if device_plan.is_client: + continue + if device_plan.required_ports <= 0: + raise ValueError(f"switch {device_plan.name} must define required_ports") + if device_plan.bgp_asn is None or device_plan.bgp_router_id is None: + raise ValueError(f"switch {device_plan.name} must define BGP identity") + + def device_plan(self, name: str) -> DevicePlan | None: + return next((device_plan for device_plan in self.device_plans if device_plan.name == name), None) + + +__all__ = ["BGPNeighborPlan", "DevicePlan", "FabricPlan", "RenderSettings"] diff --git a/netopsbench/platform/topology/planning.py b/netopsbench/platform/topology/planning.py new file mode 100644 index 0000000..2742f8a --- /dev/null +++ b/netopsbench/platform/topology/planning.py @@ -0,0 +1,109 @@ +"""Shared address allocation and render-setting helpers for fabric builders.""" + +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass + +from netopsbench.models.profiles import get_scale_profile +from netopsbench.models.topology import PingmeshPolicy + +from .plan import RenderSettings + + +@dataclass(frozen=True, slots=True) +class ManagementAddressing: + network: ipaddress.IPv4Network + allocated_offsets: frozenset[int] + + @classmethod + def create(cls, subnet: str, allocated_offsets: set[int]) -> ManagementAddressing: + network = ipaddress.ip_network(subnet, strict=False) + if not isinstance(network, ipaddress.IPv4Network): + raise ValueError(f"Only IPv4 management subnets are supported: {subnet}") + addressing = cls(network=network, allocated_offsets=frozenset(allocated_offsets)) + for offset in allocated_offsets: + addressing.host_ip(offset) + return addressing + + @property + def allocated_ips(self) -> set[ipaddress.IPv4Address]: + return {ipaddress.ip_address(int(self.network.network_address) + offset) for offset in self.allocated_offsets} + + def host_ip(self, offset: int) -> str: + if offset <= 0: + raise ValueError(f"Management host offset must be positive, got {offset}") + candidate = ipaddress.ip_address(int(self.network.network_address) + offset) + if candidate not in self.network: + raise ValueError(f"Management host offset {offset} falls outside subnet {self.network}") + if candidate in {self.network.network_address, self.network.broadcast_address}: + raise ValueError(f"Management host offset {offset} resolves to reserved address {candidate}") + return str(candidate) + + def validate_collector(self, collector_ip: str) -> str: + candidate = ipaddress.ip_address(collector_ip) + if candidate not in self.network: + raise ValueError(f"Collector IP {collector_ip} falls outside management subnet {self.network}") + if candidate in {self.network.network_address, self.network.broadcast_address}: + raise ValueError(f"Collector IP {collector_ip} is reserved in management subnet {self.network}") + if candidate in self.allocated_ips: + raise ValueError(f"Collector IP {collector_ip} overlaps a generated device management address") + return str(candidate) + + def default_collector(self, last_device_offset: int, prefer_offset_200: bool) -> str: + if prefer_offset_200 and last_device_offset < 200: + preferred = ipaddress.ip_address(int(self.network.network_address) + 200) + if preferred in self.network and preferred not in { + self.network.network_address, + self.network.broadcast_address, + }: + return self.validate_collector(str(preferred)) + + fallback = ipaddress.ip_address(int(self.network.network_address) + last_device_offset + 1) + if fallback not in self.network or fallback in { + self.network.network_address, + self.network.broadcast_address, + }: + fallback = ipaddress.ip_address(int(self.network.broadcast_address) - 1) + return self.validate_collector(str(fallback)) + + +def build_render_settings( + addressing: ManagementAddressing, + collector_ip: str, +) -> RenderSettings: + return RenderSettings(syslog_collector=addressing.validate_collector(collector_ip)) + + +def sonic_port_name(eth_index: int) -> str: + return f"Ethernet{(eth_index - 1) * 4}" + + +def client_commands(client_ip: str, gateway: str) -> tuple[str, ...]: + return ( + "ip link set dev eth1 mtu 9232", + f"ip addr add {client_ip}/30 dev eth1", + f"ip route add 192.168.0.0/16 via {gateway}", + "mkdir -p /var/log/pingmesh", + "iperf3 -s -D", + "ethtool -K eth1 rx off tx off tso off gso off gro off sg off tx-udp-segmentation off", + ) + + +def pingmesh_policy_for_scale(scale: str) -> PingmeshPolicy: + profile = get_scale_profile(scale) + return PingmeshPolicy( + destination_batch_size=profile.pingmesh_destination_batch_size, + rtt_port_pool_size=profile.pingmesh_rtt_port_pool_size, + rtt_ports_per_cycle=profile.pingmesh_rtt_ports_per_cycle, + cycle_interval_seconds=profile.pingmesh_cycle_interval_seconds, + ) + + +__all__ = [ + "ManagementAddressing", + "build_render_settings", + "client_commands", + "pingmesh_policy_for_scale", + "sonic_port_name", +] diff --git a/netopsbench/platform/topology/renderer.py b/netopsbench/platform/topology/renderer.py new file mode 100644 index 0000000..dc735ed --- /dev/null +++ b/netopsbench/platform/topology/renderer.py @@ -0,0 +1,242 @@ +"""Shared renderer for CLOS and fat-tree fabric plans.""" + +from __future__ import annotations + +import hashlib +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + +import yaml + +from .config import ( + SONIC_BASE_CONFIG_DB, + SONIC_HWSKU, + SONIC_LANEMAP_PATH, + SONIC_PLATFORM, + SONIC_PORT_CONFIG_PATH, + SONIC_START_WRAPPER_SOURCE, +) +from .plan import DevicePlan, FabricPlan + + +def _sonic_port_attrs(port_idx: int) -> dict[str, str]: + eth = port_idx * 4 + first_lane = eth + 1 + lanes = ",".join(str(first_lane + offset) for offset in range(4)) + return { + "alias": f"fortyGigE0/{eth}", + "index": str(port_idx), + "lanes": lanes, + "speed": "100000", + "subport": "0", + "admin_status": "up", + } + + +def _port_entries(required_ports: int) -> dict[str, dict[str, str]]: + return {f"Ethernet{port_idx * 4}": _sonic_port_attrs(port_idx) for port_idx in range(required_ports)} + + +def _port_config_ini(required_ports: int) -> str: + lines = ["# name lanes alias index speed"] + for port_idx in range(required_ports): + name = f"Ethernet{port_idx * 4}" + attrs = _sonic_port_attrs(port_idx) + lines.append(f"{name:<15} {attrs['lanes']:<17} {attrs['alias']:<17} {attrs['index']:<11} {attrs['speed']}") + return "\n".join(lines) + "\n" + + +def _lanemap_ini(required_ports: int) -> str: + lines = [] + for port_idx in range(required_ports): + attrs = _sonic_port_attrs(port_idx) + lines.append(f"eth{port_idx + 1}:{attrs['lanes']}") + return "\n".join(lines) + "\n" + + +def _device_mac(device: str) -> str: + digest = hashlib.sha256(device.encode("utf-8")).digest() + return "02:" + ":".join(f"{byte:02x}" for byte in digest[:5]) + + +def _load_base_config_db() -> dict[str, Any]: + with SONIC_BASE_CONFIG_DB.open(encoding="utf-8") as handle: + return deepcopy(json.load(handle)) + + +def _interface_sort_key(name: str) -> int: + return int(name[8:]) if name.startswith("Ethernet") else 0 + + +def _build_config_db(plan: FabricPlan, device_plan: DevicePlan) -> dict[str, Any]: + config_db = _load_base_config_db() + metadata = config_db.setdefault("DEVICE_METADATA", {}).setdefault("localhost", {}) + metadata["hostname"] = device_plan.name + metadata["hwsku"] = SONIC_HWSKU + metadata["platform"] = SONIC_PLATFORM + metadata["mac"] = _device_mac(device_plan.name) + + ports = _port_entries(device_plan.required_ports) + config_db["PORT"] = ports + config_db["CABLE_LENGTH"] = {"AZURE": {name: "0m" for name in ports}} + config_db["BREAKOUT_CFG"] = {name: {"brkout_mode": "1x100G[40G]"} for name in ports} + config_db["SYSLOG_SERVER"] = {"telegraf": {"server": plan.render_settings.syslog_collector}} + + interface_table: dict[str, dict[str, str]] = {} + for interface_name in sorted(device_plan.configdb_interface_cidrs, key=_interface_sort_key): + interface_table[interface_name] = {} + for cidr in device_plan.configdb_interface_cidrs[interface_name]: + interface_table[f"{interface_name}|{cidr}"] = {} + config_db["INTERFACE"] = interface_table + + return config_db + + +def _frr_config(device_plan: DevicePlan) -> str: + lines = [ + "frr version 10.3", + "frr defaults traditional", + f"hostname {device_plan.name}", + "log syslog informational", + "no ipv6 forwarding", + "service integrated-vtysh-config", + "!", + "route-map RM-ALLOW permit 10", + "!", + f"router bgp {device_plan.bgp_asn}", + f" bgp router-id {device_plan.bgp_router_id}", + " no bgp ebgp-requires-policy", + " bgp bestpath as-path multipath-relax", + ] + for neighbor in device_plan.bgp_neighbors: + lines.append(f" neighbor {neighbor.peer_ip} remote-as {neighbor.remote_as}") + lines.extend([" !", " address-family ipv4 unicast"]) + lines.append(" maximum-paths 64") + for neighbor in device_plan.bgp_neighbors: + lines.append(f" neighbor {neighbor.peer_ip} activate") + lines.append(f" neighbor {neighbor.peer_ip} route-map RM-ALLOW out") + for prefix in device_plan.bgp_networks: + lines.append(f" network {prefix} route-map RM-ALLOW") + lines.extend([" exit-address-family", "!", "line vty", "!"]) + return "\n".join(lines) + "\n" + + +def _containerlab_topology(plan: FabricPlan) -> dict[str, Any]: + topology: dict[str, Any] = { + "name": plan.manifest.name, + "mgmt": { + "network": plan.manifest.management.network, + "ipv4-subnet": plan.manifest.management.ipv4_subnet, + }, + "topology": { + "kinds": { + plan.nos_kind: { + "image": plan.nos_image, + "binds": [ + "configs/sonic/__clabNodeName__/config_db.json:/etc/sonic/config_db.json:rw", + f"configs/sonic/__clabNodeName__/port_config.ini:{SONIC_PORT_CONFIG_PATH}:rw", + f"configs/sonic/__clabNodeName__/lanemap.ini:{SONIC_LANEMAP_PATH}:rw", + "configs/sonic/start.sh:/usr/bin/start.sh:ro", + "configs/frr/__clabNodeName__.conf:/etc/frr/frr.conf:rw", + ], + }, + "linux": { + "image": plan.client_image, + "binds": ["configs/pingmesh:/tmp/pingmesh:ro"], + }, + }, + "nodes": {}, + "links": [], + }, + } + nodes = topology["topology"]["nodes"] + for device_plan in plan.device_plans: + device = device_plan.device + if device_plan.is_client: + nodes[device.name] = { + "kind": "linux", + "group": "client", + "mgmt-ipv4": device.mgmt_ip, + "exec": list(device_plan.client_commands), + } + else: + nodes[device.name] = { + "kind": plan.nos_kind, + "group": device.role.value, + "mgmt-ipv4": device.mgmt_ip, + } + topology["topology"]["links"] = [ + { + "endpoints": [f"{endpoint.device}:{endpoint.interface}" for endpoint in link.endpoints], + "mtu": link.mtu, + } + for link in plan.manifest.links + ] + return topology + + +def render_fabric_plan(plan: FabricPlan, output_dir: str | Path) -> dict[str, Any]: + """Write every topology artifact from one canonical fabric plan.""" + root = Path(output_dir) + sonic_root = root / "configs" / "sonic" + frr_root = root / "configs" / "frr" + pingmesh_root = root / "configs" / "pingmesh" + sonic_root.mkdir(parents=True, exist_ok=True) + frr_root.mkdir(parents=True, exist_ok=True) + pingmesh_root.mkdir(parents=True, exist_ok=True) + + sonic_start_wrapper = sonic_root / "start.sh" + if not SONIC_START_WRAPPER_SOURCE.is_file(): + raise FileNotFoundError(f"SONiC startup wrapper source not found: {SONIC_START_WRAPPER_SOURCE}") + sonic_start_wrapper.write_bytes(SONIC_START_WRAPPER_SOURCE.read_bytes()) + sonic_start_wrapper.chmod(0o755) + + yaml_path = root / f"{plan.manifest.name}.clab.yaml" + with yaml_path.open("w", encoding="utf-8") as handle: + handle.write(plan.yaml_header) + yaml.dump(_containerlab_topology(plan), handle, default_flow_style=False, sort_keys=False) + + config_paths: list[str] = [] + frr_paths: list[str] = [] + for device_plan in plan.device_plans: + if device_plan.is_client: + continue + sonic_dir = sonic_root / device_plan.name + sonic_dir.mkdir(parents=True, exist_ok=True) + config_db_path = sonic_dir / "config_db.json" + config_db_path.write_text( + json.dumps(_build_config_db(plan, device_plan), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (sonic_dir / "port_config.ini").write_text( + _port_config_ini(device_plan.required_ports), + encoding="utf-8", + ) + (sonic_dir / "lanemap.ini").write_text(_lanemap_ini(device_plan.required_ports), encoding="utf-8") + frr_path = frr_root / f"{device_plan.name}.conf" + frr_path.write_text(_frr_config(device_plan), encoding="utf-8") + config_paths.append(str(config_db_path)) + frr_paths.append(str(frr_path)) + + metadata_path = root / "topology.json" + metadata_path.write_text( + json.dumps(plan.manifest.model_dump(mode="json"), indent=2) + "\n", + encoding="utf-8", + ) + return { + "yaml_file": str(yaml_path), + "metadata_file": str(metadata_path), + "config_files": config_paths, + "startup_config_files": config_paths, + "sonic_start_wrapper_file": str(sonic_start_wrapper), + "frr_config_files": frr_paths, + "metadata": plan.manifest.model_dump(mode="json"), + "agent_topology": plan.manifest.to_agent_topology(), + "manifest": plan.manifest, + "plan": plan, + } + + +__all__ = ["render_fabric_plan"] diff --git a/netopsbench/platform/topology/sonic_start.sh b/netopsbench/platform/topology/sonic_start.sh new file mode 100755 index 0000000..c5a1a98 --- /dev/null +++ b/netopsbench/platform/topology/sonic_start.sh @@ -0,0 +1,253 @@ +#!/bin/bash -e + +# NetOpsBench wrapper for yyyyyt123/netopsbench-sonic-vs-202505-telemetry:202505-telemetry. +# Original /usr/bin/start.sh sha256: +NETOPSBENCH_ORIGINAL_SONIC_START_SHA256=8c5aa959f0a3ed0bf1a57f7ecfd004485d5600b9ab71b388c2b15e109b77ee12 + +wait_for_front_panel_links() { + local hwsku_dir="/usr/share/sonic/device/$PLATFORM/$HWSKU" + local lanemap_file="$hwsku_dir/lanemap.ini" + local timeout=600 + local interval=1 + local expected actual started now + + if [ ! -r "$lanemap_file" ]; then + echo "NetOpsBench SONiC startup: missing readable lanemap.ini: $lanemap_file" >&2 + return 1 + fi + + expected=$(grep -Ec '^eth[0-9]+:' "$lanemap_file" || true) + if [ "$expected" -le 0 ]; then + echo "NetOpsBench SONiC startup: no front-panel ports found in $lanemap_file" >&2 + return 1 + fi + + started=$(date +%s) + while true; do + actual=$(front_panel_links | wc -l) + if [ "$actual" -ge "$expected" ]; then + echo "NetOpsBench SONiC startup: detected $actual/$expected front-panel links" + return 0 + fi + + now=$(date +%s) + if [ $((now - started)) -ge "$timeout" ]; then + echo "NetOpsBench SONiC startup: timed out waiting for front-panel links ($actual/$expected)" >&2 + return 1 + fi + sleep "$interval" + done +} + +front_panel_links() { + local path name + + for path in /sys/class/net/eth[0-9]*; do + [ -e "$path" ] || continue + name=${path##*/} + [ "$name" = "eth0" ] || printf '%s\n' "$name" + done +} + +install_generated_config_db() { + local src="$1" + local dst="$2" + + if mv "$src" "$dst"; then + return 0 + fi + + # /etc/sonic/config_db.json is a NetOpsBench bind mount. Linux will not + # replace a mounted file with rename(2), so preserve the mount and update + # the contents in place. + cat "$src" > "$dst" + rm -f "$src" +} + +# Generate configuration + +# NOTE: 'PLATFORM' and 'HWSKU' environment variables are set +# in the Dockerfile so that they persist for the life of the container + +ln -sf /usr/share/sonic/device/$PLATFORM /usr/share/sonic/platform +ln -sf /usr/share/sonic/device/$PLATFORM/$HWSKU /usr/share/sonic/hwsku + +SWITCH_TYPE=switch +PLATFORM_CONF=platform.json +if [[ $HWSKU == "DPU-2P" ]]; then + SWITCH_TYPE=dpu + PLATFORM_CONF=platform-dpu-2p.json +fi + +wait_for_front_panel_links + +pushd /usr/share/sonic/hwsku + +# filter available front panel ports in lanemap.ini +[ -f lanemap.ini.orig ] || cp lanemap.ini lanemap.ini.orig +for p in $(front_panel_links); do + grep ^$p: lanemap.ini.orig +done > lanemap.ini + +# filter available sonic front panel ports in port_config.ini +[ -f port_config.ini.orig ] || cp port_config.ini port_config.ini.orig +grep ^# port_config.ini.orig > port_config.ini +for lanes in $(awk -F ':' '{print $2}' lanemap.ini); do + grep -E "\s$lanes\s" port_config.ini.orig +done >> port_config.ini + +popd + +[ -d /etc/sonic ] || mkdir -p /etc/sonic + +# Note: libswsscommon requires a dabase_config file in /var/run/redis/sonic-db/ +# Prepare this file before any dependent application, such as sonic-cfggen +mkdir -p /var/run/redis/sonic-db +cp /etc/default/sonic-db/database_config.json /var/run/redis/sonic-db/ + +SYSTEM_MAC_ADDRESS=$(cat /sys/class/net/eth0/address) +sonic-cfggen -t /usr/share/sonic/templates/init_cfg.json.j2 -a "{\"system_mac\": \"$SYSTEM_MAC_ADDRESS\", \"switch_type\": \"$SWITCH_TYPE\"}" > /etc/sonic/init_cfg.json + +if [[ -f /usr/share/sonic/virtual_chassis/default_config.json ]]; then + sonic-cfggen -j /etc/sonic/init_cfg.json -j /usr/share/sonic/virtual_chassis/default_config.json --print-data > /tmp/init_cfg.json + mv /tmp/init_cfg.json /etc/sonic/init_cfg.json +fi + +if [ -f /etc/sonic/config_db.json ]; then + sonic-cfggen -j /etc/sonic/init_cfg.json -j /etc/sonic/config_db.json --print-data > /tmp/config_db.json + install_generated_config_db /tmp/config_db.json /etc/sonic/config_db.json +else + # generate and merge buffers configuration into config file + if [ -f /usr/share/sonic/hwsku/buffers.json.j2 ]; then + sonic-cfggen -k $HWSKU -p /usr/share/sonic/device/$PLATFORM/$PLATFORM_CONF -t /usr/share/sonic/hwsku/buffers.json.j2 > /tmp/buffers.json + buffers_cmd="-j /tmp/buffers.json" + fi + if [ -f /usr/share/sonic/hwsku/qos.json.j2 ]; then + sonic-cfggen -j /etc/sonic/init_cfg.json -t /usr/share/sonic/hwsku/qos.json.j2 > /tmp/qos.json + qos_cmd="-j /tmp/qos.json" + fi + + sonic-cfggen -p /usr/share/sonic/device/$PLATFORM/$PLATFORM_CONF -k $HWSKU --print-data > /tmp/ports.json + # change admin_status from up to down; Test cases dependent + sed -i "s/up/down/g" /tmp/ports.json + sonic-cfggen -j /etc/sonic/init_cfg.json $buffers_cmd $qos_cmd -j /tmp/ports.json --print-data > /etc/sonic/config_db.json +fi + +sonic-cfggen -t /usr/share/sonic/templates/copp_cfg.j2 > /etc/sonic/copp_cfg.json + +if [ "$HWSKU" == "Mellanox-SN2700" ]; then + cp /usr/share/sonic/hwsku/sai_mlnx.profile /usr/share/sonic/hwsku/sai.profile +elif [ "$HWSKU" == "DPU-2P" ]; then + cp /usr/share/sonic/hwsku/sai_dpu_2p.profile /usr/share/sonic/hwsku/sai.profile +fi + +mkdir -p /etc/swss/config.d/ + +rm -f /var/run/rsyslogd.pid + +supervisorctl start rsyslogd + +supervisord_cfg="/etc/supervisor/conf.d/supervisord.conf" +chassisdb_cfg_file="/usr/share/sonic/virtual_chassis/default_config.json" +chassisdb_cfg_file_default="/etc/default/sonic-db/default_chassis_cfg.json" +host_template="/usr/share/sonic/templates/hostname.j2" +db_cfg_file="/var/run/redis/sonic-db/database_config.json" +db_cfg_file_tmp="/var/run/redis/sonic-db/database_config.json.tmp" + +if [ -r "$chassisdb_cfg_file" ]; then + echo $(sonic-cfggen -j $chassisdb_cfg_file -t $host_template) >> /etc/hosts +else + chassisdb_cfg_file="$chassisdb_cfg_file_default" + echo "10.8.1.200 redis_chassis.server" >> /etc/hosts +fi + +supervisorctl start redis-server + +start_chassis_db=`sonic-cfggen -v DEVICE_METADATA.localhost.start_chassis_db -y $chassisdb_cfg_file` +if [[ "$HOSTNAME" == *"supervisor"* ]] || [ "$start_chassis_db" == "1" ]; then + supervisorctl start redis-chassis +fi + +conn_chassis_db=`sonic-cfggen -v DEVICE_METADATA.localhost.connect_to_chassis_db -y $chassisdb_cfg_file` +if [ "$start_chassis_db" != "1" ] && [ "$conn_chassis_db" != "1" ]; then + cp $db_cfg_file $db_cfg_file_tmp + update_chassisdb_config -j $db_cfg_file_tmp -d + cp $db_cfg_file_tmp $db_cfg_file +fi + +if [ "$conn_chassis_db" == "1" ]; then + if [ -f /usr/share/sonic/virtual_chassis/coreportindexmap.ini ]; then + cp /usr/share/sonic/virtual_chassis/coreportindexmap.ini /usr/share/sonic/hwsku/ + + pushd /usr/share/sonic/hwsku + + # filter available front panel ports in coreportindexmap.ini + [ -f coreportindexmap.ini.orig ] || cp coreportindexmap.ini coreportindexmap.ini.orig + for p in $(front_panel_links); do + grep ^$p: coreportindexmap.ini.orig + done > coreportindexmap.ini + + popd + fi +fi + +/usr/bin/configdb-load.sh + +if [ "$HWSKU" = "brcm_gearbox_vs" ]; then + supervisorctl start gbsyncd + supervisorctl start gearsyncd +fi + +supervisorctl start syncd + +supervisorctl start portsyncd + +supervisorctl start orchagent + +supervisorctl start coppmgrd + +supervisorctl start neighsyncd + +supervisorctl start fdbsyncd + +supervisorctl start teamsyncd + +supervisorctl start fpmsyncd + +supervisorctl start teammgrd + +supervisorctl start vrfmgrd + +supervisorctl start portmgrd + +supervisorctl start intfmgrd + +supervisorctl start vlanmgrd + +supervisorctl start zebra + +supervisorctl start mgmtd + +supervisorctl start staticd + +supervisorctl start buffermgrd + +supervisorctl start nbrmgrd + +supervisorctl start vxlanmgrd + +supervisorctl start natmgrd + +supervisorctl start natsyncd + +supervisorctl start tunnelmgrd + +supervisorctl start fabricmgrd + +supervisorctl start rebootbackend + +# Start arp_update when VLAN exists +VLAN=`sonic-cfggen -d -v 'VLAN.keys() | join(" ") if VLAN'` +if [ "$VLAN" != "" ]; then + supervisorctl start arp_update +fi diff --git a/netopsbench/platform/topology/sonic_vs_base_config_db.json b/netopsbench/platform/topology/sonic_vs_base_config_db.json new file mode 100644 index 0000000..f7e4426 --- /dev/null +++ b/netopsbench/platform/topology/sonic_vs_base_config_db.json @@ -0,0 +1,54 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "mac": "2a:69:12:f9:25:59", + "switch_type": "switch", + "buffer_model": "traditional", + "hwsku": "Force10-S6000" + } + }, + "FEATURE": { + "swss": { + "state": "enabled" + }, + "bgp": { + "state": "enabled" + }, + "teamd": { + "state": "enabled" + }, + "nat": { + "state": "enabled" + }, + "database": { + "state": "enabled" + }, + "lldp": { + "state": "enabled" + }, + "dhcp_relay": { + "state": "enabled" + }, + "macsec": { + "state": "enabled" + } + }, + "FLEX_COUNTER_TABLE": { + "PORT": { + "FLEX_COUNTER_STATUS": "enable", + "POLL_INTERVAL": "10000" + } + }, + "GNMI": { + "gnmi": { + "port": "50051", + "client_auth": "false", + "log_level": "2" + }, + "certs": { + "ca_crt": "/etc/sonic/telemetry/dsmsroot.cer", + "server_crt": "/etc/sonic/telemetry/streamingtelemetryserver.cer", + "server_key": "/etc/sonic/telemetry/streamingtelemetryserver.key" + } + } +} diff --git a/netopsbench/platform/topology/topology_utils.py b/netopsbench/platform/topology/topology_utils.py index 8007e61..9ea9d17 100644 --- a/netopsbench/platform/topology/topology_utils.py +++ b/netopsbench/platform/topology/topology_utils.py @@ -1,329 +1,68 @@ -"""Shared topology helpers used by runtime and tooling components.""" +"""Canonical topology loading and Containerlab naming helpers.""" from __future__ import annotations -import copy -import glob import json -import os -import re -from dataclasses import dataclass, field +from pathlib import Path from typing import Any -from netopsbench.config import config -from netopsbench.logging_utils import get_logger -from netopsbench.platform.utils.proc import safe_run, sudo_prefix +from pydantic import ValidationError -logger = get_logger(__name__) +from netopsbench.models.topology import SCHEMA_VERSION, TopologyManifest + +NETWORK_DEVICE_PREFIXES: tuple[str, ...] = ("spine", "leaf", "core", "agg", "edge") +TopologyInput = TopologyManifest | dict[str, Any] def clab_container_name(lab_name: str, device_name: str) -> str: - """Return the Containerlab container name for a device.""" return f"clab-{lab_name}-{device_name}" -@dataclass -class TopologyState: - """Normalized topology state shared by the injector and agent toolkit.""" - - topology_name: str = "dcn" - container_names: dict[str, str] = field(default_factory=dict) - topology_metadata: dict[str, Any] = field(default_factory=dict) - device_mgmt_ips: dict[str, str] = field(default_factory=dict) - clients: list[dict[str, Any]] = field(default_factory=list) - clients_by_leaf: dict[str, list[dict[str, Any]]] = field(default_factory=dict) - - def apply_to(self, target: Any) -> None: - """Copy the normalized topology state onto a runtime object.""" - target.topology_name = self.topology_name - target.container_names = dict(self.container_names) - target.topology_metadata = copy.deepcopy(self.topology_metadata) - target.device_mgmt_ips = dict(self.device_mgmt_ips) - target.clients = [copy.deepcopy(client) for client in self.clients] - target.clients_by_leaf = { - leaf: [copy.deepcopy(client) for client in clients] for leaf, clients in self.clients_by_leaf.items() - } - - -def discover_topology_dir(base_dir: str) -> str: - """Find the most likely generated topology directory for the current runtime.""" - env_dir = config.topology_dir - if env_dir and os.path.exists(os.path.join(env_dir, "topology.json")): - return env_dir - - generated_dirs = sorted( - glob.glob(os.path.join(base_dir, "lab-topology", "generated_topology_*")) - + glob.glob(os.path.join(base_dir, "lab-topology", "benchmarks", "generated_topology_*")), - key=os.path.getmtime, - reverse=True, - ) - for candidate in generated_dirs: - if os.path.exists(os.path.join(candidate, "topology.json")): - return candidate - - return os.path.join(base_dir, "lab-topology") - - -def _extract_clients_by_leaf(clients: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: - grouped: dict[str, list[dict[str, Any]]] = {} - for client in clients: - leaf = client.get("leaf") - if not leaf: - continue - grouped.setdefault(str(leaf), []).append(client) - return grouped - - -def build_topology_state_from_metadata(metadata: dict[str, Any]) -> TopologyState: - """Normalize generated topology metadata into a runtime state object.""" - metadata = metadata or {} - topology_name = metadata.get("name", "dcn") - devices = metadata.get("devices", {}) or {} - - container_names: dict[str, str] = {} - device_mgmt_ips: dict[str, str] = {} - clients: list[dict[str, Any]] = [] - - for role in ("spines", "leafs", "clients"): - for entry in devices.get(role, []) or []: - name = entry.get("name") - if not name: - continue - name = str(name) - container_names[name] = clab_container_name(topology_name, name) - mgmt_ip = entry.get("mgmt_ip") - if mgmt_ip: - device_mgmt_ips[name] = str(mgmt_ip) - if role == "clients": - clients.append(dict(entry)) - - return TopologyState( - topology_name=topology_name, - container_names=container_names, - topology_metadata=metadata, - device_mgmt_ips=device_mgmt_ips, - clients=clients, - clients_by_leaf=_extract_clients_by_leaf(clients), - ) - - -def enrich_topology_metadata(topology: dict[str, Any], default_sonic_port_mtu: int = 9100) -> dict[str, Any]: - """Add runtime semantics that help agents interpret generated topology metadata.""" - enriched = copy.deepcopy(topology or {}) - defaults = enriched.setdefault("defaults", {}) - link_mtu = defaults.get("link_mtu") or enriched.get("link_mtu") - if isinstance(link_mtu, int) and link_mtu > 0: - defaults["link_mtu"] = link_mtu - defaults.setdefault("sonic_port_mtu", default_sonic_port_mtu) - - enriched.setdefault( - "mtu_semantics", - { - "link_mtu_scope": "containerlab/client link MTU", - "sonic_port_mtu_scope": "SONiC front-panel interface MTU", - "note": ( - "Treat topology defaults.link_mtu as the client/container link budget. " - "Healthy SONiC ports normally report MTU 9100 in sonic-vs, so 9232 vs 9100 " - "alone is not evidence of mtu_mismatch." - ), - }, - ) - return enriched - - -def resolve_interface_metric_identities(interface: str) -> dict[str, list[str]]: - """Return identity variants used across CLI, Linux, and InfluxDB tags.""" - safe_interface = str(interface or "").strip() - if not safe_interface: - return {"names": [], "paths": []} - - interface_names = {safe_interface} - - lower_name = safe_interface.lower() - if lower_name.startswith("eth") and lower_name[3:].isdigit(): - eth_idx = int(lower_name[3:]) - if eth_idx >= 1: - interface_names.add(f"Ethernet{(eth_idx - 1) * 4}") - elif lower_name.startswith("ethernet") and lower_name[8:].isdigit(): - port_idx = int(lower_name[8:]) - interface_names.add(f"Ethernet{port_idx}") - if port_idx % 4 == 0: - interface_names.add(f"eth{(port_idx // 4) + 1}") - - sonic_names = sorted(name for name in interface_names if name.startswith("Ethernet")) - paths: list[str] = [] - for name in sonic_names: - paths.extend([f"COUNTERS/{name}", f"/COUNTERS/{name}"]) - return { - "names": sorted(interface_names), - "paths": paths, - } - - -def _find_local_topology_dir(default_dir: str) -> str | None: - repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - candidates = [] - for raw in [default_dir, "lab-topology"]: - candidate = raw if os.path.isabs(raw) else os.path.join(repo_root, raw) - candidates.append(candidate) - candidates.extend( - sorted( - glob.glob(os.path.join(repo_root, "lab-topology", "generated_topology_*")), - key=os.path.getmtime, - reverse=True, +def coerce_topology_manifest(topology: TopologyInput) -> TopologyManifest: + """Validate a canonical topology value without legacy fallback.""" + if isinstance(topology, TopologyManifest): + return topology + if not isinstance(topology, dict): + raise TypeError("Topology must be a TopologyManifest or schema-v3 dictionary") + schema_version = topology.get("schema_version") + if schema_version != SCHEMA_VERSION: + rendered = "missing" if schema_version is None else repr(schema_version) + raise ValueError( + f"Unsupported topology schema_version {rendered}; expected {SCHEMA_VERSION!r}. " + "Regenerate topology.json with the current topology generator." ) - ) - seen = set() - ordered = [] - for candidate in candidates: - if candidate in seen: - continue - seen.add(candidate) - ordered.append(candidate) - for candidate in ordered: - if not os.path.isdir(candidate): - continue - if os.path.exists(os.path.join(candidate, "dcn.clab.yaml")) and os.path.exists( - os.path.join(candidate, "topology.json") - ): - return candidate - for candidate in ordered: - if not os.path.isdir(candidate): - continue - has_clab = bool(glob.glob(os.path.join(candidate, "*.clab.y*ml"))) - if has_clab and os.path.exists(os.path.join(candidate, "topology.json")): - return candidate - return None - - -def _is_worker_pool_topology_dir(path: str) -> bool: - normalized = os.path.normpath(path) - parts = normalized.split(os.sep) - return "pools" in parts and "workers" in parts - - -def resolve_topology_dir(explicit_topology_dir: str | None, default_dir: str) -> str: - if explicit_topology_dir: - return explicit_topology_dir try: - detect_cmd = [ - *sudo_prefix(), - "docker", - "ps", - "-a", - "--filter", - "label=containerlab", - "--format", - '{{.Label "clab-topo-file"}}', - ] - detect_result = safe_run(detect_cmd, capture_output=True, text=True, timeout=5) - lines = [line.strip() for line in detect_result.stdout.splitlines() if line.strip()] - detected_file = lines[0] if lines else "" - if detected_file and os.path.exists(detected_file): - topo_dir = os.path.dirname(detected_file) - if not _is_worker_pool_topology_dir(topo_dir): - return topo_dir - except Exception: - logger.debug("clab inspect-based topology detection failed", exc_info=True) - local_topology_dir = _find_local_topology_dir(default_dir) - if local_topology_dir: - return local_topology_dir - return default_dir - + return TopologyManifest.model_validate(topology) + except ValidationError as exc: + raise ValueError(f"Invalid topology schema_version {SCHEMA_VERSION!r} manifest: {exc}") from exc -def resolve_topology_file(topo_dir: str) -> str | None: - preferred = os.path.join(topo_dir, "dcn.clab.yaml") - if os.path.exists(preferred): - return preferred - candidates = sorted(glob.glob(os.path.join(topo_dir, "*.clab.y*ml"))) - return candidates[0] if candidates else None - -def infer_lab_name(topology_file: str) -> str | None: +def load_topology_manifest(path: str | Path) -> TopologyManifest: + """Load ``topology.json`` from a file or generated topology directory.""" + source = Path(path) + if source.is_dir(): + source = source / "topology.json" try: - with open(topology_file, encoding="utf-8") as f: - for raw in f: - line = raw.strip() - match = re.match(r"^name:\s*(\S+)", line) - if match: - return match.group(1) - except Exception: - logger.debug("failed to read topology file %s for lab name", topology_file, exc_info=True) - base = os.path.basename(topology_file) - if base.endswith(".clab.yaml"): - return base[: -len(".clab.yaml")] - if base.endswith(".clab.yml"): - return base[: -len(".clab.yml")] - return None - - -def load_topology_metadata_file(topo_dir: str) -> dict | None: - metadata_path = os.path.join(topo_dir, "topology.json") - if not os.path.exists(metadata_path): - return None + payload = json.loads(source.read_text(encoding="utf-8")) + except FileNotFoundError: + raise + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"Failed to read canonical topology manifest {source}: {exc}") from exc try: - with open(metadata_path, encoding="utf-8") as handle: - return json.load(handle) - except Exception: - logger.debug("failed to load topology metadata %s", metadata_path, exc_info=True) - return None + return coerce_topology_manifest(payload) + except (TypeError, ValueError) as exc: + raise ValueError(f"Failed to load canonical topology manifest {source}: {exc}") from exc -def preview_topology_items(items: list[str], limit: int = 4) -> str: - if not items: - return "none" - if len(items) <= limit: - return ", ".join(items) - return ", ".join(items[:limit]) + f", ... (+{len(items) - limit} more)" +def is_network_device_name(device: str) -> bool: + return str(device or "").startswith(NETWORK_DEVICE_PREFIXES) -def check_topology_deployed(topo_dir: str) -> tuple[bool, str]: - topo_file = resolve_topology_file(topo_dir) - if not topo_file: - return False, f"No topology file found under '{topo_dir}'" - lab_name = infer_lab_name(topo_file) - if not lab_name: - return False, f"Could not infer lab name from topology file: {topo_file}" - try: - result = safe_run( - [*sudo_prefix(), "docker", "ps", "--filter", f"label=containerlab={lab_name}", "--format", "{{.Names}}"], - capture_output=True, - text=True, - timeout=8, - check=False, - ) - except Exception as e: - return False, f"Failed to query docker runtime: {e}" - if result.returncode != 0: - stderr = (result.stderr or "").strip() - return False, f"Docker query failed for lab '{lab_name}': {stderr or 'unknown error'}" - running = [line.strip() for line in result.stdout.splitlines() if line.strip()] - if not running: - return False, ( - f"No running containers found for lab '{lab_name}'. " - "Deploy a lab with `bash scripts/runtime/deploy.sh ` or use the SDK-managed runtime flow." - ) - metadata = load_topology_metadata_file(topo_dir) - if metadata: - expected = [] - devices = metadata.get("devices", {}) or {} - for group in ("spines", "leafs", "clients"): - for device in devices.get(group, []) or []: - name = str((device or {}).get("name") or "").strip() - if name: - expected.append(clab_container_name(lab_name, name)) - expected_set = set(expected) - running_set = set(running) - if expected_set: - missing = sorted(expected_set - running_set) - unexpected = sorted(running_set - expected_set) - if missing or unexpected: - details = [] - if missing: - details.append(f"missing={preview_topology_items(missing)}") - if unexpected: - details.append(f"unexpected={preview_topology_items(unexpected)}") - return False, ( - f"Running containers do not match topology metadata for lab '{lab_name}': " + "; ".join(details) - ) - return True, f"Detected {len(running)} running container(s) for lab '{lab_name}'" +__all__ = [ + "NETWORK_DEVICE_PREFIXES", + "TopologyInput", + "clab_container_name", + "coerce_topology_manifest", + "is_network_device_name", + "load_topology_manifest", +] diff --git a/netopsbench/platform/traffic/__init__.py b/netopsbench/platform/traffic/__init__.py index f61e399..f46fb00 100644 --- a/netopsbench/platform/traffic/__init__.py +++ b/netopsbench/platform/traffic/__init__.py @@ -1,39 +1 @@ -"""Platform traffic subsystem.""" - -from .controller import TrafficController, TrafficFlow -from .generator import ( - BASE_SWITCH_PPS_LIMIT, - IPERF_SERVER_PORT_BASE, - IPERF_SERVER_PORT_POOL_SIZE, - SWITCH_PPS_LIMIT, - TOPOLOGY_SPECS, - TopologySpec, - TrafficProfile, - estimate_client_pps, - estimate_switch_pps, - generate_traffic_config, - generate_traffic_config_from_topology, - get_traffic_profile, - validate_traffic_config, -) -from .scenario_execution import setup_traffic, stop_traffic - -__all__ = [ - "TrafficController", - "TrafficFlow", - "TrafficProfile", - "TopologySpec", - "SWITCH_PPS_LIMIT", - "BASE_SWITCH_PPS_LIMIT", - "TOPOLOGY_SPECS", - "IPERF_SERVER_PORT_BASE", - "IPERF_SERVER_PORT_POOL_SIZE", - "generate_traffic_config", - "generate_traffic_config_from_topology", - "get_traffic_profile", - "validate_traffic_config", - "estimate_switch_pps", - "estimate_client_pps", - "setup_traffic", - "stop_traffic", -] +"""Internal background traffic subsystem.""" diff --git a/netopsbench/platform/traffic/commands.py b/netopsbench/platform/traffic/commands.py new file mode 100644 index 0000000..4729d7c --- /dev/null +++ b/netopsbench/platform/traffic/commands.py @@ -0,0 +1,142 @@ +"""Pure iperf3 command construction for traffic execution.""" + +from __future__ import annotations + +import shlex +from collections.abc import Iterable +from typing import Protocol + + +class TrafficFlowLike(Protocol): + flow_id: str + dst_ip: str + dst_port: int + parallel: int + protocol: str + bandwidth: str + udp_payload_len: int + tcp_mss: int + duration: int + + +class IperfCommandBuilder: + """Build shell-safe server, client, and cleanup commands.""" + + _RUNTIME_DIR = "/tmp/netopsbench-traffic" + + @staticmethod + def _flow_running_function() -> list[str]: + return [ + "flow_running() {", + " pid_file=$1", + ' [ -r "$pid_file" ] || return 1', + ' pid="$(cat "$pid_file" 2>/dev/null || true)"', + ' [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null || return 1', + ' [ "$(cat "/proc/$pid/comm" 2>/dev/null || true)" = "iperf3" ] || return 1', + ' tr "\\000" " " < "/proc/$pid/cmdline" 2>/dev/null | grep -q "^iperf3 -c "', + "}", + ] + + def client_args(self, flow: TrafficFlowLike) -> list[str]: + args = [ + "iperf3", + "-c", + flow.dst_ip, + "-p", + str(flow.dst_port), + "-P", + str(flow.parallel), + ] + if flow.protocol == "udp": + args.extend(["-u", "-b", flow.bandwidth, "-l", str(flow.udp_payload_len)]) + elif flow.bandwidth: + args.extend(["-b", flow.bandwidth]) + if flow.tcp_mss > 0: + args.extend(["-M", str(flow.tcp_mss)]) + args.extend(["-t", str(flow.duration)]) + return args + + def client_command(self, flow: TrafficFlowLike) -> str: + return " ".join(shlex.quote(part) for part in self.client_args(flow)) + + def source_batch_script(self, flows: Iterable[TrafficFlowLike]) -> str: + flow_list = list(flows) + commands = ["set -e", f"runtime_dir={shlex.quote(self._RUNTIME_DIR)}", 'mkdir -p "$runtime_dir"'] + commands.extend(self._flow_running_function()) + checks = [] + pid_files = [] + for flow in flow_list: + command = self.client_command(flow) + pid_file = f"{self._RUNTIME_DIR}/{flow.flow_id}.pid" + quoted_pid_file = shlex.quote(pid_file) + pid_files.append(quoted_pid_file) + commands.extend( + [ + f"if ! flow_running {quoted_pid_file}; then", + f" rm -f {quoted_pid_file}", + f" nohup {command} >/dev/null 2>&1 {quoted_pid_file}", + "fi", + ] + ) + checks.append(f" flow_running {quoted_pid_file} || missing=1") + commands.extend(["for _attempt in 1 2 3 4 5; do", " missing=0"]) + commands.extend(checks) + commands.extend( + [ + ' [ "$missing" -eq 0 ] && exit 0', + " sleep 0.1", + "done", + f"for pid_file in {' '.join(pid_files)}; do", + ' if flow_running "$pid_file"; then kill "$(cat "$pid_file")" 2>/dev/null || true; fi', + ' rm -f "$pid_file"', + "done", + "exit 1", + ] + ) + return "\n".join(commands) + + def server_batch_script(self, ports: Iterable[int]) -> str: + required_ports = " ".join(str(port) for port in sorted(set(ports))) + return "\n".join( + [ + "set -e", + f"required_ports={shlex.quote(required_ports)}", + 'listeners="$(ss -lntH 2>/dev/null || true)"', + "for port in $required_ports; do", + ' if ! printf \'%s\\n\' "$listeners" | grep -q ":${port} "; then', + ' iperf3 -s -D -p "$port"', + " fi", + "done", + "for _attempt in 1 2 3 4 5; do", + ' listeners="$(ss -lntH 2>/dev/null || true)"', + " missing=0", + " for port in $required_ports; do", + ' printf \'%s\\n\' "$listeners" | grep -q ":${port} " || missing=1', + " done", + ' [ "$missing" -eq 0 ] && exit 0', + " sleep 0.1", + "done", + "exit 1", + ] + ) + + @staticmethod + def stop_clients_script() -> str: + runtime_dir = shlex.quote(IperfCommandBuilder._RUNTIME_DIR) + lines = [f"runtime_dir={runtime_dir}", '[ -d "$runtime_dir" ] || exit 0'] + lines.extend(IperfCommandBuilder._flow_running_function()) + lines.extend( + [ + 'for pid_file in "$runtime_dir"/*.pid; do', + ' [ -e "$pid_file" ] || continue', + ' if flow_running "$pid_file"; then kill "$(cat "$pid_file")" 2>/dev/null || true; fi', + ' rm -f "$pid_file"', + "done", + 'rmdir "$runtime_dir" 2>/dev/null || true', + ] + ) + return "\n".join(lines) + + +__all__ = ["IperfCommandBuilder"] diff --git a/netopsbench/platform/traffic/controller.py b/netopsbench/platform/traffic/controller.py index f13ffe1..afb8dca 100644 --- a/netopsbench/platform/traffic/controller.py +++ b/netopsbench/platform/traffic/controller.py @@ -4,13 +4,43 @@ import subprocess import uuid -from dataclasses import dataclass, field +from collections import defaultdict +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass, field +from time import monotonic +from typing import Any from netopsbench.logging_utils import get_logger -from netopsbench.platform.utils.events import emit as _emit -from netopsbench.platform.utils.proc import safe_run, sudo_prefix +from netopsbench.platform.utils.proc import docker_prefix, safe_run + +from .commands import IperfCommandBuilder +from .settings import DEFAULT_TRAFFIC_PARALLELISM, TrafficSettings logger = get_logger(__name__) +_DEFAULT_TRAFFIC_PARALLELISM = DEFAULT_TRAFFIC_PARALLELISM + + +def _traffic_parallelism() -> int: + return TrafficSettings.from_env().parallelism + + +def _flow_summary(flow: "TrafficFlow") -> str: + return ( + f"src={flow.src} dst={flow.dst} dst_ip={flow.dst_ip} " + f"protocol={flow.protocol} port={flow.dst_port} bandwidth={flow.bandwidth}" + ) + + +def _error_detail(exc: Exception) -> str: + parts = [str(exc)] + stderr = getattr(exc, "stderr", None) + stdout = getattr(exc, "stdout", None) + if stderr: + parts.append(f"stderr={str(stderr).strip()}") + if stdout: + parts.append(f"stdout={str(stdout).strip()}") + return "; ".join(part for part in parts if part) @dataclass @@ -30,6 +60,33 @@ class TrafficFlow: flow_id: str = field(default_factory=lambda: str(uuid.uuid4())) +@dataclass(frozen=True) +class TrafficStartStats: + """Timings and retry counters for the latest matrix start.""" + + server_duration_seconds: float = 0.0 + source_duration_seconds: float = 0.0 + server_first_attempt_successes: int = 0 + server_first_attempt_failures: int = 0 + retry_count: int = 0 + timeout_count: int = 0 + started_flow_count: int = 0 + failed_flow_count: int = 0 + + def to_dict(self) -> dict[str, float | int]: + return asdict(self) + + +@dataclass(frozen=True) +class _BatchPhaseResult: + failures: dict[str, Exception] + duration_seconds: float + first_attempt_successes: int + first_attempt_failures: int + retry_count: int + timeout_count: int + + class TrafficController: """ Controls iperf3 traffic flows between client containers. @@ -37,7 +94,12 @@ class TrafficController: Manages starting/stopping background traffic for benchmark realism. """ - def __init__(self, container_names: dict[str, str]): + def __init__( + self, + container_names: dict[str, str], + command_builder: IperfCommandBuilder | None = None, + parallelism: int | None = None, + ): """ Initialize traffic controller. @@ -46,127 +108,89 @@ def __init__(self, container_names: dict[str, str]): e.g., {"client1": "clab-dcn-client1"} """ self.container_names = container_names + self.command_builder = command_builder or IperfCommandBuilder() + self.parallelism = max(1, parallelism or _traffic_parallelism()) + self.server_parallelism = min(16, max(1, (self.parallelism + 1) // 2)) + self.retry_parallelism = min(4, self.server_parallelism) self.active_flows: dict[str, TrafficFlow] = {} self.started_server_ports: dict[str, set] = {} + self.last_start_stats = TrafficStartStats() # Bound external command latency to avoid benchmark hangs. self.command_timeout_seconds = 15 self.start_retries = 1 - def _ensure_iperf_server(self, dst_client: str, dst_port: int): - """Ensure destination has an iperf3 server listening on dst_port.""" - dst_container = self.container_names.get(dst_client) - if not dst_container: - raise ValueError(f"Unknown destination client: {dst_client}") - + def _ensure_iperf_servers_batch(self, dst_container: str, dst_ports: set[int]) -> None: + """Ensure a destination container has all required iperf3 server ports.""" started_ports = self.started_server_ports.setdefault(dst_container, set()) - if dst_port in started_ports: + missing_ports = sorted(port for port in dst_ports if port not in started_ports) + if not missing_ports: return - check_cmd = f"ss -lnt 2>/dev/null | grep -q ':{dst_port} ' " f"|| iperf3 -s -D -p {dst_port}" + script = self.command_builder.server_batch_script(missing_ports) safe_run( - [*sudo_prefix(), "docker", "exec", dst_container, "sh", "-lc", check_cmd], + [*docker_prefix(), "docker", "exec", dst_container, "sh", "-lc", script], check=True, capture_output=True, - timeout=self.command_timeout_seconds, + timeout=max(self.command_timeout_seconds, 15), ) - started_ports.add(dst_port) - - def start_flow(self, flow: TrafficFlow) -> str: - """ - Start a single iperf3 traffic flow. + started_ports.update(missing_ports) - Args: - flow: TrafficFlow specification + def _start_source_flows_batch(self, src_container: str, flows: list[TrafficFlow]) -> None: + script = self.command_builder.source_batch_script(flows) + safe_run( + [*docker_prefix(), "docker", "exec", src_container, "sh", "-lc", script], + check=True, + capture_output=True, + timeout=max(self.command_timeout_seconds, 15), + ) - Returns: - flow_id of the started flow - """ - src_container = self.container_names.get(flow.src) - if not src_container: - raise ValueError(f"Unknown client: {flow.src}") - - # Build iperf3 command - cmd_parts = [ - *sudo_prefix(), - "docker", - "exec", - "-d", - src_container, - "iperf3", - "-c", - flow.dst_ip, - "-p", - str(flow.dst_port), - "-P", - str(flow.parallel), - ] - - if flow.protocol == "udp": - cmd_parts.extend(["-u", "-b", flow.bandwidth, "-l", str(flow.udp_payload_len)]) - elif flow.bandwidth: - cmd_parts.extend(["-b", flow.bandwidth]) - if flow.tcp_mss > 0: - cmd_parts.extend(["-M", str(flow.tcp_mss)]) - - # Keep continuous semantics explicit - cmd_parts.extend(["-t", str(flow.duration)]) - - # Start flow in background - last_error: Exception | None = None + def _run_batch_phase( + self, + groups: dict[str, Any], + operation: Callable[[str, Any], None], + initial_parallelism: int, + ) -> _BatchPhaseResult: + """Run one grouped Docker phase, retrying transient failures at lower pressure.""" + started_at = monotonic() + pending = dict(groups) + failures: dict[str, Exception] = {} + first_attempt_successes = 0 + first_attempt_failures = 0 + retry_count = 0 + timeout_count = 0 for attempt in range(self.start_retries + 1): - try: - self._ensure_iperf_server(flow.dst, flow.dst_port) - safe_run( - cmd_parts, - check=True, - capture_output=True, - timeout=self.command_timeout_seconds, - ) - self.active_flows[flow.flow_id] = flow - _emit(f" Started flow: {flow.src} -> {flow.dst} ({flow.bandwidth})") - return flow.flow_id - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - last_error = e - if attempt < self.start_retries: - _emit( - f" Warning: flow start retry {attempt + 1}/{self.start_retries} " - f"for {flow.src} -> {flow.dst}" - ) - continue + if not pending: break - - _emit(f" Failed to start flow {flow.src} -> {flow.dst}: {last_error}") - raise RuntimeError(f"Failed to start flow {flow.src}->{flow.dst}: {last_error}") - - def stop_flow(self, flow_id: str): - """ - Stop a specific traffic flow. - - Args: - flow_id: ID of flow to stop - """ - if flow_id not in self.active_flows: - _emit(f" Warning: Flow {flow_id} not found") - return - - flow = self.active_flows[flow_id] - src_container = self.container_names.get(flow.src) - - if src_container: - try: - # Kill iperf3 client processes - safe_run( - [*sudo_prefix(), "docker", "exec", src_container, "pkill", "-f", f"iperf3 -c {flow.dst_ip}"], - check=False, # Don't fail if process already stopped - capture_output=True, - timeout=15, - ) - _emit(f" Stopped flow: {flow.src} -> {flow.dst}") - - except Exception as e: - _emit(f" Warning: Failed to stop flow {flow_id}: {e}", level="warning") - - del self.active_flows[flow_id] + max_workers = initial_parallelism if attempt == 0 else self.retry_parallelism + current = pending + pending = {} + failures = {} + if attempt > 0: + retry_count += len(current) + with ThreadPoolExecutor(max_workers=min(max_workers, len(current))) as executor: + future_map = { + executor.submit(operation, container, value): container for container, value in current.items() + } + for future in as_completed(future_map): + container = future_map[future] + try: + future.result() + except Exception as exc: + pending[container] = current[container] + failures[container] = exc + if isinstance(exc, subprocess.TimeoutExpired): + timeout_count += 1 + if attempt == 0: + first_attempt_failures = len(pending) + first_attempt_successes = len(current) - first_attempt_failures + return _BatchPhaseResult( + failures=failures, + duration_seconds=monotonic() - started_at, + first_attempt_successes=first_attempt_successes, + first_attempt_failures=first_attempt_failures, + retry_count=retry_count, + timeout_count=timeout_count, + ) def start_matrix(self, flows: list[TrafficFlow]) -> list[str]: """ @@ -178,27 +202,129 @@ def start_matrix(self, flows: list[TrafficFlow]) -> list[str]: Returns: List of flow_ids """ - flow_ids = [] - + if not flows: + self.last_start_stats = TrafficStartStats() + logger.info("Started 0/0 flows") + return [] + + valid_flows: list[TrafficFlow] = [] + dst_ports_by_container: dict[str, set[int]] = defaultdict(set) + flows_by_dst_container: dict[str, list[TrafficFlow]] = defaultdict(list) for flow in flows: - try: - flow_id = self.start_flow(flow) - flow_ids.append(flow_id) - except Exception as e: - _emit(f" Warning: Skipping flow due to error: {e}", level="warning") - - _emit(f"\n Started {len(flow_ids)}/{len(flows)} flows") + src_container = self.container_names.get(flow.src) + dst_container = self.container_names.get(flow.dst) + if not src_container: + logger.warning("Skipping flow with unknown source: %s", _flow_summary(flow)) + continue + if not dst_container: + logger.warning("Skipping flow with unknown destination: %s", _flow_summary(flow)) + continue + valid_flows.append(flow) + dst_ports_by_container[dst_container].add(flow.dst_port) + flows_by_dst_container[dst_container].append(flow) + + failed_flows: set[str] = set() + server_result = self._run_batch_phase( + dst_ports_by_container, + self._ensure_iperf_servers_batch, + self.server_parallelism, + ) + for dst_container, exc in server_result.failures.items(): + for flow in flows_by_dst_container.get(dst_container, []): + failed_flows.add(flow.flow_id) + logger.warning("Failed to ensure server for %s: %s", _flow_summary(flow), _error_detail(exc)) + + source_groups: dict[str, list[TrafficFlow]] = defaultdict(list) + for flow in valid_flows: + if flow.flow_id in failed_flows: + continue + source_groups[self.container_names[flow.src]].append(flow) + + source_result = self._run_batch_phase( + source_groups, + self._start_source_flows_batch, + self.parallelism, + ) + for src_container, src_flows in source_groups.items(): + source_error = source_result.failures.get(src_container) + if source_error is not None: + for flow in src_flows: + failed_flows.add(flow.flow_id) + logger.warning( + "Failed to start flow on %s: %s: %s", + src_container, + _flow_summary(flow), + _error_detail(source_error), + ) + continue + for flow in src_flows: + self.active_flows[flow.flow_id] = flow + logger.debug("Started %d flow(s) from %s", len(src_flows), src_flows[0].src) + + flow_ids = [flow.flow_id for flow in valid_flows if flow.flow_id in self.active_flows] + self.last_start_stats = TrafficStartStats( + server_duration_seconds=server_result.duration_seconds, + source_duration_seconds=source_result.duration_seconds, + server_first_attempt_successes=server_result.first_attempt_successes, + server_first_attempt_failures=server_result.first_attempt_failures, + retry_count=server_result.retry_count + source_result.retry_count, + timeout_count=server_result.timeout_count + source_result.timeout_count, + started_flow_count=len(flow_ids), + failed_flow_count=len(flows) - len(flow_ids), + ) + logger.info( + "Traffic phases: servers=%.1fs sources=%.1fs retries=%d timeouts=%d", + server_result.duration_seconds, + source_result.duration_seconds, + self.last_start_stats.retry_count, + self.last_start_stats.timeout_count, + ) + logger.info("Started %d/%d flows", len(flow_ids), len(flows)) return flow_ids def stop_all(self): """Stop all active traffic flows.""" flow_ids = list(self.active_flows.keys()) - + flows_by_src_container: dict[str, list[TrafficFlow]] = defaultdict(list) for flow_id in flow_ids: - self.stop_flow(flow_id) - - _emit(f" Stopped all {len(flow_ids)} flows") - - def get_active_flows(self) -> list[TrafficFlow]: - """Get list of all active flows.""" - return list(self.active_flows.values()) + flow = self.active_flows[flow_id] + src_container = self.container_names.get(flow.src) + if src_container: + flows_by_src_container[src_container].append(flow) + else: + del self.active_flows[flow_id] + + def stop_container(src_container: str) -> None: + safe_run( + [ + *docker_prefix(), + "docker", + "exec", + src_container, + "sh", + "-lc", + self.command_builder.stop_clients_script(), + ], + check=False, + capture_output=True, + timeout=15, + ) + + parallelism = min(self.parallelism, max(len(flows_by_src_container), 1)) + with ThreadPoolExecutor(max_workers=parallelism) as executor: + future_map = { + executor.submit(stop_container, src_container): (src_container, src_flows) + for src_container, src_flows in flows_by_src_container.items() + } + for future in as_completed(future_map): + src_container, src_flows = future_map[future] + try: + future.result() + logger.debug("Stopped %d flow(s) from %s", len(src_flows), src_flows[0].src) + except Exception as exc: + logger.warning("Failed to stop flows on %s: %s", src_container, _error_detail(exc)) + finally: + for flow in src_flows: + self.active_flows.pop(flow.flow_id, None) + + logger.info("Stopped all %d flows", len(flow_ids)) diff --git a/netopsbench/platform/traffic/generator.py b/netopsbench/platform/traffic/generator.py index 4bc6a84..e39d071 100644 --- a/netopsbench/platform/traffic/generator.py +++ b/netopsbench/platform/traffic/generator.py @@ -1,98 +1,53 @@ -"""Traffic generation strategy for generated topologies.""" +"""Generate the canonical standard background-traffic matrix.""" from __future__ import annotations -import json -import os -from dataclasses import dataclass from functools import partial +from netopsbench.models.profiles import get_scale_profile +from netopsbench.models.topology import DEFAULT_LINK_MTU +from netopsbench.platform.topology.topology_utils import coerce_topology_manifest, load_topology_manifest + from .estimator import estimate_client_pps as _estimate_client_pps from .estimator import estimate_flow_pps as _estimate_flow_pps from .estimator import estimate_switch_pps as _estimate_switch_pps -from .estimator import ( - infer_topology_link_mtu, -) +from .estimator import infer_topology_link_mtu from .planner import build_candidate_flow as _build_candidate_flow -from .planner import generate_traffic_config_from_topology as _plan_traffic - - -def _parse_switch_pps_limit(value: str): - """Parse optional switch PPS limit from env (None means unlimited).""" - if value is None: - return None - raw = str(value).strip().lower() - if raw in {"", "0", "none", "unlimited", "inf", "infinite"}: - return None - return int(raw) - +from .planner import generate_traffic_config_from_topology as plan_standard_traffic +from .settings import TrafficSettings -SWITCH_PPS_LIMIT = _parse_switch_pps_limit(os.getenv("NETOPSBENCH_SWITCH_PPS_LIMIT", "5000")) BASE_SWITCH_PPS_LIMIT = 1000 -DEFAULT_LINK_MTU_BYTES = 9232 +DEFAULT_LINK_MTU_BYTES = DEFAULT_LINK_MTU UDP_PAYLOAD_LEN_BYTES = 1400 TCP_MSS_BYTES = 1360 IPERF_SERVER_PORT_BASE = 5201 -IPERF_SERVER_PORT_POOL_SIZE = 8 - - -@dataclass -class TopologySpec: - scale: str - num_clients: int - num_leafs: int - num_spines: int - max_pps_per_client: int - - -TOPOLOGY_SPECS = { - "xs": TopologySpec(scale="xs", num_clients=4, num_leafs=2, num_spines=2, max_pps_per_client=250), - "small": TopologySpec(scale="small", num_clients=8, num_leafs=4, num_spines=2, max_pps_per_client=250), - "medium": TopologySpec(scale="medium", num_clients=16, num_leafs=8, num_spines=4, max_pps_per_client=200), - "large": TopologySpec(scale="large", num_clients=32, num_leafs=16, num_spines=8, max_pps_per_client=150), -} - - -def _scale_max_pps_per_client() -> None: - if SWITCH_PPS_LIMIT is None: - return - scale_factor = SWITCH_PPS_LIMIT / BASE_SWITCH_PPS_LIMIT - for spec in TOPOLOGY_SPECS.values(): - scaled = int(round(spec.max_pps_per_client * scale_factor)) - spec.max_pps_per_client = max(1, scaled) - - -_scale_max_pps_per_client() - - -@dataclass -class TrafficProfile: - name: str - description: str - pattern: str - udp_bandwidth_per_flow: str - tcp_bandwidth_per_flow: str - tcp_connections: int - flows_per_client: int - cross_leaf_target_ratio: float - target_client_utilization: float +IPERF_SERVER_PORT_POOL_SIZE = 4 +FLOWS_PER_CLIENT = 4 def _format_bandwidth_from_pps(pps: float, packet_size_bytes: int) -> str: bits_per_sec = max(pps, 1.0) * packet_size_bytes * 8 if bits_per_sec >= 1_000_000: - mbps = max(int(bits_per_sec / 1_000_000), 1) - return f"{mbps}M" - kbps = max(int(bits_per_sec / 1_000), 100) - return f"{kbps}K" + return f"{max(int(bits_per_sec / 1_000_000), 1)}M" + return f"{max(int(bits_per_sec / 1_000), 100)}K" + +def _max_pps_per_client(scale: str, switch_pps_limit: int | None) -> int: + base = get_scale_profile(scale).traffic_max_pps_per_client + if switch_pps_limit is None: + return base + return max(1, int(round(base * switch_pps_limit / BASE_SWITCH_PPS_LIMIT))) -def _get_flow_bandwidth(profile: TrafficProfile, protocol: str) -> str: - return profile.udp_bandwidth_per_flow if protocol == "udp" else profile.tcp_bandwidth_per_flow + +def _standard_bandwidths(max_pps_per_client: int) -> dict[str, str]: + pps_per_flow = max_pps_per_client / FLOWS_PER_CLIENT + return { + "udp": _format_bandwidth_from_pps(pps_per_flow, UDP_PAYLOAD_LEN_BYTES + 28), + "tcp": _format_bandwidth_from_pps(pps_per_flow, TCP_MSS_BYTES + 40), + } def estimate_flow_pps(flow: dict) -> float: - """Estimate PPS for a single flow using module-level packet size constants.""" return _estimate_flow_pps(flow, udp_payload_len_bytes=UDP_PAYLOAD_LEN_BYTES, tcp_mss_bytes=TCP_MSS_BYTES) @@ -104,99 +59,100 @@ def estimate_client_pps(flows: list) -> dict: return _estimate_client_pps(flows, estimate_flow_pps_fn=estimate_flow_pps) -def get_traffic_profile( - scale: str, profile_type: str = "standard", link_mtu_bytes: int = DEFAULT_LINK_MTU_BYTES -) -> TrafficProfile: - spec = TOPOLOGY_SPECS.get(scale) - if not spec: - raise ValueError(f"Unknown scale: {scale}") - profile_utilization = {"light": 0.60, "standard": 1.00, "stress": 1.00} - flows_by_profile = {"light": 2, "standard": 4, "stress": 6} - cross_leaf_targets = {"light": 0.50, "standard": 0.65, "stress": 0.75} - if profile_type not in profile_utilization: - raise ValueError(f"Unknown profile type: {profile_type}") - flows_per_client = flows_by_profile[profile_type] - udp_flows_per_client = max(flows_per_client, 1) - target_pps_per_client = spec.max_pps_per_client * profile_utilization[profile_type] - target_pps_per_udp_flow = target_pps_per_client / udp_flows_per_client - udp_packet_size = UDP_PAYLOAD_LEN_BYTES + 28 - tcp_packet_size = TCP_MSS_BYTES + 40 - udp_bandwidth_per_flow = _format_bandwidth_from_pps(target_pps_per_udp_flow, udp_packet_size) - tcp_bandwidth_per_flow = _format_bandwidth_from_pps(target_pps_per_udp_flow, tcp_packet_size) - tcp_connections = spec.num_clients if profile_type == "stress" else spec.num_clients // 2 - return TrafficProfile( - name=f"{scale}_{profile_type}", - description=f"{profile_type.capitalize()} traffic profile for {scale} topology", - pattern="full_mesh", - udp_bandwidth_per_flow=udp_bandwidth_per_flow, - tcp_bandwidth_per_flow=tcp_bandwidth_per_flow, - tcp_connections=tcp_connections, - flows_per_client=flows_per_client, - cross_leaf_target_ratio=cross_leaf_targets[profile_type], - target_client_utilization=profile_utilization[profile_type], - ) - - -def generate_traffic_config_from_topology(topology: dict, scale: str, profile_type: str = "standard") -> dict: - link_mtu_bytes = infer_topology_link_mtu(topology, DEFAULT_LINK_MTU_BYTES) - profile = get_traffic_profile(scale, profile_type, link_mtu_bytes=link_mtu_bytes) - spec = TOPOLOGY_SPECS[scale] - - bound_build_flow = partial( +def generate_traffic_config_from_topology( + topology: dict, + scale: str, + profile_type: str = "standard", + *, + settings: TrafficSettings | None = None, +) -> dict: + if profile_type != "standard": + raise ValueError(f"Only the standard traffic profile is supported, got: {profile_type}") + get_scale_profile(scale) + projected = coerce_topology_manifest(topology).to_agent_topology() + settings = settings or TrafficSettings.from_env() + max_pps_per_client = _max_pps_per_client(scale, settings.switch_pps_limit) + bandwidths = _standard_bandwidths(max_pps_per_client) + link_mtu_bytes = infer_topology_link_mtu(projected, DEFAULT_LINK_MTU_BYTES) + + build_flow = partial( _build_candidate_flow, udp_payload_len_bytes=UDP_PAYLOAD_LEN_BYTES, tcp_mss_bytes=TCP_MSS_BYTES, - get_bandwidth=_get_flow_bandwidth, ) - - config = _plan_traffic( - topology=topology, + traffic = plan_standard_traffic( + topology=projected, scale=scale, - profile=profile, - spec=spec, + flows_per_client=FLOWS_PER_CLIENT, + max_pps_per_client=max_pps_per_client, + bandwidth_by_protocol=bandwidths, link_mtu_bytes=link_mtu_bytes, - switch_pps_limit=SWITCH_PPS_LIMIT, + switch_pps_limit=settings.switch_pps_limit, iperf_server_port_base=IPERF_SERVER_PORT_BASE, iperf_server_port_pool_size=IPERF_SERVER_PORT_POOL_SIZE, - build_candidate_flow_fn=bound_build_flow, + build_candidate_flow_fn=build_flow, estimate_flow_pps_fn=estimate_flow_pps, estimate_client_pps_fn=estimate_client_pps, estimate_switch_pps_fn=estimate_switch_pps, ) - config["profile"].update( + traffic["profile"].update( { "udp_payload_len_bytes": UDP_PAYLOAD_LEN_BYTES, "tcp_mss_bytes": TCP_MSS_BYTES, "tcp_payload_len_bytes_estimate": TCP_MSS_BYTES, } ) - return config - - -def generate_traffic_config(topology_file: str, scale: str, profile_type: str = "standard") -> dict: - with open(topology_file, encoding="utf-8") as f: - topology = json.load(f) - return generate_traffic_config_from_topology(topology, scale, profile_type) + return traffic + + +def generate_traffic_config( + topology_file: str, + scale: str, + profile_type: str = "standard", + *, + settings: TrafficSettings | None = None, +) -> dict: + return generate_traffic_config_from_topology( + load_topology_manifest(topology_file).model_dump(mode="json"), + scale, + profile_type, + settings=settings, + ) -def validate_traffic_config(config: dict, scale: str) -> bool: - spec = TOPOLOGY_SPECS.get(scale) - if not spec: - raise ValueError(f"Unknown scale: {scale}") +def validate_traffic_config(config: dict, scale: str, *, settings: TrafficSettings | None = None) -> bool: + settings = settings or TrafficSettings.from_env() + max_allowed_client_pps = _max_pps_per_client(scale, settings.switch_pps_limit) stats = config.get("stats", {}) estimated_clients = stats.get("estimated_pps_per_client") or stats.get("estimated_udp_pps_per_client", {}) max_client_pps = stats.get("estimated_max_pps_per_client") or stats.get("estimated_max_udp_pps_per_client", 0.0) switch_pps = stats.get("estimated_switch_pps", {}) max_leaf_pps = switch_pps.get("max_leaf_pps", 0.0) max_spine_pps = switch_pps.get("max_spine_pps", 0.0) - if max_client_pps > spec.max_pps_per_client: + if max_client_pps > max_allowed_client_pps: raise ValueError( - f"Estimated PPS per client too high ({max_client_pps:.2f} > {spec.max_pps_per_client}). " + f"Estimated PPS per client too high ({max_client_pps:.2f} > {max_allowed_client_pps}). " f"Details: {estimated_clients}" ) - if SWITCH_PPS_LIMIT is not None and (max_leaf_pps > SWITCH_PPS_LIMIT or max_spine_pps > SWITCH_PPS_LIMIT): + if settings.switch_pps_limit is not None and ( + max_leaf_pps > settings.switch_pps_limit or max_spine_pps > settings.switch_pps_limit + ): raise ValueError( f"Estimated switch PPS too high (leaf max={max_leaf_pps:.2f}, spine max={max_spine_pps:.2f}, " - f"limit={SWITCH_PPS_LIMIT})." + f"limit={settings.switch_pps_limit})." ) return True + + +__all__ = [ + "BASE_SWITCH_PPS_LIMIT", + "FLOWS_PER_CLIENT", + "IPERF_SERVER_PORT_BASE", + "IPERF_SERVER_PORT_POOL_SIZE", + "estimate_client_pps", + "estimate_flow_pps", + "estimate_switch_pps", + "generate_traffic_config", + "generate_traffic_config_from_topology", + "validate_traffic_config", +] diff --git a/netopsbench/platform/traffic/planner.py b/netopsbench/platform/traffic/planner.py index 9c40ca8..1741f9e 100644 --- a/netopsbench/platform/traffic/planner.py +++ b/netopsbench/platform/traffic/planner.py @@ -18,12 +18,11 @@ def build_candidate_flow( src_client: dict, dst_client: dict, protocol: str, - profile, + bandwidth_by_protocol: dict[str, str], link_mtu_bytes: int, dst_port: int, udp_payload_len_bytes: int, tcp_mss_bytes: int, - get_bandwidth, ) -> dict: src_ip = src_client.get("data_ip") or src_client.get("ip") dst_ip = dst_client.get("data_ip") or dst_client.get("ip") @@ -40,7 +39,7 @@ def build_candidate_flow( "dst_ip": dst_ip, "dst_port": dst_port, "protocol": protocol, - "bandwidth": get_bandwidth(profile, protocol), + "bandwidth": bandwidth_by_protocol[protocol], "duration": 0, "parallel": 1, } @@ -57,8 +56,9 @@ def generate_traffic_config_from_topology( *, topology: dict, scale: str, - profile, - spec, + flows_per_client: int, + max_pps_per_client: int, + bandwidth_by_protocol: dict[str, str], link_mtu_bytes: int, switch_pps_limit, iperf_server_port_base: int, @@ -69,25 +69,23 @@ def generate_traffic_config_from_topology( estimate_switch_pps_fn, ) -> dict: clients = [d for d in topology["devices"]["clients"]] - client_to_leaf = {client["name"]: client.get("leaf") for client in clients} - leaf_to_clients: dict[str, list[dict]] = {} - for client in clients: - leaf_name = client.get("leaf") - if leaf_name: - leaf_to_clients.setdefault(leaf_name, []).append(client) - - flows = [] + client_to_leaf: dict[str, str] = { + client["name"]: attached_switch + for client in clients + if isinstance((attached_switch := client.get("leaf")), str) + } + flows: list[dict] = [] flow_id = 0 - rejected_candidates = 0 leaf_pps = {leaf["name"]: 0.0 for leaf in topology.get("devices", {}).get("leafs", [])} spine_pps = {spine["name"]: 0.0 for spine in topology.get("devices", {}).get("spines", [])} spine_count = len(spine_pps) client_pps = {client["name"]: 0.0 for client in clients} source_flow_count = {client["name"]: 0 for client in clients} - source_used_dsts = {client["name"]: set() for client in clients} + source_used_dsts: dict[str, set[str]] = {client["name"]: set() for client in clients} incoming_flow_count = {client["name"]: 0 for client in clients} cross_leaf_flows = 0 intra_leaf_flows = 0 + rejected = {"source_budget": 0, "destination_budget": 0, "pps_budget": 0} def flow_type(src_name: str, dst_name: str) -> str: src_leaf = client_to_leaf.get(src_name) @@ -96,32 +94,33 @@ def flow_type(src_name: str, dst_name: str) -> str: return "cross" return "intra" - def can_admit(flow: dict) -> bool: - nonlocal rejected_candidates + def admission_error(flow: dict) -> str | None: src_name = flow["src"] dst_name = flow["dst"] + if source_flow_count[src_name] >= flows_per_client: + return "source_budget" + if incoming_flow_count[dst_name] >= flows_per_client: + return "destination_budget" pps = estimate_flow_pps_fn(flow) - if client_pps.get(src_name, 0.0) + pps > spec.max_pps_per_client: - return False + if client_pps.get(src_name, 0.0) + pps > max_pps_per_client: + return "pps_budget" src_leaf = client_to_leaf.get(src_name) dst_leaf = client_to_leaf.get(dst_name) if switch_pps_limit is not None: leaf_additions: dict[str, float] = {} - if src_leaf in leaf_pps: + if src_leaf is not None and src_leaf in leaf_pps: leaf_additions[src_leaf] = leaf_additions.get(src_leaf, 0.0) + pps - if dst_leaf in leaf_pps: + if dst_leaf is not None and dst_leaf in leaf_pps: leaf_additions[dst_leaf] = leaf_additions.get(dst_leaf, 0.0) + pps for leaf_name, added_pps in leaf_additions.items(): if (leaf_pps[leaf_name] + added_pps) > switch_pps_limit: - return False + return "pps_budget" if switch_pps_limit is not None and src_leaf and dst_leaf and src_leaf != dst_leaf and spine_count > 0: per_spine_pps = pps / spine_count for current in spine_pps.values(): if (current + per_spine_pps) > switch_pps_limit: - return False - if source_flow_count[src_name] >= profile.flows_per_client: - return False - return True + return "pps_budget" + return None def admit_flow(flow: dict): nonlocal flow_id, cross_leaf_flows, intra_leaf_flows @@ -149,7 +148,7 @@ def admit_flow(flow: dict): flows.append(flow) flow_id += 1 - def candidate_pools(src_idx: int) -> tuple[list[dict], list[dict]]: + def candidate_pools(src_idx: int) -> dict[str, list[dict]]: src_client = clients[src_idx] src_leaf = src_client.get("leaf") cross_candidates: list[dict] = [] @@ -162,81 +161,75 @@ def candidate_pools(src_idx: int) -> tuple[list[dict], list[dict]]: cross_candidates.append(dst_client) else: intra_candidates.append(dst_client) - return cross_candidates, intra_candidates + return {"cross": cross_candidates, "intra": intra_candidates} def try_add_from_pool(src_idx: int, pool_type: str) -> bool: - nonlocal rejected_candidates src_client = clients[src_idx] - cross_pool, intra_pool = candidate_pools(src_idx) - candidate_pool = cross_pool if pool_type == "cross" else intra_pool - for dst_client in candidate_pool: - protocol = "udp" if (flow_id % 2 == 0) else "tcp" + candidate_pool = candidate_pools(src_idx)[pool_type] + cyclic_rank = {candidate["name"]: rank for rank, candidate in enumerate(candidate_pool)} + ordered_candidates = sorted( + candidate_pool, + key=lambda candidate: ( + incoming_flow_count[candidate["name"]], + cyclic_rank[candidate["name"]], + ), + ) + for dst_client in ordered_candidates: dst_name = dst_client["name"] - dst_port = iperf_server_port_base + (incoming_flow_count.get(dst_name, 0) % iperf_server_port_pool_size) + incoming_slot = incoming_flow_count[dst_name] + if incoming_slot >= iperf_server_port_pool_size: + rejected["destination_budget"] += 1 + continue + protocol = "udp" if source_flow_count[src_client["name"]] % 2 == 0 else "tcp" candidate_flow = build_candidate_flow_fn( flow_id=flow_id, src_client=src_client, dst_client=dst_client, protocol=protocol, - profile=profile, + bandwidth_by_protocol=bandwidth_by_protocol, link_mtu_bytes=link_mtu_bytes, - dst_port=dst_port, + dst_port=iperf_server_port_base + incoming_slot, ) - if can_admit(candidate_flow): + if error := admission_error(candidate_flow): + rejected[error] += 1 + else: admit_flow(candidate_flow) return True - rejected_candidates += 1 return False - for src_idx, src_client in enumerate(clients): - src_leaf = src_client.get("leaf") - same_leaf_peers = [c for c in leaf_to_clients.get(src_leaf, []) if c["name"] != src_client["name"]] - has_cross_candidates = len(clients) > len(leaf_to_clients.get(src_leaf, [])) - if has_cross_candidates: - try_add_from_pool(src_idx, "cross") - if same_leaf_peers: - try_add_from_pool(src_idx, "intra") - - progress = True - while progress: - progress = False + preferred_pools = ["cross", "cross", "cross", "intra"] + for round_index in range(flows_per_client): + preferred_pool = preferred_pools[min(round_index, len(preferred_pools) - 1)] + fallback_pool = "intra" if preferred_pool == "cross" else "cross" for src_idx, src_client in enumerate(clients): - src_name = src_client["name"] - if source_flow_count[src_name] >= profile.flows_per_client: + if source_flow_count[src_client["name"]] >= flows_per_client: + rejected["source_budget"] += 1 continue - src_total_flows = source_flow_count[src_name] - src_cross_flows = sum(1 for f in flows if f["src"] == src_name and flow_type(f["src"], f["dst"]) == "cross") - preferred_pool = ( - "cross" - if src_total_flows == 0 or (src_cross_flows / src_total_flows) < profile.cross_leaf_target_ratio - else "intra" - ) if try_add_from_pool(src_idx, preferred_pool): - progress = True continue - fallback_pool = "intra" if preferred_pool == "cross" else "cross" - if try_add_from_pool(src_idx, fallback_pool): - progress = True + try_add_from_pool(src_idx, fallback_pool) total_path_flows = cross_leaf_flows + intra_leaf_flows cross_leaf_ratio = (cross_leaf_flows / total_path_flows) if total_path_flows else 0.0 per_client_pps = estimate_client_pps_fn(flows) - per_client_udp_pps = {} + per_client_udp_pps: dict[str, float] = {} for flow in flows: if flow["protocol"] == "udp": src_name = flow["src"] per_client_udp_pps[src_name] = per_client_udp_pps.get(src_name, 0.0) + estimate_flow_pps_fn(flow) switch_pps = estimate_switch_pps_fn(topology, flows) + required_listener_ports = sorted({int(flow["dst_port"]) for flow in flows}) + incoming_values = list(incoming_flow_count.values()) return { "profile": { - "name": profile.name, - "description": profile.description, + "name": f"{scale}_standard", + "description": f"Canonical standard traffic for {scale} topology", "scale": scale, - "max_pps_per_client": spec.max_pps_per_client, + "max_pps_per_client": max_pps_per_client, "switch_pps_limit": switch_pps_limit, - "cross_leaf_target_ratio": profile.cross_leaf_target_ratio, - "target_client_utilization": profile.target_client_utilization, + "cross_leaf_target_ratio": 0.75, + "target_client_utilization": 1.0, "link_mtu_bytes": link_mtu_bytes, }, "flows": flows, @@ -244,7 +237,15 @@ def try_add_from_pool(src_idx: int, pool_type: str) -> bool: "total_flows": len(flows), "udp_flows": sum(1 for f in flows if f["protocol"] == "udp"), "tcp_flows": sum(1 for f in flows if f["protocol"] == "tcp"), - "rejected_candidates": rejected_candidates, + "rejected_candidates": sum(rejected.values()), + "rejected_by_source_budget": rejected["source_budget"], + "rejected_by_destination_budget": rejected["destination_budget"], + "rejected_by_pps_budget": rejected["pps_budget"], + "outgoing_flows_per_client": dict(sorted(source_flow_count.items())), + "incoming_flows_per_client": dict(sorted(incoming_flow_count.items())), + "min_incoming_flows": min(incoming_values) if incoming_values else 0, + "max_incoming_flows": max(incoming_values) if incoming_values else 0, + "required_listener_ports": required_listener_ports, "estimated_pps_per_client": {client: round(pps, 2) for client, pps in per_client_pps.items()}, "estimated_max_pps_per_client": round(max(per_client_pps.values()) if per_client_pps else 0.0, 2), "estimated_udp_pps_per_client": {client: round(pps, 2) for client, pps in per_client_udp_pps.items()}, @@ -253,7 +254,16 @@ def try_add_from_pool(src_idx: int, pool_type: str) -> bool: ), "cross_leaf_flows": cross_leaf_flows, "intra_leaf_flows": intra_leaf_flows, + "cross_switch_flows": cross_leaf_flows, + "same_switch_flows": intra_leaf_flows, "cross_leaf_flow_ratio": round(cross_leaf_ratio, 3), "estimated_switch_pps": switch_pps, }, } + + +__all__ = [ + "build_candidate_flow", + "candidate_destinations", + "generate_traffic_config_from_topology", +] diff --git a/netopsbench/platform/traffic/scenario_execution.py b/netopsbench/platform/traffic/scenario_execution.py index 0b2b083..97f3ec5 100644 --- a/netopsbench/platform/traffic/scenario_execution.py +++ b/netopsbench/platform/traffic/scenario_execution.py @@ -2,14 +2,12 @@ from __future__ import annotations -import json from pathlib import Path from netopsbench.logging_utils import get_logger -from netopsbench.platform.topology.topology_utils import clab_container_name +from netopsbench.platform.topology.topology_utils import clab_container_name, load_topology_manifest from netopsbench.platform.traffic.controller import TrafficController, TrafficFlow from netopsbench.platform.traffic.generator import generate_traffic_config, validate_traffic_config -from netopsbench.platform.utils.events import emit as _emit logger = get_logger(__name__) @@ -17,7 +15,7 @@ def setup_traffic(runner, scale: str, profile: str) -> dict: """Generate topology-driven traffic, start the controller, and return the config.""" - _emit(f"\n[Traffic Setup] Generating {profile} traffic for {scale} topology...") + logger.info(f"\n[Traffic Setup] Generating {profile} traffic for {scale} topology...") topology_file = Path(runner.topology_dir) / "topology.json" if not topology_file.exists(): @@ -26,9 +24,9 @@ def setup_traffic(runner, scale: str, profile: str) -> dict: traffic_config = generate_traffic_config(str(topology_file), scale, profile) validate_traffic_config(traffic_config, scale) - _emit(f" Total flows: {traffic_config['stats']['total_flows']}") - _emit(f" UDP flows: {traffic_config['stats']['udp_flows']}") - _emit(f" TCP flows: {traffic_config['stats']['tcp_flows']}") + logger.info(f" Total flows: {traffic_config['stats']['total_flows']}") + logger.info(f" UDP flows: {traffic_config['stats']['udp_flows']}") + logger.info(f" TCP flows: {traffic_config['stats']['tcp_flows']}") estimated_client_pps = traffic_config["stats"].get("estimated_max_pps_per_client") estimated_udp_client_pps = traffic_config["stats"].get("estimated_max_udp_pps_per_client") @@ -41,15 +39,15 @@ def setup_traffic(runner, scale: str, profile: str) -> dict: cross_leaf_ratio = traffic_config["stats"].get("cross_leaf_flow_ratio", 0.0) if estimated_client_pps is not None: - _emit(f" Estimated max PPS per client (all protocols): {estimated_client_pps}") + logger.info(f" Estimated max PPS per client (all protocols): {estimated_client_pps}") if estimated_udp_client_pps is not None: - _emit(f" Estimated max UDP PPS per client: {estimated_udp_client_pps}") + logger.info(f" Estimated max UDP PPS per client: {estimated_udp_client_pps}") limit_label = "unlimited" if switch_limit in (None, "") else switch_limit - _emit(f" Estimated switch PPS limit: {limit_label}") - _emit(f" Estimated max leaf PPS: {max_leaf_pps}") - _emit(f" Estimated max spine PPS: {max_spine_pps}") - _emit( + logger.info(f" Estimated switch PPS limit: {limit_label}") + logger.info(f" Estimated max leaf PPS: {max_leaf_pps}") + logger.info(f" Estimated max spine PPS: {max_spine_pps}") + logger.info( " Path mix: " f"cross-leaf={cross_leaf_flows}, intra-leaf={intra_leaf_flows}, " f"cross-leaf-ratio={cross_leaf_ratio}" @@ -58,12 +56,11 @@ def setup_traffic(runner, scale: str, profile: str) -> dict: leafs = switch_pps.get("leafs", {}) spines = switch_pps.get("spines", {}) if leafs: - _emit(f" Leaf PPS breakdown: {leafs}") + logger.debug("Leaf PPS breakdown: %s", leafs) if spines: - _emit(f" Spine PPS breakdown: {spines}") + logger.debug("Spine PPS breakdown: %s", spines) - with open(topology_file, encoding="utf-8") as f: - topology = json.load(f) + topology = load_topology_manifest(topology_file).to_agent_topology() lab_name = topology.get("name", "dcn") container_names = {} @@ -90,8 +87,9 @@ def setup_traffic(runner, scale: str, profile: str) -> dict: ) ) - _emit(f"\n[Traffic Start] Starting {len(flows)} flows...") + logger.info(f"\n[Traffic Start] Starting {len(flows)} flows...") runner.traffic_controller.start_matrix(flows) + traffic_config["runtime"] = runner.traffic_controller.last_start_stats.to_dict() return traffic_config @@ -99,6 +97,6 @@ def stop_traffic(runner) -> None: """Stop all active traffic flows.""" if runner.traffic_controller: - _emit("\n[Traffic Stop] Stopping all traffic flows...") + logger.info("\n[Traffic Stop] Stopping all traffic flows...") runner.traffic_controller.stop_all() runner.traffic_controller = None diff --git a/netopsbench/platform/traffic/settings.py b/netopsbench/platform/traffic/settings.py new file mode 100644 index 0000000..10f15ae --- /dev/null +++ b/netopsbench/platform/traffic/settings.py @@ -0,0 +1,36 @@ +"""Environment boundary for traffic runtime settings.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +DEFAULT_TRAFFIC_PARALLELISM = 32 +DEFAULT_SWITCH_PPS_LIMIT = 5000 + + +def parse_parallelism(value: str | None, default: int = DEFAULT_TRAFFIC_PARALLELISM) -> int: + try: + return max(1, int(str(value or default).strip())) + except (TypeError, ValueError): + return default + + +@dataclass(frozen=True) +class TrafficSettings: + parallelism: int = DEFAULT_TRAFFIC_PARALLELISM + switch_pps_limit: int | None = DEFAULT_SWITCH_PPS_LIMIT + + @classmethod + def from_env(cls) -> TrafficSettings: + return cls( + parallelism=parse_parallelism(os.environ.get("NETOPSBENCH_TRAFFIC_PARALLELISM")), + ) + + +__all__ = [ + "DEFAULT_SWITCH_PPS_LIMIT", + "DEFAULT_TRAFFIC_PARALLELISM", + "TrafficSettings", + "parse_parallelism", +] diff --git a/netopsbench/platform/utils/__init__.py b/netopsbench/platform/utils/__init__.py index e23b79f..478206b 100644 --- a/netopsbench/platform/utils/__init__.py +++ b/netopsbench/platform/utils/__init__.py @@ -1,19 +1 @@ -"""Platform utility subsystem.""" - -from .interface_names import ( - are_interfaces_equivalent, - interface_aliases, - normalize_interface_name, - to_linux_interface, - to_sonic_interface, -) -from .proc import sudo_prefix - -__all__ = [ - "are_interfaces_equivalent", - "interface_aliases", - "normalize_interface_name", - "to_linux_interface", - "to_sonic_interface", - "sudo_prefix", -] +"""Small helpers shared by platform subsystems.""" diff --git a/netopsbench/platform/utils/events.py b/netopsbench/platform/utils/events.py deleted file mode 100644 index 3fc5005..0000000 --- a/netopsbench/platform/utils/events.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Unified structured-logging helper for platform modules.""" - -from __future__ import annotations - -import logging - - -def emit(*args, level: str = "info", logger: logging.Logger | None = None, **kwargs) -> None: - """Log a message at the given level, joining all positional args with spaces.""" - target = logger or logging.getLogger(__name__) - message = " ".join(str(arg) for arg in args) - getattr(target, level, target.info)(message) diff --git a/netopsbench/platform/utils/interface_names.py b/netopsbench/platform/utils/interface_names.py index 29c02a0..191bfcb 100644 --- a/netopsbench/platform/utils/interface_names.py +++ b/netopsbench/platform/utils/interface_names.py @@ -106,3 +106,11 @@ def are_interfaces_equivalent(left: str | None, right: str | None) -> bool: if normalize_interface_name(left) == normalize_interface_name(right): return True return bool(interface_aliases(left) & interface_aliases(right)) + + +def resolve_interface_metric_identities(interface_name: str | None) -> dict[str, list[str]]: + """Return equivalent interface tags and gNMI counter paths.""" + names = sorted(interface_aliases(interface_name)) + sonic_names = sorted(name for name in names if _SONIC_INTERFACE_RE.fullmatch(name)) + paths = [path for name in sonic_names for path in (f"COUNTERS/{name}", f"/COUNTERS/{name}")] + return {"names": names, "paths": paths} diff --git a/netopsbench/platform/utils/proc.py b/netopsbench/platform/utils/proc.py index 8ba73e1..a10c092 100644 --- a/netopsbench/platform/utils/proc.py +++ b/netopsbench/platform/utils/proc.py @@ -6,6 +6,7 @@ import os import subprocess from collections.abc import Sequence +from pathlib import Path logger = logging.getLogger(__name__) @@ -19,8 +20,8 @@ def sudo_prefix() -> list[str]: """Return the sudo prefix for privileged subprocess calls. - By default returns ``["sudo", "-n"]`` so that Docker and Containerlab - commands work on machines where the user is not in the ``docker`` group. + By default returns ``["sudo", "-n"]`` for Containerlab and privileged + host operations. Set the environment variable ``NETOPSBENCH_NO_SUDO=1`` to suppress the prefix (e.g. in CI containers where the process already runs as root or @@ -31,6 +32,21 @@ def sudo_prefix() -> list[str]: return ["sudo", "-n"] +def docker_prefix() -> list[str]: + """Return a privilege prefix only when the Docker socket requires it. + + Docker commands dominate the large-topology control path. Users with + direct access to the local Docker socket should not pay for a separate + sudo/PAM process on every ``docker exec``. + """ + if os.environ.get("NETOPSBENCH_NO_SUDO", "").strip() == "1" or os.geteuid() == 0: + return [] + socket = Path("/var/run/docker.sock") + if socket.exists() and os.access(socket, os.R_OK | os.W_OK): + return [] + return ["sudo", "-n"] + + def safe_run( cmd: Sequence[str], *, diff --git a/netopsbench/platform/utils/result.py b/netopsbench/platform/utils/result.py deleted file mode 100644 index 8c845c7..0000000 --- a/netopsbench/platform/utils/result.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Unified result type for platform operations.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - - -@dataclass -class OperationResult: - """Standard result for platform operations that may succeed or fail. - - Use for orchestration-level operations (topology reload, deployment, etc.). - Fault handlers return richer domain-specific dicts and are not required - to use this type. - - Supports dict-style access (``result["success"]``) for backward - compatibility with call sites that previously consumed raw dicts. - """ - - success: bool - error: str | None = None - data: dict[str, Any] = field(default_factory=dict) - - def get(self, key: str, default: Any = None) -> Any: - if key == "success": - return self.success - if key == "error": - return self.error - return self.data.get(key, default) - - def __getitem__(self, key: str) -> Any: - if key == "success": - return self.success - if key == "error": - return self.error - if key in self.data: - return self.data[key] - raise KeyError(key) - - def to_dict(self) -> dict[str, Any]: - payload = {"success": self.success, "error": self.error} - payload.update(dict(self.data or {})) - return payload diff --git a/netopsbench/platform/worker/__init__.py b/netopsbench/platform/worker/__init__.py deleted file mode 100644 index f848349..0000000 --- a/netopsbench/platform/worker/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Internal helpers for worker runtime lifecycle.""" diff --git a/netopsbench/platform/worker/common.py b/netopsbench/platform/worker/common.py deleted file mode 100644 index f903b3d..0000000 --- a/netopsbench/platform/worker/common.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Shared helpers for worker runtime orchestration.""" - -from __future__ import annotations - -from netopsbench.logging_utils import get_logger - -logger = get_logger(__name__) - - -class WorkerRuntimeError(RuntimeError): - """Raised for user-facing worker runtime setup errors.""" - - -# Alias retained for existing imports. -ParallelBenchmarkError = WorkerRuntimeError - - -def _safe_label(value: str) -> str: - text = (value or "unknown").strip().lower() - return "".join(ch if ch.isalnum() or ch in {"-", "_"} else "_" for ch in text) or "unknown" - - -# Third-octet base for worker management subnets: 172.31.{base+worker_idx}.0/24 -# Each scale tier gets a 20-address range so workers from different scales -# never overlap even if multiple tiers are deployed simultaneously. -# xs: 172.31.100-119.0/24 -# small: 172.31.120-139.0/24 -# medium: 172.31.140-159.0/24 -# large: 172.31.160-179.0/24 -_SCALE_SUBNET_BASE = { - "xs": 100, - "small": 120, - "medium": 140, - "large": 160, -} diff --git a/netopsbench/platform/worker/health.py b/netopsbench/platform/worker/health.py deleted file mode 100644 index f99bea2..0000000 --- a/netopsbench/platform/worker/health.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Worker health check: container count, BGP convergence, connectivity, Pingmesh, observability. - -Replaces ``scripts/runtime/check_worker_health.sh`` with a pure-Python -implementation providing structured error reporting and retry logic. - -CLI usage:: - - python -m netopsbench.platform.worker.health - -Programmatic usage:: - - from netopsbench.platform.worker.health import check_worker_health - errors = check_worker_health("/path/to/topology_dir") -""" - -from __future__ import annotations - -import json -import os -import re -import subprocess -import sys -import time -from collections.abc import Sequence - -from netopsbench.config import config -from netopsbench.logging_utils import get_logger -from netopsbench.platform.topology.topology_utils import clab_container_name -from netopsbench.platform.utils.proc import safe_run, sudo_prefix - -logger = get_logger(__name__) - - -def _docker_exec(container: str, *cmd: str, check: bool = False, capture: bool = True) -> subprocess.CompletedProcess: - prefix = sudo_prefix() - return safe_run( - [*prefix, "docker", "exec", container, *cmd], - check=check, - capture_output=capture, - text=True, - timeout=60, - ) - - -def _running_container_count(lab_name: str) -> int: - result = safe_run( - [*sudo_prefix(), "docker", "ps", "--filter", f"label=containerlab={lab_name}", "--format", "{{.Names}}"], - capture_output=True, - text=True, - check=False, - timeout=30, - ) - return len([line for line in result.stdout.strip().splitlines() if line]) - - -def _load_topology_metadata(topology_dir: str) -> dict: - path = os.path.join(topology_dir, "topology.json") - if not os.path.isfile(path): - raise FileNotFoundError(f"topology metadata not found: {path}") - with open(path, encoding="utf-8") as fh: - return json.load(fh) - - -def _parse_bgp_established(bgp_output: str) -> int: - """Count established BGP sessions from ``vtysh -c 'show ip bgp summary'``.""" - count = 0 - for line in bgp_output.splitlines(): - parts = line.split() - if not parts: - continue - # First field is neighbor IP (a.b.c.d), 10th field is state/pfxrcd (int if established) - if re.match(r"^\d+\.\d+\.\d+\.\d+$", parts[0]) and len(parts) >= 10: - try: - int(parts[9]) - count += 1 - except ValueError: - pass - return count - - -def check_worker_health( - topology_dir: str, - influxdb_url: str | None = None, - influxdb_token: str | None = None, - influxdb_org: str | None = None, - influxdb_bucket: str | None = None, - health_retries: int | None = None, - health_delay: int | None = None, -) -> list[str]: - """Run all health checks and return a list of error messages (empty = healthy). - - Checks performed: - 1. Worker telegraf container is running - 2. Expected container count matches running containers - 3. BGP convergence on spine1 - 4. Client-to-client connectivity + Pingmesh agent - 5. InfluxDB observability path (via validation module) - """ - errors: list[str] = [] - topo = _load_topology_metadata(topology_dir) - lab_name = (topo.get("name") or "dcn").strip() - devices = topo.get("devices", {}) or {} - spines = devices.get("spines", []) or [] - leafs = devices.get("leafs", []) or [] - clients = devices.get("clients", []) or [] - collector_ip = ((topo.get("collector", {}) or {}).get("ipv4") or "").strip() - - topology_id = config.topology_id or os.path.basename(topology_dir) - influxdb_url = influxdb_url or config.influxdb_url - influxdb_token = influxdb_token or config.influxdb_token - influxdb_org = influxdb_org or config.influxdb_org - - telegraf_config = os.path.join(topology_dir, f"telegraf-{lab_name}.conf") - if influxdb_bucket is None: - influxdb_bucket = config.influxdb_bucket - if not influxdb_bucket and os.path.isfile(telegraf_config): - # Extract bucket from telegraf config (awk equivalent) - with open(telegraf_config, encoding="utf-8") as fh: - for line in fh: - match = re.match(r'\s*bucket\s*=\s*"([^"]+)"', line) - if match: - influxdb_bucket = match.group(1) - break - influxdb_bucket = influxdb_bucket or "netopsbench" - - retries = health_retries or config.worker_health_retries - delay = health_delay or config.worker_health_delay_seconds - - if len(clients) < 2: - raise RuntimeError("need at least two clients in topology metadata for health check") - - logger.info("=== Worker Health Check ===") - logger.info("Lab name: %s", lab_name) - logger.info("Topology dir: %s", topology_dir) - - # [1/5] Worker telegraf - logger.info("[1/5] Checking worker telegraf...") - telegraf_container = f"telegraf-{lab_name}" - ret = safe_run( - [*sudo_prefix(), "docker", "ps", "--format", "{{.Names}}"], - capture_output=True, - text=True, - check=False, - timeout=30, - ) - if telegraf_container not in ret.stdout.strip().splitlines(): - errors.append(f"worker telegraf container is not running: {telegraf_container}") - # Return early — remaining checks depend on the infra - return errors - - # [2/5] Container count - logger.info("[2/5] Checking container count...") - expected_nodes = len(spines) + len(leafs) + len(clients) - running_nodes = _running_container_count(lab_name) - if running_nodes < expected_nodes: - errors.append( - f"running node count mismatch for {lab_name}: " f"got {running_nodes} expected at least {expected_nodes}" - ) - return errors - - # [3/5] BGP convergence - logger.info("[3/5] Checking BGP convergence on spine1...") - spine_container = clab_container_name(lab_name, "spine1") - leaf_count = len(leafs) - established = 0 - for _ in range(retries): - ret = _docker_exec(spine_container, "vtysh", "-c", "show ip bgp summary") - established = _parse_bgp_established(ret.stdout or "") - if established >= leaf_count: - break - time.sleep(delay) - if established < leaf_count: - errors.append(f"BGP not converged on {spine_container}: " f"established={established} expected>={leaf_count}") - return errors - - # [4/5] Client connectivity + Pingmesh agent - logger.info("[4/5] Checking client connectivity and Pingmesh agent...") - src_client = clients[0] - src_name = str(src_client.get("name", "")) - src_leaf = str(src_client.get("leaf", "")) - dst_ip = "" - # Prefer cross-rack destination - for other in clients[1:]: - other_ip = str(other.get("data_ip", "")).strip() - other_leaf = str(other.get("leaf", "")).strip() - if other_leaf != src_leaf and other_ip: - dst_ip = other_ip - break - if not dst_ip and other_ip: - dst_ip = other_ip - if not dst_ip: - errors.append("could not determine destination client IP for health check") - return errors - - src_container = clab_container_name(lab_name, src_name) - connectivity_ok = False - for _ in range(retries): - ret = _docker_exec(src_container, "ping", "-c", "1", "-W", "2", dst_ip) - if ret.returncode == 0: - connectivity_ok = True - break - time.sleep(delay) - if not connectivity_ok: - errors.append(f"client connectivity failed from {src_container} to {dst_ip}") - return errors - - agent_running = False - for _ in range(retries): - ret = _docker_exec(src_container, "ps", "aux") - if "run_pingmesh_agent.py" in (ret.stdout or ""): - agent_running = True - break - time.sleep(delay) - if not agent_running: - errors.append(f"Pingmesh agent is not running in {src_container}") - return errors - - # [5/5] InfluxDB observability path - logger.info("[5/5] Checking InfluxDB observability path...") - obs_device = "leaf1" if leaf_count > 0 else "spine1" - obs_container = clab_container_name(lab_name, obs_device) - - # Get active interfaces - ret = _docker_exec(obs_container, "bash", "-lc", "show interfaces status") - active_ifaces: list[str] = [] - for line in (ret.stdout or "").splitlines(): - parts = line.split() - if parts and parts[0].startswith("Ethernet") and len(parts) >= 9: - if parts[7].lower() == "up" and parts[8].lower() == "up": - active_ifaces.append(parts[0]) - - # Send syslog marker - syslog_marker = f"NETOPSBENCH_HEALTH_{lab_name}_{int(time.time())}" - if collector_ip: - _docker_exec( - clab_container_name(lab_name, "spine1"), - "bash", - "-lc", - f"logger -n '{collector_ip}' -P 514 -d '{syslog_marker}'", - ) - - # Delegate to the observability validation module - from netopsbench.platform.observability.validation import check_observability, run_query - - def query_runner(query: str) -> str: - return run_query(influxdb_url, influxdb_token, influxdb_org, query) - - obs_errors = check_observability( - query_runner, - bucket=influxdb_bucket, - obs_device=obs_device, - bgp_device="spine1", - topology_id=topology_id, - syslog_marker=syslog_marker, - active_interfaces=active_ifaces, - min_active_coverage_ratio=config.active_interface_coverage_min_ratio, - ) - errors.extend(obs_errors) - - if not errors: - logger.info("Worker health check passed: %s", lab_name) - return errors - - -def main(argv: Sequence[str] | None = None) -> int: - """CLI entry point (backward-compatible with shell script args).""" - args = list(argv if argv is not None else sys.argv[1:]) - if not args: - print("usage: python -m netopsbench.platform.worker.health ", file=sys.stderr) - return 1 - - topology_dir = args[0] - try: - errors = check_worker_health(topology_dir) - except (FileNotFoundError, RuntimeError) as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 1 - except Exception as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 1 - - if errors: - for err in errors: - print(f"ERROR: {err}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/netopsbench/platform/worker/lifecycle.py b/netopsbench/platform/worker/lifecycle.py deleted file mode 100644 index f999448..0000000 --- a/netopsbench/platform/worker/lifecycle.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Worker deployment, reuse, and teardown helpers.""" - -from __future__ import annotations - -import logging -import os -import subprocess -from collections.abc import Sequence -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path - -from netopsbench.config import config -from netopsbench.platform.utils.events import emit as _emit -from netopsbench.platform.utils.proc import safe_run -from netopsbench.platform.worker.pool import WorkerSpec - -logger = logging.getLogger(__name__) - - -def _parallel_job_count(env_var: str, total: int, default: int | None = None) -> int: - resolved_default = total if default is None else default - raw_value = str(os.environ.get(env_var, "")).strip() - if raw_value: - try: - resolved_default = int(raw_value) - except ValueError: - resolved_default = default or total - return max(1, min(total, resolved_default)) - - -def _build_worker_env(worker: WorkerSpec, repo_root: str | None = None) -> dict[str, str]: - env = os.environ.copy() - env.update( - { - "NETOPSBENCH_TOPOLOGY_DIR": worker.topology_dir, - "NETOPSBENCH_TOPOLOGY_ID": Path(worker.topology_dir).name, - "NETOPSBENCH_INFLUXDB_URL": env.get("NETOPSBENCH_INFLUXDB_URL", config.influxdb_url), - "NETOPSBENCH_INFLUXDB_TOKEN": env.get("NETOPSBENCH_INFLUXDB_TOKEN", config.influxdb_token), - "NETOPSBENCH_INFLUXDB_ORG": env.get("NETOPSBENCH_INFLUXDB_ORG", config.influxdb_org), - "NETOPSBENCH_INFLUXDB_BUCKET": worker.bucket, - "NETOPSBENCH_RUN_ID_SUFFIX": f"worker_{worker.index:02d}", - } - ) - if repo_root: - existing_pythonpath = str(env.get("PYTHONPATH", "")).strip() - pythonpath_parts = [repo_root] - if existing_pythonpath: - pythonpath_parts.append(existing_pythonpath) - env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts) - worker_timeout = str(env.get("NETOPSBENCH_WORKER_AGENT_TIMEOUT_SECONDS", "")).strip() - if worker_timeout: - env["NETOPSBENCH_AGENT_TIMEOUT_SECONDS"] = worker_timeout - else: - env.setdefault("NETOPSBENCH_AGENT_TIMEOUT_SECONDS", "600") - disable_langsmith = str(env.get("NETOPSBENCH_WORKER_DISABLE_LANGSMITH", "")).strip().lower() - if disable_langsmith in {"1", "true", "yes", "on"}: - env["LANGSMITH_API_KEY"] = "" - env["LANGSMITH_TRACING"] = "false" - env["LANGCHAIN_TRACING_V2"] = "false" - return env - - -def _ensure_observability_core(repo_root: str) -> None: - safe_run( - ["bash", "scripts/observability/start_observability_core.sh"], - cwd=repo_root, - check=True, - timeout=600, - ) - - -def _append_worker_log_header(path: str, label: str) -> None: - with open(path, "a", encoding="utf-8") as handle: - handle.write(f"\n=== {label} ===\n") - - -def deploy_workers( - workers: Sequence[WorkerSpec], scale: str, repo_root: str, observability_core_ready: bool = False -) -> None: - if not workers: - return - if not observability_core_ready: - _ensure_observability_core(repo_root) - job_count = _parallel_job_count("NETOPSBENCH_WORKER_DEPLOY_JOBS", len(workers)) - if job_count <= 1: - for worker in workers: - deploy_worker(worker, scale, repo_root, total_workers=len(workers), skip_observability_core_start=True) - return - _emit(f"Deploying {len(workers)} workers with {job_count} concurrent job(s)") - failures: list[tuple[WorkerSpec, Exception]] = [] - with ThreadPoolExecutor(max_workers=job_count) as executor: - future_map = { - executor.submit(deploy_worker, worker, scale, repo_root, len(workers), True): worker for worker in workers - } - for future in as_completed(future_map): - worker = future_map[future] - try: - future.result() - except Exception as exc: - failures.append((worker, exc)) - if failures: - worker, exc = failures[0] - _emit(f"ERROR: Worker deployment failed for {worker.lab_name}; see {worker.deploy_log_path}") - raise exc - - -def validate_worker_health(worker: WorkerSpec, repo_root: str) -> None: - from netopsbench.platform.worker.health import check_worker_health - - _append_worker_log_header(worker.deploy_log_path, "worker health validation") - errors = check_worker_health(worker.topology_dir) - if errors: - msg = "; ".join(errors) - with open(worker.deploy_log_path, "a", encoding="utf-8") as log_fp: - log_fp.write(f"Health check errors: {msg}\n") - raise RuntimeError(f"Worker health check failed: {msg}") - - -def ensure_worker_pool_ready(workers: Sequence[WorkerSpec], scale: str, repo_root: str) -> dict[str, int]: - if not workers: - return {"reused": 0, "redeployed": 0} - _ensure_observability_core(repo_root) - job_count = _parallel_job_count("NETOPSBENCH_WORKER_REUSE_JOBS", len(workers)) - unhealthy: list[WorkerSpec] = [] - if job_count > 1: - _emit(f"Revalidating {len(workers)} prepared workers with {job_count} concurrent job(s)") - with ThreadPoolExecutor(max_workers=job_count) as executor: - future_map = {executor.submit(validate_worker_health, worker, repo_root): worker for worker in workers} - for future in as_completed(future_map): - worker = future_map[future] - try: - future.result() - worker.reused_existing = True - worker.redeployed = False - _emit(f"[Worker Reuse Ready] {worker.lab_name}") - except Exception: - logger.warning("Worker %s health check failed; marking for redeploy", worker.lab_name, exc_info=True) - worker.reused_existing = False - unhealthy.append(worker) - if not unhealthy: - return {"reused": len(workers), "redeployed": 0} - _emit(f"Worker pool health check failed for {len(unhealthy)} worker(s); redeploying only the unhealthy set") - for worker in unhealthy: - worker.redeployed = True - deploy_workers(unhealthy, scale, repo_root, observability_core_ready=True) - return {"reused": len(workers) - len(unhealthy), "redeployed": len(unhealthy)} - - -def deploy_worker( - worker: WorkerSpec, - scale: str, - repo_root: str, - total_workers: int | None = None, - skip_observability_core_start: bool = False, -) -> None: - total = total_workers or 1 - _emit( - f"[Worker Deploy {worker.index}/{total}] {worker.lab_name} subnet={worker.mgmt_subnet} bucket={worker.bucket} scenarios={len(worker.scenarios)}" - ) - env = _build_worker_env(worker, repo_root=repo_root) - if skip_observability_core_start: - env["NETOPSBENCH_SKIP_OBSERVABILITY_CORE_START"] = "1" - _append_worker_log_header(worker.deploy_log_path, f"worker deploy {worker.lab_name}") - try: - with open(worker.deploy_log_path, "a", encoding="utf-8") as log_fp: - safe_run( - [ - "bash", - "scripts/runtime/deploy_worker.sh", - scale, - worker.topology_dir, - worker.lab_name, - worker.mgmt_subnet, - worker.bucket, - ], - cwd=repo_root, - env=env, - stdout=log_fp, - stderr=subprocess.STDOUT, - check=True, - timeout=1800, - ) - except subprocess.CalledProcessError: - _emit(f"[Worker Deploy Failed] {worker.lab_name} (see {worker.deploy_log_path})") - raise - - -def teardown_workers(workers: Sequence[WorkerSpec], repo_root: str) -> None: - for worker in workers: - teardown_worker(worker, repo_root) - - -def teardown_worker(worker: WorkerSpec, repo_root: str) -> None: - try: - safe_run( - ["bash", "scripts/runtime/teardown_worker.sh", worker.topology_dir], - cwd=repo_root, - check=False, - timeout=600, - ) - except Exception: - logger.warning("teardown_worker.sh failed for %s", worker.lab_name, exc_info=True) - return diff --git a/netopsbench/platform/worker/pool.py b/netopsbench/platform/worker/pool.py deleted file mode 100644 index 63cb7ec..0000000 --- a/netopsbench/platform/worker/pool.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Worker pool runtime helpers.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from netopsbench.platform.worker.common import ( - _SCALE_SUBNET_BASE, - WorkerRuntimeError, - _safe_label, -) - - -@dataclass -class WorkerSpec: - """Runtime description for one isolated worker lab.""" - - index: int - lab_name: str - topology_dir: str - mgmt_subnet: str - bucket: str - shard_dir: str - report_path: str - log_path: str - deploy_log_path: str - id: str | None = None - name: str | None = None - root_dir: Path | None = None - reuse_source: str | None = None - reused_existing: bool = False - redeployed: bool = False - scenarios: list[str] = field(default_factory=list) - metadata: dict[str, Any] = field(default_factory=dict) - - def __post_init__(self) -> None: - if self.id is None: - self.id = self.name or f"worker-{self.index}" - if self.name is None: - self.name = self.id - if self.root_dir is not None and not isinstance(self.root_dir, Path): - self.root_dir = Path(self.root_dir) - - -def _worker_bucket(scale: str, worker_index: int) -> str: - return f"network_data_{_safe_label(scale)}_w{worker_index:02d}" - - -def _worker_mgmt_subnet(scale: str, worker_index: int) -> str: - third_octet = _SCALE_SUBNET_BASE.get(scale, 200) + worker_index - if third_octet > 254: - raise WorkerRuntimeError(f"Worker index {worker_index} exceeds available management subnet range") - return f"172.31.{third_octet}.0/24" diff --git a/netopsbench/platform/worker/runtime_agent_input.py b/netopsbench/platform/worker/runtime_agent_input.py deleted file mode 100644 index 17735a7..0000000 --- a/netopsbench/platform/worker/runtime_agent_input.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Runtime agent input contract helpers. - -This module builds the public diagnostic payload passed to agents during -runtime execution. It intentionally strips any fault-injection labels/targets -so agents only see observable symptoms. -""" - -from __future__ import annotations - -import hashlib -from typing import Any - -_EPISODE_ALLOWED_KEYS = { - "episode_id", - "duration_seconds", - "stabilization_time", -} - - -def build_public_case_id(*, scenario_id: str, episode_result: dict[str, Any]) -> str: - """Return a stable, non-semantic case id for agent context.""" - episode = episode_result.get("episode", {}) if isinstance(episode_result, dict) else {} - episode_id = episode.get("episode_id") if isinstance(episode, dict) else None - source = f"{scenario_id}:{episode_id or 'unknown'}" - digest = hashlib.sha1(source.encode("utf-8")).hexdigest()[:12] - return f"case-{digest}" - - -def build_public_symptoms(*, episode_result: dict[str, Any], pingmesh_query_window: dict[str, Any]) -> dict[str, Any]: - """Build sanitized symptoms payload for agents.""" - episode = episode_result.get("episode", {}) if isinstance(episode_result, dict) else {} - observations = episode_result.get("observations", {}) if isinstance(episode_result, dict) else {} - - safe_episode = {} - if isinstance(episode, dict): - safe_episode = {key: episode.get(key) for key in _EPISODE_ALLOWED_KEYS if key in episode} - - return { - "episode": safe_episode, - "observations": observations if isinstance(observations, dict) else {}, - "pingmesh_query_window": pingmesh_query_window if isinstance(pingmesh_query_window, dict) else {}, - "observation_type": "scenario_episode", - } diff --git a/netopsbench/sdk/__init__.py b/netopsbench/sdk/__init__.py index 1880405..ac091af 100644 --- a/netopsbench/sdk/__init__.py +++ b/netopsbench/sdk/__init__.py @@ -9,6 +9,7 @@ "EpisodeSpec": ("netopsbench.sdk.types", "EpisodeSpec"), "ScenarioHandle": ("netopsbench.sdk.scenarios", "ScenarioHandle"), "ScenarioManager": ("netopsbench.sdk.scenarios", "ScenarioManager"), + "supported_scales": ("netopsbench.sdk.scenarios", "supported_scales"), "ScenarioEvaluator": ("netopsbench.sdk.types", "ScenarioEvaluator"), "DiagnosticAgent": ("netopsbench.sdk.agents", "DiagnosticAgent"), "SyncDiagnosticAgent": ("netopsbench.sdk.agents", "SyncDiagnosticAgent"), @@ -31,7 +32,7 @@ "simple_fault": ("netopsbench.sdk.faults", "simple_fault"), "RuntimeManager": ("netopsbench.sdk.runtimes", "RuntimeManager"), "RuntimePool": ("netopsbench.sdk.runtimes", "RuntimePool"), - "RuntimeWorker": ("netopsbench.sdk.runtimes", "RuntimeWorker"), + "RuntimeIdentity": ("netopsbench.sdk.runtimes", "RuntimeIdentity"), "SessionManager": ("netopsbench.sdk.sessions", "SessionManager"), "RunHandle": ("netopsbench.sdk.reports", "RunHandle"), "ArtifactManager": ("netopsbench.sdk.artifacts", "ArtifactManager"), diff --git a/netopsbench/sdk/artifacts.py b/netopsbench/sdk/artifacts.py index 3b03a43..1bedffa 100644 --- a/netopsbench/sdk/artifacts.py +++ b/netopsbench/sdk/artifacts.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from netopsbench.platform.session.tracing import export_traces, load_trace_index, load_trace_results +from netopsbench.platform.session.harbor_export import export_traces, load_trace_index, load_trace_results class ArtifactManager: diff --git a/netopsbench/sdk/core.py b/netopsbench/sdk/core.py index 0ffd002..d2b7463 100644 --- a/netopsbench/sdk/core.py +++ b/netopsbench/sdk/core.py @@ -1,8 +1,6 @@ """Public NetOpsBench SDK root.""" import logging -import os -from collections.abc import Mapping from pathlib import Path from typing import Any @@ -36,18 +34,9 @@ class NetOpsBench: def __init__( self, workspace: str = ".", - defaults: Mapping[str, object] | None = None, - env: Mapping[str, str] | None = None, - auto_load_env: bool = True, ): configure_logging() self.workspace = Path(workspace) - self.defaults = dict(defaults or {}) - self.auto_load_env = auto_load_env - if env is None and auto_load_env: - self.env = dict(os.environ) - else: - self.env = dict(env or {}) self._closed = False self.scenarios = self._bind_manager(ScenarioManager(workspace=self.workspace), "scenarios") diff --git a/netopsbench/sdk/faults.py b/netopsbench/sdk/faults.py index 38e8b17..9803171 100644 --- a/netopsbench/sdk/faults.py +++ b/netopsbench/sdk/faults.py @@ -13,10 +13,7 @@ FaultExecutor, FaultPack, FaultSpec, - canonicalize_fault_name, - get_fault_spec, - load_builtin_fault_specs, - register_fault_spec, + create_fault_registry, ) from netopsbench.sdk.exceptions import ( FaultNotFoundError, @@ -166,6 +163,7 @@ class FaultManager: def __init__(self, workspace: str = "."): self.workspace = Path(workspace) + self.spec_registry = create_fault_registry() self._entries: dict[str, _FaultRecord] = {} self._builtin_loaded = False @@ -225,9 +223,8 @@ def _recover_active_fault(injector: Any, fault_info: dict[str, Any]) -> dict[str aliases=list(spec.aliases), scenario_supported=spec.scenario_supported, ) - register_fault_spec(patched_spec) - canonical_name = canonicalize_fault_name(spec.name) or spec.name - registered_spec = get_fault_spec(canonical_name) or patched_spec + registered_spec = self.spec_registry.register(patched_spec) + canonical_name = self.spec_registry.canonicalize(spec.name) or spec.name self._entries[canonical_name] = _FaultRecord(spec=registered_spec, executor=executor) def register_fault( @@ -274,8 +271,8 @@ def register_pack(self, pack: FaultPack) -> None: def load_builtin(self) -> None: if self._builtin_loaded: return - for spec in load_builtin_fault_specs(): - canonical_name = canonicalize_fault_name(spec.name) or spec.name + for spec in self.spec_registry.get_builtin_specs(): + canonical_name = self.spec_registry.canonicalize(spec.name) or spec.name if canonical_name in self._entries: continue self._entries[canonical_name] = _FaultRecord(spec=spec, executor=_StaticFaultExecutor(spec)) @@ -297,7 +294,7 @@ def list(self) -> builtins.list[FaultSpec]: return [record.spec for _, record in sorted(self._entries.items(), key=lambda item: item[0])] def get(self, name: str) -> FaultSpec: - canonical_name = canonicalize_fault_name(name) + canonical_name = self.spec_registry.canonicalize(name) if not canonical_name: raise FaultNotFoundError("fault name must be a non-empty string") record = self._entries.get(canonical_name) @@ -306,7 +303,7 @@ def get(self, name: str) -> FaultSpec: return record.spec def get_executor(self, name: str) -> FaultExecutor: - canonical_name = canonicalize_fault_name(name) + canonical_name = self.spec_registry.canonicalize(name) if not canonical_name: raise FaultNotFoundError("fault name must be a non-empty string") record = self._entries.get(canonical_name) @@ -315,7 +312,7 @@ def get_executor(self, name: str) -> FaultExecutor: return record.executor def validate_parameters(self, fault_type: str, parameters: dict[str, object]) -> builtins.list[str]: - canonical_name = canonicalize_fault_name(fault_type) + canonical_name = self.spec_registry.canonicalize(fault_type) if not canonical_name: return ["Unsupported fault type: "] record = self._entries.get(canonical_name) diff --git a/netopsbench/sdk/mcp.py b/netopsbench/sdk/mcp.py index cb2ac98..2ba5ab7 100644 --- a/netopsbench/sdk/mcp.py +++ b/netopsbench/sdk/mcp.py @@ -10,6 +10,34 @@ from pathlib import Path from typing import Any +_MCP_ENV_ALLOWLIST = frozenset( + { + "NETOPSBENCH_INFLUXDB_BUCKET", + "NETOPSBENCH_INFLUXDB_ORG", + "NETOPSBENCH_INFLUXDB_TOKEN", + "NETOPSBENCH_INFLUXDB_URL", + "NETOPSBENCH_LOG_LEVEL", + "NETOPSBENCH_NO_SUDO", + "NETOPSBENCH_PINGMESH_CONTEXT_FILE", + "NETOPSBENCH_PINGMESH_END_TIME", + "NETOPSBENCH_PINGMESH_START_TIME", + "NETOPSBENCH_TOPOLOGY_DIR", + "NETOPSBENCH_TOPOLOGY_ID", + } +) +_PROCESS_ENV_ALLOWLIST = frozenset( + { + "HOME", + "LANG", + "LC_ALL", + "LD_LIBRARY_PATH", + "PATH", + "TMPDIR", + "USER", + "VIRTUAL_ENV", + } +) + @dataclass class BuiltinMCPServerHandle: @@ -49,21 +77,20 @@ def builtin_mcp_server_config( ) -> dict[str, dict[str, Any]]: """Return stdio MCP server config used by NetOpsBench built-in tools.""" - repo_root = Path(workspace).resolve() - passthrough_env = {key: value for key, value in os.environ.items() if key.startswith("NETOPSBENCH_") and value} + resolved_workspace = Path(workspace).resolve() + passthrough_env = {key: os.environ[key] for key in _MCP_ENV_ALLOWLIST if os.environ.get(key)} if env: - passthrough_env.update({k: v for k, v in env.items() if isinstance(v, str) and v}) + passthrough_env.update( + {key: value for key, value in env.items() if key in _MCP_ENV_ALLOWLIST and isinstance(value, str) and value} + ) return { "netopsbench": { "transport": "stdio", "command": python_executable or sys.executable, - "args": ["scripts/toolkit/run_fastmcp_server.py"], - "cwd": str(repo_root), - "env": { - **passthrough_env, - "PYTHONPATH": str(repo_root), - }, + "args": ["-m", "netopsbench.platform.toolkit.fastmcp_server"], + "cwd": str(resolved_workspace), + "env": passthrough_env, } } @@ -94,10 +121,11 @@ def start_builtin_mcp_server( config = builtin_mcp_server_config(workspace=workspace, env=env, python_executable=python_executable) server = config["netopsbench"] + process_env = {key: os.environ[key] for key in _PROCESS_ENV_ALLOWLIST if os.environ.get(key)} process = subprocess.Popen( [server["command"], *server["args"]], cwd=server["cwd"], - env={**os.environ, **server["env"]}, + env={**process_env, **server["env"]}, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/netopsbench/sdk/reports.py b/netopsbench/sdk/reports.py index 97c564e..4b432cf 100644 --- a/netopsbench/sdk/reports.py +++ b/netopsbench/sdk/reports.py @@ -4,12 +4,15 @@ import json import textwrap -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from typing import Any +from netopsbench.models.profiles import supported_scales from netopsbench.sdk.exceptions import RunFailedError +SUPPORTED_REPORT_SCALES = set(supported_scales()) + class BenchmarkReport: """Thin serializable benchmark report wrapper.""" @@ -164,7 +167,7 @@ def _short_scenario_id(scenario_id: str) -> str: sid = sid[len("generated_") :] # Strip trailing scale + numeric suffix like "_xs_001". parts = sid.rsplit("_", 2) - if len(parts) == 3 and parts[-2] in {"xs", "small", "medium", "large"} and parts[-1].isdigit(): + if len(parts) == 3 and parts[-2] in SUPPORTED_REPORT_SCALES and parts[-1].isdigit(): sid = parts[0] return sid @@ -486,8 +489,8 @@ def __init__( self.id = id self.mode = mode self.status = status - self.started_at = started_at - self.completed_at = completed_at + self.started_at = _coerce_datetime(started_at) + self.completed_at = _coerce_datetime(completed_at) if completed_at is not None else None self.artifact_dir = str(artifact_dir) self.scenario_ids = list(scenario_ids) self.runtime_id = runtime_id @@ -543,13 +546,12 @@ def refresh(self) -> RunHandle: def cancel(self) -> None: if self.status not in {"completed", "failed", "cancelled"}: self.status = "cancelled" - self.completed_at = self.completed_at or datetime.now(self.started_at.tzinfo) + self.completed_at = self.completed_at or datetime.now(UTC) def _coerce_datetime(value: Any) -> datetime: - if isinstance(value, datetime): - return value - return datetime.fromisoformat(str(value)) + parsed = value if isinstance(value, datetime) else datetime.fromisoformat(str(value)) + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC) def _report_succeeded(report: BenchmarkReport) -> bool: diff --git a/netopsbench/sdk/runtimes.py b/netopsbench/sdk/runtimes.py index 089d48a..b205955 100644 --- a/netopsbench/sdk/runtimes.py +++ b/netopsbench/sdk/runtimes.py @@ -1,15 +1,13 @@ """Public runtime manager exports.""" +from netopsbench.models.runtime import RuntimeIdentity from netopsbench.platform.runtime.manager import RuntimeManager as _RuntimeManager from netopsbench.platform.runtime.manager import RuntimePool as _RuntimePool -from netopsbench.platform.worker.pool import WorkerSpec as _RuntimeWorker RuntimeManager = _RuntimeManager RuntimePool = _RuntimePool -RuntimeWorker = _RuntimeWorker RuntimeManager.__module__ = __name__ RuntimePool.__module__ = __name__ -RuntimeWorker.__module__ = __name__ -__all__ = ["RuntimeManager", "RuntimePool", "RuntimeWorker"] +__all__ = ["RuntimeIdentity", "RuntimeManager", "RuntimePool"] diff --git a/netopsbench/sdk/scenarios.py b/netopsbench/sdk/scenarios.py index 28351c6..8dde0bf 100644 --- a/netopsbench/sdk/scenarios.py +++ b/netopsbench/sdk/scenarios.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any +from netopsbench.models.profiles import supported_scales from netopsbench.platform.scenario.executor import Episode, Scenario from netopsbench.platform.scenario.parser import ( episode_from_dict, @@ -88,6 +89,8 @@ def create( metadata: dict[str, Any] | None = None, parameters: dict[str, Any] | None = None, ) -> ScenarioHandle: + if traffic_profile != "standard": + raise ValueError(f"Only the standard traffic profile is supported, got: {traffic_profile}") payload = { "scenario_id": id, "name": name, @@ -110,7 +113,9 @@ def save(self, handle: ScenarioHandle | Scenario, path: str | Path) -> Path: def validate(self, handle: ScenarioHandle | Scenario) -> list[str]: scenario = self._coerce_scenario(handle) - return validate_scenario(scenario) + fault_manager = getattr(getattr(self, "platform", None), "faults", None) + registry = getattr(fault_manager, "spec_registry", None) + return validate_scenario(scenario, fault_registry=registry) def _coerce_scenario(self, handle: ScenarioHandle | Scenario) -> Scenario: if isinstance(handle, ScenarioHandle): @@ -125,3 +130,6 @@ def _coerce_episode_data(self, value: dict[str, Any] | Episode) -> dict[str, Any if isinstance(value, dict): return episode_to_dict(episode_from_dict(value)) raise TypeError(f"Unsupported episode value: {type(value)!r}") + + +__all__ = ["ScenarioHandle", "ScenarioManager", "supported_scales"] diff --git a/netopsbench/sdk/types.py b/netopsbench/sdk/types.py index 2f8892a..0e68d6d 100644 --- a/netopsbench/sdk/types.py +++ b/netopsbench/sdk/types.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Protocol +from typing import Any, Literal, Protocol from netopsbench.agents.base import DiagnosticContext @@ -43,7 +43,7 @@ class ScenarioSpec: name: str description: str = "" topology_scale: str = "xs" - traffic_profile: str = "standard" + traffic_profile: Literal["standard"] = "standard" episodes: list[EpisodeSpec] = field(default_factory=list) metadata: dict[str, Any] = field(default_factory=dict) parameters: dict[str, Any] = field(default_factory=dict) diff --git a/pyproject.toml b/pyproject.toml index 76f9b2f..0348eed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,6 @@ dependencies = [ "requests>=2.25.0", "pyyaml>=5.4.0", "pydantic>=2.0.0", - "influxdb-client>=1.30.0", - "pygnmi>=0.8.0", - "scapy>=2.5.0", - "pandas>=1.5.0", "harbor>=0.13.1,<0.14", ] @@ -38,6 +34,9 @@ Issues = "https://github.com/NetX-lab/NetOpsBench/issues" [project.scripts] netopsbench = "netopsbench.cli.main:main" netopsbench-pingmesh-agent = "netopsbench.platform.pingmesh.cli:main" +netopsbench-bgp-collector = "netopsbench.platform.observability.bgp_collector:main" +netopsbench-mcp-server = "netopsbench.platform.toolkit.fastmcp_server:main" +netopsbench-runtime = "netopsbench.platform.runtime.cli:main" [project.optional-dependencies] agent = [ @@ -50,6 +49,9 @@ agent = [ "fastmcp>=2.0.0", "json-repair>=0.54.0", ] +plots = [ + "matplotlib>=3.8.0", +] dev = [ "pytest>=6.0.0", "pytest-asyncio>=0.21.0", @@ -63,6 +65,17 @@ dev = [ [tool.setuptools.packages.find] include = ["netopsbench*"] +[tool.setuptools.package-data] +netopsbench = [ + "platform/topology/*.json", + "platform/topology/*.sh", + "platform/observability/assets/**/*", + "platform/scenario/specs/*.yaml", +] + +[tool.setuptools.exclude-package-data] +"*" = ["**/__pycache__", "**/__pycache__/*", "**/*.pyc"] + [tool.pytest.ini_options] markers = [ "real: tests requiring deployed real environment and external services", @@ -82,7 +95,7 @@ known_first_party = ["netopsbench"] [tool.ruff] line-length = 120 target-version = "py312" -extend-exclude = ["lab-topology", "scenario_results", "scenarios/generated"] +extend-exclude = ["build", "lab-topology", "scenario_results", "scenarios/generated"] [tool.ruff.lint] select = ["E", "F", "W", "I", "B", "UP"] diff --git a/scenarios/README.md b/scenarios/README.md index 817015c..7414b39 100644 --- a/scenarios/README.md +++ b/scenarios/README.md @@ -2,9 +2,9 @@ This directory contains the benchmark scenario corpus for NetOpsBench. -Only the campaign specs under `scenarios/specs/` are checked into git. The -`scenarios/generated/` tree is a generated artifact and is absent on a fresh -clone until you create it locally. +The canonical campaign spec is packaged under +`netopsbench/platform/scenario/specs/`. The `scenarios/generated/` tree is a +generated artifact and is absent on a fresh clone until you create it locally. All scenarios are generated from campaign specs and organized under `scenarios/generated/` by topology scale: @@ -13,7 +13,7 @@ All scenarios are generated from campaign specs and organized under `scenarios/g - `scenarios/generated/medium/` - `scenarios/generated/large/` -Campaign spec files live under `scenarios/specs/` (e.g. `fault_campaign.yaml`). +Pass `--spec` to use a custom campaign instead of the packaged default. ## Supported Fault Types @@ -103,8 +103,8 @@ name: "Human Readable Name" description: | Detailed description of what this scenario tests -topology_scale: xs # xs, small, medium, large -traffic_profile: standard # light, standard, stress +topology_scale: xs # xs, small, medium, large, xlarge, fat-tree-k8, fat-tree-k12 +traffic_profile: standard # canonical background traffic model metadata: difficulty: easy # easy, medium, hard @@ -147,26 +147,18 @@ episodes: 4. **Recovery** - Restore to normal 5. **Verification** (optional) - Verify recovery -## Traffic Profiles +## Traffic Profile -### Light Profile -- 25% of link capacity -- Minimal flows per client -- Good for initial testing - -### Standard Profile (Default) -- 50% of link capacity -- Moderate number of flows -- Realistic production load - -### Stress Profile -- 80% of link capacity -- Maximum safe flows -- Tests under pressure +All benchmark scenarios use the canonical `standard` profile: at most four +continuous flows per client, split evenly between UDP and TCP. Keeping one +background traffic model avoids changing load as an untracked variable between +fault types. ## PPS Limits by Topology -Switch PPS limits are configurable via `NETOPSBENCH_SWITCH_PPS_LIMIT` (default: **5000**). +The canonical switch PPS limit is **5000**. Per-client PPS caps are part of the +scale profile so benchmark load cannot change through an undocumented process +environment override. Per-client PPS caps scale linearly from the 1000 PPS baseline: | Scale | Clients | Base Max PPS/Client (1000 PPS) | diff --git a/scripts/observability/start_observability_core.sh b/scripts/observability/start_observability_core.sh deleted file mode 100755 index 00b4bb0..0000000 --- a/scripts/observability/start_observability_core.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Start shared observability core services needed by parallel workers. - -set -e - -if [ "$(id -u)" -eq 0 ]; then - SUDO=() -else - SUDO=("sudo" "-n") -fi - -BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "$BASE_DIR/observability" - -echo "=== Starting Observability Core ===" -"${SUDO[@]}" docker compose up -d influxdb grafana diff --git a/scripts/observability/start_worker_telegraf.sh b/scripts/observability/start_worker_telegraf.sh deleted file mode 100755 index 6ddcdc4..0000000 --- a/scripts/observability/start_worker_telegraf.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# Start a worker-local telegraf instance attached to one lab management network. - -set -e - -TOPOLOGY_DIR=${1:?usage: start_worker_telegraf.sh [container_name]} -CONTAINER_NAME=${2:-} - -if [ "$(id -u)" -eq 0 ]; then - SUDO=() -else - SUDO=("sudo" "-n") -fi - -BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "$BASE_DIR" - -# Locate project Python interpreter (prefers repo venv, falls back to system python3). -# shellcheck source=scripts/lib/find_python.sh -source "$BASE_DIR/scripts/lib/find_python.sh" - -METADATA_FILE="$TOPOLOGY_DIR/topology.json" -if [ ! -f "$METADATA_FILE" ]; then - echo "ERROR: topology metadata not found: $METADATA_FILE" >&2 - exit 1 -fi - -readarray -t WORKER_META < <($PYTHON - <&2 - exit 1 -fi - -$PYTHON -m netopsbench.platform.observability.telegraf \ - "$METADATA_FILE" \ - --output "$CONFIG_PATH" \ - --influxdb-url "${NETOPSBENCH_TELEGRAF_INFLUXDB_URL:-http://influxdb:8086}" \ - --influxdb-token "${NETOPSBENCH_INFLUXDB_TOKEN:-replace-me}" \ - --influxdb-org "${NETOPSBENCH_INFLUXDB_ORG:-netopsbench}" \ - --bucket "${NETOPSBENCH_INFLUXDB_BUCKET:-netopsbench}" \ - --topology-id "${NETOPSBENCH_TOPOLOGY_ID:-$LAB_NAME}" - -: > "$BGP_FILE_PATH" -chmod 755 "$CONFIG_DIR" -chmod 644 "$CONFIG_PATH" "$BGP_FILE_PATH" - -"${SUDO[@]}" docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true -"${SUDO[@]}" docker run -d \ - --name "$CONTAINER_NAME" \ - --restart unless-stopped \ - --network "$MGMT_NETWORK" \ - --ip "$COLLECTOR_IP" \ - -v "$CONFIG_PATH:/etc/telegraf/telegraf.conf:ro" \ - -v "$CONFIG_DIR:/var/lib/netopsbench:ro" \ - telegraf:latest >/dev/null - -echo "Started worker telegraf: $CONTAINER_NAME" -echo " network: $MGMT_NETWORK" -echo " collector ip: $COLLECTOR_IP" diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh index 42e29b0..d6f98e6 100755 --- a/scripts/run_all_benchmarks.sh +++ b/scripts/run_all_benchmarks.sh @@ -12,6 +12,7 @@ # small: 3 workers (15 scenarios, 3×14=42 containers) # medium: 2 workers (28 scenarios, 2×28=56 containers) # large: 2 workers (52 scenarios, 2×84=168 containers) +# xlarge/fat-tree: 1 worker by default # # 跑完后汇总 CSV 写入 scenario_results/benchmark_summary_.csv # ───────────────────────────────────────────────────────────── @@ -34,11 +35,10 @@ declare -A SCALE_WORKERS=( [small]="${BENCH_WORKERS_SMALL:-3}" [medium]="${BENCH_WORKERS_MEDIUM:-2}" [large]="${BENCH_WORKERS_LARGE:-2}" + [xlarge]="${BENCH_WORKERS_XLARGE:-1}" + [fat-tree-k8]="${BENCH_WORKERS_FAT_TREE_K8:-1}" + [fat-tree-k12]="${BENCH_WORKERS_FAT_TREE_K12:-1}" ) -# Limit worker deployment fan-out to avoid simultaneous SONiC-VS boot pressure. -declare -A SCALE_DEPLOY_JOBS=( [xs]=2 [small]=2 [medium]=2 [large]=1 ) -# Health check BGP retries: large needs more time (16 leafs vs 4 for medium) -declare -A SCALE_HEALTH_RETRIES=( [xs]=12 [small]=12 [medium]=12 [large]=36 ) # Ordered list of scales to run (override with BENCH_SCALES="xs small") IFS=' ' read -ra SCALES <<< "${BENCH_SCALES:-xs small medium large}" @@ -121,9 +121,7 @@ for scale in "${SCALES[@]}"; do echo "[$(date '+%H:%M:%S')] >>> ${label}: workers=${workers}" list_run_dirs > "$runs_before" - if NETOPSBENCH_WORKER_DEPLOY_JOBS="${SCALE_DEPLOY_JOBS[$scale]:-${workers}}" \ - NETOPSBENCH_WORKER_HEALTH_RETRIES="${SCALE_HEALTH_RETRIES[$scale]:-12}" \ - PYTHONPATH="$REPO_ROOT" "$PYTHON" examples/03_run_scale_benchmark.py \ + if PYTHONPATH="$REPO_ROOT" "$PYTHON" examples/03_run_scale_benchmark.py \ --vendor "$VENDOR" --scale "$scale" --workers "$workers" \ > "$log_file" 2>&1; then echo "[$(date '+%H:%M:%S')] ✓ ${label} — 完成" @@ -236,7 +234,7 @@ for rj in report_paths: if not sid: raw_ids = (data.get("raw") or {}).get("scenario_ids") or [] sid = raw_ids[0] if raw_ids else "" - for sc in ("xs", "small", "medium", "large"): + for sc in ("xs", "small", "medium", "large", "xlarge", "fat-tree-k8", "fat-tree-k12"): if f"_{sc}_" in sid: scale = sc break @@ -272,7 +270,7 @@ if not reports: sys.exit(0) # Sort by vendor, scale -scale_order = {"xs": 0, "small": 1, "medium": 2, "large": 3} +scale_order = {"xs": 0, "small": 1, "medium": 2, "large": 3, "xlarge": 4, "fat-tree-k8": 5, "fat-tree-k12": 6} reports.sort(key=lambda r: (r["vendor"], scale_order.get(r["scale"], 99), r["started_at"])) # Write CSV diff --git a/scripts/runtime/deploy.sh b/scripts/runtime/deploy.sh index 1d18c1a..dd12e2d 100755 --- a/scripts/runtime/deploy.sh +++ b/scripts/runtime/deploy.sh @@ -1,169 +1,26 @@ #!/bin/bash -# NetOpsBench One-Command Deployment Script -# Deploys complete system: topology + observability stack - -set -e +# Thin standalone deployment wrapper over the Python worker lifecycle. +set -euo pipefail TOPO_SCALE=${1:-xs} TOPO_DIR=${2:-lab-topology} LAB_NAME=${NETOPSBENCH_LAB_NAME:-dcn} MGMT_SUBNET=${NETOPSBENCH_MGMT_SUBNET:-} MGMT_NETWORK=${NETOPSBENCH_MGMT_NETWORK:-} +INFLUXDB_BUCKET=${NETOPSBENCH_INFLUXDB_BUCKET:-netopsbench} -if [ "$(id -u)" -eq 0 ]; then - SUDO=() -else - # Non-interactive sudo: fail fast instead of prompting for password. - SUDO=("sudo" "-n") -fi - -BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "$BASE_DIR" - -# Locate project Python interpreter (prefers repo venv, falls back to system python3). -# Override via NETOPSBENCH_PYTHON env var if needed. -# shellcheck source=scripts/lib/find_python.sh -source "$BASE_DIR/scripts/lib/find_python.sh" - -echo "=== NetOpsBench Deployment Start ===" -echo "Topology scale: $TOPO_SCALE" -echo "Topology directory: $TOPO_DIR" -echo "Lab name: $LAB_NAME" -if [ -n "$MGMT_SUBNET" ]; then - echo "Management subnet override: $MGMT_SUBNET" -fi -echo "" - -# Resolve topology directory (always-generate mode). -# Rule: -# - If a base dir is provided, generated topology goes to: -# /generated_topology_ -# - If an explicit generated dir is provided, use it as-is. -ACTUAL_TOPO_DIR="$TOPO_DIR" if [ "$TOPO_DIR" = "generated_topology" ]; then ACTUAL_TOPO_DIR="lab-topology/generated_topology_${TOPO_SCALE}" -elif echo "$TOPO_DIR" | grep -Eq '(^|/)generated_topology_(xs|small|medium|large)$'; then +elif [[ "$TOPO_DIR" =~ (^|/)generated_topology_[^/]+$ ]]; then ACTUAL_TOPO_DIR="$TOPO_DIR" else ACTUAL_TOPO_DIR="$TOPO_DIR/generated_topology_${TOPO_SCALE}" fi -mkdir -p "$ACTUAL_TOPO_DIR" - -TOPOLOGY_FILE="$ACTUAL_TOPO_DIR/${LAB_NAME}.clab.yaml" - -# [1/7] Generate topology for requested scale -echo "[1/7] Generating topology (scale=$TOPO_SCALE) into: $ACTUAL_TOPO_DIR" -$PYTHON -c " -from netopsbench.platform.topology.generator import generate_topology -result = generate_topology( - '$TOPO_SCALE', - '$ACTUAL_TOPO_DIR', - name='$LAB_NAME', - mgmt_subnet='${MGMT_SUBNET}', - mgmt_network='${MGMT_NETWORK}', -) -print(f'Generated topology: {result[\"yaml_file\"]}') -print(f'Generated metadata: {result[\"metadata_file\"]}') -" -echo "" - -# [2/7] Deploy Containerlab topology -echo "[2/7] Deploying Containerlab topology..." -if [ ! -f "$TOPOLOGY_FILE" ]; then - echo "ERROR: Topology file not found: $TOPOLOGY_FILE" - exit 1 -fi - -"${SUDO[@]}" containerlab deploy -t "$TOPOLOGY_FILE" --reconfigure -echo "" - -# [2.5/7] Apply SONiC configurations using fast parallel application -echo "[2.5/7] Applying device configurations (SONiC)..." -$PYTHON -m netopsbench.platform.runtime.apply_configs "$ACTUAL_TOPO_DIR" "" "$LAB_NAME" -echo "" - -# [3/7] Generate or verify topology metadata -echo "[3/7] Checking topology metadata..." -METADATA_FILE="$ACTUAL_TOPO_DIR/topology.json" - -if [ ! -f "$METADATA_FILE" ]; then - echo " Topology metadata not found, generating from YAML..." - $PYTHON -c " -from netopsbench.platform.topology.metadata_generator import generate_metadata_file -generate_metadata_file('$TOPOLOGY_FILE', '$METADATA_FILE') -" - if [ ! -f "$METADATA_FILE" ]; then - echo "ERROR: Failed to generate topology metadata" - exit 1 - fi -else - echo " Using existing metadata: $METADATA_FILE" -fi -echo "" - -# [4/7] Generate Telegraf configuration -echo "[4/7] Generating Telegraf configuration..." -$PYTHON -m netopsbench.platform.observability.telegraf "$METADATA_FILE" -echo "" - -# [5/7] Start observability stack -echo "[5/7] Starting observability stack..." -cd observability -"${SUDO[@]}" docker compose up -d -cd "$BASE_DIR" -echo "" - -# Attach shared InfluxDB to the lab management network so lab containers can -# resolve and reach it via the stable hostname "influxdb". -MGMT_NETWORK_NAME=$($PYTHON - </dev/null 2>&1 || true - echo " Attached influxdb to $MGMT_NETWORK_NAME" - echo "" -fi - -# [6/7] Restart Telegraf to ensure proper data collection -echo "[6/7] Restarting Telegraf for configuration reload..." -"${SUDO[@]}" docker restart telegraf -sleep 5 -echo "" - -# [7/7] Deploy Pingmesh agents -echo "[7/7] Deploying Pingmesh agents..." -$PYTHON -m netopsbench.platform.pingmesh.deploy "$ACTUAL_TOPO_DIR/pinglist.json" "$ACTUAL_TOPO_DIR" -sleep 5 -echo "" - -# Verify deployment -echo "=== Deployment Verification ===" -echo "" -echo "Network devices:" -"${SUDO[@]}" docker ps --filter "name=clab-${LAB_NAME}" --format "table {{.Names}}\t{{.Status}}" | head -10 -echo "" -echo "Observability stack:" -"${SUDO[@]}" docker ps --filter "name=influxdb" --format "table {{.Names}}\t{{.Status}}" -"${SUDO[@]}" docker ps --filter "name=grafana" --format "table {{.Names}}\t{{.Status}}" -"${SUDO[@]}" docker ps --filter "name=telegraf" --format "table {{.Names}}\t{{.Status}}" -echo "" +BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=scripts/lib/find_python.sh +source "$BASE_DIR/scripts/lib/find_python.sh" -echo "=== Deployment Complete ===" -echo "" -echo "Access points:" -echo " Grafana: http://localhost:3000" -echo " InfluxDB: http://localhost:8086" -echo "" -echo "Next steps:" -echo " 1. Wait ~30s for Pingmesh metrics to start flowing" -echo " 2. Open Grafana to view dashboards and Pingmesh monitoring" -echo " 3. Export NETOPSBENCH_TOPOLOGY_DIR to this directory so AgentToolkit / SDK resolve topology.json" -echo " 4. Run scenarios via the Python SDK (see README + examples/integration/sdk_e2e_runtime_*.py)" -echo " 5. CLI (optional): netopsbench scenario list / netopsbench scenario validate " -echo "" +exec "$PYTHON" -m netopsbench.platform.runtime.cli deploy \ + "$TOPO_SCALE" "$ACTUAL_TOPO_DIR" "$LAB_NAME" "$MGMT_SUBNET" "$INFLUXDB_BUCKET" \ + --mgmt-network "$MGMT_NETWORK" diff --git a/scripts/runtime/deploy_worker.sh b/scripts/runtime/deploy_worker.sh index 241d8eb..08b31e1 100755 --- a/scripts/runtime/deploy_worker.sh +++ b/scripts/runtime/deploy_worker.sh @@ -1,144 +1,9 @@ #!/bin/bash -# Deploy one benchmark worker lab with dedicated management network, bucket, and telegraf. - -set -e - -SCALE=${1:?usage: deploy_worker.sh [bucket]} -TOPOLOGY_DIR=${2:?usage: deploy_worker.sh [bucket]} -LAB_NAME=${3:?usage: deploy_worker.sh [bucket]} -MGMT_SUBNET=${4:?usage: deploy_worker.sh [bucket]} -INFLUXDB_BUCKET=${5:-${NETOPSBENCH_INFLUXDB_BUCKET:-netopsbench}} -MGMT_NETWORK="clab-mgmt-${LAB_NAME}" -TOPOLOGY_ID=${NETOPSBENCH_TOPOLOGY_ID:-$LAB_NAME} - -if [ "$(id -u)" -eq 0 ]; then - SUDO=() -else - SUDO=("sudo" "-n") -fi +# Thin wrapper around the Python-owned worker lifecycle. +set -euo pipefail BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "$BASE_DIR" - -# Locate project Python interpreter (prefers repo venv, falls back to system python3). -# Override via NETOPSBENCH_PYTHON env var if needed (e.g. for git worktrees). # shellcheck source=scripts/lib/find_python.sh source "$BASE_DIR/scripts/lib/find_python.sh" -mkdir -p "$TOPOLOGY_DIR" - -echo "=== Deploying Worker Lab ===" -echo "Scale: $SCALE" -echo "Topology dir: $TOPOLOGY_DIR" -echo "Lab name: $LAB_NAME" -echo "Management subnet: $MGMT_SUBNET" -echo "InfluxDB bucket: $INFLUXDB_BUCKET" - -if [ "${NETOPSBENCH_SKIP_OBSERVABILITY_CORE_START:-0}" = "1" ]; then - echo "[1/8] Observability core already ensured by caller, skipping shared startup" -else - echo "[1/8] Starting observability core..." - bash scripts/observability/start_observability_core.sh -fi -$PYTHON -m netopsbench.platform.observability.influxdb \ - --url "${NETOPSBENCH_INFLUXDB_URL:-http://localhost:8086}" \ - --token "${NETOPSBENCH_INFLUXDB_TOKEN:-replace-me}" \ - --org "${NETOPSBENCH_INFLUXDB_ORG:-netopsbench}" \ - --bucket "$INFLUXDB_BUCKET" - -echo "[2/8] Generating worker topology..." -# Clean stale configs/clab dirs from any previous run that may have left -# root-owned files behind (containerlab creates these). -"${SUDO[@]}" rm -rf "$TOPOLOGY_DIR/configs" "$TOPOLOGY_DIR/clab-"* 2>/dev/null || true -$PYTHON - <&2 - exit 1 -fi - -echo "[3/8] Deploying containerlab topology..." -"${SUDO[@]}" containerlab deploy -t "$TOPOLOGY_FILE" --reconfigure - -echo "[4/8] Applying SONiC configs..." -$PYTHON -m netopsbench.platform.runtime.apply_configs "$TOPOLOGY_DIR" 4 "$LAB_NAME" - -readarray -t WORKER_META < <($PYTHON - <&2 - exit 1 -fi - -echo "[5/8] Attaching InfluxDB and starting worker telegraf..." -if ! "${SUDO[@]}" docker inspect influxdb >/dev/null 2>&1; then - echo "ERROR: shared influxdb container is not running" >&2 - exit 1 -fi -"${SUDO[@]}" docker network connect "$MGMT_NETWORK" influxdb --alias influxdb >/dev/null 2>&1 || true -NETOPSBENCH_TOPOLOGY_ID="$TOPOLOGY_ID" \ -NETOPSBENCH_INFLUXDB_BUCKET="$INFLUXDB_BUCKET" \ -NETOPSBENCH_TELEGRAF_INFLUXDB_URL="http://influxdb:8086" \ - bash scripts/observability/start_worker_telegraf.sh "$TOPOLOGY_DIR" "telegraf-${LAB_NAME}" - -echo "[6/8] Starting BGP collector..." -BGP_COLLECTOR_PID_FILE="$TOPOLOGY_DIR/bgp_collector.pid" -BGP_COLLECTOR_LOG_FILE="$TOPOLOGY_DIR/bgp_collector.log" -BGP_COLLECTOR_OUTPUT_FILE="$TOPOLOGY_DIR/bgp_neighbors.lp" -rm -f "$BGP_COLLECTOR_PID_FILE" -$PYTHON scripts/runtime/run_bgp_collector.py \ - "$TOPOLOGY_DIR/topology.json" \ - --output "$BGP_COLLECTOR_OUTPUT_FILE" \ - --interval "${NETOPSBENCH_BGP_POLL_INTERVAL_SECONDS:-10}" \ - >> "$BGP_COLLECTOR_LOG_FILE" 2>&1 & -echo $! > "$BGP_COLLECTOR_PID_FILE" - -echo "[7/8] Deploying Pingmesh agents..." -NETOPSBENCH_TOPOLOGY_ID="$TOPOLOGY_ID" \ -NETOPSBENCH_INFLUXDB_BUCKET="$INFLUXDB_BUCKET" \ -NETOPSBENCH_PINGMESH_INFLUXDB_URL="http://influxdb:8086" \ - $PYTHON -m netopsbench.platform.pingmesh.deploy "$TOPOLOGY_DIR/pinglist.json" "$TOPOLOGY_DIR" - -echo "[8/8] Validating worker health..." -# Accept either NETOPSBENCH_WORKER_HEALTH_RETRIES (set by run_all_benchmarks.sh) or -# the legacy NETOPSBENCH_WORKER_HEALTH_ATTEMPTS name, defaulting to 5. -HEALTH_ATTEMPTS=${NETOPSBENCH_WORKER_HEALTH_RETRIES:-${NETOPSBENCH_WORKER_HEALTH_ATTEMPTS:-5}} -HEALTH_RETRY_DELAY_SECONDS=${NETOPSBENCH_WORKER_HEALTH_RETRY_DELAY_SECONDS:-20} -HEALTH_OK=0 -for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do - if NETOPSBENCH_INFLUXDB_BUCKET="$INFLUXDB_BUCKET" \ - NETOPSBENCH_TOPOLOGY_ID="$TOPOLOGY_ID" \ - $PYTHON -m netopsbench.platform.worker.health "$TOPOLOGY_DIR"; then - HEALTH_OK=1 - break - fi - if [ "$attempt" -lt "$HEALTH_ATTEMPTS" ]; then - echo "Worker health check attempt $attempt/$HEALTH_ATTEMPTS failed; retrying in ${HEALTH_RETRY_DELAY_SECONDS}s..." - sleep "$HEALTH_RETRY_DELAY_SECONDS" - fi -done - -if [ "$HEALTH_OK" -ne 1 ]; then - echo "ERROR: worker health check failed after $HEALTH_ATTEMPTS attempt(s)" >&2 - exit 1 -fi - -echo "Worker deployment complete: $LAB_NAME" +exec "$PYTHON" -m netopsbench.platform.runtime.cli deploy "$@" diff --git a/scripts/runtime/run_bgp_collector.py b/scripts/runtime/run_bgp_collector.py deleted file mode 100644 index d243772..0000000 --- a/scripts/runtime/run_bgp_collector.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -"""Poll BGP summary from SONiC nodes and emit Influx line protocol snapshots.""" - -from __future__ import annotations - -import argparse -import json -import os -import signal -import subprocess -import sys -import time -from collections.abc import Iterable -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[2] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -from netopsbench.platform.toolkit._core.device.bgp_parsers import parse_bgp_summary # noqa: E402 - - -def _docker_prefix() -> list[str]: - if os.geteuid() == 0: - return [] - return ["sudo", "-n"] - - -def _escape_tag(value: str) -> str: - return str(value).replace("\\", "\\\\").replace(",", "\\,").replace(" ", "\\ ").replace("=", "\\=") - - -def _escape_string_field(value: str) -> str: - return str(value).replace("\\", "\\\\").replace('"', '\\"') - - -def normalize_bgp_state(value: str | None) -> str: - if not value: - return "UNKNOWN" - return str(value).strip().upper() - - -def _int_field(name: str, value: object) -> str | None: - if value is None or value == "": - return None - return f"{name}={int(value)}i" - - -def build_bgp_lines(device: str, rows: Iterable[dict], timestamp_ns: int, topology_id: str = "") -> list[str]: - lines: list[str] = [] - source = _escape_tag(device) - topology_tag = _escape_tag(topology_id) - for row in rows: - neighbor = row.get("neighbor") - if not neighbor: - continue - tags = [f"source={source}", f"neighbor_address={_escape_tag(str(neighbor))}"] - if topology_tag: - tags.append(f"topology_id={topology_tag}") - fields = [f'session_state="{_escape_string_field(normalize_bgp_state(row.get("state")))}"'] - for key in ("asn", "prefixes_received", "msg_rcvd", "msg_sent", "in_q", "out_q"): - field = _int_field(key, row.get(key)) - if field: - fields.append(field) - up_down = row.get("up_down") - if up_down: - fields.append(f'up_down="{_escape_string_field(str(up_down))}"') - lines.append(f"bgp_neighbors,{','.join(tags)} {','.join(fields)} {timestamp_ns}") - return lines - - -def _read_topology(metadata_file: Path) -> tuple[str, list[str]]: - payload = json.loads(metadata_file.read_text(encoding="utf-8")) - lab_name = str(payload.get("name") or "dcn").strip() - devices = payload.get("devices", {}) or {} - names = [item.get("name") for item in devices.get("spines", []) + devices.get("leafs", []) if item.get("name")] - return lab_name, names - - -def collect_bgp_lines(metadata_file: Path, timestamp_ns: int | None = None) -> list[str]: - lab_name, devices = _read_topology(metadata_file) - topology_id = os.environ.get("NETOPSBENCH_TOPOLOGY_ID", lab_name) - docker_prefix = _docker_prefix() - resolved_timestamp = time.time_ns() if timestamp_ns is None else int(timestamp_ns) - lines: list[str] = [] - for device in devices: - container = f"clab-{lab_name}-{device}" # matches clab_container_name() convention - result = subprocess.run( - [*docker_prefix, "docker", "exec", container, "vtysh", "-c", "show ip bgp summary"], - capture_output=True, - text=True, - check=False, - timeout=30, - ) - if result.returncode != 0: - continue - rows = parse_bgp_summary(result.stdout) - lines.extend(build_bgp_lines(device, rows, resolved_timestamp, topology_id=topology_id)) - return lines - - -def run_loop(metadata_file: Path, output_file: Path, interval_seconds: float) -> int: - running = True - - def _stop(_signum, _frame): - nonlocal running - running = False - - signal.signal(signal.SIGTERM, _stop) - signal.signal(signal.SIGINT, _stop) - - output_file.parent.mkdir(parents=True, exist_ok=True) - output_file.touch(exist_ok=True) - - while running: - try: - lines = collect_bgp_lines(metadata_file) - if lines: - with output_file.open("a", encoding="utf-8") as handle: - handle.write("\n".join(lines) + "\n") - except Exception as exc: - print(f"WARN: bgp collector iteration failed: {exc}", file=sys.stderr) - if not running: - break - time.sleep(max(1.0, interval_seconds)) - return 0 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Emit BGP neighbor snapshots as Influx line protocol") - parser.add_argument("metadata_file", help="Path to topology.json") - parser.add_argument("--output", required=True, help="Output line protocol file") - parser.add_argument("--interval", type=float, default=10.0, help="Polling interval in seconds") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - return run_loop(Path(args.metadata_file), Path(args.output), args.interval) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/runtime/run_pingmesh_agent.py b/scripts/runtime/run_pingmesh_agent.py deleted file mode 100644 index ac25d4f..0000000 --- a/scripts/runtime/run_pingmesh_agent.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -"""Thin shim that delegates to :mod:`netopsbench.platform.pingmesh.cli`. - -This file is copied into each client container by -:mod:`netopsbench.platform.pingmesh.deploy` so the agent can be launched -with ``python3 /tmp/pingmesh/run_pingmesh_agent.py [interval]`` -even when the package layout is not on ``sys.path``. -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -try: - if __package__ in {None, ""}: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - from netopsbench.platform.pingmesh.cli import main -except Exception: - sys.path.insert(0, str(Path(__file__).resolve().parent)) - from cli import main # type: ignore[no-redef] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/runtime/teardown.sh b/scripts/runtime/teardown.sh index 5064b44..40dd6be 100755 --- a/scripts/runtime/teardown.sh +++ b/scripts/runtime/teardown.sh @@ -1,120 +1,10 @@ #!/bin/bash -# NetOpsBench Teardown Script -# Cleans up complete environment: topology + observability stack - -set -e +# Thin standalone teardown wrapper over the Python worker lifecycle. +set -euo pipefail TOPO_DIR=${1:-clab-topology} -STOP_OBSERVABILITY_CORE=${NETOPSBENCH_STOP_OBSERVABILITY_CORE:-0} - -if [ "$(id -u)" -eq 0 ]; then - SUDO=() -else - # Non-interactive sudo: fail fast instead of prompting for password. - SUDO=("sudo" "-n") -fi - BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "$BASE_DIR" - -# Locate project Python interpreter (prefers repo venv, falls back to system python3). # shellcheck source=scripts/lib/find_python.sh source "$BASE_DIR/scripts/lib/find_python.sh" -echo "=== NetOpsBench Teardown Start ===" -echo "Topology directory: $TOPO_DIR" -echo "" - -# [1/2] Destroy Containerlab topology -echo "[1/2] Destroying Containerlab topology..." - -TOPOLOGY_FILE="$TOPO_DIR/dcn.clab.yaml" -if [ ! -f "$TOPOLOGY_FILE" ]; then - # Try to auto-detect a topology file in the directory. - TOPOLOGY_FILE="$(ls -1 "$TOPO_DIR"/*.clab.y*ml 2>/dev/null | head -n 1 || true)" -fi - -LAB_NAME="" -MGMT_NETWORK="" -if [ -n "$TOPOLOGY_FILE" ] && [ -f "$TOPOLOGY_FILE" ]; then - # Prefer explicit lab name from the topology file. - LAB_NAME="$(grep -E '^name:' "$TOPOLOGY_FILE" 2>/dev/null | head -n 1 | sed -E 's/^name:[[:space:]]*//')" - if [ -z "$LAB_NAME" ]; then - # Derive lab name from filename (strip .clab.yaml/.clab.yml). - base="$(basename "$TOPOLOGY_FILE")" - LAB_NAME="${base%.clab.yaml}" - LAB_NAME="${LAB_NAME%.clab.yml}" - fi - - # If the lab already exists, prefer the exact topology file path it was deployed with. - # Containerlab matches containers by labels, including clab-topo-file, so path mismatches can lead to - # "no containerlab containers found" even when containers exist. - deployed_topo_file="" - existing_id="$("${SUDO[@]}" docker ps -a --filter "label=containerlab=${LAB_NAME}" --format '{{.ID}}' | head -n 1 || true)" - if [ -n "$existing_id" ]; then - deployed_topo_file="$("${SUDO[@]}" docker inspect -f '{{ index .Config.Labels "clab-topo-file" }}' "$existing_id" 2>/dev/null || true)" - fi - if [ -n "$deployed_topo_file" ] && [ -f "$deployed_topo_file" ]; then - echo "Detected deployed topology file: $deployed_topo_file" - TOPOLOGY_FILE="$deployed_topo_file" - fi - - echo "Using topology file: $TOPOLOGY_FILE" - echo "Lab name: $LAB_NAME" - - METADATA_FILE="$TOPO_DIR/topology.json" - if [ -f "$METADATA_FILE" ]; then - MGMT_NETWORK="$($PYTHON -c 'import json, sys; topo = json.load(open(sys.argv[1])); print((topo.get("management", {}) or {}).get("network", ""))' "$METADATA_FILE")" - fi - TELEGRAF_CONTAINER="telegraf-${LAB_NAME}" - "${SUDO[@]}" docker rm -f "$TELEGRAF_CONTAINER" >/dev/null 2>&1 || true - if [ -n "$MGMT_NETWORK" ]; then - "${SUDO[@]}" docker network disconnect "$MGMT_NETWORK" influxdb >/dev/null 2>&1 || true - fi - - # Primary: destroy by topology file. - destroy_out="$("${SUDO[@]}" containerlab destroy -t "$TOPOLOGY_FILE" --cleanup 2>&1 || true)" - echo "$destroy_out" - - # Fallback: if the lab was deployed from a different path, destroy by lab name. - if echo "$destroy_out" | grep -qi "no containerlab containers found"; then - echo "WARN: containerlab didn't match any containers by -t; falling back to --name $LAB_NAME" - "${SUDO[@]}" containerlab destroy --name "$LAB_NAME" --cleanup || true - fi - - echo "Topology destroy attempted" - - # If there are no containerlab containers left, remove the shared management network if present. - if [ -z "$("${SUDO[@]}" docker ps -a --filter label=containerlab --format '{{.ID}}' | head -n 1 || true)" ]; then - if "${SUDO[@]}" docker network ls --format '{{.Name}}' | grep -qx "clab"; then - "${SUDO[@]}" docker network rm clab >/dev/null 2>&1 || true - fi - fi - if [ -n "$MGMT_NETWORK" ] && "${SUDO[@]}" docker network ls --format '{{.Name}}' | grep -qx "$MGMT_NETWORK"; then - "${SUDO[@]}" docker network rm "$MGMT_NETWORK" >/dev/null 2>&1 || true - fi -else - echo "WARNING: No topology file found in: $TOPO_DIR" - echo "Attempting cleanup of all containerlab labs..." - "${SUDO[@]}" containerlab destroy --all --cleanup --yes || true -fi -echo "" - -# [2/2] Stop observability stack only when explicitly requested -if [ "$STOP_OBSERVABILITY_CORE" = "1" ]; then - echo "[2/2] Stopping observability stack..." - cd observability - docker compose down --volumes || true - cd "$BASE_DIR" - echo "" -else - echo "[2/2] Leaving observability stack running (use 'netopsbench cleanup --core' or '--all' to stop it)" - echo "" -fi - -echo "=== Teardown Complete ===" -echo "" -echo "Target lab containers stopped and removed." -echo "To verify cleanup:" -echo " docker ps -a | grep -E 'clab|influx|grafana|telegraf'" -echo "" +exec "$PYTHON" -m netopsbench.platform.runtime.cli teardown "$TOPO_DIR" diff --git a/scripts/runtime/teardown_worker.sh b/scripts/runtime/teardown_worker.sh index dcdfc4e..8189792 100755 --- a/scripts/runtime/teardown_worker.sh +++ b/scripts/runtime/teardown_worker.sh @@ -1,70 +1,9 @@ #!/bin/bash -# Teardown one benchmark worker lab and its dedicated telegraf sidecar. - -set -e - -TOPOLOGY_DIR=${1:?usage: teardown_worker.sh } - -if [ "$(id -u)" -eq 0 ]; then - SUDO=() -else - SUDO=("sudo" "-n") -fi +# Thin wrapper around the Python-owned worker lifecycle. +set -euo pipefail BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "$BASE_DIR" - -# Locate project Python interpreter (prefers repo venv, falls back to system python3). # shellcheck source=scripts/lib/find_python.sh source "$BASE_DIR/scripts/lib/find_python.sh" -METADATA_FILE="$TOPOLOGY_DIR/topology.json" -if [ ! -f "$METADATA_FILE" ]; then - echo "Skipping worker teardown; topology metadata not found: $METADATA_FILE" - exit 0 -fi - -readarray -t WORKER_META < <($PYTHON - </dev/null || true) - if [ -n "$BGP_PID" ] && kill -0 "$BGP_PID" >/dev/null 2>&1; then - kill "$BGP_PID" >/dev/null 2>&1 || true - fi - rm -f "$BGP_COLLECTOR_PID_FILE" -fi - -"${SUDO[@]}" docker rm -f "$TELEGRAF_CONTAINER" >/dev/null 2>&1 || true -"${SUDO[@]}" docker network disconnect "$MGMT_NETWORK" influxdb >/dev/null 2>&1 || true - -if [ -f "$TOPOLOGY_FILE" ]; then - destroy_out=$("${SUDO[@]}" containerlab destroy -t "$TOPOLOGY_FILE" --cleanup 2>&1 || true) - echo "$destroy_out" - if echo "$destroy_out" | grep -qi "no containerlab containers found"; then - "${SUDO[@]}" containerlab destroy --name "$LAB_NAME" --cleanup || true - fi -else - "${SUDO[@]}" containerlab destroy --name "$LAB_NAME" --cleanup || true -fi - -if "${SUDO[@]}" docker network ls --format '{{.Name}}' | grep -qx "$MGMT_NETWORK"; then - "${SUDO[@]}" docker network rm "$MGMT_NETWORK" >/dev/null 2>&1 || true -fi - -echo "Worker teardown complete: $LAB_NAME" +exec "$PYTHON" -m netopsbench.platform.runtime.cli teardown "$@" diff --git a/scripts/toolkit/run_fastmcp_server.py b/scripts/toolkit/run_fastmcp_server.py index d92360b..edeb030 100644 --- a/scripts/toolkit/run_fastmcp_server.py +++ b/scripts/toolkit/run_fastmcp_server.py @@ -1,13 +1,5 @@ #!/usr/bin/env python3 -from __future__ import annotations - -import sys -from pathlib import Path - -if __package__ in {None, ""}: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from netopsbench.platform.toolkit.fastmcp_server import run_server +from netopsbench.platform.toolkit.fastmcp_server import main if __name__ == "__main__": - run_server() + raise SystemExit(main()) diff --git a/tests/test_api_faults.py b/tests/test_api_faults.py index fcf0471..6fbf564 100644 --- a/tests/test_api_faults.py +++ b/tests/test_api_faults.py @@ -3,8 +3,10 @@ from dataclasses import dataclass from types import SimpleNamespace +import pytest + import netopsbench.sdk.faults as faults_api -from netopsbench.platform.faults.specs import FaultSpec, unregister_fault_spec +from netopsbench.platform.faults.specs import FaultSpec @dataclass @@ -37,15 +39,12 @@ def test_fault_manager_registers_custom_fault_spec_and_executor(): manager = faults_api.FaultManager() spec = FaultSpec(name="custom_fault") - try: - manager.register(spec=spec, executor=_FakeExecutor()) + manager.register(spec=spec, executor=_FakeExecutor()) - loaded = manager.get("custom_fault") - assert loaded.name == "custom_fault" - assert "custom_fault" in {item.name for item in manager.list()} - assert manager.validate_parameters("custom_fault", {}) == [] - finally: - unregister_fault_spec("custom_fault") + loaded = manager.get("custom_fault") + assert loaded.name == "custom_fault" + assert "custom_fault" in {item.name for item in manager.list()} + assert manager.validate_parameters("custom_fault", {}) == [] def test_fault_manager_load_builtin_exposes_builtin_faults(): @@ -61,13 +60,10 @@ def test_fault_manager_load_builtin_exposes_builtin_faults(): def test_fault_manager_register_pack_adds_pack_faults(): manager = faults_api.FaultManager() - try: - manager.register_pack(_FakeFaultPack()) + manager.register_pack(_FakeFaultPack()) - names = {item.name for item in manager.list()} - assert "pack_fault" in names - finally: - unregister_fault_spec("pack_fault") + names = {item.name for item in manager.list()} + assert "pack_fault" in names def test_fault_manager_validate_parameters_uses_minimal_adapter(): @@ -84,19 +80,16 @@ def _validator(episode): manager = faults_api.FaultManager() spec = FaultSpec(name="shape_fault", episode_validator=_validator) - try: - manager.register(spec=spec, executor=_FakeExecutor()) + manager.register(spec=spec, executor=_FakeExecutor()) - assert manager.validate_parameters("shape_fault", {}) == [] - assert seen == { - "has_target_device": False, - "has_target_interface": True, - "has_target_prefix": True, - "has_parameters": True, - "has_metadata": True, - } - finally: - unregister_fault_spec("shape_fault") + assert manager.validate_parameters("shape_fault", {}) == [] + assert seen == { + "has_target_device": False, + "has_target_interface": True, + "has_target_prefix": True, + "has_parameters": True, + "has_metadata": True, + } def test_fault_manager_rejects_invalid_plugin_style_registration(monkeypatch): @@ -107,14 +100,8 @@ def test_fault_manager_rejects_invalid_plugin_style_registration(monkeypatch): lambda module_path: SimpleNamespace(BrokenPlugin=object()), ) - try: - try: - manager.load_plugin("fake.module:BrokenPlugin") - raise AssertionError("expected TypeError") - except TypeError as exc: - assert "register()" in str(exc) - finally: - unregister_fault_spec("BrokenPlugin") + with pytest.raises(TypeError, match=r"register\(\)"): + manager.load_plugin("fake.module:BrokenPlugin") def test_custom_fault_example_registers_and_executes(tmp_path, monkeypatch): @@ -147,7 +134,6 @@ def test_custom_fault_example_registers_and_executes(tmp_path, monkeypatch): def test_simple_fault_creates_dispatchable_pack(tmp_path): - from netopsbench.platform.faults.specs import unregister_fault_spec from netopsbench.sdk import FaultContext, NetOpsBench, simple_fault calls = [] @@ -168,34 +154,30 @@ def _recover(ctx: FaultContext) -> dict: required_parameters=("delay_ms",), ) bench = NetOpsBench(workspace=str(tmp_path)) - try: - bench.faults.register_pack(pack) - - spec = bench.faults.get("test_simple_latency") - assert spec.name == "test_simple_latency" - assert spec.requires_interface is True - - executor = bench.faults.get_executor("test_simple_latency") - ctx = FaultContext( - fault_type="test_simple_latency", - target_device="leaf2", - target_interface="eth0", - parameters={"delay_ms": 10}, - ) - inject_result = executor.inject(ctx) - assert inject_result.success is True - recover_result = executor.recover(ctx) - assert recover_result.success is True - assert calls == [ - ("inject", "test_simple_latency", "leaf2"), - ("recover", "test_simple_latency"), - ] - finally: - unregister_fault_spec("test_simple_latency") + bench.faults.register_pack(pack) + + spec = bench.faults.get("test_simple_latency") + assert spec.name == "test_simple_latency" + assert spec.requires_interface is True + + executor = bench.faults.get_executor("test_simple_latency") + ctx = FaultContext( + fault_type="test_simple_latency", + target_device="leaf2", + target_interface="eth0", + parameters={"delay_ms": 10}, + ) + inject_result = executor.inject(ctx) + assert inject_result.success is True + recover_result = executor.recover(ctx) + assert recover_result.success is True + assert calls == [ + ("inject", "test_simple_latency", "leaf2"), + ("recover", "test_simple_latency"), + ] def test_register_fault_shortcut(tmp_path): - from netopsbench.platform.faults.specs import unregister_fault_spec from netopsbench.sdk import FaultContext, NetOpsBench def _inject(ctx: FaultContext) -> dict: @@ -205,20 +187,51 @@ def _recover(ctx: FaultContext) -> dict: return {"success": True} bench = NetOpsBench(workspace=str(tmp_path)) - try: - bench.faults.register_fault( - "test_shortcut_fault", - _inject, - _recover, - requires_interface=False, - ) + bench.faults.register_fault( + "test_shortcut_fault", + _inject, + _recover, + requires_interface=False, + ) - spec = bench.faults.get("test_shortcut_fault") - assert spec.name == "test_shortcut_fault" + spec = bench.faults.get("test_shortcut_fault") + assert spec.name == "test_shortcut_fault" - executor = bench.faults.get_executor("test_shortcut_fault") - ctx = FaultContext(fault_type="test_shortcut_fault", target_device="spine1") - assert executor.inject(ctx).success is True - assert executor.recover(ctx).success is True - finally: - unregister_fault_spec("test_shortcut_fault") + executor = bench.faults.get_executor("test_shortcut_fault") + ctx = FaultContext(fault_type="test_shortcut_fault", target_device="spine1") + assert executor.inject(ctx).success is True + assert executor.recover(ctx).success is True + + +def test_custom_fault_registry_is_isolated_per_netopsbench_instance(tmp_path): + from netopsbench.sdk import NetOpsBench + from netopsbench.sdk.exceptions import FaultNotFoundError + + first = NetOpsBench(workspace=str(tmp_path / "first")) + second = NetOpsBench(workspace=str(tmp_path / "second")) + first.faults.register_fault( + "isolated_fault", + lambda context: {"success": True}, + lambda context: {"success": True}, + ) + + assert first.faults.get("isolated_fault").name == "isolated_fault" + with pytest.raises(FaultNotFoundError): + second.faults.get("isolated_fault") + + scenario = first.scenarios.create( + id="isolated", + name="isolated", + scale="xs", + metadata={"difficulty": "easy", "expected_diagnosis": "isolated_fault"}, + episodes=[ + { + "episode_id": "ep1", + "description": "custom", + "fault_type": "isolated_fault", + "target_device": "leaf1", + } + ], + ) + assert first.scenarios.validate(scenario) == [] + assert any("Unsupported fault_type" in error for error in second.scenarios.validate(scenario)) diff --git a/tests/test_api_mcp.py b/tests/test_api_mcp.py index 3707059..1d517d3 100644 --- a/tests/test_api_mcp.py +++ b/tests/test_api_mcp.py @@ -16,9 +16,30 @@ def test_builtin_mcp_server_config_has_expected_shape(tmp_path): assert "netopsbench" in config server = config["netopsbench"] assert server["transport"] == "stdio" - assert server["args"] == ["scripts/toolkit/run_fastmcp_server.py"] + assert server["args"] == ["-m", "netopsbench.platform.toolkit.fastmcp_server"] assert Path(server["cwd"]) == repo.resolve() - assert server["env"]["PYTHONPATH"] == str(repo.resolve()) + assert "PYTHONPATH" not in server["env"] + + +def test_builtin_mcp_server_config_only_forwards_tool_environment(monkeypatch, tmp_path): + monkeypatch.setenv("NETOPSBENCH_TOPOLOGY_DIR", "/tmp/topology") + monkeypatch.setenv("NETOPSBENCH_INFLUXDB_BUCKET", "worker-bucket") + monkeypatch.setenv("NETOPSBENCH_WORKER_DEPLOY_JOBS", "99") + monkeypatch.setenv("OPENAI_API_KEY", "secret") + + config = builtin_mcp_server_config( + workspace=tmp_path, + env={ + "NETOPSBENCH_PINGMESH_CONTEXT_FILE": "/tmp/window.json", + "DEEPSEEK_API_KEY": "secret", + }, + )["netopsbench"]["env"] + + assert config == { + "NETOPSBENCH_INFLUXDB_BUCKET": "worker-bucket", + "NETOPSBENCH_PINGMESH_CONTEXT_FILE": "/tmp/window.json", + "NETOPSBENCH_TOPOLOGY_DIR": "/tmp/topology", + } def test_builtin_mcp_server_command_uses_python_and_script(tmp_path): @@ -27,7 +48,7 @@ def test_builtin_mcp_server_command_uses_python_and_script(tmp_path): command = builtin_mcp_server_command(workspace=repo) - assert command["args"] == ["scripts/toolkit/run_fastmcp_server.py"] + assert command["args"] == ["-m", "netopsbench.platform.toolkit.fastmcp_server"] assert command["cwd"] == str(repo.resolve()) assert command["command"] @@ -58,13 +79,15 @@ def wait(self, timeout=None): def kill(self): self._returncode = -9 + monkeypatch.setenv("OPENAI_API_KEY", "must-not-reach-tools") monkeypatch.setattr("netopsbench.sdk.mcp.subprocess.Popen", _FakePopen) handle = start_builtin_mcp_server(workspace=repo) assert handle.pid == 12345 - assert calls["args"][1:] == ["scripts/toolkit/run_fastmcp_server.py"] + assert calls["args"][1:] == ["-m", "netopsbench.platform.toolkit.fastmcp_server"] assert calls["cwd"] == str(repo.resolve()) + assert "OPENAI_API_KEY" not in calls["env"] handle.stop() assert handle.poll() == 0 diff --git a/tests/test_api_platform.py b/tests/test_api_platform.py index 171eb66..45ba54c 100644 --- a/tests/test_api_platform.py +++ b/tests/test_api_platform.py @@ -44,47 +44,18 @@ def test_sdk_managers_namespace_is_no_longer_public(): importlib.import_module("netopsbench.sdk.managers") -def test_netopsbench_root_preserves_constructor_state(tmp_path): +def test_netopsbench_root_only_persists_workspace(tmp_path): from netopsbench.sdk import NetOpsBench - defaults = {"scale": "xs"} - env = {"NETOPSBENCH_ENV": "test"} - - bench = NetOpsBench(workspace=str(tmp_path), defaults=defaults, env=env, auto_load_env=False) + bench = NetOpsBench(workspace=str(tmp_path)) assert bench.workspace == tmp_path - assert bench.defaults == defaults - assert bench.defaults is not defaults - assert bench.env == env - assert bench.env is not env - assert bench.auto_load_env is False - - defaults["scale"] = "large" - env["NETOPSBENCH_ENV"] = "mutated" - - assert bench.defaults["scale"] == "xs" - assert bench.env["NETOPSBENCH_ENV"] == "test" - - -def test_netopsbench_loads_env_from_process_when_requested(monkeypatch): - from netopsbench.sdk import NetOpsBench - - monkeypatch.setenv("NETOPSBENCH_AUTO_ENV", "from-process") - - bench = NetOpsBench(env=None, auto_load_env=True) - - assert bench.env["NETOPSBENCH_AUTO_ENV"] == "from-process" - assert isinstance(bench.env, dict) - - -def test_netopsbench_keeps_empty_env_when_auto_load_disabled(monkeypatch): - from netopsbench.sdk import NetOpsBench - - monkeypatch.setenv("NETOPSBENCH_AUTO_ENV", "from-process") - - bench = NetOpsBench(env=None, auto_load_env=False) + assert not hasattr(bench, "defaults") + assert not hasattr(bench, "env") + assert not hasattr(bench, "auto_load_env") - assert bench.env == {} + with pytest.raises(TypeError): + NetOpsBench(env={}) def test_public_api_exports_shared_types(): @@ -160,9 +131,9 @@ def test_session_orchestrator_is_available_under_platform_session_package(): def test_worker_health_check_tracks_the_deployed_pingmesh_agent_path(): repo = Path(__file__).resolve().parents[1] deploy_py = (repo / "netopsbench" / "platform" / "pingmesh" / "deploy.py").read_text(encoding="utf-8") - health_py = (repo / "netopsbench" / "platform" / "worker" / "health.py").read_text(encoding="utf-8") + health_py = (repo / "netopsbench" / "platform" / "runtime" / "health.py").read_text(encoding="utf-8") - expected_path = "/tmp/pingmesh/run_pingmesh_agent.py" + expected_path = "/tmp/pingmesh/netopsbench/platform/pingmesh/cli.py" assert expected_path in deploy_py - assert "run_pingmesh_agent.py" in health_py + assert "netopsbench.platform.pingmesh.cli" in health_py diff --git a/tests/test_api_runtimes.py b/tests/test_api_runtimes.py index fab96c5..6a1120a 100644 --- a/tests/test_api_runtimes.py +++ b/tests/test_api_runtimes.py @@ -1,5 +1,52 @@ """Tests for the public runtime pool SDK manager.""" +import json +from contextlib import nullcontext +from pathlib import Path + +import pytest + + +def _record_lifecycle_operations(monkeypatch, calls, *, fail_stage=None): + import netopsbench.platform.runtime.lifecycle as lifecycle + + def record(stage, runtime_id): + calls.append((stage, runtime_id)) + if stage == fail_stage: + raise RuntimeError(f"{stage} failed") + + monkeypatch.setattr(lifecycle, "runtime_deploy_lock", nullcontext) + monkeypatch.setattr( + lifecycle, + "allocate_management_subnets", + lambda _scale, count: [f"172.31.{100 + index}.0/24" for index in range(count)], + ) + monkeypatch.setattr( + lifecycle, + "deploy_workers", + lambda workers, _scale, _root: record("deploy", workers[0].runtime_id), + ) + monkeypatch.setattr( + lifecycle, + "ensure_worker_observability", + lambda worker: record("observability", worker.runtime_id), + ) + monkeypatch.setattr( + lifecycle, + "ensure_worker_pingmesh", + lambda worker: record("pingmesh", worker.runtime_id), + ) + monkeypatch.setattr( + lifecycle, + "validate_worker_health", + lambda worker, _root: record("warm", worker.runtime_id), + ) + monkeypatch.setattr( + lifecycle, + "teardown_workers", + lambda workers: record("teardown", workers[0].runtime_id), + ) + def test_runtime_manager_create_workers_1_returns_runtime_pool(tmp_path): from netopsbench.sdk.runtimes import RuntimeManager, RuntimePool @@ -11,15 +58,42 @@ def test_runtime_manager_create_workers_1_returns_runtime_pool(tmp_path): assert runtime.id == runtime.name assert runtime.scale == "small" assert len(runtime.workers) == 1 - assert runtime.workers[0].name == "worker-1" - assert runtime.workers[0].index == 1 + assert runtime.workers[0].worker_id == "worker-1" + assert runtime.workers[0].worker_index == 1 assert runtime.workers[0].lab_name == runtime.name - assert runtime.workers[0].topology_dir == str(runtime.root_dir / "worker-1") + assert runtime.workers[0].topology_dir == runtime.root_dir / "worker-1" assert runtime.root_dir.exists() assert runtime.root_dir == tmp_path / ".netopsbench" / "runtimes" / runtime.name assert runtime.status() == {"id": runtime.id, "name": runtime.name, "scale": "small", "state": "created"} +def test_xlarge_runtime_workers_use_non_overlapping_23_subnets(tmp_path): + from netopsbench.sdk.runtimes import RuntimeManager + + runtime = RuntimeManager(workspace=tmp_path).create(scale="xlarge", workers=2, name="runtime-xlarge") + + subnets = [worker.mgmt_subnet for worker in runtime.workers] + assert subnets[0].endswith("/23") + assert subnets[1].endswith("/23") + assert subnets[0] != subnets[1] + assert int(subnets[0].split(".")[2]) % 2 == 0 + assert int(subnets[1].split(".")[2]) == int(subnets[0].split(".")[2]) + 2 + + +def test_fat_tree_k12_runtime_workers_use_non_overlapping_23_subnets(tmp_path): + from netopsbench.sdk.runtimes import RuntimeManager + + runtime = RuntimeManager(workspace=tmp_path).create(scale="fat-tree-k12", workers=2, name="runtime-ft12") + + subnets = [worker.mgmt_subnet for worker in runtime.workers] + assert subnets[0].endswith("/23") + assert subnets[1].endswith("/23") + assert subnets[0] != subnets[1] + assert int(subnets[0].split(".")[2]) >= 220 + assert int(subnets[0].split(".")[2]) % 2 == 0 + assert int(subnets[1].split(".")[2]) == int(subnets[0].split(".")[2]) + 2 + + def test_runtime_manager_attach_list_get_roundtrip(tmp_path): from netopsbench.sdk.runtimes import RuntimeManager @@ -31,22 +105,261 @@ def test_runtime_manager_attach_list_get_roundtrip(tmp_path): assert attached.name == "runtime-xs" assert attached.id == "runtime-xs" - assert attached.workers[0].root_dir == runtime.root_dir / "worker-1" - assert attached.workers[0].topology_dir == str(runtime.root_dir / "worker-1") + assert attached.workers[0].topology_dir == runtime.root_dir / "worker-1" assert attached.status()["state"] == "created" assert attached_manager.get("runtime-xs").root_dir == runtime.root_dir assert [item.name for item in attached_manager.list()] == ["runtime-xs"] -def test_runtime_pool_exposes_required_lifecycle_surface(tmp_path): +def test_same_scale_runtime_identities_are_isolated_and_persisted(tmp_path): + from netopsbench.sdk.runtimes import RuntimeManager + + manager = RuntimeManager(workspace=tmp_path) + first = manager.create(scale="xlarge", workers=1, name="run-a") + second = manager.create(scale="xlarge", workers=1, name="run-b") + + first_identity = first.workers[0] + second_identity = second.workers[0] + assert first_identity.topology_id == "run-a" + assert second_identity.topology_id == "run-b" + assert first_identity.bucket == "network_data_run-a_w01" + assert second_identity.bucket == "network_data_run-b_w01" + assert first_identity.bucket != second_identity.bucket + assert first_identity.mgmt_network == "clab-mgmt-run-a" + + payload = json.loads((first.root_dir / "runtime.json").read_text(encoding="utf-8")) + assert payload["schema_version"] == "3" + assert payload["workers"][0] == first_identity.model_dump(mode="json") + + +def test_runtime_identity_has_no_process_environment_projection(tmp_path): + from netopsbench.sdk.runtimes import RuntimeManager + + worker = RuntimeManager(workspace=tmp_path).create(scale="xs", workers=1, name="identity-lab").workers[0] + assert worker.topology_id == "identity-lab" + assert worker.bucket == "network_data_identity-lab_w01" + assert worker.mgmt_network == "clab-mgmt-identity-lab" + assert not hasattr(worker, "as_env") + + +def test_worker_observability_restarts_stale_bgp_collector(tmp_path, monkeypatch): + from netopsbench.platform.observability import lifecycle + from netopsbench.sdk.runtimes import RuntimeManager + + worker = RuntimeManager(workspace=tmp_path).create(scale="xs", workers=1, name="collector-lab").workers[0] + topology_dir = Path(worker.topology_dir) + (topology_dir / "topology.json").write_text("{}", encoding="utf-8") + (topology_dir / "bgp_collector.pid").write_text("999999\n", encoding="utf-8") + started = [] + + class Process: + pid = 4242 + + monkeypatch.setattr(lifecycle, "_bgp_collector_is_running", lambda *_args: False) + monkeypatch.setattr( + lifecycle.subprocess, + "Popen", + lambda command, **kwargs: started.append((command, kwargs)) or Process(), + ) + + lifecycle.ensure_worker_bgp_collector(worker) + + assert (topology_dir / "bgp_neighbors.lp").is_file() + assert (topology_dir / "bgp_collector.pid").read_text(encoding="utf-8") == "4242\n" + assert "netopsbench.platform.observability.bgp_collector" in started[0][0] + assert str(topology_dir / "topology.json") in started[0][0] + + +def test_python_worker_deploy_owns_topology_containerlab_and_activation(tmp_path, monkeypatch): + from types import SimpleNamespace + + from netopsbench.platform.runtime import deployment + from netopsbench.sdk.runtimes import RuntimeManager + + worker = RuntimeManager(workspace=tmp_path).create(scale="xs", workers=1, name="deploy-lab").workers[0] + topology_dir = Path(worker.topology_dir) + calls = [] + + def fake_generate_topology(**kwargs): + calls.append(("generate", kwargs)) + (topology_dir / "deploy-lab.clab.yaml").write_text("name: deploy-lab\n", encoding="utf-8") + + def fake_safe_run(command, **kwargs): + calls.append(("command", [str(part) for part in command], kwargs)) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(deployment, "generate_topology", fake_generate_topology) + monkeypatch.setattr(deployment, "safe_run", fake_safe_run) + monkeypatch.setattr(deployment, "apply_configs", lambda *args: SimpleNamespace(failed=[])) + deployment.deploy_worker_lab(worker, "xs") + + generated = next(item for item in calls if item[0] == "generate")[1] + commands = [item[1] for item in calls if item[0] == "command"] + assert generated["name"] == "deploy-lab" + assert generated["mgmt_subnet"] == worker.mgmt_subnet + assert any("containerlab" in command and "deploy" in command for command in commands) + assert not any("telegraf" in " ".join(command) or "pingmesh" in " ".join(command) for command in commands) + + +def test_k12_worker_deploy_uses_profile_containerlab_parallelism(tmp_path, monkeypatch): + from types import SimpleNamespace + + from netopsbench.platform.runtime import deployment from netopsbench.sdk.runtimes import RuntimeManager + worker = RuntimeManager(workspace=tmp_path).create(scale="fat-tree-k12", workers=1, name="deploy-k12").workers[0] + topology_dir = Path(worker.topology_dir) + commands = [] + + def fake_generate_topology(**_kwargs): + topology_dir.mkdir(parents=True, exist_ok=True) + (topology_dir / "deploy-k12.clab.yaml").write_text("name: deploy-k12\n", encoding="utf-8") + + def fake_safe_run(command, **_kwargs): + commands.append([str(part) for part in command]) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(deployment, "generate_topology", fake_generate_topology) + monkeypatch.setattr(deployment, "safe_run", fake_safe_run) + monkeypatch.setattr(deployment, "apply_configs", lambda *args: SimpleNamespace(failed=[])) + deployment.deploy_worker_lab(worker, "fat-tree-k12") + + deploy_command = next(command for command in commands if "containerlab" in command and "deploy" in command) + assert deploy_command[-2:] == ["--max-workers", "1"] + + +def test_removed_containerlab_env_does_not_override_scale_profile(tmp_path, monkeypatch): + from types import SimpleNamespace + + from netopsbench.platform.runtime import deployment + from netopsbench.sdk.runtimes import RuntimeManager + + worker = RuntimeManager(workspace=tmp_path).create(scale="fat-tree-k12", workers=1, name="deploy-k12").workers[0] + topology_dir = Path(worker.topology_dir) + commands = [] + + def fake_generate_topology(**_kwargs): + topology_dir.mkdir(parents=True, exist_ok=True) + (topology_dir / "deploy-k12.clab.yaml").write_text("name: deploy-k12\n", encoding="utf-8") + + def fake_safe_run(command, **_kwargs): + commands.append([str(part) for part in command]) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(deployment, "generate_topology", fake_generate_topology) + monkeypatch.setattr(deployment, "safe_run", fake_safe_run) + monkeypatch.setattr(deployment, "apply_configs", lambda *args: SimpleNamespace(failed=[])) + monkeypatch.setenv("NETOPSBENCH_CONTAINERLAB_MAX_WORKERS", "3") + + deployment.deploy_worker_lab(worker, "fat-tree-k12") + + deploy_command = next(command for command in commands if "containerlab" in command and "deploy" in command) + assert deploy_command[-2:] == ["--max-workers", "1"] + + +def test_management_subnet_allocation_batches_docker_inspection(monkeypatch): + from types import SimpleNamespace + + from netopsbench.platform.runtime import deployment + + commands = [] + + def fake_safe_run(command, **_kwargs): + commands.append([str(part) for part in command]) + if "ls" in command: + return SimpleNamespace(returncode=0, stdout="network-a\nnetwork-b\n", stderr="") + return SimpleNamespace(returncode=0, stdout="172.31.101.0/24\n10.0.0.0/8\n", stderr="") + + monkeypatch.setattr(deployment, "safe_run", fake_safe_run) + + assert deployment.allocate_management_subnets("xs", 2) == ["172.31.102.0/24", "172.31.103.0/24"] + assert len(commands) == 2 + inspect_command = commands[1] + assert inspect_command[-2:] == ["network-a", "network-b"] + + +@pytest.mark.parametrize("failed_operation", ["ls", "inspect"]) +def test_management_subnet_allocation_fails_when_docker_query_fails(monkeypatch, failed_operation): + from types import SimpleNamespace + + from netopsbench.platform.runtime import deployment + + def fake_safe_run(command, **_kwargs): + operation = "ls" if "ls" in command else "inspect" + if operation == failed_operation: + return SimpleNamespace(returncode=1, stdout="", stderr=f"{operation} failed") + return SimpleNamespace(returncode=0, stdout="network-a\n", stderr="") + + monkeypatch.setattr(deployment, "safe_run", fake_safe_run) + + verb = "list" if failed_operation == "ls" else "inspect" + with pytest.raises(RuntimeError, match=f"Unable to {verb} Docker networks"): + deployment.allocate_management_subnets("xs", 1) + + +def test_runtime_identity_resolves_relative_workspace_paths(tmp_path, monkeypatch): + from netopsbench.sdk.runtimes import RuntimeManager + + monkeypatch.chdir(tmp_path) + + runtime = RuntimeManager(workspace=".").create(scale="xs", workers=1, name="absolute-path-lab") + + assert Path(runtime.workers[0].topology_dir).is_absolute() + assert runtime.workers[0].topology_dir == ( + tmp_path / ".netopsbench" / "runtimes" / "absolute-path-lab" / "worker-1" + ) + + +def test_teardown_worker_identity_uses_manifest_management_network(tmp_path): + from netopsbench.platform.runtime.deployment import worker_from_topology + from netopsbench.platform.topology.generator import generate_topology + + topology_dir = tmp_path / "topology" + generate_topology( + "xs", + str(topology_dir), + name="custom-lab", + mgmt_subnet="172.30.40.0/24", + mgmt_network="custom-management", + ) + + worker = worker_from_topology(str(topology_dir)) + + assert worker.lab_name == "custom-lab" + assert worker.mgmt_subnet == "172.30.40.0/24" + assert worker.mgmt_network == "custom-management" + assert worker.bucket == "network_data_custom-lab_w01" + assert worker.topology_dir == topology_dir.resolve() + + +def test_teardown_removal_barrier_waits_until_docker_container_set_is_empty(monkeypatch): + from netopsbench.platform.runtime import deployment + + observations = [["clab-test-leaf1"], ["clab-test-leaf1"], []] + sleeps = [] + removals = [] + monkeypatch.setattr(deployment, "_lab_container_names", lambda *_args: observations.pop(0)) + monkeypatch.setattr(deployment, "safe_run", lambda command, **_kwargs: removals.append(command)) + monkeypatch.setattr(deployment.time, "sleep", sleeps.append) + + deployment._wait_for_lab_removal([], "test") + + assert sleeps == [deployment.LAB_REMOVAL_POLL_SECONDS] * 2 + assert removals == [["docker", "rm", "-f", "clab-test-leaf1"]] * 2 + + +def test_runtime_pool_exposes_required_lifecycle_surface(tmp_path, monkeypatch): + from netopsbench.sdk.runtimes import RuntimeManager + + calls = [] + _record_lifecycle_operations(monkeypatch, calls) runtime = RuntimeManager(workspace=tmp_path).create(scale="medium", workers=1, name="runtime-medium") deployed = runtime.deploy() observed = runtime.ensure_observability() pingmesh_ready = runtime.ensure_pingmesh() warmed = runtime.warm() + runtime.warm() assert deployed is runtime assert observed is runtime @@ -55,12 +368,34 @@ def test_runtime_pool_exposes_required_lifecycle_surface(tmp_path): assert callable(runtime.status) assert callable(runtime.teardown) assert runtime.status()["state"] == "warm" + assert [stage for stage, _ in calls] == ["deploy", "observability", "pingmesh", "warm"] + assert set(runtime.stage_results) == {"deploy", "observability", "pingmesh", "warm"} torn_down = runtime.teardown() assert torn_down is runtime assert runtime.status()["state"] == "torn_down" assert not runtime.root_dir.exists() + assert [stage for stage, _ in calls][-1] == "teardown" + + +def test_runtime_stage_failure_is_persisted_without_advancing_state(tmp_path, monkeypatch): + from netopsbench.platform.runtime.lifecycle import RuntimeLifecycleError + from netopsbench.sdk.runtimes import RuntimeManager + + calls = [] + _record_lifecycle_operations(monkeypatch, calls, fail_stage="observability") + manager = RuntimeManager(workspace=tmp_path) + runtime = manager.create(scale="xs", workers=1, name="runtime-failure") + runtime.deploy() + + with pytest.raises(RuntimeLifecycleError, match="observability"): + runtime.ensure_observability() + + assert runtime.state == "deployed" + assert runtime.stage_results["observability"].status == "failed" + attached = manager.attach(runtime.root_dir) + assert attached.stage_results["observability"].error == "RuntimeError: observability failed" def test_runtime_manager_attach_rejects_missing_runtime_metadata(tmp_path): @@ -91,34 +426,24 @@ def test_runtime_manager_attach_rejects_malformed_runtime_metadata(tmp_path): try: manager.attach(runtime_dir) except ValueError as exc: - assert "missing required runtime metadata" in str(exc) + assert "Unsupported runtime.json schema" in str(exc) else: raise AssertionError("expected ValueError for malformed runtime metadata") -def test_runtime_manager_provision_single_worker_uses_worker_pool_deploy_hook(tmp_path, monkeypatch): +def test_runtime_manager_provision_composes_lifecycle_stages(tmp_path, monkeypatch): import netopsbench.platform.runtime.manager as runtimes_mod - captured = {} - - def fake_deploy_workers(workers, scale, repo_root, observability_core_ready=False): - captured["scale"] = scale - captured["worker_count"] = len(workers) - captured["topology_dir"] = workers[0].topology_dir - captured["lab_name"] = workers[0].lab_name - captured["mgmt_subnet"] = workers[0].mgmt_subnet - - monkeypatch.setattr(runtimes_mod, "deploy_workers", fake_deploy_workers) - + calls = [] + _record_lifecycle_operations(monkeypatch, calls) manager = runtimes_mod.RuntimeManager(workspace=tmp_path) runtime = manager.provision(scale="xs", workers=1, name="runtime-xs") - assert runtime.state == "deployed" + assert runtime.state == "warm" assert runtime.metadata["provisioning_mode"] == "worker_pool" - assert captured["scale"] == "xs" - assert captured["lab_name"] == "runtime-xs" - assert captured["topology_dir"] == str(runtime.root_dir / "worker-1") - assert runtime.workers[0].topology_dir == str(runtime.root_dir / "worker-1") + assert [stage for stage, _ in calls] == ["deploy", "observability", "pingmesh", "warm"] + assert runtime.workers[0].topology_id == "runtime-xs" + assert runtime.workers[0].topology_dir == runtime.root_dir / "worker-1" def test_runtime_pool_teardown_uses_worker_pool_teardown_hook(tmp_path, monkeypatch): @@ -130,18 +455,12 @@ def test_runtime_pool_teardown_uses_worker_pool_teardown_hook(tmp_path, monkeypa runtime._write_metadata() (runtime.root_dir / "worker-1" / "topology.json").write_text('{"name":"runtime-xs"}', encoding="utf-8") - called = {} - - def fake_teardown_workers(workers, repo_root): - called["worker_count"] = len(workers) - called["topology_dir"] = workers[0].topology_dir - - monkeypatch.setattr(runtimes_mod, "teardown_workers", fake_teardown_workers) + calls = [] + _record_lifecycle_operations(monkeypatch, calls) runtime.teardown() - assert called["topology_dir"] == str(runtime.root_dir / "worker-1") - assert called["worker_count"] == 1 + assert calls == [("teardown", "runtime-xs")] assert runtime.state == "torn_down" assert not runtime.root_dir.exists() @@ -149,11 +468,8 @@ def fake_teardown_workers(workers, repo_root): def test_provision_cleans_up_metadata_on_deploy_failure(tmp_path, monkeypatch): import netopsbench.platform.runtime.manager as runtimes_mod - def failing_deploy_workers(workers, scale, repo_root, observability_core_ready=False): - raise RuntimeError("deploy failed") - - monkeypatch.setattr(runtimes_mod, "deploy_workers", failing_deploy_workers) - + calls = [] + _record_lifecycle_operations(monkeypatch, calls, fail_stage="deploy") manager = runtimes_mod.RuntimeManager(workspace=tmp_path) runtime_dir = tmp_path / ".netopsbench" / "runtimes" / "will-fail" diff --git a/tests/test_api_scenarios.py b/tests/test_api_scenarios.py index 072f6fb..2b25010 100644 --- a/tests/test_api_scenarios.py +++ b/tests/test_api_scenarios.py @@ -1,6 +1,8 @@ """Tests for the public scenario authoring API.""" -from netopsbench.platform.faults.specs import FaultSpec, register_fault_spec, unregister_fault_spec +import pytest + +from netopsbench.platform.faults.specs import FaultSpec from netopsbench.platform.scenario.models import Episode, Scenario from netopsbench.platform.scenario.validator import validate_scenario @@ -43,38 +45,53 @@ def test_scenario_manager_can_create_and_roundtrip_yaml(tmp_path): assert loaded.metadata["expected_diagnosis"] == "static_route_misconfig" -def test_scenario_validation_uses_fault_registry(tmp_path): +@pytest.mark.parametrize("profile", ["light", "stress"]) +def test_scenario_manager_rejects_nonstandard_traffic_profile(tmp_path, profile): from netopsbench.sdk.scenarios import ScenarioManager - register_fault_spec( - FaultSpec( - name="public_registry_fault", - required_parameters=("probe",), - ) + manager = ScenarioManager(workspace=tmp_path) + + with pytest.raises(ValueError, match="Only the standard traffic profile is supported"): + manager.create(id="legacy_profile", name="Legacy Profile", traffic_profile=profile) + + +def test_supported_scales_are_available_from_public_sdk(): + from netopsbench.sdk import supported_scales + + assert supported_scales() == ("xs", "small", "medium", "large", "xlarge", "fat-tree-k8", "fat-tree-k12") + + +def test_scenario_validation_uses_fault_registry(tmp_path): + from netopsbench.sdk import NetOpsBench + + bench = NetOpsBench(workspace=str(tmp_path)) + bench.faults.register( + spec=FaultSpec(name="public_registry_fault", required_parameters=("probe",)), + executor=type( + "Executor", + (), + {"inject": lambda self, context: {}, "recover": lambda self, context: {}}, + )(), ) - try: - manager = ScenarioManager(workspace=tmp_path) - scenario = manager.create( - id="registry_case", - name="Registry Case", - description="desc", - scale="xs", - traffic_profile="standard", - episodes=[ - { - "episode_id": "ep001", - "description": "fault", - "fault_type": "public_registry_fault", - "target_device": "leaf1", - "parameters": {"probe": "icmp"}, - } - ], - metadata={"difficulty": "easy", "expected_diagnosis": "public_registry_fault"}, - ) - - assert manager.validate(scenario) == [] - finally: - unregister_fault_spec("public_registry_fault") + scenario = bench.scenarios.create( + id="registry_case", + name="Registry Case", + description="desc", + scale="xs", + traffic_profile="standard", + episodes=[ + { + "episode_id": "ep001", + "description": "fault", + "fault_type": "public_registry_fault", + "target_device": "leaf1", + "parameters": {"probe": "icmp"}, + } + ], + metadata={"difficulty": "easy", "expected_diagnosis": "public_registry_fault"}, + ) + + assert bench.scenarios.validate(scenario) == [] def test_validate_scenario_does_not_mutate_episode_fault_type(): diff --git a/tests/test_api_sessions.py b/tests/test_api_sessions.py index c2907c4..5cae891 100644 --- a/tests/test_api_sessions.py +++ b/tests/test_api_sessions.py @@ -60,8 +60,14 @@ def generate_report(self, results, agent_name="unknown", topology_scale="unknown def _install_real_runtime_mocks(monkeypatch): + import netopsbench.platform.session.diagnosis as diagnosis_mod + import netopsbench.platform.session.dispatch as dispatch_mod import netopsbench.platform.session.orchestrator as sessions_mod + class FakeToolkit: + def set_pingmesh_time_window(self, start_time, end_time): + self.pingmesh_window = (start_time, end_time) + class FakeScenarioExecutor: def __init__( self, @@ -75,6 +81,8 @@ def __init__( influxdb_org=None, influxdb_bucket=None, topology_id=None, + persist_results=True, + fault_registry=None, ): self.topology_dir = topology_dir self.topology_metadata = topology_metadata @@ -111,19 +119,28 @@ def run_scenario(self, scenario, diagnosis_callback=None): ], } - monkeypatch.setattr(sessions_mod, "ScenarioExecutor", FakeScenarioExecutor) + monkeypatch.setattr(dispatch_mod, "ScenarioExecutor", FakeScenarioExecutor) + monkeypatch.setattr(dispatch_mod, "load_topology_metadata", lambda _topology_dir: None) + monkeypatch.setattr(dispatch_mod, "_create_evaluator", _FakeEvaluator) + monkeypatch.setattr( + dispatch_mod, + "score_scenario_fault_episodes", + lambda *args, **kwargs: [_FakeEvalResult(1.0)], + ) monkeypatch.setattr(sessions_mod, "Evaluator", _FakeEvaluator) - monkeypatch.setattr(sessions_mod, "score_scenario_fault_episodes", lambda *args, **kwargs: [_FakeEvalResult(1.0)]) - monkeypatch.setattr(sessions_mod, "_build_toolkit_for_topology", lambda topology_dir: object()) + monkeypatch.setattr(diagnosis_mod, "_build_toolkit_for_topology", lambda topology_dir: FakeToolkit()) monkeypatch.setattr( - sessions_mod, + diagnosis_mod, "build_topology_snapshot", lambda toolkit: {"devices": {"spines": [], "leafs": [], "clients": []}, "links": []}, ) def _install_platform_runtime_mocks(monkeypatch): + import shutil + import netopsbench.platform.runtime.manager as runtimes_mod + from netopsbench.platform.topology.generator import generate_topology def fake_provision(self, *, scale, workers=1, name=None, root_dir=None): runtime = self._build_runtime(scale=scale, workers=workers, name=name, root_dir=root_dir) @@ -132,13 +149,23 @@ def fake_provision(self, *, scale, workers=1, name=None, root_dir=None): for worker in runtime.workers: worker_dir = Path(worker.topology_dir or worker.root_dir) worker_dir.mkdir(parents=True, exist_ok=True) - (worker_dir / "topology.json").write_text( - '{"name":"%s"}' % (worker.lab_name or worker.name), encoding="utf-8" + generate_topology( + scale, + str(worker_dir), + name=worker.lab_name, + mgmt_subnet=worker.mgmt_subnet, + mgmt_network=worker.mgmt_network, ) runtime._write_metadata() return runtime + def fake_teardown(self): + self.state = "torn_down" + shutil.rmtree(self.root_dir, ignore_errors=True) + return self + monkeypatch.setattr(runtimes_mod.RuntimeManager, "provision", fake_provision) + monkeypatch.setattr(runtimes_mod.RuntimePool, "teardown", fake_teardown) def _make_scenario(*, scenario_id: str, scale: str = "xs"): @@ -361,7 +388,7 @@ def diagnose(self, context): def test_runtime_agent_failure_trace_is_linked_from_results_sidecar(tmp_path, monkeypatch): _install_real_runtime_mocks(monkeypatch) - import netopsbench.platform.session.orchestrator as sessions_mod + import netopsbench.platform.session.dispatch as dispatch_mod class LinkedEvalResult: score = 0.0 @@ -373,7 +400,7 @@ def to_dict(self): "details": {"scenario_id": "failure-scenario", "episode_id": "ep1"}, } - monkeypatch.setattr(sessions_mod, "score_scenario_fault_episodes", lambda *args, **kwargs: [LinkedEvalResult()]) + monkeypatch.setattr(dispatch_mod, "score_scenario_fault_episodes", lambda *args, **kwargs: [LinkedEvalResult()]) bench = NetOpsBench(workspace=str(tmp_path)) runtime = bench.runtimes.create(scale="xs", workers=1, name="trace-failure-runtime") @@ -624,6 +651,7 @@ def test_run_handle_exposes_report_wait_refresh_and_cancel_contract(tmp_path): assert pending.cancel() is None assert pending.status == "cancelled" assert isinstance(pending.completed_at, datetime) + assert pending.completed_at.tzinfo is not None def test_keep_runtime_false_tears_down_runtime_and_true_preserves_it(tmp_path, monkeypatch): diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py new file mode 100644 index 0000000..4a93d45 --- /dev/null +++ b/tests/test_apply_configs.py @@ -0,0 +1,292 @@ +import subprocess + +import pytest + +from netopsbench.platform.runtime import apply_configs + + +def _completed(args, returncode=0, stdout="", stderr=""): + return subprocess.CompletedProcess(args=args, returncode=returncode, stdout=stdout, stderr=stderr) + + +def test_wait_for_sonic_requires_configdb_after_vtysh(monkeypatch): + calls = [] + + def fake_safe_run(cmd, **kwargs): + calls.append(cmd) + if "supervisorctl" in cmd and "status" in cmd and "start.sh" in cmd: + return _completed(cmd, stdout="start.sh EXITED Jun 16 06:32 AM\n") + if "supervisorctl" in cmd and "status" in cmd: + return _completed( + cmd, + stdout=("redis-server RUNNING pid 10\n" "orchagent RUNNING pid 11\n" "zebra RUNNING pid 12\n"), + ) + if "sonic-cfggen" in cmd: + return _completed(cmd, returncode=0) + if "vtysh" in cmd: + return _completed(cmd, returncode=0) + raise AssertionError(f"unexpected command: {cmd}") + + monkeypatch.setattr(apply_configs, "docker_prefix", lambda: []) + monkeypatch.setattr(apply_configs, "safe_run", fake_safe_run) + + assert apply_configs._wait_for_sonic("spine1", "clab-demo-spine1", max_tries=1, delay=0) + assert any("sonic-cfggen" in cmd for cmd in calls) + assert any("vtysh" in cmd for cmd in calls) + + +def test_wait_for_sonic_ignores_fatal_startup_when_services_are_ready(monkeypatch): + calls = [] + + def fake_safe_run(cmd, **kwargs): + calls.append(cmd) + if "supervisorctl" in cmd and "status" in cmd and "start.sh" in cmd: + return _completed(cmd, returncode=3, stdout="start.sh FATAL Exited too quickly\n") + if "supervisorctl" in cmd and "status" in cmd: + return _completed( + cmd, + stdout=( + "redis-server RUNNING pid 10\n" + "orchagent RUNNING pid 11\n" + "zebra RUNNING pid 12\n" + "bgpd RUNNING pid 13\n" + ), + ) + if "sonic-cfggen" in cmd: + return _completed(cmd, returncode=0) + if "vtysh" in cmd: + return _completed(cmd, returncode=0) + raise AssertionError(f"unexpected command: {cmd}") + + monkeypatch.setattr(apply_configs, "docker_prefix", lambda: []) + monkeypatch.setattr(apply_configs, "safe_run", fake_safe_run) + + assert apply_configs._wait_for_sonic("spine1", "clab-demo-spine1", max_tries=1, delay=0) + repair_calls = [cmd for cmd in calls if "bash" in cmd and any("start.sh" in part for part in cmd)] + assert repair_calls == [] + + +def test_wait_for_sonic_does_not_repair_fatal_startup_without_configdb(monkeypatch): + calls = [] + + def fake_safe_run(cmd, **kwargs): + calls.append(cmd) + if "supervisorctl" in cmd and "status" in cmd and "start.sh" in cmd: + return _completed(cmd, returncode=3, stdout="start.sh FATAL Exited too quickly\n") + if "supervisorctl" in cmd and "status" in cmd: + return _completed(cmd, stdout="") + if "sonic-cfggen" in cmd: + return _completed(cmd, returncode=1, stderr="redis unavailable") + raise AssertionError(f"unexpected command: {cmd}") + + monkeypatch.setattr(apply_configs, "docker_prefix", lambda: []) + monkeypatch.setattr(apply_configs, "safe_run", fake_safe_run) + + assert not apply_configs._wait_for_sonic("spine12", "clab-demo-spine12", max_tries=2, delay=0) + repair_calls = [cmd for cmd in calls if "bash" in cmd and any("start.sh" in part for part in cmd)] + assert repair_calls == [] + + +def test_wait_for_sonic_requires_key_supervisor_services(monkeypatch): + def fake_safe_run(cmd, **kwargs): + if "supervisorctl" in cmd and "status" in cmd and "start.sh" in cmd: + return _completed(cmd, stdout="start.sh FATAL Exited too quickly\n") + if "sonic-cfggen" in cmd: + return _completed(cmd, returncode=0) + if "vtysh" in cmd: + return _completed(cmd, returncode=0) + if "supervisorctl" in cmd and "status" in cmd: + return _completed( + cmd, + returncode=3, + stdout=( + "redis-server RUNNING pid 10\n" + "orchagent STOPPED Not started\n" + "zebra RUNNING pid 12\n" + "bgpd RUNNING pid 13\n" + ), + ) + raise AssertionError(f"unexpected command: {cmd}") + + monkeypatch.setattr(apply_configs, "docker_prefix", lambda: []) + monkeypatch.setattr(apply_configs, "safe_run", fake_safe_run) + + assert not apply_configs._wait_for_sonic("spine1", "clab-demo-spine1", max_tries=1, delay=0) + + +def test_wait_for_sonic_requires_complete_counters_port_map(monkeypatch): + def fake_safe_run(cmd, **kwargs): + if "supervisorctl" in cmd and "status" in cmd and "start.sh" in cmd: + return _completed(cmd, stdout="start.sh EXITED Jun 16 06:32 AM\n") + if "supervisorctl" in cmd and "status" in cmd: + return _completed( + cmd, + stdout=("redis-server RUNNING pid 10\n" "orchagent RUNNING pid 11\n" "zebra RUNNING pid 12\n"), + ) + if "sonic-cfggen" in cmd or "vtysh" in cmd: + return _completed(cmd, returncode=0) + if "redis-cli" in cmd: + return _completed(cmd, stdout="7\n") + raise AssertionError(f"unexpected command: {cmd}") + + monkeypatch.setattr(apply_configs, "docker_prefix", lambda: []) + monkeypatch.setattr(apply_configs, "safe_run", fake_safe_run) + + assert not apply_configs._wait_for_sonic( + "edge1", + "clab-demo-edge1", + max_tries=1, + delay=0, + expected_port_count=8, + ) + + +def test_wait_for_sonic_accepts_complete_counters_port_map(monkeypatch): + def fake_safe_run(cmd, **kwargs): + if "supervisorctl" in cmd and "status" in cmd and "start.sh" in cmd: + return _completed(cmd, stdout="start.sh EXITED Jun 16 06:32 AM\n") + if "supervisorctl" in cmd and "status" in cmd: + return _completed( + cmd, + stdout=("redis-server RUNNING pid 10\n" "orchagent RUNNING pid 11\n" "zebra RUNNING pid 12\n"), + ) + if "sonic-cfggen" in cmd or "vtysh" in cmd: + return _completed(cmd, returncode=0) + if "redis-cli" in cmd: + return _completed(cmd, stdout="8\n") + raise AssertionError(f"unexpected command: {cmd}") + + monkeypatch.setattr(apply_configs, "docker_prefix", lambda: []) + monkeypatch.setattr(apply_configs, "safe_run", fake_safe_run) + + assert apply_configs._wait_for_sonic( + "edge1", + "clab-demo-edge1", + max_tries=1, + delay=0, + expected_port_count=8, + ) + + +def test_discover_devices_uses_preseed_layout_only(tmp_path): + sonic_dir = tmp_path / "configs" / "sonic" + (sonic_dir / "spine1").mkdir(parents=True) + (sonic_dir / "leaf1").mkdir(parents=True) + (sonic_dir / "spine1" / "config_db.json").write_text("{}\n", encoding="utf-8") + (sonic_dir / "leaf1" / "config_db.json").write_text("{}\n", encoding="utf-8") + stale_dir = tmp_path / "configs" + (stale_dir / "spine9.sh").write_text("# stale generated file from an old run\n", encoding="utf-8") + + assert apply_configs._discover_devices(str(tmp_path)) == ["spine1", "leaf1"] + + +def test_apply_single_device_preseed_activates_without_shell_copy(monkeypatch, tmp_path): + config_dir = tmp_path / "configs" / "sonic" / "spine1" + config_dir.mkdir(parents=True) + (config_dir / "config_db.json").write_text('{"PORT": {"Ethernet0": {}}}\n', encoding="utf-8") + (config_dir / "port_config.ini").write_text("# ports\n", encoding="utf-8") + (tmp_path / "configs" / "sonic" / "start.sh").write_text("# wrapper\n", encoding="utf-8") + calls = [] + + def fake_safe_run(cmd, **kwargs): + calls.append(cmd) + return _completed(cmd, returncode=0) + + monkeypatch.setattr(apply_configs, "docker_prefix", lambda: []) + monkeypatch.setattr(apply_configs, "safe_run", fake_safe_run) + monkeypatch.setattr(apply_configs, "_wait_for_sonic", lambda *args, **kwargs: True) + + device, success, message, _elapsed, _ready_elapsed, _activation_elapsed = apply_configs._apply_single_device( + "spine1", str(tmp_path), "demo", 1 + ) + + assert (device, success, message) == ("spine1", True, "post-deploy activation") + assert not any(cmd[:2] == ["docker", "cp"] for cmd in calls) + assert not any(cmd[-2:] == ["bash", "/tmp/config.sh"] for cmd in calls) + joined = "\n".join(" ".join(cmd) for cmd in calls) + assert "fib_multipath_hash_policy=1" in joined + assert "sysctl -n net.ipv4.fib_multipath_hash_policy" in joined + assert "fib_multipath_hash_policy=1 >/dev/null || true" not in joined + assert "vtysh -b" in joined + assert "counterpoll port interval 10000" in joined + assert "/usr/sbin/telemetry -port 50051 -noTLS -client_auth none" in joined + assert "pkill -x telemetry" in joined + + +def test_apply_single_device_requires_preseed_config(monkeypatch, tmp_path): + wait_called = False + + def fake_wait(*_args, **_kwargs): + nonlocal wait_called + wait_called = True + return True + + monkeypatch.setattr(apply_configs, "_wait_for_sonic", fake_wait) + + device, success, message, _elapsed, _ready_elapsed, _activation_elapsed = apply_configs._apply_single_device( + "spine1", str(tmp_path), "demo", 1 + ) + + assert (device, success) == ("spine1", False) + assert "preseed config not found" in message + assert wait_called is False + + +def test_apply_single_device_requires_startup_wrapper(monkeypatch, tmp_path): + config_dir = tmp_path / "configs" / "sonic" / "spine1" + config_dir.mkdir(parents=True) + (config_dir / "config_db.json").write_text("{}\n", encoding="utf-8") + wait_called = False + + def fake_wait(*_args, **_kwargs): + nonlocal wait_called + wait_called = True + return True + + monkeypatch.setattr(apply_configs, "_wait_for_sonic", fake_wait) + + device, success, message, _elapsed, _ready_elapsed, _activation_elapsed = apply_configs._apply_single_device( + "spine1", str(tmp_path), "demo", 1 + ) + + assert (device, success) == ("spine1", False) + assert "startup wrapper not found" in message + assert wait_called is False + + +def test_activate_preseed_device_fails_when_hash_policy_cannot_be_applied(monkeypatch): + calls = [] + + def fake_safe_run(cmd, **kwargs): + calls.append(cmd) + return _completed(cmd, returncode=1, stderr="sysctl: permission denied") + + monkeypatch.setattr(apply_configs, "safe_run", fake_safe_run) + + assert apply_configs._activate_preseed_device([], "clab-demo-agg1", 0) is False + command = calls[0][-1] + assert "fib_multipath_hash_policy=0" in command + assert "sysctl -n net.ipv4.fib_multipath_hash_policy" in command + + +def test_ecmp_hash_policies_are_loaded_from_manifest_roles(tmp_path): + from netopsbench.platform.topology.generator import generate_topology + from netopsbench.platform.topology.topology_utils import load_topology_manifest + + generate_topology("fat-tree-k12", str(tmp_path)) + manifest = load_topology_manifest(tmp_path) + + policies = apply_configs._ecmp_hash_policies(manifest, ["core1", "agg1", "edge1"]) + + assert policies == {"core1": 1, "agg1": 0, "edge1": 1} + + +def test_ecmp_hash_policies_reject_preseed_device_missing_from_manifest(tmp_path): + from netopsbench.platform.topology.generator import generate_topology + from netopsbench.platform.topology.topology_utils import load_topology_manifest + + generate_topology("xs", str(tmp_path)) + manifest = load_topology_manifest(tmp_path) + + with pytest.raises(RuntimeError, match="stale1"): + apply_configs._ecmp_hash_policies(manifest, ["spine1", "stale1"]) diff --git a/tests/test_architecture.py b/tests/test_architecture.py new file mode 100644 index 0000000..c5aab9c --- /dev/null +++ b/tests/test_architecture.py @@ -0,0 +1,299 @@ +"""Static dependency rules for the modular NetOpsBench implementation.""" + +from __future__ import annotations + +import ast +import subprocess +import sys +import tomllib +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +PACKAGE_ROOT = PROJECT_ROOT / "netopsbench" + + +def _python_files(root: Path = PACKAGE_ROOT) -> list[Path]: + return sorted(root.rglob("*.py")) + + +def _module_name(path: Path) -> str: + module = ".".join(path.relative_to(PROJECT_ROOT).with_suffix("").parts) + return module.removesuffix(".__init__") + + +def _imports(path: Path) -> set[str]: + module = _module_name(path) + package = module if path.name == "__init__.py" else module.rsplit(".", 1)[0] + imported: set[str] = set() + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.level: + package_parts = package.split(".") + prefix = package_parts[: len(package_parts) - node.level + 1] + base = ".".join(prefix + ([node.module] if node.module else [])) + else: + base = node.module or "" + if base: + imported.add(base) + if not node.module: + imported.update(f"{base}.{alias.name}" if base else alias.name for alias in node.names) + return imported + + +def test_platform_does_not_depend_on_public_sdk(): + offenders = { + str(path.relative_to(PROJECT_ROOT)): sorted( + name for name in _imports(path) if name.startswith("netopsbench.sdk") + ) + for path in _python_files(PACKAGE_ROOT / "platform") + } + assert not {path: imports for path, imports in offenders.items() if imports} + + +def test_internal_module_graph_has_no_import_cycles(): + files_by_module = {_module_name(path): path for path in _python_files()} + graph = { + module: {target for target in _imports(path) if target in files_by_module and target != module} + for module, path in files_by_module.items() + } + visiting: list[str] = [] + visited: set[str] = set() + + def visit(module: str) -> list[str] | None: + if module in visiting: + start = visiting.index(module) + return visiting[start:] + [module] + if module in visited: + return None + visiting.append(module) + for dependency in graph[module]: + if cycle := visit(dependency): + return cycle + visiting.pop() + visited.add(module) + return None + + for module in graph: + cycle = visit(module) + assert cycle is None, " -> ".join(cycle or []) + + +def test_domain_and_service_modules_do_not_read_process_environment(): + roots = [ + PACKAGE_ROOT / "models", + PACKAGE_ROOT / "platform" / "faults" / "handlers", + PACKAGE_ROOT / "platform" / "faults" / "services", + ] + files = [path for root in roots for path in _python_files(root)] + files.extend( + PROJECT_ROOT / relative + for relative in ( + "netopsbench/platform/faults/models.py", + "netopsbench/platform/scenario/generator.py", + "netopsbench/platform/scenario/validator.py", + "netopsbench/platform/topology/clos_builder.py", + "netopsbench/platform/topology/fat_tree_builder.py", + "netopsbench/platform/topology/plan.py", + "netopsbench/platform/topology/renderer.py", + "netopsbench/platform/traffic/commands.py", + "netopsbench/platform/traffic/generator.py", + "netopsbench/platform/traffic/planner.py", + "netopsbench/platform/session/diagnosis.py", + "netopsbench/platform/session/reporting.py", + ) + ) + forbidden = ("os.environ", "os.getenv", "netopsbench.config") + offenders = { + str(path.relative_to(PROJECT_ROOT)): [token for token in forbidden if token in path.read_text(encoding="utf-8")] + for path in files + } + assert not {path: tokens for path, tokens in offenders.items() if tokens} + + +def test_removed_environment_controls_do_not_return(): + removed = { + "NETOPSBENCH_ACTIVE_INTERFACE_COVERAGE_MIN_RATIO", + "NETOPSBENCH_AGENT_TIMEOUT_SECONDS", + "NETOPSBENCH_APPLY_CONFIG_PARALLEL", + "NETOPSBENCH_BGP_COLLECTOR_MAX_BYTES", + "NETOPSBENCH_BGP_COLLECTOR_PARALLELISM", + "NETOPSBENCH_BGP_POLL_INTERVAL_SECONDS", + "NETOPSBENCH_CONTAINERLAB_MAX_WORKERS", + "NETOPSBENCH_CONTAINERLAB_TIMEOUT", + "NETOPSBENCH_GRAFANA_DEFAULT_BUCKET", + "NETOPSBENCH_GRAFANA_INFLUXDB_URL", + "NETOPSBENCH_GRAFANA_PASSWORD", + "NETOPSBENCH_GRAFANA_URL", + "NETOPSBENCH_PINGMESH_DEPLOY_PARALLELISM", + "NETOPSBENCH_PINGMESH_INFLUXDB_URL", + "NETOPSBENCH_RUNTIME_ID", + "NETOPSBENCH_SFLOW_", + "NETOPSBENCH_SONIC_LINK_WAIT_INTERVAL", + "NETOPSBENCH_SONIC_LINK_WAIT_TIMEOUT", + "NETOPSBENCH_SONIC_WAIT_TRIES", + "NETOPSBENCH_SWITCH_PPS_LIMIT", + "NETOPSBENCH_SYSLOG_COLLECTOR", + "NETOPSBENCH_TELEGRAF_INFLUXDB_URL", + "NETOPSBENCH_TRACE", + "NETOPSBENCH_WORKER_AGENT_TIMEOUT_SECONDS", + "NETOPSBENCH_WORKER_DEPLOY_JOBS", + "NETOPSBENCH_WORKER_DEPLOY_TIMEOUT", + "NETOPSBENCH_WORKER_DISABLE_LANGSMITH", + "NETOPSBENCH_WORKER_HEALTH_DELAY_SECONDS", + "NETOPSBENCH_WORKER_HEALTH_RETRIES", + "NETOPSBENCH_WORKER_ID", + "PINGMESH_CYCLE_INTERVAL", + "SONIC_GNMI_", + } + roots = [ + PACKAGE_ROOT, + PROJECT_ROOT / "docs", + PROJECT_ROOT / "examples", + PROJECT_ROOT / "scenarios", + PROJECT_ROOT / "scripts", + ] + files = [path for root in roots for path in root.rglob("*") if path.is_file()] + files.append(PROJECT_ROOT / ".env.example") + offenders: dict[str, list[str]] = {} + for path in files: + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + matches = sorted(token for token in removed if token in text) + if matches: + offenders[str(path.relative_to(PROJECT_ROOT))] = matches + assert not offenders + + +def test_library_modules_do_not_call_sys_exit(): + cli_boundaries = { + PACKAGE_ROOT / "cli" / "main.py", + PACKAGE_ROOT / "platform" / "pingmesh" / "cli.py", + } + offenders: list[str] = [] + for path in _python_files(): + if path in cli_boundaries: + continue + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "sys" + and node.func.attr == "exit" + ): + offenders.append(f"{path.relative_to(PROJECT_ROOT)}:{node.lineno}") + assert not offenders + + +def test_source_uses_timezone_aware_utc_clock(): + forbidden = ("datetime.utcnow(", "datetime.now()") + offenders = { + str(path.relative_to(PROJECT_ROOT)): [token for token in forbidden if token in path.read_text(encoding="utf-8")] + for path in _python_files() + } + offenders = {path: tokens for path, tokens in offenders.items() if tokens} + assert not offenders + + +def test_runtime_lifecycle_cli_imports_without_package_cycle(): + result = subprocess.run( + [sys.executable, "-m", "netopsbench.platform.runtime.cli", "--help"], + cwd=PROJECT_ROOT, + text=True, + capture_output=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + +def test_removed_internal_compatibility_trees_do_not_return(): + removed = [ + PACKAGE_ROOT / "resources", + PACKAGE_ROOT / "screenshots", + PACKAGE_ROOT / "platform" / "worker", + PROJECT_ROOT / "observability", + PROJECT_ROOT / "scripts" / "observability", + ] + assert not [path for path in removed if path.exists()] + + +def test_observability_does_not_depend_on_runtime_or_toolkit(): + forbidden_prefixes = ( + "netopsbench.platform.runtime", + "netopsbench.platform.toolkit", + ) + offenders = { + str(path.relative_to(PROJECT_ROOT)): sorted( + name for name in _imports(path) if name.startswith(forbidden_prefixes) + ) + for path in _python_files(PACKAGE_ROOT / "platform" / "observability") + } + assert not {path: imports for path, imports in offenders.items() if imports} + + +def test_removed_platform_state_and_features_do_not_return(): + forbidden = ( + "TopologyState", + "TopologySpec", + "WorkerSpec", + "_discover_topology", + "NETOPSBENCH_ALLOW_TOPOLOGY_FALLBACK", + "sflow", + "screenshot", + ) + offenders = { + str(path.relative_to(PROJECT_ROOT)): [ + token for token in forbidden if token.lower() in path.read_text(encoding="utf-8").lower() + ] + for path in _python_files(PACKAGE_ROOT / "platform") + } + assert not {path: tokens for path, tokens in offenders.items() if tokens} + + +def test_platform_main_functions_match_declared_console_scripts(): + project = tomllib.loads((PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + declared_modules = { + target.split(":", 1)[0] + for target in project["project"]["scripts"].values() + if target.startswith("netopsbench.platform.") + } + entry_modules = { + _module_name(path) + for path in _python_files(PACKAGE_ROOT / "platform") + if any( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "main" + for node in ast.parse(path.read_text(encoding="utf-8")).body + ) + } + assert entry_modules == declared_modules + + +def test_runtime_code_does_not_infer_a_source_checkout(): + forbidden = ("NETOPSBENCH_REPO_ROOT", "runtime_asset_root", "materialize_resource") + offenders = { + str(path.relative_to(PROJECT_ROOT)): [token for token in forbidden if token in path.read_text(encoding="utf-8")] + for path in _python_files() + } + assert not {path: tokens for path, tokens in offenders.items() if tokens} + + +def test_packaged_asset_trees_contain_only_runtime_inputs(): + observability_root = PACKAGE_ROOT / "platform" / "observability" / "assets" + observability_files = { + str(path.relative_to(observability_root)) for path in observability_root.rglob("*") if path.is_file() + } + assert observability_files == { + "docker-compose.yaml", + "telegraf.conf.template", + "grafana/dashboards/network_overview.json", + "grafana/dashboards/pingmesh.json", + "grafana/provisioning/dashboards/default.yaml", + "grafana/provisioning/datasources/default.yaml", + } + + scenario_specs = PACKAGE_ROOT / "platform" / "scenario" / "specs" + assert {path.name for path in scenario_specs.iterdir() if path.is_file()} == {"fault_campaign.yaml"} diff --git a/tests/test_bgp_collector.py b/tests/test_bgp_collector.py index 5dbf05b..4a1fb7f 100644 --- a/tests/test_bgp_collector.py +++ b/tests/test_bgp_collector.py @@ -1,6 +1,58 @@ from pathlib import Path -from scripts.runtime.run_bgp_collector import build_bgp_lines, collect_bgp_lines, normalize_bgp_state +from netopsbench.models import topology as topology_models +from netopsbench.models.topology import Collector, Device, DeviceRole, Management, TopologyManifest +from netopsbench.platform.observability.bgp_collector import ( + _collect_bgp_lines_paced, + _write_lines, + build_bgp_collection_line, + build_bgp_lines, + collect_bgp_lines, + normalize_bgp_state, + run_once, +) + + +def _write_topology( + path: Path, + *, + name: str = "demo", + family: str = "clos", + devices: list[Device] | None = None, +) -> None: + devices = devices or [ + Device(name="spine1", role=DeviceRole.SPINE), + Device(name="leaf1", role=DeviceRole.LEAF), + ] + manifest = TopologyManifest( + topology_id=name, + name=name, + scale="xs" if family == "clos" else "fat-tree-k8", + family=family, + management=Management(network=f"clab-{name}", ipv4_subnet="172.20.20.0/24"), + collector=Collector(ipv4="172.20.20.200"), + defaults=topology_models.TopologyDefaults(), + facts=topology_models.TopologyFacts( + num_spines=sum(device.role is DeviceRole.SPINE for device in devices), + num_leafs=sum(device.role is DeviceRole.LEAF for device in devices), + num_cores=sum(device.role is DeviceRole.CORE for device in devices), + num_aggs=sum(device.role is DeviceRole.AGG for device in devices), + num_edges=sum(device.role is DeviceRole.EDGE for device in devices), + num_pods=2 if family == "fat-tree" else 0, + clients_per_attached_switch=1, + total_clients=sum(device.role is DeviceRole.CLIENT for device in devices), + total_switches=sum(device.role is not DeviceRole.CLIENT for device in devices), + fat_tree_k=2 if family == "fat-tree" else None, + full_density_clients_per_attached_switch=1 if family == "fat-tree" else None, + host_density="standard" if family == "fat-tree" else None, + ), + routing=topology_models.RoutingMetadata( + ecmp_hash_policy_by_role={device.role: 1 for device in devices if device.role is not DeviceRole.CLIENT} + ), + devices=devices, + links=[], + ) + path.write_text(manifest.model_dump_json(), encoding="utf-8") def test_build_bgp_lines_normalizes_states_and_fields(): @@ -36,12 +88,20 @@ def test_normalize_bgp_state_defaults_to_unknown(): assert normalize_bgp_state("Idle") == "IDLE" +def test_build_bgp_collection_line_records_success_and_failure(): + success = build_bgp_collection_line("leaf1", 7, "runtime-xs", True, 2, 41, "") + failure = build_bgp_collection_line("leaf1", 8, "runtime-xs", False, 0, 30000, "timeout") + + assert success.startswith("bgp_collection,source=leaf1,topology_id=runtime-xs ") + assert "collection_ok=true" in success + assert "neighbor_count=2i" in success + assert 'error_type="timeout"' in failure + assert "collection_ok=false" in failure + + def test_collect_bgp_lines_reads_topology_and_executes_docker(monkeypatch, tmp_path): metadata_file = tmp_path / "topology.json" - metadata_file.write_text( - '{"name":"demo","devices":{"spines":[{"name":"spine1"}],"leafs":[{"name":"leaf1"}]}}', - encoding="utf-8", - ) + _write_topology(metadata_file) calls = [] @@ -57,15 +117,159 @@ def fake_run(args, capture_output, text, check, timeout): 192.168.11.2 4 65011 310 309 20 0 0 04:54:34 2 16 N/A """) - monkeypatch.setattr("scripts.runtime.run_bgp_collector.subprocess.run", fake_run) - monkeypatch.setattr("scripts.runtime.run_bgp_collector._docker_prefix", lambda: []) + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.subprocess.run", fake_run) + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.docker_prefix", lambda: []) - monkeypatch.setenv("NETOPSBENCH_TOPOLOGY_ID", "runtime-xs") + lines = collect_bgp_lines(Path(metadata_file), timestamp_ns=7, topology_id="runtime-xs") - lines = collect_bgp_lines(Path(metadata_file), timestamp_ns=7) - - assert len(lines) == 2 + assert len(lines) == 4 assert calls[0][:3] == ["docker", "exec", "clab-demo-spine1"] assert calls[1][:3] == ["docker", "exec", "clab-demo-leaf1"] - assert all('session_state="ESTABLISHED"' in line for line in lines) + assert sum(line.startswith("bgp_neighbors,") for line in lines) == 2 + assert sum(line.startswith("bgp_collection,") for line in lines) == 2 assert all(",topology_id=runtime-xs " in line for line in lines) + + +def test_collect_bgp_lines_supports_parallelism(monkeypatch, tmp_path): + metadata_file = tmp_path / "topology.json" + _write_topology( + metadata_file, + devices=[ + Device(name="spine1", role=DeviceRole.SPINE), + Device(name="spine2", role=DeviceRole.SPINE), + Device(name="leaf1", role=DeviceRole.LEAF), + ], + ) + + calls = [] + + class _Result: + returncode = 0 + stdout = """ +Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt Desc +192.168.11.2 4 65011 310 309 20 0 0 04:54:34 2 16 N/A +""" + + def fake_run(args, capture_output, text, check, timeout): + calls.append(args[2]) + return _Result() + + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.subprocess.run", fake_run) + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.docker_prefix", lambda: []) + + lines = collect_bgp_lines(Path(metadata_file), timestamp_ns=9, parallelism=2) + + assert len(lines) == 6 + assert sorted(calls) == ["clab-demo-leaf1", "clab-demo-spine1", "clab-demo-spine2"] + assert all(line.endswith(" 9") for line in lines) + + +def test_loop_collection_spreads_device_starts_over_interval(monkeypatch, tmp_path): + metadata_file = tmp_path / "topology.json" + _write_topology( + metadata_file, + devices=[ + Device(name="spine1", role=DeviceRole.SPINE), + Device(name="spine2", role=DeviceRole.SPINE), + Device(name="leaf1", role=DeviceRole.LEAF), + ], + ) + waits = [] + + class _StopEvent: + @staticmethod + def wait(seconds): + waits.append(seconds) + return False + + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.time.monotonic", lambda: 0.0) + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.time.time_ns", lambda: 9) + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.docker_prefix", lambda: []) + monkeypatch.setattr( + "netopsbench.platform.observability.bgp_collector._collect_device_bgp", + lambda _lab, device, _prefix, timestamp, _topology: [f"{device} {timestamp}"], + ) + + lines = _collect_bgp_lines_paced(metadata_file, interval_seconds=9, parallelism=3, stop_event=_StopEvent()) + + assert lines == ["spine1 9", "spine2 9", "leaf1 9"] + assert waits == [3, 6] + + +def test_collect_bgp_lines_writes_collection_failure_without_fake_neighbor(monkeypatch, tmp_path): + metadata_file = tmp_path / "topology.json" + _write_topology(metadata_file, devices=[Device(name="leaf1", role=DeviceRole.LEAF)]) + + class _Result: + returncode = 1 + stdout = "" + stderr = "vtysh failed" + + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.subprocess.run", lambda *a, **k: _Result()) + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.docker_prefix", lambda: []) + + lines = collect_bgp_lines(metadata_file, timestamp_ns=9) + + assert len(lines) == 1 + assert lines[0].startswith("bgp_collection,source=leaf1") + assert "collection_ok=false" in lines[0] + assert 'error_type="command_failed"' in lines[0] + + +def test_collect_bgp_lines_reads_native_fat_tree_routing_devices_once(monkeypatch, tmp_path): + metadata_file = tmp_path / "topology.json" + _write_topology( + metadata_file, + name="ft", + family="fat-tree", + devices=[ + Device(name="core1", role=DeviceRole.CORE), + Device(name="agg1", role=DeviceRole.AGG), + Device(name="edge1", role=DeviceRole.EDGE), + Device(name="client1", role=DeviceRole.CLIENT, attached_switch="edge1"), + ], + ) + + calls = [] + + class _Result: + returncode = 0 + stdout = """ +Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt Desc +192.168.11.2 4 65011 310 309 20 0 0 04:54:34 2 16 N/A +""" + + def fake_run(args, capture_output, text, check, timeout): + calls.append(args) + return _Result() + + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.subprocess.run", fake_run) + monkeypatch.setattr("netopsbench.platform.observability.bgp_collector.docker_prefix", lambda: []) + + collect_bgp_lines(Path(metadata_file), timestamp_ns=7) + + containers = [call[2] for call in calls] + assert containers == ["clab-ft-core1", "clab-ft-agg1", "clab-ft-edge1"] + + +def test_run_once_writes_snapshot_and_exits(monkeypatch, tmp_path): + metadata_file = tmp_path / "topology.json" + metadata_file.write_text('{"name":"demo","devices":{"spines":[],"leafs":[]}}', encoding="utf-8") + output_file = tmp_path / "bgp.lp" + + monkeypatch.setattr( + "netopsbench.platform.observability.bgp_collector.collect_bgp_lines", + lambda metadata, parallelism=1, topology_id=None: ["bgp_neighbors,source=spine1 value=1i 7"], + ) + + assert run_once(metadata_file, output_file, parallelism=4) == 0 + assert output_file.read_text(encoding="utf-8") == "bgp_neighbors,source=spine1 value=1i 7\n" + + +def test_write_lines_truncates_existing_bgp_file_when_size_limit_would_be_exceeded(tmp_path): + output_file = tmp_path / "bgp.lp" + output_file.write_text("old_snapshot value=1i 1\n" * 4, encoding="utf-8") + + _write_lines(output_file, ["new_snapshot value=2i 2"], max_bytes=32) + + assert output_file.read_text(encoding="utf-8") == "new_snapshot value=2i 2\n" diff --git a/tests/test_bgp_observability.py b/tests/test_bgp_observability.py new file mode 100644 index 0000000..9ffc4cb --- /dev/null +++ b/tests/test_bgp_observability.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import subprocess + +from netopsbench.platform.toolkit.toolkit import AgentToolkit +from netopsbench.platform.topology.generator import generate_topology + + +def _toolkit(tmp_path) -> AgentToolkit: + metadata = generate_topology("xs", str(tmp_path))["metadata"] + return AgentToolkit(topology_metadata=metadata) + + +def test_query_bgp_events_classifies_session_transitions(monkeypatch, tmp_path): + toolkit = _toolkit(tmp_path) + rows = [ + { + "_measurement": "bgp_neighbors", + "_time": "2026-07-11T00:00:00Z", + "source": "leaf1", + "neighbor_address": "10.0.0.1", + "session_state": "ESTABLISHED", + "prefixes_received": 4, + }, + { + "_measurement": "bgp_neighbors", + "_time": "2026-07-11T00:00:10Z", + "source": "leaf1", + "neighbor_address": "10.0.0.1", + "session_state": "IDLE", + }, + { + "_measurement": "bgp_neighbors", + "_time": "2026-07-11T00:00:20Z", + "source": "leaf1", + "neighbor_address": "10.0.0.1", + "session_state": "ESTABLISHED", + "prefixes_received": 3, + }, + {"_measurement": "bgp_collection", "_time": "2026-07-11T00:00:20Z", "source": "leaf1", "collection_ok": True}, + ] + monkeypatch.setattr( + toolkit, + "_query_influx_rows", + lambda query, **kwargs: rows[:1] if "range(start: -30d" in query else rows[1:], + ) + + result = toolkit.query_bgp_events(start_time="2026-07-11T00:00:05Z", end_time="2026-07-11T00:00:30Z") + + assert result.success is True + event = result.data["events"][0] + assert event["event_type"] == "session_flap" + assert event["previous_state"] == "ESTABLISHED" + assert event["latest_state"] == "ESTABLISHED" + assert event["states_observed"] == ["IDLE", "ESTABLISHED"] + + +def test_query_bgp_events_reports_non_established_and_collection_gap(monkeypatch, tmp_path): + toolkit = _toolkit(tmp_path) + rows = [ + { + "_measurement": "bgp_neighbors", + "_time": "2026-07-11T00:00:10Z", + "source": "leaf1", + "neighbor_address": "10.0.0.1", + "session_state": "IDLE", + }, + { + "_measurement": "bgp_collection", + "_time": "2026-07-11T00:00:10Z", + "source": "leaf2", + "collection_ok": False, + "error_type": "timeout", + }, + ] + monkeypatch.setattr(toolkit, "_query_influx_rows", lambda *a, **k: rows) + + result = toolkit.query_bgp_events(start_time="2026-07-11T00:00:00Z", end_time="2026-07-11T00:00:30Z") + + kinds = {(event["device"], event["event_type"]) for event in result.data["events"]} + assert ("leaf1", "non_established_observed") in kinds + assert ("leaf2", "collection_gap") in kinds + + +def test_query_bgp_events_classifies_down_and_recovery(monkeypatch, tmp_path): + toolkit = _toolkit(tmp_path) + prior = [ + { + "_measurement": "bgp_neighbors", + "_time": "2026-07-10T23:59:50Z", + "source": "leaf1", + "neighbor_address": "10.0.0.1", + "session_state": "ESTABLISHED", + } + ] + window = [ + { + "_measurement": "bgp_neighbors", + "_time": "2026-07-11T00:00:10Z", + "source": "leaf1", + "neighbor_address": "10.0.0.1", + "session_state": "IDLE", + }, + {"_measurement": "bgp_collection", "_time": "2026-07-11T00:00:10Z", "source": "leaf1", "collection_ok": True}, + {"_measurement": "bgp_collection", "_time": "2026-07-11T00:00:10Z", "source": "leaf2", "collection_ok": True}, + {"_measurement": "bgp_collection", "_time": "2026-07-11T00:00:10Z", "source": "spine1", "collection_ok": True}, + {"_measurement": "bgp_collection", "_time": "2026-07-11T00:00:10Z", "source": "spine2", "collection_ok": True}, + ] + monkeypatch.setattr(toolkit, "_query_influx_rows", lambda query, **kwargs: prior if "-30d" in query else window) + + down = toolkit.query_bgp_events(start_time="2026-07-11T00:00:00Z", end_time="2026-07-11T00:00:30Z") + assert down.data["events"][0]["event_type"] == "session_down" + + prior[0]["session_state"] = "IDLE" + window[0]["session_state"] = "ESTABLISHED" + recovered = toolkit.query_bgp_events( + start_time="2026-07-11T00:00:00Z", end_time="2026-07-11T00:00:30Z", state="all" + ) + assert recovered.data["events"][0]["event_type"] == "session_recovered" + + +def test_query_bgp_events_filters_role_and_limit(monkeypatch, tmp_path): + toolkit = _toolkit(tmp_path) + rows = [ + { + "_measurement": "bgp_neighbors", + "_time": "2026-07-11T00:00:10Z", + "source": leaf, + "neighbor_address": f"10.0.0.{index}", + "session_state": "IDLE", + } + for index, leaf in enumerate(("leaf1", "leaf2"), 1) + ] + rows += [ + {"_measurement": "bgp_collection", "_time": "2026-07-11T00:00:10Z", "source": device, "collection_ok": True} + for device in ("spine1", "spine2", "leaf1", "leaf2") + ] + monkeypatch.setattr(toolkit, "_query_influx_rows", lambda *args, **kwargs: rows) + + result = toolkit.query_bgp_events(time_range_minutes=10, role="leaf", limit=1) + + assert result.success is True + assert result.data["returned_events"] == 1 + assert result.data["truncated"] is True + assert result.data["events"][0]["role"] == "leaf" + + +def test_query_bgp_events_query_is_centralized_and_topology_scoped(monkeypatch, tmp_path): + toolkit = _toolkit(tmp_path) + queries = [] + monkeypatch.setattr(toolkit, "_query_influx_rows", lambda query, **kwargs: queries.append(query) or []) + monkeypatch.setattr( + toolkit, "_docker_exec", lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("live query")) + ) + + result = toolkit.query_bgp_events(time_range_minutes=10, device="leaf1", peer="10.0.0.1") + + assert result.success is True + assert len(queries) == 2 + assert f'r.topology_id == "{toolkit.topology_id}"' in queries[0] + assert 'r._measurement == "bgp_neighbors"' in queries[0] + assert 'r._measurement == "bgp_collection"' in queries[1] + assert 'r.source == "leaf1"' in queries[0] + assert 'r.neighbor_address == "10.0.0.1"' in queries[0] + + +def test_query_bgp_events_propagates_influx_failure(monkeypatch, tmp_path): + toolkit = _toolkit(tmp_path) + monkeypatch.setattr(toolkit, "_query_influx_rows", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("down"))) + + result = toolkit.query_bgp_events(time_range_minutes=10) + + assert result.success is False + assert "down" in result.error + + +def test_get_bgp_neighbor_only_queries_requested_device_and_peer(monkeypatch, tmp_path): + toolkit = _toolkit(tmp_path) + calls = [] + + def fake_exec(container, args, timeout): + calls.append((container, args)) + if "neighbors" in args[-1]: + stdout = "BGP neighbor is 10.0.0.1, remote AS 65100, local AS 65001\n BGP state = Idle\n Last reset due to Bad Peer AS\n" + else: + stdout = "Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd\n10.0.0.1 4 65100 3 4 0 0 0 never Idle\n" + return subprocess.CompletedProcess(args, 0, stdout, "") + + monkeypatch.setattr(toolkit, "_docker_exec", fake_exec) + result = toolkit.get_bgp_neighbor("leaf1", "10.0.0.1") + + assert result.success is True + assert result.data["state"] == "Idle" + assert result.data["peer_as"] == 65100 + assert "Bad Peer AS" in result.data["last_reset"] + assert all(call[0].endswith("leaf1") for call in calls) + assert all("10.0.0.1" in call[1][-1] or "summary" in call[1][-1] for call in calls) diff --git a/tests/test_canonical_models.py b/tests/test_canonical_models.py new file mode 100644 index 0000000..1148ede --- /dev/null +++ b/tests/test_canonical_models.py @@ -0,0 +1,421 @@ +"""Focused contracts for the architecture refactor's canonical models.""" + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from netopsbench.models import topology as topology_models +from netopsbench.models.profiles import get_scale_profile, supported_scales +from netopsbench.models.runtime import RuntimeIdentity +from netopsbench.models.topology import ( + Collector, + Device, + DeviceRole, + Link, + LinkEndpoint, + Management, + TopologyManifest, +) + + +def _manifest(*, devices: list[Device], links: list[Link]) -> TopologyManifest: + return TopologyManifest( + topology_id="demo-topology", + name="demo", + scale="xs", + family="clos", + management=Management(network="clab-demo", ipv4_subnet="172.20.20.0/24"), + collector=Collector(ipv4="172.20.20.200"), + defaults=topology_models.TopologyDefaults(), + facts=topology_models.TopologyFacts( + num_spines=2, + num_leafs=1, + clients_per_attached_switch=1, + total_clients=sum(device.role is DeviceRole.CLIENT for device in devices), + total_switches=sum(device.role is not DeviceRole.CLIENT for device in devices), + ), + routing=topology_models.RoutingMetadata( + ecmp_hash_policy_by_role={device.role: 1 for device in devices if device.role is not DeviceRole.CLIENT} + ), + devices=devices, + links=links, + ) + + +def test_topology_manifest_validates_device_and_link_references(): + spine = Device(name="spine1", role=DeviceRole.SPINE) + leaf = Device(name="leaf1", role=DeviceRole.LEAF) + client = Device(name="client1", role=DeviceRole.CLIENT, attached_switch="leaf1") + link = Link( + kind="spine-leaf", + endpoints=( + LinkEndpoint(device="spine1", interface="Ethernet0"), + LinkEndpoint(device="leaf1", interface="Ethernet0"), + ), + ) + + manifest = _manifest(devices=[spine, leaf, client], links=[link]) + + assert manifest.device("leaf1") == leaf + assert manifest.devices_by_role("spine") == [spine] + assert manifest.switches() == [spine, leaf] + assert manifest.routing_devices() == [spine, leaf] + assert manifest.edge_devices() == [leaf] + assert manifest.clients() == [client] + assert manifest.client_attached_devices() == [leaf] + + with pytest.raises(ValidationError, match="unique"): + _manifest(devices=[spine, Device(name="spine1", role=DeviceRole.LEAF)], links=[]) + + with pytest.raises(ValidationError, match="unknown device"): + _manifest( + devices=[spine], + links=[ + Link( + kind="invalid", + endpoints=( + LinkEndpoint(device="spine1", interface="Ethernet0"), + LinkEndpoint(device="missing", interface="Ethernet0"), + ), + ) + ], + ) + + +def test_topology_manifest_requires_clients_to_attach_to_known_non_clients(): + leaf = Device(name="leaf1", role=DeviceRole.LEAF) + + with pytest.raises(ValidationError, match="must define attached_switch"): + _manifest(devices=[leaf, Device(name="client1", role=DeviceRole.CLIENT)], links=[]) + + with pytest.raises(ValidationError, match="unknown device"): + _manifest( + devices=[leaf, Device(name="client1", role=DeviceRole.CLIENT, attached_switch="missing")], + links=[], + ) + + +def test_topology_manifest_requires_exact_ecmp_policy_for_switch_roles(): + spine = Device(name="spine1", role=DeviceRole.SPINE) + leaf = Device(name="leaf1", role=DeviceRole.LEAF) + payload = _manifest(devices=[spine, leaf], links=[]).model_dump(mode="json") + + del payload["routing"]["ecmp_hash_policy_by_role"]["leaf"] + with pytest.raises(ValidationError, match="missing roles: leaf"): + TopologyManifest.model_validate(payload) + + payload = _manifest(devices=[spine], links=[]).model_dump(mode="json") + payload["routing"]["ecmp_hash_policy_by_role"]["leaf"] = 1 + with pytest.raises(ValidationError, match="unexpected roles: leaf"): + TopologyManifest.model_validate(payload) + + payload = _manifest(devices=[spine], links=[]).model_dump(mode="json") + payload["routing"]["ecmp_hash_policy_by_role"]["spine"] = 2 + with pytest.raises(ValidationError, match="literal_error"): + TopologyManifest.model_validate(payload) + + with pytest.raises(ValidationError, match="non-client"): + _manifest( + devices=[ + leaf, + Device(name="client1", role=DeviceRole.CLIENT, attached_switch="leaf1"), + Device(name="client2", role=DeviceRole.CLIENT, attached_switch="client1"), + ], + links=[], + ) + + +def test_agent_projection_keeps_grouped_fields_and_adapts_fat_tree_roles(): + core = Device(name="core1", role="core", mgmt_ip="172.20.20.11", asn=65001) + agg = Device(name="agg1", role="agg", mgmt_ip="172.20.20.12", asn=65101) + edge = Device(name="edge1", role="edge", mgmt_ip="172.20.20.13", asn=65201) + client = Device(name="client1", role="client", attached_switch="edge1", data_ip="192.168.1.2") + manifest = TopologyManifest( + topology_id="fat-demo", + name="fat-demo", + scale="fat-tree-k8", + family="fat-tree", + management=Management(network="clab-fat-demo", ipv4_subnet="172.20.20.0/24"), + collector=Collector(ipv4="172.20.20.200"), + defaults=topology_models.TopologyDefaults(), + facts=topology_models.TopologyFacts( + num_cores=1, + num_aggs=1, + num_edges=1, + num_pods=1, + clients_per_attached_switch=1, + total_clients=1, + total_switches=3, + fat_tree_k=2, + full_density_clients_per_attached_switch=1, + host_density="standard", + ), + routing=topology_models.RoutingMetadata( + core_asn_range="65001-65001", + agg_asn_range="65101-65101", + edge_asn_range="65201-65201", + ecmp_hash_policy_by_role={DeviceRole.CORE: 1, DeviceRole.AGG: 0, DeviceRole.EDGE: 1}, + ), + devices=[core, agg, edge, client], + links=[], + ) + + projected = manifest.to_agent_topology() + + assert [device["name"] for device in projected["devices"]["spines"]] == ["core1"] + assert [device["name"] for device in projected["devices"]["leafs"]] == ["edge1"] + assert [device["name"] for device in projected["devices"]["cores"]] == ["core1"] + assert [device["name"] for device in projected["devices"]["aggs"]] == ["agg1"] + assert [device["name"] for device in projected["devices"]["edges"]] == ["edge1"] + projected_client = projected["devices"]["clients"][0] + assert projected_client["name"] == "client1" + assert projected_client["data_ip"] == "192.168.1.2" + assert projected_client["attached_switch"] == "edge1" + assert projected_client["edge"] == "edge1" + assert projected_client["leaf"] == "edge1" + assert projected["defaults"] == {"link_mtu": 9232, "sonic_port_mtu": 9100} + assert projected["fat_tree_k"] == 2 + assert projected["scale"] == { + "name": "fat-tree-k8", + "num_core": 1, + "num_agg": 1, + "num_edge": 1, + "num_pods": 1, + "clients_per_edge": 1, + "full_density_clients_per_edge": 1, + "host_density": "standard", + "total_clients": 1, + "total_devices": 3, + "num_spines": 1, + "num_leafs": 1, + } + assert projected["routing"]["core_asn_range"] == "65001-65001" + assert projected["routing"]["ecmp_hash_policy_by_role"] == {"core": 1, "agg": 0, "edge": 1} + assert [device["role"] for device in manifest.model_dump(mode="json")["devices"]] == [ + "core", + "agg", + "edge", + "client", + ] + + +def test_agent_projection_metadata_cannot_overwrite_canonical_device_fields(): + spine = Device( + name="spine1", + role=DeviceRole.SPINE, + mgmt_ip="172.20.20.11", + asn=65001, + metadata={ + "name": "metadata-name", + "role": "client", + "mgmt_ip": "192.0.2.1", + "asn": 99999, + "site": "lab-a", + }, + ) + + projected = _manifest(devices=[spine], links=[]).to_agent_topology()["devices"]["spines"][0] + + assert projected["name"] == "spine1" + assert projected["mgmt_ip"] == "172.20.20.11" + assert projected["asn"] == 65001 + assert "role" not in projected + assert projected["site"] == "lab-a" + + +def test_topology_defaults_facts_and_links_are_typed_and_strict(): + defaults = topology_models.TopologyDefaults() + facts = topology_models.TopologyFacts( + num_spines=16, + num_leafs=128, + clients_per_attached_switch=1, + total_clients=128, + total_switches=144, + ) + link = Link( + kind="spine-leaf", + mtu=9232, + endpoints=( + LinkEndpoint(device="spine1", interface="eth1"), + LinkEndpoint(device="leaf1", interface="eth1"), + ), + ) + + assert defaults.model_dump() == {"link_mtu": 9232, "sonic_port_mtu": 9100} + assert facts.clients_per_attached_switch == 1 + assert facts.num_leafs == 128 + assert link.mtu == 9232 + + with pytest.raises(ValidationError, match="extra"): + topology_models.TopologyDefaults(link_mtu=9232, sonic_port_mtu=9100, legacy_mtu=1500) + + with pytest.raises(ValidationError, match="extra"): + topology_models.TopologyFacts( + clients_per_attached_switch=1, + total_clients=2, + total_switches=4, + num_spines=2, + num_leafs=2, + legacy_count=4, + ) + + +def test_runtime_identity_defaults_are_isolated_by_runtime_id(): + first = RuntimeIdentity.create( + runtime_id="Run A/1", + worker_id="worker-1", + worker_index=1, + lab_name="lab-a", + topology_dir=Path("/tmp/lab-a"), + mgmt_subnet="172.31.100.0/24", + mgmt_network="clab-mgmt-lab-a", + ) + second = RuntimeIdentity.create( + runtime_id="Run B/1", + worker_id="worker-1", + worker_index=1, + lab_name="lab-b", + topology_dir=Path("/tmp/lab-b"), + mgmt_subnet="172.31.101.0/24", + mgmt_network="clab-mgmt-lab-b", + ) + + assert first.topology_id == "lab-a" + assert first.bucket == "network_data_run_a_1_w01" + assert first.bucket != second.bucket + assert not hasattr(first, "as_env") + with pytest.raises(ValidationError): + RuntimeIdentity.create( + runtime_id="runtime", + worker_id="worker", + worker_index=0, + lab_name="lab", + topology_dir=Path("/tmp/lab"), + mgmt_subnet="172.31.100.0/24", + mgmt_network="clab-mgmt-lab", + ) + + +def test_runtime_identity_normal_constructor_applies_defaults(): + identity = RuntimeIdentity( + runtime_id="Run Direct/1", + worker_id="worker-2", + worker_index=2, + lab_name="lab-direct", + topology_dir=Path("/tmp/lab-direct"), + mgmt_subnet="172.31.102.0/24", + mgmt_network="clab-mgmt-lab-direct", + ) + + assert identity.schema_version == "3" + assert identity.topology_id == "lab-direct" + assert identity.bucket == "network_data_run_direct_1_w02" + + +def test_runtime_identity_rejects_unknown_schema_version(): + with pytest.raises(ValidationError) as exc_info: + RuntimeIdentity( + schema_version="1", + runtime_id="runtime", + worker_id="worker-1", + worker_index=1, + lab_name="lab", + topology_id="lab", + topology_dir=Path("/tmp/lab"), + bucket="bucket", + mgmt_subnet="172.31.100.0/24", + mgmt_network="clab-mgmt-lab", + ) + + assert {error["loc"]: error["type"] for error in exc_info.value.errors()}[("schema_version",)] == "literal_error" + + +def test_runtime_identity_schema_version_defaults_to_v3(): + identity = RuntimeIdentity( + runtime_id="runtime", + worker_id="worker-1", + worker_index=1, + lab_name="lab", + topology_id="lab", + topology_dir=Path("/tmp/lab"), + bucket="bucket", + mgmt_subnet="172.31.100.0/24", + mgmt_network="clab-mgmt-lab", + ) + + assert identity.schema_version == "3" + + +@pytest.mark.parametrize( + ( + "name", + "family", + "spines", + "leafs", + "k", + "clients", + "prefix", + "pingmesh", + "pps", + "timeout", + "deploy_jobs", + "health_timeout", + ), + [ + ("xs", "clos", 2, 2, None, 1, 24, None, 250, 1800, 2, 60), + ("small", "clos", 2, 4, None, 2, 24, None, 250, 1800, 2, 60), + ("medium", "clos", 4, 8, None, 2, 24, None, 200, 1800, 2, 60), + ("large", "clos", 4, 16, None, 4, 24, None, 150, 2700, 1, 180), + ("xlarge", "clos", 16, 128, None, 1, 23, 16, 100, 3600, 1, 240), + ("fat-tree-k8", "fat-tree", None, None, 8, 4, 24, 16, 100, 3600, 1, 240), + ("fat-tree-k12", "fat-tree", None, None, 12, 2, 23, 16, 50, 5400, 1, 300), + ], +) +def test_scale_profiles_keep_exact_runtime_values( + name: str, + family: str, + spines: int | None, + leafs: int | None, + k: int | None, + clients: int, + prefix: int, + pingmesh: int | None, + pps: int, + timeout: int, + deploy_jobs: int, + health_timeout: int, +): + profile = get_scale_profile(name) + + assert profile.family == family + assert profile.num_spines == spines + assert profile.num_leafs == leafs + assert profile.fat_tree_k == k + assert profile.clients_per_attached_switch == clients + assert profile.management_prefix == prefix + assert profile.pingmesh_destination_batch_size == pingmesh + assert profile.traffic_max_pps_per_client == pps + assert profile.deploy_timeout_seconds == timeout + assert profile.worker_deploy_parallelism == deploy_jobs + assert profile.health_timeout_seconds == health_timeout + + +def test_scale_profile_registry_is_complete_and_rejects_unknown_scales(): + assert supported_scales() == ( + "xs", + "small", + "medium", + "large", + "xlarge", + "fat-tree-k8", + "fat-tree-k12", + ) + with pytest.raises(ValueError, match="Unknown scale"): + get_scale_profile("not-a-scale") + + +def test_fat_tree_profiles_use_bounded_containerlab_parallelism(): + assert get_scale_profile("fat-tree-k8").containerlab_max_workers == 1 + assert get_scale_profile("fat-tree-k12").containerlab_max_workers == 1 + assert get_scale_profile("xlarge").containerlab_max_workers == 16 diff --git a/tests/test_command_common.py b/tests/test_command_common.py deleted file mode 100644 index 19c4346..0000000 --- a/tests/test_command_common.py +++ /dev/null @@ -1,82 +0,0 @@ -import json -import subprocess - -from netopsbench.platform.topology.topology_utils import check_topology_deployed - - -def _write_topology(tmp_path, lab_name="dcn", devices=None): - topology_dir = tmp_path / "topology" - topology_dir.mkdir() - (topology_dir / f"{lab_name}.clab.yaml").write_text(f"name: {lab_name}\n") - (topology_dir / "topology.json").write_text(json.dumps({"name": lab_name, "devices": devices or {}})) - return topology_dir - - -def test_check_topology_deployed_rejects_mismatched_running_containers(monkeypatch, tmp_path): - topology_dir = _write_topology( - tmp_path, - devices={ - "spines": [{"name": "spine1"}, {"name": "spine2"}], - "leafs": [{"name": "leaf1"}, {"name": "leaf2"}], - "clients": [{"name": "client1"}, {"name": "client2"}], - }, - ) - - monkeypatch.setattr( - subprocess, - "run", - lambda *args, **kwargs: subprocess.CompletedProcess( - args=args[0], - returncode=0, - stdout="\n".join( - [ - "clab-dcn-spine1", - "clab-dcn-spine2", - "clab-dcn-leaf1", - "clab-dcn-leaf2", - "clab-dcn-leaf3", - "clab-dcn-leaf4", - ] - ), - stderr="", - ), - ) - - ok, message = check_topology_deployed(str(topology_dir)) - - assert ok is False - assert "do not match topology metadata" in message - assert "missing=clab-dcn-client1" in message - - -def test_check_topology_deployed_accepts_exact_metadata_match(monkeypatch, tmp_path): - topology_dir = _write_topology( - tmp_path, - devices={ - "spines": [{"name": "spine1"}], - "leafs": [{"name": "leaf1"}], - "clients": [{"name": "client1"}], - }, - ) - - monkeypatch.setattr( - subprocess, - "run", - lambda *args, **kwargs: subprocess.CompletedProcess( - args=args[0], - returncode=0, - stdout="\n".join( - [ - "clab-dcn-spine1", - "clab-dcn-leaf1", - "clab-dcn-client1", - ] - ), - stderr="", - ), - ) - - ok, message = check_topology_deployed(str(topology_dir)) - - assert ok is True - assert "Detected 3 running container(s)" in message diff --git a/tests/test_commands_cli.py b/tests/test_commands_cli.py index 9deec32..fb4ad7e 100644 --- a/tests/test_commands_cli.py +++ b/tests/test_commands_cli.py @@ -192,7 +192,8 @@ def fake_generate(workspace, *, scale, spec, topology_dir=None, out=None, seed=4 out = capsys.readouterr().out assert "generated scenarios" in out assert calls["scale"] == "xs" - assert calls["spec"] == tmp_path / "scenarios/specs/fault_campaign.yaml" + assert calls["spec"].name == "fault_campaign.yaml" + assert calls["spec"].is_file() assert calls["topology_dir"] is None assert calls["out"] is None assert calls["seed"] == 42 @@ -476,5 +477,6 @@ def fake_generate_scenarios(workspace, *, scale, spec, topology_dir=None, out=No assert "scenarios:medium" in out assert [call[1] for call in topology_calls] == ["xs", "medium"] assert [call[1] for call in scenario_calls] == ["xs", "medium"] - assert scenario_calls[0][2] == tmp_path / "scenarios/specs/fault_campaign.yaml" + assert scenario_calls[0][2].name == "fault_campaign.yaml" + assert scenario_calls[0][2].is_file() assert scenario_calls[0][5] == 7 diff --git a/tests/test_e2e.py b/tests/test_e2e.py index fa597e1..58a5999 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -7,28 +7,35 @@ import json import os +import stat import sys import tempfile from pathlib import Path +from types import SimpleNamespace import pytest +import yaml # Add parent directory to path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from netopsbench.evaluator.scorer import AgentOutput, EvaluationResult, Evaluator +from netopsbench.models.profiles import supported_scales +from netopsbench.models.topology import TopologyManifest from netopsbench.platform.faults.injector import FaultInjector -from netopsbench.platform.faults.specs import get_builtin_fault_specs -from netopsbench.platform.pingmesh.generator import PinglistGenerator +from netopsbench.platform.faults.services.topology_runtime import TopologyRuntime +from netopsbench.platform.faults.specs import create_fault_registry +from netopsbench.platform.pingmesh.generator import PinglistGenerator, generate_pinglist_from_topology +from netopsbench.platform.scenario.generator import parse_bgp_config, parse_network_interfaces from netopsbench.platform.scenario.parser import parse_scenario_file from netopsbench.platform.scenario.validator import validate_scenario, validate_scenario_topology -from netopsbench.platform.session.scoring import resolve_scenario_files, score_scenario_fault_episodes +from netopsbench.platform.session.scoring import score_scenario_fault_episodes from netopsbench.platform.toolkit import fastmcp_server from netopsbench.platform.toolkit.mcp.registry import load_tool_specs # Internal test path: direct toolkit import keeps implementation-level e2e checks fast. from netopsbench.platform.toolkit.toolkit import AgentToolkit, ToolResult -from netopsbench.platform.topology.generator import TOPOLOGY_SCALES, generate_topology +from netopsbench.platform.topology.generator import generate_topology from netopsbench.platform.utils.interface_names import are_interfaces_equivalent @@ -42,56 +49,169 @@ def _generated_scenario_path(filename: str) -> str: return str(matches[0]) -class TestTopologyGenerator: - """Tests for topology generation.""" +def _write_config_db(topology_dir: str | Path, device: str, interfaces: dict[str, list[str]]) -> Path: + interface_table: dict[str, dict] = {} + for interface_name, cidrs in interfaces.items(): + interface_table[interface_name] = {} + for cidr in cidrs: + interface_table[f"{interface_name}|{cidr}"] = {} - def test_topology_scales_defined(self): - """Test that all expected topology scales are defined.""" - expected_scales = ["xs", "small", "medium", "large"] - for scale in expected_scales: - assert scale in TOPOLOGY_SCALES, f"Scale {scale} not defined" + path = Path(topology_dir) / "configs" / "sonic" / device / "config_db.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"INTERFACE": interface_table}), encoding="utf-8") + return path - def test_generate_xs_topology(self): - """Test generating an xs topology.""" - with tempfile.TemporaryDirectory() as tmpdir: - result = generate_topology("xs", tmpdir) - assert os.path.exists(result["yaml_file"]), "YAML file not created" - assert os.path.exists(result["metadata_file"]), "Metadata file not created" - assert len(result["config_files"]) == 4, "Expected 4 config files (2 spines + 2 leafs)" +def _generated_metadata(scale: str = "xs") -> dict: + with tempfile.TemporaryDirectory() as tmpdir: + return generate_topology(scale, tmpdir)["metadata"] - # Verify metadata - metadata = result["metadata"] - assert metadata["scale"]["num_spines"] == 2 - assert metadata["scale"]["num_leafs"] == 2 - assert metadata["scale"]["clients_per_leaf"] == 1 - assert metadata["scale"]["total_clients"] == 2 - rendered_yaml = Path(result["yaml_file"]).read_text(encoding="utf-8") - assert "yyyyyt123/netopsbench-sonic-vs-202505-telemetry:202505-telemetry" in rendered_yaml +class TestTopologyGeneration: + """Tests for topology generation.""" + + @pytest.mark.parametrize("scale", supported_scales()) + def test_all_scales_use_bind_based_preseed_artifacts(self, scale): + """Every scale should use the same bind-mounted SONiC startup artifact path.""" + with tempfile.TemporaryDirectory() as tmpdir: + result = generate_topology(scale, tmpdir) + rendered = yaml.safe_load(Path(result["yaml_file"]).read_text(encoding="utf-8")) + + binds = rendered["topology"]["kinds"]["sonic-vs"]["binds"] + linux_binds = rendered["topology"]["kinds"]["linux"]["binds"] + assert "configs/sonic/__clabNodeName__/config_db.json:/etc/sonic/config_db.json:rw" in binds + assert ( + "configs/sonic/__clabNodeName__/port_config.ini:" + "/usr/share/sonic/device/x86_64-kvm_x86_64-r0/Force10-S6000/port_config.ini:rw" + ) in binds + assert ( + "configs/sonic/__clabNodeName__/lanemap.ini:" + "/usr/share/sonic/device/x86_64-kvm_x86_64-r0/Force10-S6000/lanemap.ini:rw" + ) in binds + assert "configs/sonic/start.sh:/usr/bin/start.sh:ro" in binds + assert "configs/frr/__clabNodeName__.conf:/etc/frr/frr.conf:rw" in binds + assert "configs/pingmesh:/tmp/pingmesh:ro" in linux_binds + + assert not list(Path(tmpdir, "configs").glob("*.sh")) + assert not list(Path(tmpdir, "configs").glob("*.configdb.json")) + assert Path(tmpdir, "configs", "pingmesh").is_dir() + + manifest = TopologyManifest.model_validate(result["metadata"]) + first_routing = manifest.routing_devices()[0].name + first_attached = manifest.client_attached_devices()[0].name + first_routing_config = Path(tmpdir, "configs", "sonic", first_routing, "config_db.json") + first_attached_config = Path(tmpdir, "configs", "sonic", first_attached, "config_db.json") + assert first_routing_config.exists() + assert first_attached_config.exists() + assert Path(tmpdir, "configs", "sonic", first_routing, "port_config.ini").exists() + assert Path(tmpdir, "configs", "sonic", first_routing, "lanemap.ini").exists() + start_wrapper = Path(tmpdir, "configs", "sonic", "start.sh") + assert start_wrapper.exists() + assert start_wrapper.stat().st_mode & stat.S_IXUSR + wrapper_text = start_wrapper.read_text(encoding="utf-8") + assert ( + "NETOPSBENCH_ORIGINAL_SONIC_START_SHA256=" + "8c5aa959f0a3ed0bf1a57f7ecfd004485d5600b9ab71b388c2b15e109b77ee12" + ) in wrapper_text + assert "wait_for_front_panel_links" in wrapper_text + assert "/sys/class/net/eth[0-9]*" in wrapper_text + assert "ip link show" not in wrapper_text + assert "local timeout=600" in wrapper_text + assert "local interval=1" in wrapper_text + assert "install_generated_config_db" in wrapper_text + assert 'cat "$src" > "$dst"' in wrapper_text + assert Path(tmpdir, "configs", "frr", f"{first_routing}.conf").exists() + + routing_config = json.loads(first_routing_config.read_text(encoding="utf-8")) + assert routing_config["DEVICE_METADATA"]["localhost"]["platform"] == "x86_64-kvm_x86_64-r0" + assert routing_config["DEVICE_METADATA"]["localhost"]["mac"].startswith("02:") + assert routing_config["PORT"] + assert routing_config["INTERFACE"] + assert str(first_routing_config) in result["config_files"] + + attached_config = json.loads(first_attached_config.read_text(encoding="utf-8")) + assert ( + attached_config["DEVICE_METADATA"]["localhost"]["mac"] + != routing_config["DEVICE_METADATA"]["localhost"]["mac"] + ) + assert attached_config["PORT"] + assert str(first_attached_config) in result["config_files"] def test_generated_switch_configs_seed_gnmi_defaults(self): - """Generated SONiC configs should seed the GNMI config expected by 202505 images.""" + """Generated SONiC startup configs should seed telemetry tables expected by 202505 images.""" with tempfile.TemporaryDirectory() as tmpdir: result = generate_topology("xs", tmpdir) for config_path in result["config_files"]: - text = Path(config_path).read_text(encoding="utf-8") - assert "sonic-db-cli CONFIG_DB hmset 'GNMI|gnmi' port 50051" in text - assert "sonic-db-cli CONFIG_DB hmset 'GNMI|certs'" in text - assert "pgrep -x telemetry" in text - assert "/usr/sbin/telemetry -port 50051 -noTLS -client_auth none" in text - - def test_generate_small_topology(self): - """Test generating a small topology.""" + payload = json.loads(Path(config_path).read_text(encoding="utf-8")) + assert payload["GNMI"]["gnmi"]["port"] == "50051" + assert payload["GNMI"]["gnmi"]["client_auth"] == "false" + assert payload["GNMI"]["certs"]["server_key"].endswith("streamingtelemetryserver.key") + assert payload["FLEX_COUNTER_TABLE"]["PORT"]["FLEX_COUNTER_STATUS"] == "enable" + assert payload["FLEX_COUNTER_TABLE"]["PORT"]["POLL_INTERVAL"] == "10000" + assert "telegraf" in payload["SYSLOG_SERVER"] + + def test_generate_xlarge_topology_metadata_and_addressing(self): + """xlarge should fit the Clos address plans and avoid collector collisions.""" with tempfile.TemporaryDirectory() as tmpdir: - result = generate_topology("small", tmpdir) + result = generate_topology("xlarge", tmpdir) + + metadata = result["agent_topology"] + devices = metadata["devices"] + assert metadata["topology_scale"] == "xlarge" + assert metadata["management"]["ipv4_subnet"] == "172.20.20.0/23" + assert metadata["scale"]["num_spines"] == 16 + assert metadata["scale"]["num_leafs"] == 128 + assert metadata["scale"]["clients_per_leaf"] == 1 + assert metadata["scale"]["total_clients"] == 128 + assert metadata["pingmesh"]["destination_batch_size"] == 16 - metadata = result["metadata"] - assert metadata["scale"]["num_spines"] == 2 - assert metadata["scale"]["num_leafs"] == 4 - assert metadata["scale"]["clients_per_leaf"] == 2 - assert metadata["scale"]["total_clients"] == 8 + mgmt_ips = {item["mgmt_ip"] for role in ("spines", "leafs", "clients") for item in devices[role]} + assert len(mgmt_ips) == 16 + 128 + 128 + assert metadata["collector"]["ipv4"] not in mgmt_ips + + spine16_config = json.loads( + Path(tmpdir, "configs", "sonic", "spine16", "config_db.json").read_text(encoding="utf-8") + ) + leaf128_config = json.loads( + Path(tmpdir, "configs", "sonic", "leaf128", "config_db.json").read_text(encoding="utf-8") + ) + spine16_ports = Path(tmpdir, "configs", "sonic", "spine16", "port_config.ini").read_text(encoding="utf-8") + spine16_lanemap = Path(tmpdir, "configs", "sonic", "spine16", "lanemap.ini").read_text(encoding="utf-8") + leaf128_frr = Path(tmpdir, "configs", "frr", "leaf128.conf").read_text(encoding="utf-8") + spine16_frr = Path(tmpdir, "configs", "frr", "spine16.conf").read_text(encoding="utf-8") + + assert not Path(tmpdir, "configs", "spine16.sh").exists() + assert not Path(tmpdir, "configs", "spine16.configdb.json").exists() + assert "neighbor 10.16.128.2 remote-as 65138" in spine16_frr + assert "bgp bestpath as-path multipath-relax" in spine16_frr + assert "maximum-paths 64" in leaf128_frr + assert len([key for key in spine16_config["INTERFACE"] if "|" not in key]) == 128 + assert spine16_config["INTERFACE"]["Ethernet508"] == {} + assert spine16_config["INTERFACE"]["Ethernet508|10.16.128.1/30"] == {} + assert spine16_config["PORT"]["Ethernet508"]["lanes"] == "509,510,511,512" + assert "Ethernet508" in spine16_ports + assert "eth1:1,2,3,4" in spine16_lanemap + assert "eth128:509,510,511,512" in spine16_lanemap + assert len([line for line in spine16_lanemap.splitlines() if line.startswith("eth")]) == 128 + assert len([key for key in leaf128_config["INTERFACE"] if "|" not in key]) == 17 + assert leaf128_config["INTERFACE"]["Ethernet60|10.16.128.2/30"] == {} + assert leaf128_config["INTERFACE"]["Ethernet64|192.168.228.1/30"] == {} + assert "network 192.168.228.0/30 route-map RM-ALLOW" in leaf128_frr + leaf128_lanemap = Path(tmpdir, "configs", "sonic", "leaf128", "lanemap.ini").read_text(encoding="utf-8") + assert len([line for line in leaf128_lanemap.splitlines() if line.startswith("eth")]) == 17 + + def test_structured_startup_artifacts_drive_scenario_parsers(self): + """Scenario parsers should use generated ConfigDB and FRR artifacts as the normal path.""" + with tempfile.TemporaryDirectory() as tmpdir: + generate_topology("xs", tmpdir) + spine_cfg = Path(tmpdir, "configs", "sonic", "spine1", "config_db.json") + spine_frr = Path(tmpdir, "configs", "frr", "spine1.conf") + + assert parse_network_interfaces(spine_cfg) == ["Ethernet0", "Ethernet4"] + bgp_info = parse_bgp_config(spine_frr) + assert bgp_info["local_as"] == 65001 + assert [neighbor["remote_as"] for neighbor in bgp_info["neighbors"]] == [65011, 65012] class TestAgentToolkit: @@ -103,7 +223,6 @@ def test_toolkit_initialization(self): topo = generate_topology("xs", tmpdir) toolkit = AgentToolkit(topology_metadata=topo["metadata"]) assert toolkit is not None - assert toolkit.grafana_url is not None assert toolkit.influxdb_url is not None def test_tool_registry_specs_complete(self): @@ -143,23 +262,11 @@ def test_toolkit_loads_dynamic_topology(self): assert "leaf1" in toolkit.container_names assert "client1" in toolkit.container_names - def test_reload_topology(self): - """Test toolkit can reload topology at runtime.""" - with tempfile.TemporaryDirectory() as tmpdir: - # Generate topology - result = generate_topology("xs", tmpdir) - - toolkit = AgentToolkit(topology_metadata=result["metadata"]) - reload_result = toolkit.reload_topology(metadata_file=result["metadata_file"]) - - assert reload_result.success is True - assert reload_result.data["topology_name"] == "dcn" - class TestPingmeshGenerator: """Tests for Pingmesh pinglist generation across topology scales.""" - @pytest.mark.parametrize("scale", list(TOPOLOGY_SCALES.keys())) + @pytest.mark.parametrize("scale", supported_scales()) def test_pinglist_scales_with_topology(self, scale): """Pinglist should be N*(N-1) for N clients across all scales.""" with tempfile.TemporaryDirectory() as tmpdir: @@ -169,11 +276,37 @@ def test_pinglist_scales_with_topology(self, scale): generator = PinglistGenerator() tasks = generator.generate(metadata) - total_clients = metadata["scale"]["total_clients"] + total_clients = TopologyManifest.model_validate(metadata).facts.total_clients assert len(tasks) == total_clients * (total_clients - 1) assert all(t.src_name != t.dst_name for t in tasks) assert {t.path_type for t in tasks}.issubset({"same_rack", "cross_rack"}) + def test_xlarge_pinglist_uses_full_universe_with_bounded_runtime_policy(self): + with tempfile.TemporaryDirectory() as tmpdir: + result = generate_topology("xlarge", tmpdir) + output_file = Path(tmpdir) / "pinglist.json" + + first = generate_pinglist_from_topology(result["metadata_file"], str(output_file)) + second = generate_pinglist_from_topology(result["metadata_file"], str(output_file)) + + assert [(task.src_name, task.dst_name) for task in first] == [ + (task.src_name, task.dst_name) for task in second + ] + assert len(first) == 128 * 127 + per_src: dict[str, int] = {} + per_dst: dict[str, int] = {} + for task in first: + per_src[task.src_name] = per_src.get(task.src_name, 0) + 1 + per_dst[task.dst_name] = per_dst.get(task.dst_name, 0) + 1 + assert task.src_name != task.dst_name + assert set(per_src.values()) == {127} + assert set(per_dst.values()) == {127} + + payload = json.loads(output_file.read_text(encoding="utf-8")) + assert payload["total_probes"] == 128 * 127 + assert payload["pingmesh_policy"]["destination_batch_size"] == 16 + assert payload["pingmesh_policy"]["coverage_epoch_cycles"] == 32 + class TestFastMCPServer: """Tests for FastMCP server tool wiring.""" @@ -188,6 +321,7 @@ def test_fastmcp_has_all_expected_tools(self): "get_topology", "get_device_interfaces", "get_bgp_neighbors", + "get_bgp_neighbor", "get_route_table", "get_device_config", "get_bgp_rib", @@ -196,7 +330,7 @@ def test_fastmcp_has_all_expected_tools(self): "traceroute", "ping_test", "get_interface_metrics", - "get_all_bgp_status", + "query_bgp_events", "get_pingmesh_summary", "get_pingmesh_hotspots", ] @@ -219,12 +353,10 @@ class TestFaultInjector: def test_injector_initialization(self): """Test fault injector initializes correctly.""" - injector = FaultInjector( - topology_metadata={"name": "dcn", "devices": {"spines": [], "leafs": [], "clients": []}} - ) + injector = FaultInjector(topology_metadata=_generated_metadata()) assert injector is not None assert injector.topology_name == "dcn" - assert injector.container_names == {} + assert injector.container_names["spine1"] == "clab-dcn-spine1" assert injector.active_faults == [] def test_injector_loads_dynamic_topology(self): @@ -238,18 +370,6 @@ def test_injector_loads_dynamic_topology(self): assert "spine1" in injector.container_names assert "leaf1" in injector.container_names - def test_reload_topology(self): - """Test injector can reload topology at runtime.""" - with tempfile.TemporaryDirectory() as tmpdir: - result = generate_topology("xs", tmpdir) - - injector = FaultInjector( - topology_metadata={"name": "dcn", "devices": {"spines": [], "leafs": [], "clients": []}} - ) - reload_result = injector.reload_topology(metadata_file=result["metadata_file"]) - - assert reload_result["success"] is True - def test_fault_types_defined(self): """Test all expected fault types are defined.""" expected_faults = [ @@ -266,12 +386,12 @@ def test_fault_types_defined(self): "static_route_misconfig", ] - defined_faults = [spec.name for spec in get_builtin_fault_specs()] + defined_faults = [spec.name for spec in create_fault_registry().get_builtin_specs()] for fault in expected_faults: assert fault in defined_faults, f"Fault type {fault} not defined" def test_inject_high_latency_uses_latency_ms_contract(self): - injector = FaultInjector(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) + injector = FaultInjector(topology_metadata=_generated_metadata()) class FakeImpairment: def __init__(self): @@ -290,7 +410,7 @@ def inject_high_latency(self, device, interface, latency_ms=100): assert fake.calls == [("leaf1", "Ethernet0", 120)] def test_inject_mtu_mismatch_uses_mtu_contract(self): - injector = FaultInjector(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) + injector = FaultInjector(topology_metadata=_generated_metadata()) class FakeImpairment: def __init__(self): @@ -309,7 +429,7 @@ def inject_mtu_mismatch(self, device, interface, mtu=1400): assert fake.calls == [("leaf1", "Ethernet0", 1450)] def test_inject_packet_corruption_uses_corruption_pct_contract(self): - injector = FaultInjector(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) + injector = FaultInjector(topology_metadata=_generated_metadata()) class FakeImpairment: def __init__(self): @@ -328,7 +448,7 @@ def inject_packet_corruption(self, device, interface, corruption_pct=20): assert fake.calls == [("leaf1", "Ethernet0", 17)] def test_inject_packet_loss_uses_loss_pct_contract(self): - injector = FaultInjector(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) + injector = FaultInjector(topology_metadata=_generated_metadata()) class FakeImpairment: def __init__(self): @@ -347,7 +467,7 @@ def inject_packet_loss(self, device, interface, loss_pct=10): assert fake.calls == [("leaf1", "Ethernet0", 11)] def test_injector_interface_alias_resolution_supports_vendor_style(self): - injector = FaultInjector(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) + injector = FaultInjector(topology_metadata=_generated_metadata()) assert injector._iface.resolve_linux("ethernet-1/2") == "eth2" assert injector._iface.resolve_sonic("ethernet-1/2") == "Ethernet4" @@ -657,40 +777,12 @@ def test_score_scenario_fault_episodes_only_scores_faults(self): assert scored[0].testcase_id == f"{scenario.scenario_id}:ep002_link_down" assert scored[0].score == 1.0 - def test_resolve_scenario_files_from_directory(self): - """Scenario file resolver should return sorted YAML files by pattern.""" - with tempfile.TemporaryDirectory() as tmpdir: - p1 = os.path.join(tmpdir, "scenario_02.yaml") - p2 = os.path.join(tmpdir, "scenario_01.yaml") - p3 = os.path.join(tmpdir, "note.txt") - for path in [p1, p2, p3]: - with open(path, "w") as f: - f.write("test") - - files = resolve_scenario_files(tmpdir) - assert files == sorted([p1, p2]) - def test_topology_guard_rejects_scale_mismatch_by_default(self): """Strict topology guard should fail on declared/actual scale mismatch.""" scenario = parse_scenario_file(_generated_scenario_path("generated_link_down_xs_001.yaml")) with tempfile.TemporaryDirectory() as tmpdir: - metadata = { - "name": "dcn", - "scale": {"total_clients": 8}, - "devices": { - "spines": [{"name": "spine1"}, {"name": "spine2"}], - "leafs": [{"name": "leaf1"}, {"name": "leaf2"}], - "clients": [{"name": "client1"}, {"name": "client2"}], - }, - } - - with open(os.path.join(tmpdir, "topology.json"), "w") as f: - json.dump(metadata, f) - - os.makedirs(os.path.join(tmpdir, "configs"), exist_ok=True) - with open(os.path.join(tmpdir, "configs", "spine1.sh"), "w") as f: - f.write("config interface startup Ethernet0\n") + generate_topology("small", tmpdir) result = validate_scenario_topology( scenario=scenario, @@ -702,26 +794,13 @@ def test_topology_guard_rejects_scale_mismatch_by_default(self): assert result["actual_scale"] == "small" def test_topology_guard_accepts_e_style_interface_aliases(self): - """Legacy e1-1 style interface labels should map to SONiC Ethernet ports.""" + """e1-1 style interface labels should map to SONiC Ethernet ports.""" scenario = parse_scenario_file(_generated_scenario_path("generated_link_down_medium_001.yaml")) with tempfile.TemporaryDirectory() as tmpdir: - metadata = { - "name": "dcn", - "scale": {"total_clients": 16}, - "devices": { - "spines": [{"name": f"spine{i}"} for i in range(1, 5)], - "leafs": [{"name": f"leaf{i}"} for i in range(1, 9)], - "clients": [{"name": f"client{i}"} for i in range(1, 17)], - }, - } + generate_topology("medium", tmpdir) - with open(os.path.join(tmpdir, "topology.json"), "w") as f: - json.dump(metadata, f) - - os.makedirs(os.path.join(tmpdir, "configs"), exist_ok=True) - with open(os.path.join(tmpdir, "configs", "spine1.sh"), "w") as f: - f.write("config interface startup Ethernet0\n") + _write_config_db(tmpdir, "spine1", {"Ethernet0": []}) result = validate_scenario_topology( scenario=scenario, @@ -735,24 +814,10 @@ def test_topology_guard_accepts_vendor_style_interface_aliases(self): scenario = parse_scenario_file(_generated_scenario_path("generated_link_down_small_001.yaml")) with tempfile.TemporaryDirectory() as tmpdir: - metadata = { - "name": "dcn", - "scale": {"total_clients": 8}, - "devices": { - "spines": [{"name": "spine1"}, {"name": "spine2"}], - "leafs": [{"name": f"leaf{i}"} for i in range(1, 5)], - "clients": [{"name": f"client{i}"} for i in range(1, 9)], - }, - } + generate_topology("small", tmpdir) - with open(os.path.join(tmpdir, "topology.json"), "w") as f: - json.dump(metadata, f) - - os.makedirs(os.path.join(tmpdir, "configs"), exist_ok=True) - with open(os.path.join(tmpdir, "configs", "spine1.sh"), "w") as f: - f.write("config interface startup Ethernet0\n") - with open(os.path.join(tmpdir, "configs", "spine2.sh"), "w") as f: - f.write("config interface startup Ethernet4\n") + _write_config_db(tmpdir, "spine1", {"Ethernet0": []}) + _write_config_db(tmpdir, "spine2", {"Ethernet4": []}) result = validate_scenario_topology( scenario=scenario, @@ -761,6 +826,44 @@ def test_topology_guard_accepts_vendor_style_interface_aliases(self): assert result["status"] == "pass" + def test_configdb_payload_interfaces_are_used_by_runtime_helpers(self): + """Runtime helpers read interface metadata from generated ConfigDB artifacts.""" + with tempfile.TemporaryDirectory() as tmpdir: + generate_topology("small", tmpdir) + spine_config = _write_config_db(tmpdir, "spine1", {"Ethernet4": ["10.0.0.2/30"]}) + _write_config_db(tmpdir, "leaf1", {"Ethernet4": ["10.0.0.1/30"]}) + + scenario = SimpleNamespace( + scenario_id="configdb_interface_case", + topology_scale="small", + episodes=[ + SimpleNamespace( + episode_id="ep001", + fault_type="link_down", + target_device="spine1", + target_interface="e1-2", + ) + ], + ) + result = validate_scenario_topology(scenario=scenario, topology_dir=tmpdir) + assert result["status"] == "pass" + assert parse_network_interfaces(spine_config) == ["Ethernet4"] + + topo_runtime = TopologyRuntime( + sonic=SimpleNamespace(), + iface=SimpleNamespace(resolve_sonic=lambda interface: interface), + ctx=SimpleNamespace(clab_dir=Path(tmpdir), clients=[]), + ) + assert topo_runtime.configured_device_interfaces("spine1") == ["Ethernet4"] + + from netopsbench.platform.session.scoring import build_episode_ground_truth + + ground_truth = build_episode_ground_truth( + {"fault_type": "link_down", "target_device": "leaf1", "target_interface": "Ethernet4"}, + topology_dir=tmpdir, + ) + assert ground_truth["equivalent_locations"] == [{"device": "spine1", "interface": "Ethernet4"}] + def test_interface_alias_helper_keeps_scale_agnostic_equivalence(self): assert are_interfaces_equivalent("e1-1", "Ethernet0") is True assert are_interfaces_equivalent("ethernet-1/2", "Ethernet4") is True @@ -791,11 +894,8 @@ def test_score_scenario_fault_episodes_accepts_link_peer_equivalence(self): } with tempfile.TemporaryDirectory() as tmpdir: - os.makedirs(os.path.join(tmpdir, "configs"), exist_ok=True) - with open(os.path.join(tmpdir, "configs", "spine1.sh"), "w") as f: - f.write("config interface ip add Ethernet0 192.168.11.1/30\n") - with open(os.path.join(tmpdir, "configs", "leaf1.sh"), "w") as f: - f.write("config interface ip add Ethernet0 192.168.11.2/30\n") + _write_config_db(tmpdir, "spine1", {"Ethernet0": ["192.168.11.1/30"]}) + _write_config_db(tmpdir, "leaf1", {"Ethernet0": ["192.168.11.2/30"]}) scored = score_scenario_fault_episodes( scenario, @@ -822,12 +922,9 @@ def test_score_scenario_interface_symmetric_fault_accepts_peer_endpoint(self, fa } with tempfile.TemporaryDirectory() as tmpdir: - os.makedirs(os.path.join(tmpdir, "configs"), exist_ok=True) # leaf1 Ethernet0 and spine1 Ethernet0 share the same /30 subnet - with open(os.path.join(tmpdir, "configs", "leaf1.sh"), "w") as f: - f.write("config interface ip add Ethernet0 10.0.0.1/30\n") - with open(os.path.join(tmpdir, "configs", "spine1.sh"), "w") as f: - f.write("config interface ip add Ethernet0 10.0.0.2/30\n") + _write_config_db(tmpdir, "leaf1", {"Ethernet0": ["10.0.0.1/30"]}) + _write_config_db(tmpdir, "spine1", {"Ethernet0": ["10.0.0.2/30"]}) gt = build_episode_ground_truth(episode_info, topology_dir=tmpdir) diff --git a/tests/test_e2e_real.py b/tests/test_e2e_real.py index 8a28435..aa36ed5 100644 --- a/tests/test_e2e_real.py +++ b/tests/test_e2e_real.py @@ -19,9 +19,8 @@ from __future__ import annotations import csv -import os import time -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from io import StringIO import pytest @@ -42,38 +41,42 @@ def _toolkit_or_skip() -> AgentToolkit: ) +def _detector_or_skip() -> AnomalyDetector: + toolkit = _toolkit_or_skip() + return AnomalyDetector( + influxdb_url=toolkit.influxdb_url, + token=toolkit.influxdb_token, + org=toolkit.influxdb_org, + bucket=toolkit.influxdb_bucket, + topology_metadata=toolkit.topology_metadata, + topology_id=toolkit.topology_id, + ) + + @pytest.mark.real class TestPingmeshIntegration: """Tests for Pingmesh anomaly detection (InfluxDB on localhost).""" - @staticmethod - def _influx_token() -> str: - return os.environ.get("NETOPSBENCH_INFLUXDB_TOKEN") or "replace-me" - def test_pingmesh_anomaly_detector(self): """ Requires: - InfluxDB running at localhost:8086 - Pingmesh data in netopsbench bucket """ - detector = AnomalyDetector( - influxdb_url="http://localhost:8086", - token=self._influx_token(), - org="netopsbench", - bucket="netopsbench", - ) + detector = _detector_or_skip() - now = datetime.utcnow().replace(microsecond=0) + now = datetime.now(UTC).replace(microsecond=0) baseline_start = (now - timedelta(minutes=10)).isoformat() + "Z" baseline_end = (now - timedelta(minutes=5)).isoformat() + "Z" current_start = baseline_end current_end = now.isoformat() + "Z" - report = detector.generate_anomaly_report( + report = detector.generate_windowed_anomaly_report( baseline_start=baseline_start, baseline_end=baseline_end, current_start=current_start, current_end=current_end, + windows=[{"name": "full", "start_time": current_start, "end_time": current_end}], ) assert "timestamp" in report @@ -89,64 +92,56 @@ def test_pingmesh_anomaly_detector(self): assert "by_dst_leaf" in report["aggregated_anomalies"] def test_latency_anomaly_detection(self): - detector = AnomalyDetector( - influxdb_url="http://localhost:8086", - token=self._influx_token(), - org="netopsbench", - bucket="netopsbench", - ) + detector = _detector_or_skip() - now = datetime.utcnow().replace(microsecond=0) + now = datetime.now(UTC).replace(microsecond=0) baseline_start = (now - timedelta(minutes=10)).isoformat() + "Z" baseline_end = (now - timedelta(minutes=5)).isoformat() + "Z" current_start = baseline_end current_end = now.isoformat() + "Z" - anomalies = detector.detect_latency_anomalies( + report = detector.generate_windowed_anomaly_report( baseline_start=baseline_start, baseline_end=baseline_end, current_start=current_start, current_end=current_end, + windows=[{"name": "full", "start_time": current_start, "end_time": current_end}], ) + anomalies = [item for item in report["anomalies"] if item["type"] == "latency_spike"] assert isinstance(anomalies, list) for anomaly in anomalies: - assert anomaly.type == "latency_spike" - assert anomaly.src_ip - assert anomaly.dst_ip - assert anomaly.value >= 0 - assert anomaly.baseline >= 0 - assert anomaly.severity in ["low", "medium", "high"] + assert anomaly["src_ip"] + assert anomaly["dst_ip"] + assert anomaly["value"] >= 0 + assert anomaly["baseline"] >= 0 + assert anomaly["severity"] in ["low", "medium", "high"] def test_packet_loss_detection(self): - detector = AnomalyDetector( - influxdb_url="http://localhost:8086", - token=self._influx_token(), - org="netopsbench", - bucket="netopsbench", - ) + detector = _detector_or_skip() - now = datetime.utcnow().replace(microsecond=0) + now = datetime.now(UTC).replace(microsecond=0) baseline_start = (now - timedelta(minutes=10)).isoformat() + "Z" baseline_end = (now - timedelta(minutes=5)).isoformat() + "Z" current_start = baseline_end current_end = now.isoformat() + "Z" - anomalies = detector.detect_packet_loss( + report = detector.generate_windowed_anomaly_report( baseline_start=baseline_start, baseline_end=baseline_end, current_start=current_start, current_end=current_end, + windows=[{"name": "full", "start_time": current_start, "end_time": current_end}], ) + anomalies = [item for item in report["anomalies"] if item["type"] == "packet_loss"] assert isinstance(anomalies, list) for anomaly in anomalies: - assert anomaly.type == "packet_loss" - assert anomaly.src_ip - assert anomaly.dst_ip - assert anomaly.value > 0 + assert anomaly["src_ip"] + assert anomaly["dst_ip"] + assert anomaly["value"] > 0 @pytest.mark.real @@ -169,19 +164,19 @@ def test_traffic_controller_basic(self): duration=5, ) - flow_id = controller.start_flow(flow) + [flow_id] = controller.start_matrix([flow]) assert flow_id == flow.flow_id assert flow_id in controller.active_flows time.sleep(6) - controller.stop_flow(flow_id) + controller.stop_all() assert flow_id not in controller.active_flows @pytest.mark.real class TestObservabilityNormal: - """Pingmesh summary, Grafana panel proxy, and datasource checks.""" + """Pingmesh summary and packaged Grafana datasource checks.""" @pytest.fixture(scope="class") def toolkit(self): @@ -209,16 +204,16 @@ def test_pingmesh_summary_has_data(self, toolkit): def test_grafana_datasource_configured(self, toolkit): health = requests.get( - f"{toolkit.grafana_url}/api/health", - auth=toolkit.grafana_auth, + "http://localhost:3000/api/health", + auth=("admin", "admin"), timeout=10, proxies={"http": "", "https": ""}, ) assert health.status_code == 200 datasource = requests.get( - f"{toolkit.grafana_url}/api/datasources/uid/InfluxDB_v2", - auth=toolkit.grafana_auth, + "http://localhost:3000/api/datasources/uid/InfluxDB_v2", + auth=("admin", "admin"), timeout=10, proxies={"http": "", "https": ""}, ) diff --git a/tests/test_episode_runner.py b/tests/test_episode_runner.py index 4e3708f..7a4a81c 100644 --- a/tests/test_episode_runner.py +++ b/tests/test_episode_runner.py @@ -1,18 +1,16 @@ -"""Tests for :class:`EpisodeRunner` extracted from ScenarioExecutor.""" +"""Tests for per-episode scenario execution.""" from __future__ import annotations +from datetime import datetime from typing import Any -from netopsbench.platform.scenario.episode_runner import ( - EpisodeExecutionPort, - EpisodeRunner, -) +from netopsbench.platform.scenario.episode_runner import run_episode from netopsbench.platform.scenario.models import Episode class StubExecutor: - """Minimal stand-in for ScenarioExecutor used by EpisodeRunner.""" + """Minimal ScenarioExecutor stand-in for exercising run_episode().""" def __init__( self, @@ -47,7 +45,11 @@ def _wait_and_observe(self, seconds, *, baseline_end_time=None): self.calls.append(f"observe:{seconds}") return {"duration": seconds, **self._observe_payload} - def _merge_observation_windows(self, windows, *, total_duration_seconds): + def _capture_observation_window(self, seconds, name): + self.calls.append(f"capture:{name}:{seconds}") + return {"name": name, "duration": seconds, **self._observe_payload} + + def _merge_observation_windows(self, windows, *, total_duration_seconds, baseline_end_time=None): self.calls.append("merge") return {"windows": list(windows), "total": total_duration_seconds} @@ -76,19 +78,19 @@ def _make_episode(**overrides) -> Episode: def test_runner_skips_baseline_when_fault_type_none(): executor = StubExecutor(skip_none_episodes=True) - runner = EpisodeRunner(executor) episode = _make_episode(fault_type="none", duration_seconds=0, metadata={}) - result = runner.run(episode) + result = run_episode(executor, episode) assert result["skipped"] is True assert "inject" not in executor.calls assert executor.calls == ["skipped"] + assert datetime.fromisoformat(result["start_time"]).tzinfo is not None + assert datetime.fromisoformat(result["end_time"]).tzinfo is not None def test_runner_returns_error_when_injection_fails(): executor = StubExecutor(injection_success=False) - runner = EpisodeRunner(executor) episode = _make_episode(duration_seconds=2, metadata={"early_observation_seconds": 0}) - result = runner.run(episode) + result = run_episode(executor, episode) assert result["success"] is False assert result["error"] == "Fault injection failed" assert "recover" not in executor.calls @@ -96,26 +98,40 @@ def test_runner_returns_error_when_injection_fails(): def test_runner_full_happy_path_invokes_inject_observe_recover(): executor = StubExecutor() - runner = EpisodeRunner(executor) episode = _make_episode(duration_seconds=2, metadata={"early_observation_seconds": 0}) - result = runner.run(episode) + result = run_episode(executor, episode) assert result["success"] is True assert "injection" in result assert "observations" in result assert "recovery" in result - assert "inject" in executor.calls - assert "recover" in executor.calls - assert "merge" in executor.calls + assert executor.calls == ["inject", "capture:steady:2", "merge", "recover"] + + +def test_runner_defers_detector_analysis_until_all_observation_windows_are_captured(): + executor = StubExecutor() + executor.sleep = lambda _seconds: None + episode = _make_episode(duration_seconds=12, stabilization_time=2, metadata={"early_observation_seconds": 4}) + + result = run_episode(executor, episode) + + assert result["success"] is True + assert executor.calls == [ + "inject", + "capture:early:4", + "capture:steady:8", + "merge", + "recover", + ] + assert all(not call.startswith("observe:") for call in executor.calls) def test_runner_uses_executor_sleep_hook(): executor = StubExecutor(post_recovery_wait_seconds=2) slept = [] executor.sleep = slept.append - runner = EpisodeRunner(executor) episode = _make_episode(duration_seconds=3, stabilization_time=1, metadata={"early_observation_seconds": 0}) - result = runner.run(episode) + result = run_episode(executor, episode) assert result["success"] is True assert slept == [1, 2] @@ -123,7 +139,6 @@ def test_runner_uses_executor_sleep_hook(): def test_runner_invokes_diagnosis_callback_for_fault_episodes(): executor = StubExecutor() - runner = EpisodeRunner(executor) episode = _make_episode(duration_seconds=2, metadata={"early_observation_seconds": 0}) captured: list[dict[str, Any]] = [] @@ -132,14 +147,13 @@ def diagnose(payload): captured.append(payload) return {"verdict": "ok", "success": True} - result = runner.run(episode, diagnosis_callback=diagnose) + result = run_episode(executor, episode, diagnosis_callback=diagnose) assert result["diagnosis"] == {"verdict": "ok", "success": True} assert captured, "diagnosis callback should have been invoked" def test_runner_does_not_invoke_diagnosis_for_baseline_episode(): executor = StubExecutor(skip_none_episodes=False) - runner = EpisodeRunner(executor) episode = _make_episode( fault_type="none", duration_seconds=2, @@ -147,56 +161,30 @@ def test_runner_does_not_invoke_diagnosis_for_baseline_episode(): ) called = [] - runner.run(episode, diagnosis_callback=lambda payload: called.append(payload) or {}) + run_episode(executor, episode, diagnosis_callback=lambda payload: called.append(payload) or {}) assert called == [] def test_runner_captures_diagnosis_exception_into_result(): executor = StubExecutor() - runner = EpisodeRunner(executor) episode = _make_episode(duration_seconds=2, metadata={"early_observation_seconds": 0}) def boom(_payload): raise RuntimeError("diagnose failed") - result = runner.run(episode, diagnosis_callback=boom) + result = run_episode(executor, episode, diagnosis_callback=boom) assert result["diagnosis"] == {"error": "diagnose failed", "success": False} assert result["success"] is True def test_runner_attempts_recovery_on_unexpected_exception(): class ExplodingExecutor(StubExecutor): - def _wait_and_observe(self, seconds, *, baseline_end_time=None): + def _capture_observation_window(self, seconds, name): raise RuntimeError("observe blew up") executor = ExplodingExecutor() - runner = EpisodeRunner(executor) episode = _make_episode(duration_seconds=2, metadata={"early_observation_seconds": 0}) - result = runner.run(episode) + result = run_episode(executor, episode) assert result["success"] is False assert "observe blew up" in result["error"] assert "recover" in executor.calls - - -def test_stub_executor_satisfies_episode_execution_port_protocol(): - """The test stub must satisfy the public Protocol so production - executors and stubs share the same typed contract.""" - assert isinstance(StubExecutor(), EpisodeExecutionPort) - - -def test_real_scenario_executor_satisfies_protocol(): - """ScenarioExecutor (the production host) must also satisfy the Protocol.""" - from netopsbench.platform.scenario.executor import ScenarioExecutor - - # We only need a class that *defines* the methods; we don't construct - # a real one because it would require a topology + influxdb setup. - # ``runtime_checkable`` Protocols verify method presence, so checking - # against a synthetic instance is sufficient. - for attr in ( - "_inject_fault", - "_wait_and_observe", - "_recover_fault", - "_merge_observation_windows", - "_build_skipped_episode_result", - ): - assert hasattr(ScenarioExecutor, attr), f"executor missing required attr: {attr}" diff --git a/tests/test_example_agents.py b/tests/test_example_agents.py index 1ea14fe..b50add6 100644 --- a/tests/test_example_agents.py +++ b/tests/test_example_agents.py @@ -747,6 +747,7 @@ async def aclose(self): fake_sdk = types.ModuleType('netopsbench.sdk') fake_sdk.NetOpsBench = NetOpsBench fake_sdk.RunFailedError = RuntimeError +fake_sdk.supported_scales = lambda: ('xs', 'small', 'medium', 'large', 'xlarge', 'fat-tree-k8', 'fat-tree-k12') sys.modules['netopsbench.sdk'] = fake_sdk fake_examples_agents = types.ModuleType('examples.agents') @@ -845,9 +846,9 @@ def test_minimal_deepagent_uses_sdk_default_mcp_server_config(monkeypatch): "netopsbench": { "transport": "stdio", "command": "python", - "args": ["scripts/toolkit/run_fastmcp_server.py"], + "args": ["-m", "netopsbench.platform.toolkit.fastmcp_server"], "cwd": str(workspace), - "env": {"PYTHONPATH": str(workspace)}, + "env": {}, } }, ) @@ -869,4 +870,4 @@ async def ainvoke(self, payload, config=None): context = DiagnosticContext(scenario_id="scenario-mcp", topology={"devices": {}}, symptoms={}) asyncio.run(agent.diagnose(context)) - assert captured_config["args"] == ["scripts/toolkit/run_fastmcp_server.py"] + assert captured_config["args"] == ["-m", "netopsbench.platform.toolkit.fastmcp_server"] diff --git a/tests/test_fat_tree_topology.py b/tests/test_fat_tree_topology.py new file mode 100644 index 0000000..7d9a1ac --- /dev/null +++ b/tests/test_fat_tree_topology.py @@ -0,0 +1,78 @@ +import ipaddress +import json +from pathlib import Path + +from netopsbench.models.topology import DeviceRole, TopologyManifest +from netopsbench.platform.topology.configdb_payload import interface_networks_for_config +from netopsbench.platform.topology.generator import generate_topology + + +def _load_config(topology_dir: Path, device: str) -> dict: + return json.loads((topology_dir / "configs" / "sonic" / device / "config_db.json").read_text(encoding="utf-8")) + + +def _assert_preseed_artifacts(topology_dir: Path, device: str) -> None: + sonic_dir = topology_dir / "configs" / "sonic" / device + assert (sonic_dir / "config_db.json").exists() + assert (sonic_dir / "port_config.ini").exists() + assert (sonic_dir / "lanemap.ini").exists() + frr_config = topology_dir / "configs" / "frr" / f"{device}.conf" + assert frr_config.exists() + rendered = frr_config.read_text(encoding="utf-8") + assert "bgp bestpath as-path multipath-relax" in rendered + assert "maximum-paths 64" in rendered + + +def test_fat_tree_k12_sparse_generates_valid_non_overlapping_link_networks(tmp_path): + result = generate_topology("fat-tree-k12", str(tmp_path)) + metadata = result["agent_topology"] + manifest = TopologyManifest.model_validate_json(Path(result["metadata_file"]).read_text(encoding="utf-8")) + + assert metadata["topology_scale"] == "fat-tree-k12" + assert metadata["fat_tree_k"] == 12 + assert metadata["management"]["ipv4_subnet"] == "172.20.20.0/23" + assert metadata["scale"]["num_core"] == 36 + assert metadata["scale"]["num_agg"] == 72 + assert metadata["scale"]["num_edge"] == 72 + assert metadata["scale"]["clients_per_edge"] == 2 + assert metadata["scale"]["total_clients"] == 144 + assert metadata["scale"]["host_density"] == "sparse" + assert metadata["scale"]["full_density_clients_per_edge"] == 6 + assert metadata["pingmesh"]["coverage_epoch_cycles"] == 36 + assert metadata["pingmesh"]["coverage_epoch_seconds"] == 72 + assert len(metadata["devices"]["clients"]) == 144 + assert manifest.facts.clients_per_attached_switch == 2 + assert manifest.facts.host_density == "sparse" + assert manifest.routing.ecmp_hash_policy_by_role == { + DeviceRole.CORE: 1, + DeviceRole.AGG: 0, + DeviceRole.EDGE: 1, + } + assert len(manifest.links) == 1008 + + for device in ("core1", "core36", "agg1", "agg72", "edge1", "edge72"): + _assert_preseed_artifacts(tmp_path, device) + + core1 = _load_config(tmp_path, "core1") + agg1 = _load_config(tmp_path, "agg1") + edge1 = _load_config(tmp_path, "edge1") + assert len([key for key in core1["INTERFACE"] if "|" not in key]) == 12 + assert len([key for key in agg1["INTERFACE"] if "|" not in key]) == 12 + assert len([key for key in edge1["INTERFACE"] if "|" not in key]) == 8 + + network_counts: dict[ipaddress.IPv4Network, int] = {} + devices = [ + *(f"core{i}" for i in range(1, 37)), + *(f"agg{i}" for i in range(1, 73)), + *(f"edge{i}" for i in range(1, 73)), + ] + for device in devices: + config_path = tmp_path / "configs" / "sonic" / device / "config_db.json" + for network in interface_networks_for_config(config_path).values(): + parsed = ipaddress.ip_network(network) + if parsed.subnet_of(ipaddress.ip_network("10.0.0.0/8")): + assert parsed.prefixlen == 30 + network_counts[parsed] = network_counts.get(parsed, 0) + 1 + + assert len(network_counts) == 864 + assert set(network_counts.values()) == {2} diff --git a/tests/test_fault_registry.py b/tests/test_fault_registry.py index fa0c667..d011f2c 100644 --- a/tests/test_fault_registry.py +++ b/tests/test_fault_registry.py @@ -1,15 +1,22 @@ """Tests for centralized fault registry dispatch.""" import importlib +import tempfile import pytest from netopsbench.platform.scenario.executor import ScenarioExecutor from netopsbench.platform.scenario.models import Episode +from netopsbench.platform.topology.generator import generate_topology + + +def _metadata() -> dict: + with tempfile.TemporaryDirectory() as tmpdir: + return generate_topology("xs", tmpdir)["metadata"] def test_scenario_runner_uses_registered_fault_spec(monkeypatch): - from netopsbench.platform.faults.specs import FaultSpec, register_fault_spec, unregister_fault_spec + from netopsbench.platform.faults.specs import FaultSpec, create_fault_registry captured = {} @@ -19,27 +26,36 @@ def inject_episode(injector, episode): captured["fault_type"] = episode.fault_type return {"success": True, "type": episode.fault_type, "source": "registry"} - register_fault_spec(FaultSpec(name="synthetic_fault", inject_episode=inject_episode)) - - try: - runner = ScenarioExecutor( - topology_dir="lab-topology", - topology_metadata={"name": "dcn", "devices": {"spines": [], "leafs": [], "clients": []}}, - ) - episode = Episode( - episode_id="ep_synth", - description="Synthetic fault via registry", - fault_type="synthetic_fault", - target_device="leaf1", - ) - - result = runner._inject_fault(episode) - - assert result["success"] is True - assert result["source"] == "registry" - assert captured["episode_id"] == "ep_synth" - finally: - unregister_fault_spec("synthetic_fault") + registry = create_fault_registry() + registry.register(FaultSpec(name="synthetic_fault", inject_episode=inject_episode)) + runner = ScenarioExecutor( + topology_dir="lab-topology", + topology_metadata=_metadata(), + fault_registry=registry, + ) + episode = Episode( + episode_id="ep_synth", + description="Synthetic fault via registry", + fault_type="synthetic_fault", + target_device="leaf1", + ) + + result = runner._inject_fault(episode) + + assert result["success"] is True + assert result["source"] == "registry" + assert captured["episode_id"] == "ep_synth" + + +def test_fault_registries_do_not_share_custom_specs(): + from netopsbench.platform.faults.specs import FaultSpec, create_fault_registry + + first = create_fault_registry() + second = create_fault_registry() + first.register(FaultSpec(name="isolated_fault")) + + assert first.get("isolated_fault") is not None + assert second.get("isolated_fault") is None def test_fault_spec_validate_episode_supports_prefix_and_required_parameters(): diff --git a/tests/test_negative_sample_scenarios.py b/tests/test_negative_sample_scenarios.py index ec42b00..b6db57e 100644 --- a/tests/test_negative_sample_scenarios.py +++ b/tests/test_negative_sample_scenarios.py @@ -1,16 +1,15 @@ +import tempfile + from netopsbench.evaluator.scorer import AgentOutput, Evaluator from netopsbench.platform.scenario.executor import ScenarioExecutor from netopsbench.platform.scenario.models import Episode, Scenario from netopsbench.platform.session.scoring import score_scenario_fault_episodes +from netopsbench.platform.topology.generator import generate_topology + -_TOPOLOGY_METADATA = { - "name": "dcn", - "devices": { - "spines": [{"name": "spine1"}], - "leafs": [{"name": "leaf1"}], - "clients": [{"name": "client1"}], - }, -} +def _metadata() -> dict: + with tempfile.TemporaryDirectory() as tmpdir: + return generate_topology("xs", tmpdir)["metadata"] def _none_episode(episode_id="ep001"): @@ -26,7 +25,7 @@ def _none_episode(episode_id="ep001"): def test_negative_sample_none_episode_observes_and_diagnoses(monkeypatch): runner = ScenarioExecutor( - topology_metadata=_TOPOLOGY_METADATA, + topology_metadata=_metadata(), skip_none_episodes=True, sleep_fn=lambda seconds: None, persist_results=False, @@ -73,7 +72,7 @@ def diagnose(episode_result): def test_regular_skipped_none_episode_does_not_observe_or_diagnose(monkeypatch): runner = ScenarioExecutor( - topology_metadata=_TOPOLOGY_METADATA, + topology_metadata=_metadata(), skip_none_episodes=True, sleep_fn=lambda seconds: None, persist_results=False, diff --git a/tests/test_observability_validation.py b/tests/test_observability_validation.py new file mode 100644 index 0000000..3bbc0d0 --- /dev/null +++ b/tests/test_observability_validation.py @@ -0,0 +1,79 @@ +from netopsbench.platform.observability.validation import check_observability, extract_interface_names + + +def test_extract_interface_names_ignores_empty_counter_identity_samples(): + csv_text = "\n".join( + [ + ",result,table,_time,_field,_value,name,path,source", + ",,0,2026-07-01T18:49:04Z,Ethernet100,,Ethernet100,/COUNTERS/Ethernet100,leaf1", + ",,0,2026-07-01T18:49:04Z,in_octets,42,Ethernet0,/COUNTERS/Ethernet0,leaf1", + ",,0,2026-07-01T18:49:04Z,out_octets,43,Ethernet4,/COUNTERS/Ethernet4,leaf1", + ] + ) + + assert extract_interface_names(csv_text) == ["Ethernet0", "Ethernet4"] + + +def test_interface_observability_queries_are_scoped_by_topology_id(): + queries: list[str] = [] + + def query_runner(query: str) -> str: + queries.append(query) + if '_measurement == "pingmesh"' in query: + return ",result,table,_time,_value\n,,0,2026-07-01T18:49:04Z,1\n" + if '_measurement == "interfaces"' in query: + return "\n".join( + [ + ",result,table,_time,_field,_value,name,path,source,topology_id", + ",,0,2026-07-01T18:49:04Z,in_octets,42,Ethernet0,/COUNTERS/Ethernet0,leaf1,lab-a", + ] + ) + return ",result,table,_time,_value\n" + + errors = check_observability( + query_runner, + bucket="netopsbench", + obs_device="leaf1", + topology_id="lab-a", + active_interfaces=["Ethernet0"], + ) + + assert errors == [] + interface_queries = [query for query in queries if '_measurement == "interfaces"' in query] + assert len(interface_queries) == 1 + assert all('r.topology_id == "lab-a"' in query for query in interface_queries) + assert 'group(columns: ["name", "path", "_field"])' in interface_queries[0] + assert "|> last()" in interface_queries[0] + assert "limit(n: 2000)" not in interface_queries[0] + + +def test_observability_existence_queries_are_globally_bounded(): + queries: list[str] = [] + + def query_runner(query: str) -> str: + queries.append(query) + if '_measurement == "interfaces"' in query: + return ",result,table,_time,_field,_value,name,path\n,,0,2026-07-01T18:49:04Z,in_octets,1,Ethernet0,/COUNTERS/Ethernet0\n" + return ",result,table,_time,_value\n,,0,2026-07-01T18:49:04Z,1\n" + + errors = check_observability( + query_runner, + bucket="netopsbench", + obs_device="leaf1", + bgp_device="spine1", + topology_id="lab-a", + syslog_marker="health-marker", + active_interfaces=["Ethernet0"], + ) + + assert errors == [] + assert len(queries) == 4 + pingmesh_query = next(query for query in queries if '_measurement == "pingmesh"' in query) + bgp_query = next(query for query in queries if '_measurement == "bgp_neighbors"' in query) + syslog_query = next(query for query in queries if '_measurement == "syslog"' in query) + assert '_field == "rtt_p99"' in pingmesh_query + assert "|> group()" in pingmesh_query + assert "|> limit(n: 1)" in pingmesh_query + assert "|> group()" in bgp_query + assert 'r.topology_id == "lab-a"' in syslog_query + assert "|> limit(n: 1)" in syslog_query diff --git a/tests/test_pingmesh_deploy.py b/tests/test_pingmesh_deploy.py new file mode 100644 index 0000000..f4b8845 --- /dev/null +++ b/tests/test_pingmesh_deploy.py @@ -0,0 +1,195 @@ +"""Unit tests for bind-mounted Pingmesh deployment.""" + +from __future__ import annotations + +import json +import subprocess +from concurrent.futures import Future +from pathlib import Path + +import pytest + +import netopsbench.platform.pingmesh.deploy as deploy_mod + + +def _write_topology(topology_dir: Path, *, clients: int = 3, include_bind: bool = True) -> list[str]: + client_names = [f"client{i}" for i in range(1, clients + 1)] + topology_dir.mkdir(parents=True, exist_ok=True) + (topology_dir / "topology.json").write_text( + json.dumps( + { + "schema_version": "3", + "topology_id": "demo", + "name": "demo", + "scale": "test", + "family": "clos", + "management": {"network": "clab-mgmt-demo", "ipv4_subnet": "172.20.20.0/24"}, + "collector": {"ipv4": "172.20.20.200"}, + "defaults": {}, + "facts": { + "num_leafs": 1, + "clients_per_attached_switch": max(1, clients), + "total_clients": clients, + "total_switches": 1, + }, + "routing": {"ecmp_hash_policy_by_role": {"leaf": 1}}, + "devices": [ + {"name": "leaf1", "role": "leaf"}, + *[ + { + "name": name, + "role": "client", + "attached_switch": "leaf1", + "data_ip": f"192.168.101.{index + 1}", + } + for index, name in enumerate(client_names, start=1) + ], + ], + "links": [], + "pingmesh": {"destination_batch_size": 16}, + } + ), + encoding="utf-8", + ) + linux_kind = {"image": "client"} + if include_bind: + linux_kind["binds"] = ["configs/pingmesh:/tmp/pingmesh:ro"] + (topology_dir / "demo.clab.yaml").write_text( + json.dumps({"name": "demo", "topology": {"kinds": {"linux": linux_kind}}}), + encoding="utf-8", + ) + return client_names + + +class _RecordingExecutor: + max_workers_seen: list[int] = [] + submitted: list[tuple] = [] + + def __init__(self, max_workers: int): + self.max_workers_seen.append(max_workers) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def submit(self, fn, *args, **kwargs): + self.submitted.append((fn, args, kwargs)) + future: Future = Future() + future.set_result(fn(*args, **kwargs)) + return future + + +@pytest.fixture(autouse=True) +def _reset_executor(): + _RecordingExecutor.max_workers_seen.clear() + _RecordingExecutor.submitted.clear() + + +def _fake_pinglist_generator(calls: list[Path]): + def _generate(metadata_file: str, output_file: str, topology_id: str | None = None): + output_path = Path(output_file) + calls.append(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps({"probes": [], "total_probes": 0, "topology_id": topology_id}), + encoding="utf-8", + ) + return [] + + return _generate + + +def test_deploy_pingmesh_stages_runtime_once_and_starts_clients_in_parallel(tmp_path, monkeypatch): + topology_dir = tmp_path / "topology" + clients = _write_topology(topology_dir, clients=3) + generator_calls: list[Path] = [] + docker_calls: list[tuple[str, ...]] = [] + + monkeypatch.setattr(deploy_mod, "ThreadPoolExecutor", _RecordingExecutor, raising=False) + monkeypatch.setattr(deploy_mod, "generate_pinglist_from_topology", _fake_pinglist_generator(generator_calls)) + monkeypatch.setattr( + deploy_mod, + "_running_containers", + lambda: [f"clab-demo-{client}" for client in clients], + ) + + def _docker(*args: str, check: bool = True, capture: bool = False, **kwargs): + docker_calls.append(args) + assert args[0] != "cp", "Pingmesh deploy should use bind-mounted runtime, not docker cp" + if args[0] == "exec" and args[2:4] == ("sh", "-c"): + command = args[4] + assert "mkdir -p /var/log/pingmesh" in command + assert "test -r /tmp/pingmesh/netopsbench/platform/pingmesh/cli.py" in command + assert "test -r /tmp/pingmesh/pinglist.json" in command + assert "pgrep -f netopsbench.platform.pingmesh.cli" in command + assert '[ "$pid" = "$$" ] && continue' in command + assert 'kill "$pid"' in command + assert "nohup python3 -m netopsbench.platform.pingmesh.cli" in command + return subprocess.CompletedProcess(["docker", *args], 0, stdout="", stderr="") + if args[0] == "exec" and args[2:] == ("ps", "aux"): + return subprocess.CompletedProcess( + ["docker", *args], 0, stdout="netopsbench.platform.pingmesh.cli", stderr="" + ) + raise AssertionError(f"unexpected docker call: {args}") + + monkeypatch.setattr(deploy_mod, "_docker", _docker) + + result = deploy_mod.deploy_pingmesh(str(topology_dir), verify=False, parallelism=7) + + runtime_dir = topology_dir / "configs" / "pingmesh" + assert generator_calls == [runtime_dir / "pinglist.json"] + assert (runtime_dir / "pinglist.json").is_file() + assert not (runtime_dir / "run_pingmesh_agent.py").exists() + for module in deploy_mod._AGENT_MODULES: + assert (runtime_dir / "netopsbench" / "platform" / "pingmesh" / module).is_file() + for module in deploy_mod._PACKAGE_MODULES: + assert (runtime_dir / "netopsbench" / module).is_file() + assert not (runtime_dir / "agent.py").exists() + + start_calls = [args for args in docker_calls if args[0] == "exec"] + assert len(start_calls) == len(clients) + assert all(args[2:4] == ("sh", "-c") for args in start_calls) + assert _RecordingExecutor.max_workers_seen == [7] + assert len(_RecordingExecutor.submitted) == len(clients) + assert result.deployed == len(clients) + assert result.failed == [] + + +def test_deploy_pingmesh_records_failed_client_when_bind_runtime_is_unreadable(tmp_path, monkeypatch): + topology_dir = tmp_path / "topology" + clients = _write_topology(topology_dir, clients=2) + generator_calls: list[Path] = [] + + monkeypatch.setattr(deploy_mod, "ThreadPoolExecutor", _RecordingExecutor, raising=False) + monkeypatch.setattr(deploy_mod, "generate_pinglist_from_topology", _fake_pinglist_generator(generator_calls)) + monkeypatch.setattr( + deploy_mod, + "_running_containers", + lambda: [f"clab-demo-{client}" for client in clients], + ) + + def _docker(*args: str, check: bool = True, capture: bool = False, **kwargs): + assert args[0] != "cp", "Pingmesh deploy should not fall back to docker cp" + if args[0] == "exec" and args[2:4] == ("sh", "-c"): + if args[1].endswith("client2"): + return subprocess.CompletedProcess(["docker", *args], 1, stdout="", stderr="missing bind") + return subprocess.CompletedProcess(["docker", *args], 0, stdout="", stderr="") + raise AssertionError(f"unexpected docker call: {args}") + + monkeypatch.setattr(deploy_mod, "_docker", _docker) + + result = deploy_mod.deploy_pingmesh(str(topology_dir), verify=False) + + assert generator_calls == [topology_dir / "configs" / "pingmesh" / "pinglist.json"] + assert result.deployed == 1 + assert result.failed == ["client2"] + + +def test_deploy_pingmesh_rejects_topology_without_pingmesh_bind(tmp_path): + topology_dir = tmp_path / "topology" + _write_topology(topology_dir, clients=1, include_bind=False) + + with pytest.raises(RuntimeError, match="configs/pingmesh:/tmp/pingmesh:ro"): + deploy_mod.deploy_pingmesh(str(topology_dir), verify=False) diff --git a/tests/test_pingmesh_time_scope.py b/tests/test_pingmesh_time_scope.py index e1c76de..f00e2c4 100644 --- a/tests/test_pingmesh_time_scope.py +++ b/tests/test_pingmesh_time_scope.py @@ -1,13 +1,32 @@ import json - +import tempfile +from types import SimpleNamespace + +from netopsbench.models.topology import ( + Collector, + Device, + DeviceRole, + Management, + PingmeshPolicy, + RoutingMetadata, + TopologyDefaults, + TopologyFacts, + TopologyManifest, +) +from netopsbench.platform.pingmesh._detector_query import SnapshotQueryResult +from netopsbench.platform.pingmesh.detector import Anomaly, AnomalyDetector +from netopsbench.platform.scenario.observation import wait_and_observe from netopsbench.platform.toolkit._core.common import ToolResult from netopsbench.platform.toolkit.mcp import context as mcp_context from netopsbench.platform.toolkit.mcp import observability as fastmcp_server from netopsbench.platform.toolkit.toolkit import AgentToolkit +from netopsbench.platform.topology.generator import generate_topology def _toolkit_with_captured_queries(monkeypatch): - toolkit = AgentToolkit(topology_metadata={"devices": {}}) + with tempfile.TemporaryDirectory() as tmpdir: + metadata = generate_topology("xs", tmpdir)["metadata"] + toolkit = AgentToolkit(topology_metadata=metadata) captured = {} def fake_query(query, require_value=True): @@ -19,6 +38,468 @@ def fake_query(query, require_value=True): return toolkit, captured +def _coverage_detector(client_count=144): + clients = [ + Device(name=f"client{i}", role=DeviceRole.CLIENT, attached_switch="leaf1") for i in range(1, client_count + 1) + ] + topology = TopologyManifest( + topology_id="k12-runtime", + name="coverage", + scale="test", + family="clos", + management=Management(network="clab-coverage", ipv4_subnet="172.31.1.0/24"), + collector=Collector(ipv4="172.31.1.200"), + defaults=TopologyDefaults(), + facts=TopologyFacts( + num_leafs=1, + clients_per_attached_switch=max(1, client_count), + total_clients=client_count, + total_switches=1, + ), + devices=[Device(name="leaf1", role=DeviceRole.LEAF), *clients], + links=[], + routing=RoutingMetadata(ecmp_hash_policy_by_role={DeviceRole.LEAF: 1}), + pingmesh=PingmeshPolicy( + destination_batch_size=16, + rtt_port_pool_size=16, + rtt_ports_per_cycle=4, + cycle_interval_seconds=2, + ), + ) + return AnomalyDetector( + "http://influxdb:8086", + "token", + "org", + "bucket", + topology_metadata=topology, + topology_id="k12-runtime", + ) + + +def _rtt_rows(values, *, ranges=None): + ranges = ranges or [0.2] * len(values) + return [ + { + "_time": f"2026-01-01T00:00:{index:02d}Z", + "_field": "rtt_avg", + "value": value, + "rtt_min": value - ranges[index] / 2, + "rtt_avg": value, + "rtt_max": value + ranges[index] / 2, + "src_ip": "192.0.2.1", + "dst_ip": "192.0.2.2", + "src_name": "client1", + "dst_name": "client2", + "src_leaf": "edge1", + "dst_leaf": "edge2", + } + for index, value in enumerate(values) + ] + + +def _snapshot_sequence(monkeypatch, detector, responses): + pending = iter(responses) + calls = [] + + def query(start_time, end_time): + calls.append((start_time, end_time)) + return SnapshotQueryResult(status="ok", rows=next(pending)) + + monkeypatch.setattr(detector, "_query_snapshot", query) + return calls + + +def test_jitter_detector_uses_per_cycle_rtt_range(): + detector = _coverage_detector(client_count=2) + anomalies = detector.analyze_snapshot_rows( + _rtt_rows([10.0, 10.1, 9.9, 10.0], ranges=[0.2, 0.3, 0.2, 0.3]), + _rtt_rows([10.0, 10.1, 9.9, 10.0], ranges=[6.0, 7.0, 6.0, 7.0]), + ).anomalies + jitter = [item for item in anomalies if item.type == "jitter_spike"] + + assert len(jitter) == 1 + assert jitter[0].type == "jitter_spike" + + +def test_jitter_detector_ignores_stable_per_cycle_rtt_range(): + detector = _coverage_detector(client_count=2) + anomalies = detector.analyze_snapshot_rows( + _rtt_rows([10.0, 10.2, 9.9, 10.1], ranges=[0.2, 0.3, 0.2, 0.3]), + _rtt_rows([10.1, 9.9, 10.2, 10.0], ranges=[0.3, 0.2, 0.3, 0.2]), + ).anomalies + + assert not [item for item in anomalies if item.type == "jitter_spike"] + + +def test_latency_detector_reports_sustained_increase(): + detector = _coverage_detector(client_count=2) + anomalies = detector.analyze_snapshot_rows( + _rtt_rows([1.0, 1.2, 0.9, 1.1, 1.0]), + _rtt_rows([8.0, 8.2, 7.9, 8.1, 8.0]), + ).anomalies + latency = [item for item in anomalies if item.type == "latency_spike"] + + assert len(latency) == 1 + assert latency[0].type == "latency_spike" + + +def test_latency_detector_preserves_one_high_impact_ecmp_sample(): + detector = _coverage_detector(client_count=2) + anomalies = detector.analyze_snapshot_rows( + _rtt_rows([1.0, 1.2, 0.9, 1.1]), + _rtt_rows([1.0, 1.1, 1.0, 101.0]), + ).anomalies + latency = [item for item in anomalies if item.type == "latency_spike"] + + assert len(latency) == 1 + assert latency[0].value == 101.0 + assert latency[0].severity == "high" + + +def test_jitter_partial_consensus_is_advisory_low_severity(): + detector = _coverage_detector(client_count=2) + anomalies = detector.analyze_snapshot_rows( + _rtt_rows([10.0] * 4, ranges=[0.2] * 4), + _rtt_rows([10.0] * 4, ranges=[6.0, 6.0, 6.0, 0.2]), + ).anomalies + jitter = [item for item in anomalies if item.type == "jitter_spike"] + + assert len(jitter) == 1 + assert jitter[0].severity == "low" + + +def _probe_sample( + *, + timestamp="2026-01-01T00:00:00Z", + src_ip="192.0.2.1", + dst_ip="192.0.2.2", + sent=4, + lost=0, + df_sent=4, + df_lost=0, +): + return { + "_time": timestamp, + "src_ip": src_ip, + "dst_ip": dst_ip, + "src_name": "client1", + "dst_name": "client2", + "src_leaf": "edge1", + "dst_leaf": "edge2", + "rtt_avg": 1.0 if lost < sent else 0.0, + "packets_sent": float(sent), + "packets_lost": float(lost), + "packet_loss": (lost / sent) * 100.0, + "df_packets_sent": float(df_sent), + "df_packets_lost": float(df_lost), + "df_loss_pct": (df_lost / df_sent) * 100.0, + "df_mtu_drops": 0.0, + } + + +def test_bounded_rotation_missing_paths_are_not_unreachable(): + detector = _coverage_detector(client_count=3) + baseline = [ + _probe_sample(dst_ip="192.0.2.2"), + _probe_sample(dst_ip="192.0.2.3"), + ] + current = [_probe_sample(dst_ip="192.0.2.2")] + + analysis = detector.analyze_snapshot_rows(baseline, current) + + assert analysis.anomalies == [] + assert analysis.quality["not_observed_paths"] == 1 + + +def test_actual_complete_probe_loss_is_path_unreachable(): + detector = _coverage_detector(client_count=2) + + analysis = detector.analyze_snapshot_rows([_probe_sample()], [_probe_sample(lost=4, df_lost=4)]) + + assert [item.type for item in analysis.anomalies] == ["path_unreachable"] + assert analysis.anomalies[0].samples_sent == 4 + assert analysis.anomalies[0].samples_lost == 4 + + +def test_df_loss_is_suppressed_when_rtt_is_also_lost(): + detector = _coverage_detector(client_count=2) + + analysis = detector.analyze_snapshot_rows([_probe_sample()], [_probe_sample(lost=4, df_lost=4)]) + + assert all(item.type != "mtu_or_fragmentation_suspect" for item in analysis.anomalies) + + +def test_df_only_loss_is_mtu_suspect(): + detector = _coverage_detector(client_count=2) + + analysis = detector.analyze_snapshot_rows([_probe_sample()], [_probe_sample(lost=0, df_lost=4)]) + + assert [item.type for item in analysis.anomalies] == ["mtu_or_fragmentation_suspect"] + + +def test_generate_report_queries_each_snapshot_once(monkeypatch): + detector = _coverage_detector(client_count=2) + calls = _snapshot_sequence(monkeypatch, detector, [[_probe_sample()], [_probe_sample()]]) + + report = detector.generate_windowed_anomaly_report( + baseline_start="2026-01-01T00:00:00Z", + baseline_end="2026-01-01T00:01:00Z", + current_start="2026-01-01T00:01:00Z", + current_end="2026-01-01T00:01:01Z", + windows=[], + ) + + assert report["query_status"]["ok"] is True + assert calls == [ + ("2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"), + ("2026-01-01T00:01:00Z", "2026-01-01T00:01:01Z"), + ] + + +def test_snapshot_query_failure_is_not_reported_as_healthy(monkeypatch): + detector = _coverage_detector(client_count=2) + monkeypatch.setattr( + detector, + "_query_snapshot", + lambda _start, _end: SnapshotQueryResult(status="error", rows=[], error="influx unavailable"), + ) + + report = detector.generate_windowed_anomaly_report( + baseline_start="baseline-start", + baseline_end="baseline-end", + current_start="current-start", + current_end="current-end", + windows=[], + ) + + assert report["query_status"] == {"ok": False, "error": "influx unavailable"} + assert report["coverage"]["coverage_status"] == "error" + + +def test_snapshot_query_failure_without_error_text_is_still_an_error(monkeypatch): + detector = _coverage_detector(client_count=2) + monkeypatch.setattr( + detector, + "_query_snapshot", + lambda _start, _end: SnapshotQueryResult(status="error", rows=[]), + ) + + report = detector.generate_windowed_anomaly_report( + baseline_start="baseline-start", + baseline_end="baseline-end", + current_start="current-start", + current_end="current-end", + windows=[], + ) + + assert report["query_status"] == {"ok": False, "error": "query_failed"} + + +def test_window_slice_handles_fractional_timestamps_at_boundaries(): + rows = [ + {"_time": "2026-01-01T00:00:00.500000000Z", "value": "inside"}, + {"_time": "2026-01-01T00:00:01Z", "value": "end"}, + ] + + selected = AnomalyDetector._slice_rows( + rows, + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:01Z", + ) + + assert [row["value"] for row in selected] == ["inside"] + + +def test_window_merge_promotes_unreachable_and_marks_persistence(): + detector = _coverage_detector(client_count=2) + common = dict( + src_ip="192.0.2.1", + src_name="client1", + dst_ip="192.0.2.2", + dst_name="client2", + src_leaf="edge1", + dst_leaf="edge2", + baseline=0.0, + threshold=5.0, + severity="high", + timestamp="2026-01-01T00:00:00Z", + ) + early = Anomaly(type="packet_loss", value=50.0, **common) + steady = Anomaly(type="path_unreachable", value=100.0, **common) + + merged = detector._merge_window_anomalies([("early", [early]), ("steady", [steady])]) + + assert len(merged) == 1 + assert merged[0].type == "path_unreachable" + assert merged[0].windows_observed == ["early", "steady"] + assert merged[0].persistence == "persistent" + + +def test_window_merge_does_not_promote_short_window_statistical_noise(): + detector = _coverage_detector(client_count=2) + anomaly = Anomaly( + type="jitter_spike", + src_ip="192.0.2.1", + src_name="client1", + dst_ip="192.0.2.2", + dst_name="client2", + src_leaf="edge1", + dst_leaf="edge2", + value=8.0, + baseline=1.0, + threshold=3.0, + severity="high", + timestamp="2026-01-01T00:00:00Z", + ) + + merged = detector._merge_window_anomalies([("full", []), ("early", [anomaly]), ("steady", [])]) + + assert merged == [] + + +def test_snapshot_csv_parser_accepts_pivoted_rows(): + csv_text = """#group,false,false,true,true,false,false,false,false +#datatype,string,string,long,dateTime:RFC3339,string,string,double,double +,result,table,_time,src_ip,dst_ip,packets_sent,packets_lost +,_result,0,2026-01-01T00:00:00Z,192.0.2.1,192.0.2.2,4,1 +""" + + rows = AnomalyDetector._parse_snapshot_csv(csv_text) + + assert rows == [ + { + "result": "_result", + "table": "0", + "_time": "2026-01-01T00:00:00Z", + "src_ip": "192.0.2.1", + "dst_ip": "192.0.2.2", + "packets_sent": 4.0, + "packets_lost": 1.0, + } + ] + + +def test_pingmesh_coverage_summary_reports_complete_epoch(): + detector = _coverage_detector() + rows = [] + for source in range(1, 145): + for offset in range(1, 144): + destination_batch = (offset - 1) // 16 + for port_batch in range(4): + cycle = destination_batch + port_batch * 9 + rows.append( + { + "probe_cycle": float(cycle), + "destination_batch_index": float(destination_batch), + "port_batch_index": float(port_batch), + "src_name": f"client{source}", + "dst_name": f"client{((source + offset - 1) % 144) + 1}", + "rtt_ports_active": 4, + "rtt_ports_total": 16, + } + ) + audit = detector.summarize_coverage(rows) + + assert audit["coverage_status"] == "complete" + assert audit["expected_epoch_cycles"] == 36 + assert audit["min_source_cycle_span"] == 36 + assert audit["destination_batches_observed"] == list(range(9)) + assert audit["port_batches_observed"] == list(range(4)) + assert audit["pair_port_combinations_observed"] == 20_592 * 4 + assert audit["missing_pair_port_combinations"] == 0 + + +def test_detector_projects_canonical_manifest_before_loading_coverage_policy(tmp_path): + from netopsbench.platform.topology.generator import generate_topology + from netopsbench.platform.topology.topology_utils import load_topology_manifest + + generate_topology("xlarge", str(tmp_path), name="coverage-xlarge") + manifest = load_topology_manifest(tmp_path) + + detector = AnomalyDetector( + "http://influxdb:8086", + "token", + "org", + "bucket", + topology_metadata=manifest.model_dump(mode="json"), + topology_id=manifest.topology_id, + ) + + assert detector._pingmesh_policy["destination_batch_count"] == 8 + assert detector._pingmesh_policy["port_batch_count"] == 4 + assert detector._pingmesh_policy["coverage_epoch_cycles"] == 32 + + +def test_xlarge_and_k8_coverage_counts_all_pairs_and_port_batches(): + client_count = 128 + rows = [] + for source in range(1, client_count + 1): + for offset in range(1, client_count): + destination_batch = (offset - 1) // 16 + destination = ((source + offset - 1) % client_count) + 1 + for port_batch in range(4): + rows.append( + { + "probe_cycle": float(destination_batch + port_batch * 8), + "destination_batch_index": float(destination_batch), + "port_batch_index": float(port_batch), + "src_name": f"client{source}", + "dst_name": f"client{destination}", + "rtt_ports_active": 4, + "rtt_ports_total": 16, + } + ) + + audit = _coverage_detector(client_count).summarize_coverage(rows) + + assert audit["coverage_status"] == "complete" + assert audit["expected_epoch_cycles"] == 32 + assert audit["destination_pairs_observed"] == 16_256 + assert audit["pair_port_combinations_observed"] == 65_024 + + +def test_coverage_rejects_incomplete_socket_pool(): + detector = _coverage_detector(2) + rows = [ + { + "probe_cycle": float(port_batch), + "destination_batch_index": 0.0, + "port_batch_index": float(port_batch), + "src_name": source, + "dst_name": destination, + "rtt_ports_active": 4, + "rtt_ports_total": 0 if source == "client1" and port_batch == 0 else 16, + } + for source, destination in (("client1", "client2"), ("client2", "client1")) + for port_batch in range(4) + ] + + audit = detector.summarize_coverage(rows) + + assert audit["coverage_status"] == "incomplete" + assert audit["invalid_socket_rows"] == 1 + + +def test_pingmesh_coverage_summary_reports_missing_batches(): + detector = _coverage_detector() + rows = [ + { + "probe_cycle": float(index), + "destination_batch_index": float(index % 8), + "port_batch_index": float(index % 3), + "src_name": "client1", + "dst_name": "client2", + } + for index in range(10) + ] + audit = detector.summarize_coverage(rows) + + assert audit["coverage_status"] == "incomplete" + assert audit["missing_destination_batches"] == [8] + assert audit["missing_port_batches"] == [3] + assert audit["missing_source_clients"] == 143 + + def test_pingmesh_time_scope_uses_explicit_window_first(monkeypatch): toolkit, captured = _toolkit_with_captured_queries(monkeypatch) toolkit.set_pingmesh_time_window("2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z") @@ -51,6 +532,18 @@ def test_pingmesh_time_scope_uses_toolkit_default_before_context_file(monkeypatc assert 'range(start: time(v: "2026-01-02T00:00:00Z")' in captured["query"] +def test_pingmesh_hotspots_applies_global_loss_first_limit(monkeypatch): + toolkit, captured = _toolkit_with_captured_queries(monkeypatch) + + result = toolkit.get_pingmesh_hotspots(limit=7) + + assert result.success is True + query = captured["query"] + assert '|> pivot(rowKey: ["src_leaf", "dst_leaf"]' in query + assert '|> group()\n |> sort(columns: ["packet_loss", "rtt_p99"], desc: true)' in query + assert "|> limit(n: 7)" in query + + def test_pingmesh_time_scope_uses_context_file_before_env(monkeypatch, tmp_path): context_file = tmp_path / "pingmesh-window.json" context_file.write_text( @@ -131,3 +624,56 @@ def test_builtin_mcp_config_passes_netopsbench_env(monkeypatch): config = builtin_mcp_server_config() assert config["netopsbench"]["env"]["NETOPSBENCH_PINGMESH_CONTEXT_FILE"] == "/tmp/window.json" + + +def test_pingmesh_detector_builds_spine_map_from_canonical_links(tmp_path): + topology = generate_topology("xs", str(tmp_path))["metadata"] + + detector = AnomalyDetector("http://influxdb:8086", "token", "org", "bucket", topology_metadata=topology) + + assert detector.leaf_to_spines == { + "leaf1": ["spine1", "spine2"], + "leaf2": ["spine1", "spine2"], + } + + +def test_pingmesh_detector_projects_canonical_fat_tree_metadata(tmp_path): + topology = generate_topology("fat-tree-k8", str(tmp_path))["metadata"] + + detector = AnomalyDetector("http://influxdb:8086", "token", "org", "bucket", topology_metadata=topology) + + assert set(detector.leaf_to_spines["edge1"]) == {f"core{index}" for index in range(1, 17)} + assert "core17" not in detector.leaf_to_spines["edge1"] + + +def test_scenario_observation_passes_current_topology_to_pingmesh_detector(monkeypatch): + with tempfile.TemporaryDirectory() as tmpdir: + topology = generate_topology("fat-tree-k8", tmpdir)["metadata"] + captured = {} + + class FakeDetector: + def __init__(self, **kwargs): + captured.update(kwargs) + + def generate_windowed_anomaly_report(self, **_kwargs): + return { + "summary": {"total_anomalies": 0}, + "query_status": {"ok": True, "error": None}, + "coverage": {"status": "ok", "coverage_status": "complete"}, + "anomalies": [], + } + + monkeypatch.setattr("netopsbench.platform.pingmesh.detector.AnomalyDetector", FakeDetector) + runner = SimpleNamespace( + influxdb_url="http://influxdb:8086", + influxdb_token="token", + influxdb_org="org", + influxdb_bucket="bucket", + topology_id="fat-tree-runtime", + topology_metadata=topology, + ) + + observations = wait_and_observe(runner, duration=0) + + assert observations["data_source_status"] == "ok" + assert captured["topology_metadata"] == topology diff --git a/tests/test_pingmesh_udp_probe.py b/tests/test_pingmesh_udp_probe.py index 577b3b6..cd3b1c6 100644 --- a/tests/test_pingmesh_udp_probe.py +++ b/tests/test_pingmesh_udp_probe.py @@ -1,18 +1,23 @@ -"""Loopback unit tests for the UDP burst probe + responder.""" +"""Loopback unit tests for the UDP cycle event loop and responder.""" from __future__ import annotations +import json +import socket +import struct import time import pytest -from netopsbench.platform.pingmesh._agent_probe import UdpProbeMixin +from netopsbench.platform.pingmesh._agent_probe import _HEADER_FMT, UdpProbeMixin from netopsbench.platform.pingmesh._agent_responder import UdpEchoResponder -from netopsbench.platform.pingmesh._detector_analysis import _aggregate_loss_pct +from netopsbench.platform.pingmesh._agent_runtime import _next_cycle_deadline +from netopsbench.platform.pingmesh._detector_analysis import _loss_stats from netopsbench.platform.pingmesh.agent import ( - _default_ports_per_cycle, + PingmeshAgent, _deterministic_startup_jitter_seconds, - _resolve_ports_per_cycle, + _probe_batch_indices, + _rotating_destination_batch, ) @@ -24,26 +29,18 @@ def __init__( dst_port: int, *, n_rtt_ports: int = 8, - n_df_ports: int = 4, rtt_src_port_base: int = 34000, - df_src_port_base: int = 34100, enable_df_probe: bool = True, - burst_timeout_s: float = 1.0, rtt_ports_per_cycle: int | None = None, - df_ports_per_cycle: int | None = None, + interval: float = 0.2, ): self.udp_dst_port = dst_port self.n_rtt_ports = n_rtt_ports - self.n_df_ports = n_df_ports self.rtt_src_port_base = rtt_src_port_base - self.df_src_port_base = df_src_port_base self.df_payload_size = 1400 - self.burst_timeout_s = burst_timeout_s + self.interval = interval self.enable_df_probe = enable_df_probe - if rtt_ports_per_cycle is not None: - self.rtt_ports_per_cycle = rtt_ports_per_cycle - if df_ports_per_cycle is not None: - self.df_ports_per_cycle = df_ports_per_cycle + self.rtt_ports_per_cycle = rtt_ports_per_cycle if rtt_ports_per_cycle is not None else n_rtt_ports class _SpyHarness(_Harness): @@ -51,27 +48,23 @@ def __init__(self, dst_port: int, **kwargs): super().__init__(dst_port, **kwargs) self.open_calls = [] - def _open_burst_sockets(self, src_ip: str, src_port_base: int, n_ports: int, set_df: bool) -> list: + def _open_probe_sockets(self, src_ip: str, src_port_base: int, n_ports: int, set_df: bool) -> list: self.open_calls.append((src_ip, src_port_base, n_ports, set_df)) - return super()._open_burst_sockets(src_ip, src_port_base, n_ports, set_df) + return super()._open_probe_sockets(src_ip, src_port_base, n_ports, set_df) -class _FanoutSpyHarness(_Harness): +class _CycleSpyHarness(_Harness): def __init__(self, dst_port: int, **kwargs): super().__init__(dst_port, **kwargs) - self.fanout_ports = [] - - def _udp_fanout(self, probes: list[dict], sockets: list, payload_size: int, timeout_s: float) -> list[dict]: - self.fanout_ports.append([src_port for src_port, _sock in sockets]) - return [ - { - "sent": len(sockets), - "received": len(sockets), - "rtts": [0.1] * len(sockets), - "mtu_drops": 0, - } - for _probe in probes + self.cycle_ports = [] + + def _run_udp_cycle(self, probes: list[dict], rtt_sockets: list, enable_df: bool): + self.cycle_ports.append(([src_port for src_port, _sock in rtt_sockets], enable_df)) + rtt = [ + {"sent": len(rtt_sockets), "received": len(rtt_sockets), "rtts": [0.1], "mtu_drops": 0} for _probe in probes ] + df = [{"sent": int(enable_df), "received": int(enable_df), "rtts": [0.1], "mtu_drops": 0} for _probe in probes] + return rtt, df @pytest.fixture() @@ -119,11 +112,19 @@ def _probe(dst_ip: str, index: int, src_ip: str = "127.0.0.1") -> dict: } -def test_udp_rtt_burst_loopback(loopback_responder): +def test_pingmesh_agent_rejects_incomplete_canonical_policy(tmp_path): + pinglist = tmp_path / "pinglist.json" + pinglist.write_text(json.dumps({"probes": [], "pingmesh_policy": {}}), encoding="utf-8") + + with pytest.raises(ValueError, match="regenerate the topology"): + PingmeshAgent(str(pinglist)) + + +def test_udp_cycle_loopback_has_no_loss(loopback_responder): port = loopback_responder harness = _Harness(dst_port=port) try: - result = harness.udp_rtt_burst(dst_ip="127.0.0.1", src_ip="127.0.0.1") + result = harness.udp_probe_cycle([_probe("127.0.0.1", 1)], 0)[0]["result"] finally: harness._close_udp_probe_sockets() @@ -136,34 +137,69 @@ def test_udp_rtt_burst_loopback(loopback_responder): assert result["loss_pct"] == 0.0 -def test_udp_df_burst_loopback(loopback_responder): - port = loopback_responder - harness = _Harness(dst_port=port) - # Loopback MTU is 65536 so a 1400-byte DF payload always fits. +def test_udp_cycle_rtt_and_df_share_one_deadline_without_responder(): + harness = _Harness(dst_port=38435, interval=0.15) + started = time.monotonic() try: - result = harness.udp_df_burst(dst_ip="127.0.0.1", src_ip="127.0.0.1") + result = harness.udp_probe_cycle([_probe("127.0.0.1", 1)], 0)[0]["result"] finally: harness._close_udp_probe_sockets() + elapsed = time.monotonic() - started - assert result["packets_sent"] == harness.n_df_ports - assert result["packets_lost"] == 0 - assert result["success"] is True - assert result["mtu_drops"] == 0 + assert result["packets_sent"] == harness.n_rtt_ports + assert result["packets_lost"] == harness.n_rtt_ports + assert result["success"] is False + assert result["loss_pct"] == pytest.approx(100.0) + assert result["df_packets_lost"] == 1 + assert elapsed < 0.3 -def test_udp_rtt_burst_no_responder(): - """Probes to an unbound port should report 100% loss without crashing.""" - harness = _Harness(dst_port=38435) +def test_udp_cycle_partial_timeout_only_affects_missing_destination(loopback_responder): + harness = _Harness(dst_port=loopback_responder, n_rtt_ports=2, enable_df_probe=False) + probes = [_probe("127.0.0.1", 1), _probe("192.0.2.1", 2)] try: - result = harness.udp_rtt_burst(dst_ip="127.0.0.1", src_ip="127.0.0.1") + results = harness.udp_probe_cycle(probes, 0) finally: harness._close_udp_probe_sockets() - assert result["packets_sent"] == harness.n_rtt_ports - # On loopback the kernel may deliver an ICMP "port unreachable" that - # causes recvfrom() to return ECONNREFUSED — those count as loss too. - assert result["packets_lost"] == harness.n_rtt_ports - assert result["success"] is False - assert result["loss_pct"] == pytest.approx(100.0) + + assert results[0]["result"]["packets_lost"] == 0 + assert results[1]["result"]["packets_lost"] == 2 + + +def test_udp_probe_requires_the_complete_source_port_pool(monkeypatch): + harness = _Harness(dst_port=38435, n_rtt_ports=4) + + class SocketStub: + closed = False + + def close(self): + self.closed = True + + socket_stub = SocketStub() + monkeypatch.setattr(harness, "_open_probe_sockets", lambda **_kwargs: [(34000, socket_stub)]) + + with pytest.raises(RuntimeError, match="complete Pingmesh source-port pool"): + harness._ensure_udp_probe_sockets("127.0.0.1") + + assert socket_stub.closed is True + assert harness._udp_rtt_sockets == [] + + +def test_udp_cycle_discards_stale_reply_before_reusing_socket(loopback_responder): + harness = _Harness(dst_port=loopback_responder, n_rtt_ports=1, enable_df_probe=False) + rtt_sockets = harness._ensure_udp_probe_sockets("127.0.0.1") + stale = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + stale_payload = struct.pack(_HEADER_FMT, time.time_ns(), 999_999) + b"stale" + stale.sendto(stale_payload, ("127.0.0.1", rtt_sockets[0][0])) + time.sleep(0.01) + result = harness.udp_probe_cycle([_probe("127.0.0.1", 1)], 0)[0]["result"] + finally: + stale.close() + harness._close_udp_probe_sockets() + + assert result["packets_sent"] == 1 + assert result["packets_lost"] == 0 def test_udp_probe_cycle_fanout_loopback(loopback_responder_set): @@ -171,14 +207,12 @@ def test_udp_probe_cycle_fanout_loopback(loopback_responder_set): harness = _Harness( dst_port=port, n_rtt_ports=3, - n_df_ports=2, rtt_src_port_base=34200, - df_src_port_base=34300, ) probes = [_probe(dst_ip, index) for index, dst_ip in enumerate(bind_ips)] try: - cycle_results = harness.udp_probe_cycle(probes) + cycle_results = harness.udp_probe_cycle(probes, 0) finally: harness._close_udp_probe_sockets() @@ -199,15 +233,13 @@ def test_udp_probe_cycle_reuses_bound_source_ports(loopback_responder): harness = _SpyHarness( dst_port=port, n_rtt_ports=2, - n_df_ports=1, rtt_src_port_base=34400, - df_src_port_base=34500, ) probes = [_probe("127.0.0.1", 1)] try: - first = harness.udp_probe_cycle(probes) - second = harness.udp_probe_cycle(probes) + first = harness.udp_probe_cycle(probes, 0) + second = harness.udp_probe_cycle(probes, 0) finally: harness._close_udp_probe_sockets() @@ -215,7 +247,6 @@ def test_udp_probe_cycle_reuses_bound_source_ports(loopback_responder): assert second[0]["result"]["packets_lost"] == 0 assert harness.open_calls == [ ("127.0.0.1", harness.rtt_src_port_base, harness.n_rtt_ports, True), - ("127.0.0.1", harness.df_src_port_base, harness.n_df_ports, True), ] @@ -224,15 +255,13 @@ def test_udp_probe_cycle_sequence_maps_shared_dst_probes(loopback_responder): harness = _Harness( dst_port=port, n_rtt_ports=3, - n_df_ports=1, rtt_src_port_base=34600, - df_src_port_base=34700, enable_df_probe=False, ) probes = [_probe("127.0.0.1", index) for index in range(3)] try: - cycle_results = harness.udp_probe_cycle(probes) + cycle_results = harness.udp_probe_cycle(probes, 0) finally: harness._close_udp_probe_sockets() @@ -246,103 +275,46 @@ def test_udp_probe_cycle_sequence_maps_shared_dst_probes(loopback_responder): def test_udp_probe_cycle_rotates_active_source_ports(): - harness = _FanoutSpyHarness( + harness = _CycleSpyHarness( dst_port=38437, n_rtt_ports=4, - n_df_ports=2, rtt_ports_per_cycle=2, - df_ports_per_cycle=1, rtt_src_port_base=34800, - df_src_port_base=34900, ) probes = [_probe("127.0.0.1", 1)] try: - first = harness.udp_probe_cycle(probes) - second = harness.udp_probe_cycle(probes) + first = harness.udp_probe_cycle(probes, 0) + second = harness.udp_probe_cycle(probes, 1) finally: harness._close_udp_probe_sockets() - assert harness.fanout_ports == [ - [34800, 34801], - [34900], - [34802, 34803], - [34901], + assert harness.cycle_ports == [ + ([34800, 34801], True), + ([34802, 34803], True), ] assert first[0]["result"]["packets_sent"] == 2 assert first[0]["result"]["rtt_ports_active"] == 2 assert first[0]["result"]["rtt_ports_total"] == 4 assert first[0]["result"]["df_packets_sent"] == 1 assert first[0]["result"]["df_ports_active"] == 1 - assert first[0]["result"]["df_ports_total"] == 2 + assert first[0]["result"]["df_ports_total"] == 4 assert second[0]["result"]["packets_sent"] == 2 assert second[0]["result"]["df_packets_sent"] == 1 -def test_udp_burst_compatibility_still_uses_full_port_pool(loopback_responder): - port = loopback_responder - harness = _Harness( - dst_port=port, - n_rtt_ports=3, - n_df_ports=2, - rtt_ports_per_cycle=1, - df_ports_per_cycle=1, - rtt_src_port_base=35000, - df_src_port_base=35100, - ) - try: - rtt_result = harness.udp_rtt_burst(dst_ip="127.0.0.1", src_ip="127.0.0.1") - df_result = harness.udp_df_burst(dst_ip="127.0.0.1", src_ip="127.0.0.1") - finally: - harness._close_udp_probe_sockets() +def test_df_probe_reuses_one_active_rtt_tuple_per_destination(): + sockets = [(33000 + index, object()) for index in range(4)] + probes = [_probe("192.0.2.1", index) for index in range(6)] - assert rtt_result["packets_sent"] == harness.n_rtt_ports - assert df_result["packets_sent"] == harness.n_df_ports - - -def test_pingmesh_ports_per_cycle_defaults_and_env(monkeypatch): - monkeypatch.delenv("PINGMESH_RTT_PORTS_PER_CYCLE", raising=False) - monkeypatch.delenv("PINGMESH_DF_PORTS_PER_CYCLE", raising=False) - - assert _default_ports_per_cycle(8) == (8, 2) - assert _default_ports_per_cycle(16) == (6, 1) - assert _default_ports_per_cycle(64) == (4, 1) - assert _resolve_ports_per_cycle( - client_count=64, - n_rtt_ports=16, - n_df_ports=4, - enable_df_probe=True, - ) == (4, 1) - assert _resolve_ports_per_cycle( - client_count=4, - n_rtt_ports=6, - n_df_ports=1, - enable_df_probe=True, - ) == (6, 1) - - monkeypatch.setenv("PINGMESH_RTT_PORTS_PER_CYCLE", "12") - monkeypatch.setenv("PINGMESH_DF_PORTS_PER_CYCLE", "3") - assert _resolve_ports_per_cycle( - client_count=64, - n_rtt_ports=16, - n_df_ports=4, - enable_df_probe=True, - ) == (12, 3) - - monkeypatch.setenv("PINGMESH_RTT_PORTS_PER_CYCLE", "999") - monkeypatch.setenv("PINGMESH_DF_PORTS_PER_CYCLE", "999") - assert _resolve_ports_per_cycle( - client_count=64, - n_rtt_ports=16, - n_df_ports=4, - enable_df_probe=True, - ) == (16, 4) - assert _resolve_ports_per_cycle( - client_count=64, - n_rtt_ports=16, - n_df_ports=4, - enable_df_probe=False, - ) == (16, 0) + queue = UdpProbeMixin._build_send_queue(probes, sockets, enable_df=True) + + for probe_index in range(len(probes)): + rtt_entries = [item for item in queue if item[0] == "rtt" and item[1] == probe_index] + df_entries = [item for item in queue if item[0] == "df" and item[1] == probe_index] + assert len(rtt_entries) == 4 + assert len(df_entries) == 1 + assert df_entries[0][2] is sockets[probe_index % len(sockets)][1] def test_pingmesh_startup_jitter_is_deterministic(): @@ -354,41 +326,105 @@ def test_pingmesh_startup_jitter_is_deterministic(): assert _deterministic_startup_jitter_seconds("client-17", 0.0) == 0.0 +def test_pingmesh_startup_jitter_spreads_sequential_client_names(): + bins = {int(_deterministic_startup_jitter_seconds(f"client{index}", 2.0) * 10) for index in range(1, 145)} + + assert len(bins) >= 16 + + +def test_fixed_rate_cycle_deadline_recovers_one_overrun(): + deadline = _next_cycle_deadline(0.0, 2.0, 2.1) + + assert deadline == 2.0 + assert max(0.0, deadline - 2.1) == 0.0 + + deadline = _next_cycle_deadline(deadline, 2.0, 3.1) + + assert deadline == 4.0 + assert max(0.0, deadline - 3.1) == pytest.approx(0.9) + + +def test_fixed_rate_cycle_deadline_drops_excessive_backlog(): + assert _next_cycle_deadline(0.0, 2.0, 5.0) == 5.0 + + +def test_k12_destination_and_port_batches_cover_joint_epoch(): + tasks = [{"dst_name": f"client{i}"} for i in range(1, 144)] + phase = 16 + combinations = set() + destinations = set() + for cycle in range(36): + destination_batch, port_batch = _probe_batch_indices(cycle, 9, 4) + active = _rotating_destination_batch(tasks, 16, destination_batch, phase) + assert 15 <= len(active) <= 16 + destinations.update(item["dst_name"] for item in active) + combinations.add((destination_batch, port_batch)) + + assert len(destinations) == 143 + assert len(combinations) == 36 + + +def test_k8_and_xlarge_cover_all_destination_port_batch_pairs(): + combinations = {_probe_batch_indices(cycle, 8, 4) for cycle in range(32)} + + assert combinations == {(destination, port) for destination in range(8) for port in range(4)} + + +def test_destination_rotation_is_deterministic_and_bounded(): + tasks = [{"dst_name": f"client{i}"} for i in range(1, 40)] + first = [_rotating_destination_batch(tasks, 16, cycle) for cycle in range(3)] + second = [_rotating_destination_batch(tasks, 16, cycle) for cycle in range(3)] + + assert first == second + assert [len(batch) for batch in first] == [13, 13, 13] + assert {item["dst_name"] for batch in first for item in batch} == {item["dst_name"] for item in tasks} + + +def test_destination_slot_phase_balances_incoming_probe_load(): + client_count = 144 + tasks_by_source = { + source: [ + {"src_name": f"client{source + 1}", "dst_name": f"client{destination + 1}"} + for destination in range(client_count) + if destination != source + ] + for source in range(client_count) + } + + for cycle in range(9): + incoming = {destination: 0 for destination in range(client_count)} + for source, tasks in tasks_by_source.items(): + batch = _rotating_destination_batch(tasks, 16, cycle, phase_offset=source) + for probe in batch: + incoming[int(probe["dst_name"].removeprefix("client")) - 1] += 1 + + assert min(incoming.values()) >= 14 + assert max(incoming.values()) <= 18 + + def test_loss_aggregation_prefers_ratio_of_sums(): points = [ - {"_time": "t1", "_field": "packets_sent", "value": 10}, - {"_time": "t1", "_field": "packets_lost", "value": 0}, - {"_time": "t1", "_field": "packet_loss", "value": 0.0}, - {"_time": "t2", "_field": "packets_sent", "value": 2}, - {"_time": "t2", "_field": "packets_lost", "value": 1}, - {"_time": "t2", "_field": "packet_loss", "value": 50.0}, + {"_time": "t1", "packets_sent": 10, "packets_lost": 0, "packet_loss": 0.0}, + {"_time": "t2", "packets_sent": 2, "packets_lost": 1, "packet_loss": 50.0}, ] - assert _aggregate_loss_pct( - points, - pct_field="packet_loss", - sent_field="packets_sent", - lost_field="packets_lost", - ) == pytest.approx((1 / 12) * 100.0) + stats = _loss_stats(points) + + assert stats is not None + assert stats["loss_pct"] == pytest.approx((1 / 12) * 100.0) def test_loss_aggregation_falls_back_to_pct_average(): points = [ - {"_time": "t1", "_field": "df_loss_pct", "value": 10.0}, - {"_time": "t2", "_field": "df_loss_pct", "value": 20.0}, - {"_time": "t3", "_field": "df_loss_pct", "value": 100.0}, + {"_time": "t1", "df_loss_pct": 10.0}, + {"_time": "t2", "df_loss_pct": 20.0}, + {"_time": "t3", "df_loss_pct": 100.0}, ] - assert _aggregate_loss_pct( - points, - pct_field="df_loss_pct", - sent_field="df_packets_sent", - lost_field="df_packets_lost", - ) == pytest.approx(130.0 / 3) - assert _aggregate_loss_pct( - points, - pct_field="df_loss_pct", - sent_field="df_packets_sent", - lost_field="df_packets_lost", - drop_unreachable=True, - ) == pytest.approx(15.0) + stats = _loss_stats(points, prefix="df_") + converged = _loss_stats(points, prefix="df_", drop_unreachable=True) + + assert stats is not None + assert converged is not None + assert stats["loss_pct"] == pytest.approx(130.0 / 3) + assert converged["loss_pct"] == pytest.approx(15.0) diff --git a/tests/test_platform_fault_active_faults.py b/tests/test_platform_fault_active_faults.py index 1665ed1..f58cbe4 100644 --- a/tests/test_platform_fault_active_faults.py +++ b/tests/test_platform_fault_active_faults.py @@ -1,7 +1,15 @@ """Regression tests for structured active fault tracking.""" +import tempfile + from netopsbench.platform.faults.injector import FaultInjector from netopsbench.platform.faults.models import ActiveFault +from netopsbench.platform.topology.generator import generate_topology + + +def _metadata() -> dict: + with tempfile.TemporaryDirectory() as tmpdir: + return generate_topology("xs", tmpdir)["metadata"] def test_active_fault_dataclass_roundtrips_metadata_to_dict(): @@ -21,8 +29,8 @@ def test_active_fault_dataclass_roundtrips_metadata_to_dict(): assert payload["linux_interface"] == "eth1" -def test_get_active_faults_preserves_public_dict_shape(): - injector = FaultInjector(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) +def test_active_fault_tracker_preserves_domain_objects(): + injector = FaultInjector(topology_metadata=_metadata()) injector.active_faults = [ ActiveFault( type="static_route_misconfig", @@ -31,25 +39,13 @@ def test_get_active_faults_preserves_public_dict_shape(): ) ] - active = injector.get_active_faults() - - assert active == [ - { - "type": "static_route_misconfig", - "device": "leaf1", - "target_ip": "192.168.1.2/32", - "wrong_nexthop": "192.168.1.1", - "success": True, - "error": None, - } - ] + assert len(injector.active_faults) == 1 + assert isinstance(injector.active_faults[0], ActiveFault) + assert injector.active_faults[0].metadata["target_ip"] == "192.168.1.2/32" def test_link_flapping_uses_python_background_control_metadata(): - injector = FaultInjector( - topology_metadata={"name": "dcn", "devices": {"spines": [{"name": "spine1"}], "leafs": [], "clients": []}} - ) - injector.container_names = {"spine1": "clab-dcn-spine1"} + injector = FaultInjector(topology_metadata=_metadata()) stub = type("R", (), {"returncode": 0, "stderr": "", "stdout": ""})() injector._sonic.config_cmd = lambda *args, **kwargs: stub @@ -61,17 +57,3 @@ def test_link_flapping_uses_python_background_control_metadata(): assert fault["orchestration"] == "python" assert "task_id" in fault assert "pid" not in fault - - -def test_fault_injector_normalizes_handler_result_to_legacy_dict(): - normalized = FaultInjector._legacy_fault_result( - {"recovered": True, "type": "link_down", "device": "leaf1", "error": None} - ) - - assert normalized == { - "success": True, - "error": None, - "recovered": True, - "type": "link_down", - "device": "leaf1", - } diff --git a/tests/test_route_parser_and_static_route.py b/tests/test_route_parser_and_static_route.py index 7343802..05e5730 100644 --- a/tests/test_route_parser_and_static_route.py +++ b/tests/test_route_parser_and_static_route.py @@ -1,17 +1,33 @@ import random import subprocess +import tempfile from datetime import UTC, datetime -from pathlib import Path import pytest from netopsbench.platform.faults.injector import FaultInjector +from netopsbench.platform.observability.bgp_parser import parse_bgp_summary +from netopsbench.platform.observability.influxdb import FluxQueryResult from netopsbench.platform.scenario import generator as _generate_scenarios_mod +from netopsbench.platform.toolkit._core.device.route_parsers import parse_route_table +from netopsbench.platform.toolkit._core.device.telemetry_parsers import ( + parse_influx_metric_rows, + parse_influx_timestamp, + query_influx, + summarize_counter_points, +) +from netopsbench.platform.toolkit._core.device.text_parsers import parse_table from netopsbench.platform.toolkit.toolkit import AgentToolkit +from netopsbench.platform.topology.generator import generate_topology +from netopsbench.platform.utils.interface_names import resolve_interface_metric_identities + + +def _metadata() -> dict: + with tempfile.TemporaryDirectory() as tmpdir: + return generate_topology("xs", tmpdir)["metadata"] def test_parse_route_table_brief_output(): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) sample = """ Codes: K - kernel route, C - connected, L - local, S - static, @@ -20,11 +36,13 @@ def test_parse_route_table_brief_output(): B>* 192.168.102.0/30 [20/0] via 192.168.11.1, Ethernet0, weight 1, 01:30:50 * via 192.168.21.1, Ethernet4, weight 1, 01:30:50 """ - routes = toolkit._parse_route_table(sample) + routes = parse_route_table(sample) assert routes[0]["prefix"] == "192.168.2.2/32" assert routes[0]["protocol"] == "static" assert routes[0]["nexthops"] == [{"via": "192.168.101.2", "interface": "Ethernet8"}] + assert routes[0]["selected"] is True + assert routes[0]["is_discard"] is False assert routes[1]["prefix"] == "192.168.102.0/30" assert routes[1]["protocol"] == "bgp" @@ -35,7 +53,6 @@ def test_parse_route_table_brief_output(): def test_parse_route_table_detailed_output(): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) sample = """ Routing entry for 192.168.102.0/30 Known via "bgp", distance 20, metric 0, best @@ -43,7 +60,7 @@ def test_parse_route_table_detailed_output(): * 192.168.11.1, via Ethernet0, weight 1 * 192.168.21.1, via Ethernet4, weight 1 """ - routes = toolkit._parse_route_table(sample) + routes = parse_route_table(sample) assert routes == [ { @@ -56,26 +73,53 @@ def test_parse_route_table_detailed_output(): ], "admin_distance": 20, "metric": 0, + "selected": True, + "is_discard": False, + "discard_interface": None, } ] -def test_build_fault_instance_static_route_targets_remote_client(): +def test_parse_route_table_marks_null0_as_discard_route(): + sample = """ +Routing entry for 192.168.105.0/30 + Known via "static", distance 1, metric 0, best + * directly connected, Null0 +""" + + routes = parse_route_table(sample) + + assert routes == [ + { + "prefix": "192.168.105.0/30", + "code": None, + "protocol": "static", + "nexthops": [{"via": None, "interface": "Null0"}], + "admin_distance": 1, + "metric": 0, + "selected": True, + "is_discard": True, + "discard_interface": "Null0", + } + ] + + +def test_parse_route_table_preserves_blackhole_discard_kind(): + sample = "S>* 192.168.105.0/30 [1/0] blackhole, weight 1, 00:00:10" + + routes = parse_route_table(sample) + + assert routes[0]["protocol"] == "static" + assert routes[0]["selected"] is True + assert routes[0]["is_discard"] is True + assert routes[0]["discard_interface"] == "blackhole" + + +def test_build_fault_instance_static_route_targets_remote_client(tmp_path): gs = _generate_scenarios_mod - topo = gs.TopologyContext( - scale="xs", - topology_dir=Path("lab-topology/generated_topology_xs"), - metadata={"scale": {"num_spines": 2}}, - spines=["spine1", "spine2"], - leafs=["leaf1", "leaf2"], - clients=[ - {"name": "client1", "leaf": "leaf1", "data_ip": "192.168.101.2"}, - {"name": "client2", "leaf": "leaf2", "data_ip": "192.168.102.2"}, - ], - device_interfaces={}, - leaf_interface_roles={}, - client_interfaces={}, - ) + topology_dir = tmp_path / "topology" + generate_topology("xs", str(topology_dir)) + topo = gs.load_topology("xs", str(topology_dir)) template = {"name": "static_route_misconfig", "device_role": "leaf", "severity": "medium"} defaults = {"seed": 42} @@ -96,18 +140,7 @@ def test_build_fault_instance_static_route_targets_remote_client(): def test_recover_static_route_misconfig_prefers_specific_nexthop(monkeypatch): - injector = FaultInjector( - topology_metadata={ - "devices": { - "spines": [], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.20.20.13"}], - "clients": [ - {"name": "client1", "leaf": "leaf1", "data_ip": "192.168.101.2"}, - {"name": "client2", "leaf": "leaf2", "data_ip": "192.168.102.2"}, - ], - } - } - ) + injector = FaultInjector(topology_metadata=_metadata()) injector.active_faults = [ { "type": "static_route_misconfig", @@ -150,19 +183,7 @@ def fake_vtysh(device, commands): def test_inject_static_route_misconfig_auto_uses_topology_clients(monkeypatch): - injector = FaultInjector( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.20.20.13"}], - "clients": [ - {"name": "client1", "leaf": "leaf1", "data_ip": "192.168.101.2"}, - {"name": "client2", "leaf": "leaf2", "data_ip": "192.168.102.2"}, - ], - }, - } - ) + injector = FaultInjector(topology_metadata=_metadata()) class Result: returncode = 0 @@ -190,15 +211,7 @@ def fake_vtysh(device, commands): def test_get_interface_mtu_falls_back_to_live_link(monkeypatch): - injector = FaultInjector( - topology_metadata={ - "devices": { - "spines": [{"name": "spine1"}], - "leafs": [], - "clients": [], - } - } - ) + injector = FaultInjector(topology_metadata=_metadata()) class Result: def __init__(self, returncode=0, stdout="", stderr=""): @@ -219,15 +232,7 @@ def fake_docker_exec(container, cmd, timeout=30): def test_recover_mtu_mismatch_normalizes_invalid_saved_mtu(monkeypatch): - injector = FaultInjector( - topology_metadata={ - "devices": { - "spines": [{"name": "spine1"}], - "leafs": [], - "clients": [], - } - } - ) + injector = FaultInjector(topology_metadata=_metadata()) captured = [] @@ -256,14 +261,7 @@ def unexpected_docker_exec(*args, **kwargs): def test_topology_view_adds_mtu_semantics(): - toolkit = AgentToolkit( - topology_metadata={ - "name": "dcn", - "defaults": {"link_mtu": 9232}, - "devices": {"spines": [], "leafs": [], "clients": []}, - "links": [], - } - ) + toolkit = AgentToolkit(topology_metadata=_metadata()) topo = toolkit.get_topology() @@ -275,16 +273,13 @@ def test_topology_view_adds_mtu_semantics(): def test_interface_metric_identities_cover_cli_and_gnmi_names(): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) + identities = resolve_interface_metric_identities("Ethernet8") - identities = toolkit._resolve_interface_metric_identities("Ethernet8") - - assert identities["names"] == ["Ethernet8", "eth3"] + assert identities["names"] == ["Ethernet8", "e1-3", "eth3", "ethernet-1/3"] assert identities["paths"] == ["COUNTERS/Ethernet8", "/COUNTERS/Ethernet8"] def test_parse_influx_metric_rows_skips_repeated_headers(): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) csv_text = """#group,false,false,true,true,true,true #datatype,string,long,dateTime:RFC3339,double,string,string,string ,result,table,_time,_value,_field,path,source @@ -294,7 +289,7 @@ def test_parse_influx_metric_rows_skips_repeated_headers(): ,_result,1,2026-03-22T07:01:00Z,8,in_discarded_packets,Ethernet0,/COUNTERS/Ethernet0,leaf1 """ - rows = toolkit._parse_influx_metric_rows(csv_text) + rows = parse_influx_metric_rows(csv_text) assert rows == [ { @@ -311,13 +306,12 @@ def test_parse_influx_metric_rows_skips_repeated_headers(): def test_interface_metric_summary_uses_window_delta_not_absolute_counter(): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) points = [ {"time": "2026-03-22T07:00:00Z", "value": 8.0}, {"time": "2026-03-22T07:01:00Z", "value": 8.0}, ] - summary = toolkit._summarize_counter_points("in_discarded_packets", points) + summary = summarize_counter_points("in_discarded_packets", points) assert summary["counter_start"] == 8.0 assert summary["counter_end"] == 8.0 @@ -328,23 +322,34 @@ def test_interface_metric_summary_uses_window_delta_not_absolute_counter(): def test_parse_influx_timestamp_supports_nanosecond_precision(): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) - - parsed = toolkit._parse_influx_timestamp("2026-03-22T07:26:29.338351792Z") + parsed = parse_influx_timestamp("2026-03-22T07:26:29.338351792Z") assert parsed is not None assert parsed.isoformat() == "2026-03-22T07:26:29.338351+00:00" +def test_influx_query_failure_is_structured_not_empty_data(monkeypatch): + toolkit = AgentToolkit(topology_metadata=_metadata()) + monkeypatch.setattr( + "netopsbench.platform.toolkit._core.device.telemetry_parsers.query_flux", + lambda *args, **kwargs: FluxQueryResult(status="error", error="ConnectionError: offline"), + ) + + result = query_influx(toolkit, 'from(bucket: "test")') + + assert result.status == "error" + assert result.rows == [] + assert "offline" in result.error + + def test_parse_bgp_summary_skips_total_footer(): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) sample = """ Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt Desc 192.168.11.2 4 65011 310 309 20 0 0 04:54:34 2 16 N/A Total number of neighbors 1 """ - rows = toolkit._parse_bgp_summary(sample) + rows = parse_bgp_summary(sample) assert rows == [ { @@ -362,20 +367,19 @@ def test_parse_bgp_summary_skips_total_footer(): def test_parse_table_skips_separator_row(): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) sample = """ IFACE STATE RX_OK RX_BPS ----------- ------- --------- -------- Ethernet0 U 318,237 N/A """ - rows = toolkit._parse_table(sample) + rows = parse_table(sample) assert rows == [{"IFACE": "Ethernet0", "STATE": "U", "RX_OK": "318,237", "RX_BPS": "N/A"}] def test_get_interface_metrics_summary_reports_windowed_rates(monkeypatch): - toolkit = AgentToolkit(topology_metadata={"devices": {"spines": [], "leafs": [], "clients": []}}) + toolkit = AgentToolkit(topology_metadata=_metadata()) csv_text = """#group,false,false,true,true,true,true,true #datatype,string,long,dateTime:RFC3339,double,string,string,string,string ,result,table,_time,_value,_field,path,source @@ -387,13 +391,9 @@ def test_get_interface_metrics_summary_reports_windowed_rates(monkeypatch): ,_result,2,2026-03-22T07:01:00Z,8,in_discarded_packets,Ethernet0,/COUNTERS/Ethernet0,leaf1 """ - class FakeResponse: - status_code = 200 - text = csv_text - monkeypatch.setattr( - "netopsbench.platform.toolkit._core.observability.metrics_ops.requests.post", - lambda *args, **kwargs: FakeResponse(), + "netopsbench.platform.toolkit._core.observability.metrics_ops.query_flux", + lambda *args, **kwargs: FluxQueryResult(status="ok", text=csv_text), ) result = toolkit.get_interface_metrics("leaf1", "Ethernet0", time_range_minutes=5) @@ -406,24 +406,11 @@ class FakeResponse: def test_get_interface_metrics_attaches_live_snapshot_when_influx_missing(monkeypatch): - toolkit = AgentToolkit( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.20.20.15"}], - "clients": [], - }, - } - ) - - class EmptyResponse: - status_code = 200 - text = "" + toolkit = AgentToolkit(topology_metadata=_metadata()) monkeypatch.setattr( - "netopsbench.platform.toolkit._core.observability.metrics_ops.requests.post", - lambda *args, **kwargs: EmptyResponse(), + "netopsbench.platform.toolkit._core.observability.metrics_ops.query_flux", + lambda *args, **kwargs: FluxQueryResult(status="ok"), ) monkeypatch.setattr( toolkit, @@ -455,36 +442,24 @@ class EmptyResponse: def test_get_interface_metrics_reports_recent_interface_coverage_gap(monkeypatch): - toolkit = AgentToolkit( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.20.20.15"}], - "clients": [], - }, - } - ) - - class FakeResponse: - def __init__(self, text): - self.status_code = 200 - self.text = text + toolkit = AgentToolkit(topology_metadata=_metadata()) - responses = iter( - [ - FakeResponse(""), - FakeResponse("""#group,false,false,true,true,true + identity_result = FluxQueryResult( + status="ok", + text="""#group,false,false,true,true,true #datatype,string,long,dateTime:RFC3339,string,string ,result,table,_time,name,path ,_result,0,2026-03-23T14:00:00Z,Ethernet24,/COUNTERS/Ethernet24 -"""), - ] +""", ) monkeypatch.setattr( - "netopsbench.platform.toolkit._core.observability.metrics_ops.requests.post", - lambda *args, **kwargs: next(responses), + "netopsbench.platform.toolkit._core.observability.metrics_ops.query_flux", + lambda *args, **kwargs: FluxQueryResult(status="ok"), + ) + monkeypatch.setattr( + "netopsbench.platform.toolkit._core.device.telemetry_parsers.query_flux", + lambda *args, **kwargs: identity_result, ) monkeypatch.setattr( toolkit, @@ -514,24 +489,11 @@ def __init__(self, text): def test_get_device_logs_falls_back_to_container_logs(monkeypatch): - toolkit = AgentToolkit( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.20.20.15"}], - "clients": [], - }, - } - ) - - class EmptyResponse: - status_code = 200 - text = "" + toolkit = AgentToolkit(topology_metadata=_metadata()) monkeypatch.setattr( - "netopsbench.platform.toolkit._core.device.log_ops.requests.post", - lambda *args, **kwargs: EmptyResponse(), + "netopsbench.platform.toolkit._core.device.log_ops.query_flux", + lambda *args, **kwargs: FluxQueryResult(status="ok"), ) monkeypatch.setattr(toolkit, "_resolve_container", lambda device: "clab-dcn-leaf1") monkeypatch.setattr( @@ -554,19 +516,7 @@ class EmptyResponse: def test_ping_test_allows_infra_source(monkeypatch): - toolkit = AgentToolkit( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [{"name": "spine1", "mgmt_ip": "172.20.20.11"}], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.20.20.12"}], - "clients": [ - {"name": "client1", "mgmt_ip": "172.20.20.21", "data_ip": "192.168.101.2"}, - {"name": "client2", "mgmt_ip": "172.20.20.22", "data_ip": "192.168.102.2"}, - ], - }, - } - ) + toolkit = AgentToolkit(topology_metadata=_metadata()) class Result: returncode = 0 @@ -583,19 +533,7 @@ class Result: def test_traceroute_allows_infra_source(monkeypatch): - toolkit = AgentToolkit( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [{"name": "spine1", "mgmt_ip": "172.20.20.11"}], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.20.20.12"}], - "clients": [ - {"name": "client1", "mgmt_ip": "172.20.20.21", "data_ip": "192.168.101.2"}, - {"name": "client2", "mgmt_ip": "172.20.20.22", "data_ip": "192.168.102.2"}, - ], - }, - } - ) + toolkit = AgentToolkit(topology_metadata=_metadata()) class Result: returncode = 0 @@ -612,19 +550,7 @@ class Result: def test_ping_test_allows_client_source(monkeypatch): - toolkit = AgentToolkit( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [], - "leafs": [], - "clients": [ - {"name": "client1", "mgmt_ip": "172.20.20.21", "data_ip": "192.168.101.2"}, - {"name": "client2", "mgmt_ip": "172.20.20.22", "data_ip": "192.168.102.2"}, - ], - }, - } - ) + toolkit = AgentToolkit(topology_metadata=_metadata()) class Result: returncode = 0 @@ -652,19 +578,7 @@ def fake_docker_exec(container, cmd, timeout): def test_ping_test_supports_large_packet_df_probe(monkeypatch): - toolkit = AgentToolkit( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [], - "leafs": [], - "clients": [ - {"name": "client1", "mgmt_ip": "172.20.20.21", "data_ip": "192.168.101.2"}, - {"name": "client2", "mgmt_ip": "172.20.20.22", "data_ip": "192.168.102.2"}, - ], - }, - } - ) + toolkit = AgentToolkit(topology_metadata=_metadata()) class Result: returncode = 1 diff --git a/tests/test_runtime_agent_input_contract.py b/tests/test_runtime_agent_input_contract.py index 9b729f8..3fe79cb 100644 --- a/tests/test_runtime_agent_input_contract.py +++ b/tests/test_runtime_agent_input_contract.py @@ -1,4 +1,4 @@ -from netopsbench.platform.worker.runtime_agent_input import ( +from netopsbench.platform.session.context import ( build_public_case_id, build_public_symptoms, ) @@ -40,3 +40,37 @@ def test_build_public_case_id_is_non_semantic_and_stable(): assert case_a == case_b assert case_a.startswith("case-") assert "link_down" not in case_a + + +def test_build_public_symptoms_bounds_anomalies_without_mutating_raw_result(): + anomalies = [ + { + "type": "packet_loss", + "src_ip": f"192.0.2.{index}", + "dst_ip": "198.51.100.1", + "src_leaf": f"leaf{index % 4}", + "dst_leaf": "leaf9", + "severity": "high", + "value": float(index), + } + for index in range(150) + ] + episode_result = { + "episode": {"episode_id": "ep002", "duration_seconds": 72, "stabilization_time": 5}, + "observations": { + "pingmesh_metrics": { + "summary": {"total_anomalies": 150}, + "anomalies": anomalies, + "aggregated_anomalies": {"by_src_leaf": {}}, + } + }, + } + + payload = build_public_symptoms(episode_result=episode_result, pingmesh_query_window={}) + metrics = payload["observations"]["pingmesh_metrics"] + + assert len(metrics["anomalies"]) == 100 + assert metrics["returned_anomalies"] == 100 + assert metrics["truncated"] is True + assert metrics["summary"]["total_anomalies"] == 150 + assert len(episode_result["observations"]["pingmesh_metrics"]["anomalies"]) == 150 diff --git a/tests/test_runtime_config_consistency.py b/tests/test_runtime_config_consistency.py index 64b030e..543a9d7 100644 --- a/tests/test_runtime_config_consistency.py +++ b/tests/test_runtime_config_consistency.py @@ -1,6 +1,12 @@ """Regression tests for streamlined runtime defaults and public examples.""" +import subprocess from pathlib import Path +from unittest.mock import patch + +import pytest + +OBSERVABILITY_ASSETS = Path("netopsbench/platform/observability/assets") def test_config_module_no_longer_exports_provider_registry_helpers(): @@ -12,76 +18,150 @@ def test_config_module_no_longer_exports_provider_registry_helpers(): assert not hasattr(config_mod.NetOpsBenchConfig, "openai_base_url") -def test_observability_defaults_are_centralized_in_config_source(): - toolkit_text = Path("netopsbench/platform/toolkit/toolkit.py").read_text(encoding="utf-8") - scenario_executor_text = Path("netopsbench/platform/scenario/executor.py").read_text(encoding="utf-8") - detector_text = Path("netopsbench/platform/pingmesh/detector.py").read_text(encoding="utf-8") +def test_update_telegraf_config_uses_packaged_template_and_central_defaults(tmp_path): + import netopsbench.platform.observability.telegraf as telegraf_mod + from netopsbench.platform.topology.generator import generate_topology - assert "my-super-secret-auth-token" not in toolkit_text - assert '"myorg"' not in toolkit_text - assert '"network_data"' not in toolkit_text + topology_dir = tmp_path / "topology" + generate_topology("xs", str(topology_dir), name="demo-runtime") + topo_path = topology_dir / "topology.json" - assert "my-super-secret-auth-token" not in scenario_executor_text - assert '"myorg"' not in scenario_executor_text - assert '"network_data"' not in scenario_executor_text + output_path = tmp_path / "telegraf.conf" + rc = telegraf_mod.update_telegraf_config(str(topo_path), output_file=str(output_path)) - assert "my-super-secret-auth-token" not in detector_text - assert '"myorg"' not in detector_text - assert '"network_data"' not in detector_text + assert rc == 0 + rendered = output_path.read_text(encoding="utf-8") + assert '"172.20.20.11:50051"' in rendered + assert '"172.20.20.13:50051"' in rendered + assert "http://influxdb:8086" in rendered + assert "replace-me" in rendered + assert "netopsbench" in rendered + assert "demo-runtime" in rendered -def test_update_telegraf_config_uses_central_defaults_without_shadowing(tmp_path, monkeypatch): +def test_update_telegraf_config_rejects_legacy_grouped_topology(tmp_path): import json + import pytest + import netopsbench.platform.observability.telegraf as telegraf_mod - topo = { - "name": "demo-runtime", - "devices": { - "spines": [{"name": "spine1", "mgmt_ip": "172.31.10.11/24"}], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.31.10.13/24"}], - }, - "scale": {"num_spines": 1, "num_leafs": 1, "clients_per_leaf": 0}, - } topo_path = tmp_path / "topology.json" - topo_path.write_text(json.dumps(topo), encoding="utf-8") - - repo_root = tmp_path / "repo" - template_dir = repo_root / "observability" - template_dir.mkdir(parents=True) - (template_dir / "telegraf.conf.template").write_text( - "{{GNMI_ADDRESSES}}\n{{INFLUXDB_URL}}\n{{INFLUXDB_TOKEN}}\n{{INFLUXDB_ORG}}\n{{INFLUXDB_BUCKET}}\n{{TOPOLOGY_ID}}\n", + topo_path.write_text( + json.dumps( + { + "name": "legacy", + "devices": {"spines": [{"name": "spine1", "mgmt_ip": "172.20.20.11"}]}, + } + ), encoding="utf-8", ) - monkeypatch.setattr(telegraf_mod, "REPO_ROOT", repo_root) + with pytest.raises(ValueError, match="schema_version.*3.*Regenerate"): + telegraf_mod.update_telegraf_config(str(topo_path), output_file=str(tmp_path / "telegraf.conf")) + + +def test_update_telegraf_config_requires_generated_configdb_artifacts(tmp_path): + import netopsbench.platform.observability.telegraf as telegraf_mod + from netopsbench.platform.topology.generator import generate_topology + + topology_dir = tmp_path / "topology" + generate_topology("xs", str(topology_dir)) + (topology_dir / "configs" / "sonic" / "spine1" / "config_db.json").unlink() + + with pytest.raises(FileNotFoundError, match="Generated ConfigDB artifact"): + telegraf_mod.update_telegraf_config( + str(topology_dir / "topology.json"), + output_file=str(tmp_path / "telegraf.conf"), + ) + + +def test_update_telegraf_config_isolates_gnmi_subscriptions_per_role(tmp_path, monkeypatch): + import netopsbench.platform.observability.telegraf as telegraf_mod + from netopsbench.platform.topology.generator import generate_topology + + topology_dir = tmp_path / "topology" + generate_topology("xlarge", str(topology_dir)) + topology_file = topology_dir / "topology.json" output_path = tmp_path / "telegraf.conf" - rc = telegraf_mod.update_telegraf_config(str(topo_path), output_file=str(output_path)) + monkeypatch.setenv("SONIC_GNMI_PORT", "59999") + monkeypatch.setenv("GNMI_SUBSCRIPTION_MODE", "sample") + + rc = telegraf_mod.update_telegraf_config(str(topology_file), output_file=str(output_path)) assert rc == 0 rendered = output_path.read_text(encoding="utf-8") - assert '"172.31.10.11:50051"' in rendered - assert '"172.31.10.13:50051"' in rendered - assert "http://influxdb:8086" in rendered - assert "replace-me" in rendered - assert "netopsbench" in rendered - assert "demo-runtime" in rendered + assert rendered.count("[[inputs.gnmi]]") == 2 + assert "# gNMI role: spine" in rendered + assert "# gNMI role: leaf" in rendered + + spine_block = rendered.split("# gNMI role: spine", 1)[1].split("# gNMI role:", 1)[0] + leaf_block = rendered.split("# gNMI role: leaf", 1)[1].split("[[processors.regex]]", 1)[0] + + assert '"172.20.20.11:50051"' in spine_block + assert '"172.20.20.27:50051"' not in spine_block + assert 'path = "COUNTERS/Ethernet508"' in spine_block + + assert '"172.20.20.27:50051"' in leaf_block + assert '"172.20.20.11:50051"' not in leaf_block + assert 'path = "COUNTERS/Ethernet64"' in leaf_block + assert 'path = "COUNTERS/Ethernet68"' not in leaf_block + assert 'subscription_mode = "on_change"' in rendered + assert "sample_interval" not in rendered + assert 'username = "admin"' in rendered + assert 'password = ""' in rendered + assert 'encoding = "json_ietf"' in rendered + assert 'target = "COUNTERS_DB"' in rendered + + +def test_update_telegraf_config_scopes_native_fat_tree_roles_from_artifacts(tmp_path, monkeypatch): + import netopsbench.platform.observability.telegraf as telegraf_mod + from netopsbench.platform.topology.generator import generate_topology + + topology_dir = tmp_path / "topology" + generate_topology("fat-tree-k12", str(topology_dir), name="ft") + + output_path = tmp_path / "telegraf.conf" + + rc = telegraf_mod.update_telegraf_config(str(topology_dir / "topology.json"), output_file=str(output_path)) + + assert rc == 0 + rendered = output_path.read_text(encoding="utf-8") + assert rendered.count("[[inputs.gnmi]]") == 3 + assert "# gNMI role: core" in rendered + assert "# gNMI role: agg" in rendered + assert "# gNMI role: edge" in rendered + core_block = rendered.split("# gNMI role: core", 1)[1].split("# gNMI role:", 1)[0] + agg_block = rendered.split("# gNMI role: agg", 1)[1].split("# gNMI role:", 1)[0] + edge_block = rendered.split("# gNMI role: edge", 1)[1].split("[[processors.regex]]", 1)[0] + assert '"172.20.20.11:50051"' in core_block + assert '"172.20.20.47:50051"' not in core_block + assert 'path = "COUNTERS/Ethernet44"' in core_block + assert '"172.20.20.47:50051"' in agg_block + assert 'path = "COUNTERS/Ethernet44"' in agg_block + assert '"172.20.20.119:50051"' in edge_block + assert 'path = "COUNTERS/Ethernet28"' in edge_block + assert 'path = "COUNTERS/Ethernet32"' not in edge_block def test_grafana_configs_use_runtime_scoping_variables(): - datasource_text = Path("observability/grafana/provisioning/datasources/default.yaml").read_text(encoding="utf-8") - compose_text = Path("observability/docker-compose.yaml").read_text(encoding="utf-8") + datasource_text = (OBSERVABILITY_ASSETS / "grafana/provisioning/datasources/default.yaml").read_text( + encoding="utf-8" + ) + compose_text = (OBSERVABILITY_ASSETS / "docker-compose.yaml").read_text(encoding="utf-8") assert "${NETOPSBENCH_INFLUXDB_TOKEN}" in datasource_text - assert "${NETOPSBENCH_GRAFANA_DEFAULT_BUCKET}" in datasource_text + assert "${NETOPSBENCH_INFLUXDB_BUCKET}" in datasource_text + assert "${NETOPSBENCH_INFLUXDB_URL}" in datasource_text assert "NETOPSBENCH_INFLUXDB_TOKEN=${NETOPSBENCH_INFLUXDB_TOKEN:-replace-me}" in compose_text - assert "NETOPSBENCH_GRAFANA_DEFAULT_BUCKET=${NETOPSBENCH_GRAFANA_DEFAULT_BUCKET:-netopsbench}" in compose_text + assert "NETOPSBENCH_INFLUXDB_BUCKET=${NETOPSBENCH_INFLUXDB_BUCKET:-netopsbench}" in compose_text + assert "NETOPSBENCH_INFLUXDB_URL=http://influxdb:8086" in compose_text def test_grafana_dashboards_parameterize_bucket_and_topology(): dashboards = [ - Path("observability/grafana/dashboards/network_overview.json"), - Path("observability/grafana/dashboards/pingmesh.json"), + OBSERVABILITY_ASSETS / "grafana/dashboards/network_overview.json", + OBSERVABILITY_ASSETS / "grafana/dashboards/pingmesh.json", ] for dashboard_path in dashboards: @@ -95,29 +175,112 @@ def test_grafana_dashboards_parameterize_bucket_and_topology(): def test_network_overview_casts_interface_counters_before_derivative(): - text = Path("observability/grafana/dashboards/network_overview.json").read_text(encoding="utf-8") + text = (OBSERVABILITY_ASSETS / "grafana/dashboards/network_overview.json").read_text(encoding="utf-8") # Telegraf's gNMI interface counters arrive as string columns in Flux frames. # The dashboard must cast them before applying derivative(), or Grafana shows "No data". assert text.count("|> toFloat()\\n |> derivative(unit: 1s, nonNegative: true)") >= 4 -def test_network_overview_removes_cpu_memory_panels_and_enables_bgp_tail_input(): - dashboard_text = Path("observability/grafana/dashboards/network_overview.json").read_text(encoding="utf-8") - telegraf_text = Path("observability/telegraf.conf.template").read_text(encoding="utf-8") - start_worker_text = Path("scripts/observability/start_worker_telegraf.sh").read_text(encoding="utf-8") +def test_packaged_observability_assets_enable_bgp_tail_input(): + dashboard_text = (OBSERVABILITY_ASSETS / "grafana/dashboards/network_overview.json").read_text(encoding="utf-8") + telegraf_text = (OBSERVABILITY_ASSETS / "telegraf.conf.template").read_text(encoding="utf-8") assert "CPU Usage (optional)" not in dashboard_text assert "Memory Utilization (optional)" not in dashboard_text assert "/var/lib/netopsbench/bgp_neighbors.lp" in telegraf_text assert "from_beginning = true" in telegraf_text assert 'watch_method = "poll"' in telegraf_text - assert 'chmod 755 "$CONFIG_DIR"' in start_worker_text - assert 'chmod 644 "$CONFIG_PATH" "$BGP_FILE_PATH"' in start_worker_text + assert "metric_batch_size = 5000" in telegraf_text + assert "metric_buffer_limit = 200000" in telegraf_text + assert "debug = false" in telegraf_text + assert "[[processors.printer]]" not in telegraf_text assert dashboard_text.count('group(columns: [\\"source\\", \\"neighbor_address\\"])\\n |> last()') >= 3 -def test_pingmesh_agent_can_run_without_repo_package_install(tmp_path): +@pytest.mark.parametrize( + ("script", "args", "expected"), + [ + ( + "scripts/runtime/deploy.sh", + ["xlarge", "runtime-root"], + [ + "-m", + "netopsbench.platform.runtime.cli", + "deploy", + "xlarge", + "runtime-root/generated_topology_xlarge", + "test-lab", + "172.31.180.0/23", + "test-bucket", + "--mgmt-network", + "test-network", + ], + ), + ( + "scripts/runtime/deploy_worker.sh", + ["xs", "/tmp/topology", "test-lab", "172.31.101.0/24", "test-bucket"], + [ + "-m", + "netopsbench.platform.runtime.cli", + "deploy", + "xs", + "/tmp/topology", + "test-lab", + "172.31.101.0/24", + "test-bucket", + ], + ), + ], +) +def test_runtime_shell_scripts_delegate_to_python_lifecycle(tmp_path, script, args, expected): + capture = tmp_path / "args.txt" + fake_python = tmp_path / "python" + fake_python.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$CAPTURE_FILE"\n', encoding="utf-8") + fake_python.chmod(0o755) + env = { + "PATH": "/usr/bin:/bin", + "NETOPSBENCH_PYTHON": str(fake_python), + "NETOPSBENCH_LAB_NAME": "test-lab", + "NETOPSBENCH_MGMT_SUBNET": "172.31.180.0/23", + "NETOPSBENCH_MGMT_NETWORK": "test-network", + "NETOPSBENCH_INFLUXDB_BUCKET": "test-bucket", + "CAPTURE_FILE": str(capture), + } + + result = subprocess.run(["bash", script, *args], text=True, capture_output=True, env=env, check=False) + + assert result.returncode == 0, result.stderr + assert capture.read_text(encoding="utf-8").splitlines() == expected + + +def test_bgp_collector_size_default_is_owned_by_collector(): + from netopsbench.platform.observability.bgp_collector import ( + DEFAULT_BGP_COLLECTOR_MAX_BYTES, + DEFAULT_BGP_COLLECTOR_PARALLELISM, + DEFAULT_BGP_POLL_INTERVAL_SECONDS, + ) + + assert DEFAULT_BGP_COLLECTOR_MAX_BYTES == 128 * 1024 * 1024 + assert DEFAULT_BGP_COLLECTOR_PARALLELISM == 16 + assert DEFAULT_BGP_POLL_INTERVAL_SECONDS == 10 + + +def test_runtime_scripts_default_sonic_apply_parallelism_to_32(): + from netopsbench.platform.runtime.deployment import APPLY_CONFIG_PARALLELISM + + assert APPLY_CONFIG_PARALLELISM == 32 + + +def test_env_example_documents_only_public_runtime_parallelism(): + env_example = Path(".env.example").read_text(encoding="utf-8") + + assert "NETOPSBENCH_TRAFFIC_PARALLELISM=32" in env_example + assert "NETOPSBENCH_BGP_COLLECTOR_PARALLELISM" not in env_example + assert "NETOPSBENCH_BGP_COLLECTOR_MAX_BYTES" not in env_example + + +def test_pingmesh_agent_runs_from_its_package_module(tmp_path): import json import os import subprocess @@ -125,14 +288,33 @@ def test_pingmesh_agent_can_run_without_repo_package_install(tmp_path): import time pinglist = tmp_path / "pinglist.json" - pinglist.write_text(json.dumps({"probes": [], "topology_id": "xs"}), encoding="utf-8") + pinglist.write_text( + json.dumps( + { + "probes": [], + "topology_id": "xs", + "pingmesh_policy": { + "destination_batch_size": None, + "rtt_port_pool_size": 16, + "rtt_ports_per_cycle": 8, + "cycle_interval_seconds": 1, + "destination_batch_count": 1, + "port_batch_count": 2, + "coverage_epoch_cycles": 2, + "coverage_epoch_seconds": 2, + "df_payload_size": 9072, + }, + } + ), + encoding="utf-8", + ) env = os.environ.copy() env.pop("PYTHONPATH", None) env["HOSTNAME"] = "client1" proc = subprocess.Popen( - [sys.executable, "scripts/runtime/run_pingmesh_agent.py", str(pinglist), "5"], + [sys.executable, "-m", "netopsbench.platform.pingmesh.cli", str(pinglist)], cwd=str(Path.cwd()), env=env, stdout=subprocess.PIPE, @@ -160,33 +342,30 @@ def test_pingmesh_agent_can_run_without_repo_package_install(tmp_path): def test_pingmesh_agent_uses_single_fanout_probe_worker(): - agent_text = Path("netopsbench/platform/pingmesh/agent.py").read_text(encoding="utf-8") - runtime_text = Path("netopsbench/platform/pingmesh/_agent_runtime.py").read_text(encoding="utf-8") - support_text = Path("netopsbench/platform/pingmesh/_agent_support.py").read_text(encoding="utf-8") + from netopsbench.platform.pingmesh._agent_runtime import PingRuntimeMixin - assert "Parallel workers" not in agent_text - assert "ThreadPoolExecutor" not in agent_text - assert "ThreadPoolExecutor" not in runtime_text - assert "ThreadPoolExecutor" not in support_text - assert "as_completed" not in runtime_text - assert "Probe worker: 1" in agent_text - assert "Concurrent flows" in agent_text - assert "Port pool" in agent_text - assert "Active ports/cycle" in agent_text - assert "udp_probe_cycle(self.tasks)" in runtime_text + class StopAfterCycle(Exception): + pass + class Runtime(PingRuntimeMixin): + min_interval = 1.0 + max_interval = 1.0 + startup_jitter_s = 0.0 + tasks = [{"src_name": "client1", "dst_name": "client2"}] -def test_plugin_agent_doc_uses_public_sdk_agent_narrative(): - doc_text = Path("docs/content/docs/build-your-agent/custom-agents.mdx").read_text(encoding="utf-8") + def __init__(self): + self.calls = [] - assert "diagnose(context)" in doc_text - assert "MinimalDeepAgent" in doc_text - assert "simple_baseline_agent.py" not in doc_text - assert "@register_agent" not in doc_text + def next_probe_batch(self): + return self.tasks, {"port_batch_index": 3} + def udp_probe_cycle(self, tasks, port_batch_index): + self.calls.append((tasks, port_batch_index)) + return [] -def test_xs_real_smoke_uses_explicit_stub_agent_name(): - smoke_text = Path("tests/test_runtime_xs_smoke_real.py").read_text(encoding="utf-8") + runtime = Runtime() + with pytest.raises(StopAfterCycle): + with patch("netopsbench.platform.pingmesh._agent_runtime.time.sleep", side_effect=StopAfterCycle): + runtime.run() - assert "_EpisodeAwareAgent" not in smoke_text - assert "_RuntimeSmokeStubAgent" in smoke_text + assert runtime.calls == [(runtime.tasks, 3)] diff --git a/tests/test_safe_run.py b/tests/test_safe_run.py index 78c8453..b25bf99 100644 --- a/tests/test_safe_run.py +++ b/tests/test_safe_run.py @@ -7,8 +7,10 @@ import pytest +from netopsbench.platform.utils import proc from netopsbench.platform.utils.proc import ( DEFAULT_SUBPROCESS_TIMEOUT_SECONDS, + docker_prefix, safe_run, sudo_prefix, ) @@ -60,3 +62,21 @@ def test_sudo_prefix_default(monkeypatch): def test_sudo_prefix_disabled_via_env(monkeypatch): monkeypatch.setenv("NETOPSBENCH_NO_SUDO", "1") assert sudo_prefix() == [] + + +def test_docker_prefix_uses_accessible_local_socket_without_sudo(monkeypatch): + monkeypatch.delenv("NETOPSBENCH_NO_SUDO", raising=False) + monkeypatch.setattr(proc.os, "geteuid", lambda: 1000) + monkeypatch.setattr(proc.Path, "exists", lambda self: True) + monkeypatch.setattr(proc.os, "access", lambda path, mode: True) + + assert docker_prefix() == [] + + +def test_docker_prefix_uses_sudo_when_socket_is_not_accessible(monkeypatch): + monkeypatch.delenv("NETOPSBENCH_NO_SUDO", raising=False) + monkeypatch.setattr(proc.os, "geteuid", lambda: 1000) + monkeypatch.setattr(proc.Path, "exists", lambda self: True) + monkeypatch.setattr(proc.os, "access", lambda path, mode: False) + + assert docker_prefix() == ["sudo", "-n"] diff --git a/tests/test_scenario_commands.py b/tests/test_scenario_commands.py index 9f1ca53..bda0c88 100644 --- a/tests/test_scenario_commands.py +++ b/tests/test_scenario_commands.py @@ -20,3 +20,21 @@ def test_scenario_cli_rejects_legacy_run_command(): parser = build_parser() with pytest.raises(SystemExit): parser.parse_args(["scenario", "run", "scenarios/xs"]) + + +def test_cli_accepts_xlarge_and_fat_tree_generation_commands(): + parser = build_parser() + + topology_args = parser.parse_args(["topology", "generate", "--scale", "xlarge"]) + scenario_args = parser.parse_args(["scenario", "generate", "--scale", "xlarge"]) + benchmark_args = parser.parse_args(["benchmark", "prepare", "--scales", "xlarge"]) + fat_tree_topology_args = parser.parse_args(["topology", "generate", "--scale", "fat-tree-k12"]) + fat_tree_scenario_args = parser.parse_args(["scenario", "generate", "--scale", "fat-tree-k8"]) + fat_tree_benchmark_args = parser.parse_args(["benchmark", "prepare", "--scales", "fat-tree-k8,fat-tree-k12"]) + + assert topology_args.scale == "xlarge" + assert scenario_args.scale == "xlarge" + assert benchmark_args.scales == "xlarge" + assert fat_tree_topology_args.scale == "fat-tree-k12" + assert fat_tree_scenario_args.scale == "fat-tree-k8" + assert fat_tree_benchmark_args.scales == "fat-tree-k8,fat-tree-k12" diff --git a/tests/test_scenario_schema.py b/tests/test_scenario_schema.py index 360662e..3d091f0 100644 --- a/tests/test_scenario_schema.py +++ b/tests/test_scenario_schema.py @@ -1,12 +1,86 @@ """Schema regression tests for normalized scenario parameters.""" +import ipaddress +import json +import random +from types import SimpleNamespace + +import pytest import yaml -from netopsbench.platform.faults.specs import FaultSpec, register_fault_spec, unregister_fault_spec +from netopsbench.models import topology as topology_models +from netopsbench.models.topology import Collector, Device, DeviceRole, Management, TopologyManifest +from netopsbench.platform.faults.specs import FaultSpec, create_fault_registry +from netopsbench.platform.pingmesh.generator import PinglistGenerator +from netopsbench.platform.scenario import generator as scenario_generator from netopsbench.platform.scenario.executor import ScenarioExecutor from netopsbench.platform.scenario.models import Episode, Scenario from netopsbench.platform.scenario.parser import parse_scenario_file -from netopsbench.platform.scenario.validator import validate_scenario +from netopsbench.platform.scenario.validator import validate_scenario, validate_scenario_topology +from netopsbench.platform.topology.generator import generate_topology + + +def _minimal_canonical_topology() -> dict: + manifest = TopologyManifest( + topology_id="dcn", + name="dcn", + scale="xs", + family="clos", + management=Management(network="clab-dcn", ipv4_subnet="172.20.20.0/24"), + collector=Collector(ipv4="172.20.20.200"), + defaults=topology_models.TopologyDefaults(), + facts=topology_models.TopologyFacts( + num_spines=1, + num_leafs=1, + clients_per_attached_switch=1, + total_clients=0, + total_switches=2, + ), + routing=topology_models.RoutingMetadata(ecmp_hash_policy_by_role={DeviceRole.SPINE: 1, DeviceRole.LEAF: 1}), + devices=[ + Device(name="spine1", role=DeviceRole.SPINE), + Device(name="leaf1", role=DeviceRole.LEAF), + ], + links=[], + ) + return manifest.model_dump(mode="json") + + +def test_all_diagnostic_observation_durations_use_the_pingmesh_epoch(): + k12 = topology_models.PingmeshPolicy( + destination_batch_size=16, + rtt_port_pool_size=16, + rtt_ports_per_cycle=4, + cycle_interval_seconds=2, + ) + manifest = TopologyManifest.model_validate( + { + **_minimal_canonical_topology(), + "facts": { + "clients_per_attached_switch": 2, + "total_clients": 144, + "total_switches": 180, + }, + "pingmesh": k12.model_dump(), + } + ) + + assert scenario_generator.diagnostic_observation_duration(20, manifest) == 72 + assert scenario_generator.diagnostic_observation_duration(30, manifest) == 72 + assert scenario_generator.diagnostic_observation_duration(90, manifest) == 90 + + xlarge = manifest.model_copy( + update={ + "facts": manifest.facts.model_copy(update={"total_clients": 128}), + "pingmesh": topology_models.PingmeshPolicy( + destination_batch_size=16, + rtt_port_pool_size=16, + rtt_ports_per_cycle=4, + cycle_interval_seconds=2, + ), + } + ) + assert scenario_generator.diagnostic_observation_duration(30, xlarge) == 64 def test_parse_scenario_allows_none_episode_without_target_device(tmp_path): @@ -39,6 +113,50 @@ def test_parse_scenario_allows_none_episode_without_target_device(tmp_path): assert validate_scenario(scenario) == [] +def test_validate_scenario_accepts_xlarge_scale(tmp_path): + scenario_path = tmp_path / "scenario.yaml" + scenario_path.write_text( + yaml.safe_dump( + { + "scenario_id": "generated_healthy_network_xlarge_001", + "name": "XLarge healthy case", + "description": "No fault episode", + "topology_scale": "xlarge", + "traffic_profile": "standard", + "metadata": {"negative_sample": True}, + "episodes": [ + { + "episode_id": "ep001", + "description": "baseline", + "fault_type": "none", + "duration_seconds": 10, + "stabilization_time": 1, + } + ], + } + ), + encoding="utf-8", + ) + + scenario = parse_scenario_file(str(scenario_path)) + + assert validate_scenario(scenario) == [] + + +@pytest.mark.parametrize("profile", ["light", "stress"]) +def test_validate_scenario_rejects_nonstandard_traffic_profile(profile): + scenario = Scenario( + scenario_id="nonstandard_traffic", + name="Nonstandard traffic", + description="Only the canonical standard profile is valid", + topology_scale="xs", + traffic_profile=profile, + episodes=[Episode(episode_id="ep001", description="baseline", fault_type="none")], + ) + + assert validate_scenario(scenario) == [f"Invalid traffic_profile: {profile}; only 'standard' is supported"] + + def test_parse_scenario_preserves_episode_parameters(tmp_path): scenario_path = tmp_path / "scenario.yaml" scenario_path.write_text( @@ -112,35 +230,33 @@ def test_validate_scenario_rejects_blackhole_route_without_target_prefix(tmp_pat def test_validate_scenario_accepts_runtime_registered_fault(): - register_fault_spec(FaultSpec(name="synthetic_schema_fault")) - try: - scenario = type( - "Scenario", - (), - { - "scenario_id": "synthetic", - "name": "Synthetic", - "episodes": [ - type( - "Episode", - (), - { - "episode_id": "ep1", - "fault_type": "synthetic_schema_fault", - "target_device": "leaf1", - "target_interface": None, - "target_prefix": None, - }, - )() - ], - "topology_scale": "xs", - "traffic_profile": "standard", - "metadata": {"difficulty": "easy", "expected_diagnosis": "synthetic_schema_fault"}, - }, - )() - assert validate_scenario(scenario) == [] - finally: - unregister_fault_spec("synthetic_schema_fault") + registry = create_fault_registry() + registry.register(FaultSpec(name="synthetic_schema_fault")) + scenario = type( + "Scenario", + (), + { + "scenario_id": "synthetic", + "name": "Synthetic", + "episodes": [ + type( + "Episode", + (), + { + "episode_id": "ep1", + "fault_type": "synthetic_schema_fault", + "target_device": "leaf1", + "target_interface": None, + "target_prefix": None, + }, + )() + ], + "topology_scale": "xs", + "traffic_profile": "standard", + "metadata": {"difficulty": "easy", "expected_diagnosis": "synthetic_schema_fault"}, + }, + )() + assert validate_scenario(scenario, fault_registry=registry) == [] def test_validate_scenario_enforces_blackhole_route_target_prefix(): @@ -182,46 +298,196 @@ def validate_episode(episode): return ["custom validator requires target_device to be leaf1"] return [] - register_fault_spec(FaultSpec(name="synthetic_validated_fault", episode_validator=validate_episode)) - try: - scenario = type( - "Scenario", - (), + registry = create_fault_registry() + registry.register(FaultSpec(name="synthetic_validated_fault", episode_validator=validate_episode)) + scenario = type( + "Scenario", + (), + { + "scenario_id": "synthetic_validator", + "name": "Synthetic Validator", + "episodes": [ + type( + "Episode", + (), + { + "episode_id": "ep1", + "fault_type": "synthetic_validated_fault", + "target_device": "leaf2", + "target_interface": None, + "target_prefix": None, + "parameters": {}, + "metadata": {}, + }, + )() + ], + "topology_scale": "xs", + "traffic_profile": "standard", + "metadata": {"difficulty": "easy", "expected_diagnosis": "synthetic_validated_fault"}, + }, + )() + + errors = validate_scenario(scenario, fault_registry=registry) + + assert "custom validator requires target_device to be leaf1" in errors + + +def test_validate_scenario_topology_accepts_fat_tree_agg_target(tmp_path): + topology_dir = tmp_path / "generated_topology_fat-tree-k8" + generate_topology("fat-tree-k8", str(topology_dir), name="ft") + scenario = SimpleNamespace( + scenario_id="fat_tree_agg_fault", + topology_scale="fat-tree-k8", + episodes=[ + SimpleNamespace( + episode_id="ep1", + fault_type="link_down", + target_device="agg1", + target_interface="Ethernet0", + ) + ], + ) + + result = validate_scenario_topology(scenario, str(topology_dir)) + + assert result["status"] == "pass" + assert result["actual_scale"] == "fat-tree-k8" + assert "agg1" in result["topology_devices"] + + +def test_validate_scenario_topology_rejects_legacy_grouped_topology(tmp_path): + topology_dir = tmp_path / "legacy_topology" + topology_dir.mkdir() + (topology_dir / "topology.json").write_text( + json.dumps( { - "scenario_id": "synthetic_validator", - "name": "Synthetic Validator", - "episodes": [ - type( - "Episode", - (), - { - "episode_id": "ep1", - "fault_type": "synthetic_validated_fault", - "target_device": "leaf2", - "target_interface": None, - "target_prefix": None, - "parameters": {}, - "metadata": {}, - }, - )() - ], + "name": "legacy", "topology_scale": "xs", - "traffic_profile": "standard", - "metadata": {"difficulty": "easy", "expected_diagnosis": "synthetic_validated_fault"}, - }, - )() + "devices": {"spines": [{"name": "spine1"}], "leafs": [{"name": "leaf1"}], "clients": []}, + } + ), + encoding="utf-8", + ) + scenario = SimpleNamespace( + scenario_id="legacy_schema", + topology_scale="xs", + episodes=[], + ) + + with pytest.raises(ValueError, match="schema_version.*3.*Regenerate"): + validate_scenario_topology(scenario, str(topology_dir)) + + +def test_scenario_generator_supports_fat_tree_roles_with_structured_artifacts(tmp_path): + topology_dir = tmp_path / "topology" + generate_topology("fat-tree-k8", str(topology_dir)) + topo = scenario_generator.load_topology("fat-tree-k8", str(topology_dir)) + + assert topo.cores[0] == "core1" + assert topo.aggs[0] == "agg1" + assert topo.edges[0] == "edge1" + assert "Ethernet0" in topo.device_interfaces["agg1"] + + spec = { + "defaults": {"count_per_fault": 1, "seed": 1}, + "fault_templates": [ + { + "name": "link_flapping_agg", + "fault_type": "link_flapping", + "device_role": "agg", + "interface_role": "uplink", + } + ], + } + generated = scenario_generator.generate(spec, topo, tmp_path / "scenarios", seed=1) + + scenario = parse_scenario_file(str(generated[0])) + assert scenario.topology_scale == "fat-tree-k8" + assert scenario.episodes[1].target_device.startswith("agg") + assert validate_scenario(scenario) == [] + + +@pytest.mark.parametrize("location", ["defaults", "template"]) +def test_scenario_generator_rejects_nonstandard_traffic_profile(tmp_path, location): + topology_dir = tmp_path / "topology" + generate_topology("xs", str(topology_dir)) + topo = scenario_generator.load_topology("xs", str(topology_dir)) + spec = { + "defaults": {"count_per_fault": 1}, + "fault_templates": [{"name": "healthy", "fault_type": "none"}], + } + if location == "defaults": + spec["defaults"]["traffic_profile"] = "stress" + else: + spec["fault_templates"][0]["traffic_profile"] = "light" + + with pytest.raises(ValueError, match="Only the standard traffic profile is supported"): + scenario_generator.generate(spec, topo, tmp_path / "scenarios", seed=1) + + +def _xlarge_topology_context(tmp_path): + topology_dir = tmp_path / "topology" + generate_topology("xlarge", str(topology_dir)) + return scenario_generator.load_topology("xlarge", str(topology_dir)) + + +def _pingmesh_destination_names(topo, src_leaf: str) -> set[str]: + tasks = PinglistGenerator().generate(topo.manifest.model_dump(mode="json")) + return {task.dst_name for task in tasks if task.src_leaf == src_leaf} + + +def _client_for_prefix(topo, prefix: str) -> dict: + network = ipaddress.ip_network(prefix, strict=False) + return next(client for client in topo.clients if ipaddress.ip_address(client["data_ip"]) in network) + + +def _client_for_host_route(topo, host_route: str) -> dict: + host_ip = host_route.split("/", 1)[0] + return next(client for client in topo.clients if client["data_ip"] == host_ip) + + +def test_xlarge_blackhole_route_targets_pingmesh_observable_destination(tmp_path): + topo = _xlarge_topology_context(tmp_path) + + scenario = scenario_generator.build_fault_instance( + "blackhole_route", + "medium", + topo, + random.Random(1), + {"seed": 1}, + {"name": "blackhole_route_leaf", "fault_type": "blackhole_route", "device_role": "leaf"}, + 1, + ) + + fault = scenario["episodes"][1] + target_client = _client_for_prefix(topo, fault["target_prefix"]) + + assert target_client["name"] in _pingmesh_destination_names(topo, fault["target_device"]) + + +def test_xlarge_static_route_targets_pingmesh_observable_destination(tmp_path): + topo = _xlarge_topology_context(tmp_path) + + scenario = scenario_generator.build_fault_instance( + "static_route_misconfig", + "medium", + topo, + random.Random(1), + {"seed": 1}, + {"name": "static_route_misconfig", "fault_type": "static_route_misconfig", "device_role": "leaf"}, + 1, + ) - errors = validate_scenario(scenario) + fault = scenario["episodes"][1] + target_client = _client_for_host_route(topo, fault["metadata"]["target_ip"]) - assert "custom validator requires target_device to be leaf1" in errors - finally: - unregister_fault_spec("synthetic_validated_fault") + assert target_client["name"] in _pingmesh_destination_names(topo, fault["target_device"]) def test_scenario_runner_prefers_parameters_over_metadata(monkeypatch): runner = ScenarioExecutor( topology_dir="lab-topology", - topology_metadata={"name": "dcn", "devices": {"spines": [], "leafs": [], "clients": []}}, + topology_metadata=_minimal_canonical_topology(), ) captured = {} @@ -255,7 +521,7 @@ def fake_inject(device, target_ip, wrong_nexthop): def test_scenario_executor_can_return_result_without_persisting_raw_file(tmp_path, monkeypatch): runner = ScenarioExecutor( topology_dir="lab-topology", - topology_metadata={"name": "dcn", "devices": {"spines": [], "leafs": [], "clients": []}}, + topology_metadata=_minimal_canonical_topology(), baseline_wait_seconds=3, sleep_fn=lambda _seconds: None, persist_results=False, diff --git a/tests/test_session_dispatch.py b/tests/test_session_dispatch.py index f692c07..a6ab22a 100644 --- a/tests/test_session_dispatch.py +++ b/tests/test_session_dispatch.py @@ -1,14 +1,16 @@ from __future__ import annotations -from datetime import UTC, datetime from pathlib import Path from types import SimpleNamespace +import pytest + +from netopsbench.models.runtime import RuntimeIdentity from netopsbench.platform.runtime.manager import RuntimePool from netopsbench.platform.scenario.models import Episode, Scenario +from netopsbench.platform.session.context import build_worker_execution_context from netopsbench.platform.session.dispatch import execute_on_runtime_pool from netopsbench.platform.session.types import ScenarioExecutionRef, WorkerExecutionContext -from netopsbench.platform.worker.pool import WorkerSpec class _FakeRunner: @@ -64,26 +66,49 @@ def _scenario_ref(scenario_id: str) -> ScenarioExecutionRef: return ScenarioExecutionRef.from_scenario(scenario) -def _worker(tmp_path: Path, index: int) -> WorkerSpec: +def _worker(tmp_path: Path, index: int) -> RuntimeIdentity: topology_dir = tmp_path / f"worker-{index}" topology_dir.mkdir() - return WorkerSpec( - id=f"worker-{index}", - name=f"worker-{index}", - root_dir=topology_dir, - index=index, + return RuntimeIdentity.create( + runtime_id="runtime-1", + worker_id=f"worker-{index}", + worker_index=index, lab_name=f"lab-{index}", - topology_dir=str(topology_dir), + topology_dir=topology_dir, mgmt_subnet=f"172.31.{index}.0/24", + mgmt_network=f"clab-mgmt-lab-{index}", bucket=f"bucket-{index}", - shard_dir=str(topology_dir / "scenarios"), - report_path=str(topology_dir / "report.json"), - log_path=str(topology_dir / "worker.log"), - deploy_log_path=str(topology_dir / "deploy.log"), ) -def test_execute_on_runtime_pool_uses_per_worker_evaluators_and_session_raw_persistence(tmp_path): +def test_worker_context_uses_lab_name_for_observability_topology_id(tmp_path): + worker = _worker(tmp_path, 1) + + context = build_worker_execution_context(worker, Path(worker.topology_dir)) + + assert context.topology_id == "lab-1" + assert context.as_env()["NETOPSBENCH_TOPOLOGY_ID"] == "lab-1" + + +def test_worker_context_uses_explicit_identity_topology_id(tmp_path): + worker = _worker(tmp_path, 1) + worker = worker.model_copy(update={"topology_id": "runtime-observability-id"}) + + context = build_worker_execution_context(worker, Path(worker.topology_dir)) + + assert context.topology_id == "runtime-observability-id" + + +def test_worker_context_rejects_topology_directory_mismatch(tmp_path): + worker = _worker(tmp_path, 1) + + with pytest.raises(ValueError, match="does not match runtime identity"): + build_worker_execution_context(worker, tmp_path / "other") + + +def test_execute_on_runtime_pool_uses_per_worker_evaluators_and_session_raw_persistence(tmp_path, monkeypatch): + import netopsbench.platform.session.dispatch as dispatch + _FakeEvaluator._next_id = 0 runtime = RuntimePool( id="runtime-1", @@ -93,59 +118,35 @@ def test_execute_on_runtime_pool_uses_per_worker_evaluators_and_session_raw_pers workers=[_worker(tmp_path, 1), _worker(tmp_path, 2)], ) scenarios = [_scenario_ref("scenario-1"), _scenario_ref("scenario-2")] - created_reports = [] def score_fault_episodes(_scenario, scenario_result, evaluator, **_kwargs): assert scenario_result["persist_results"] is False return [SimpleNamespace(score=1.0, evaluator_id=evaluator.id)] - def create_run_report(**kwargs): - created_reports.append(kwargs) - return { - "status": "completed", - "summary": {"status": "completed"}, - "raw": {"status": "completed"}, - "scenario_summaries": kwargs["scenario_summaries"], - "aggregate_report": kwargs["aggregate_report"], - } + monkeypatch.setattr(dispatch, "ScenarioExecutor", _FakeRunner) + monkeypatch.setattr(dispatch, "_create_evaluator", _FakeEvaluator) + monkeypatch.setattr(dispatch, "score_scenario_fault_episodes", score_fault_episodes) + monkeypatch.setattr(dispatch, "build_runtime_diagnosis_callback", lambda *_args: lambda _payload: {}) + monkeypatch.setattr( + dispatch, + "build_worker_execution_context", + lambda worker, topology_dir: WorkerExecutionContext( + topology_dir=topology_dir, + topology_id=f"topo-{worker.worker_index}", + influxdb_bucket=worker.bucket, + ), + ) + monkeypatch.setattr(dispatch, "load_topology_metadata", lambda _topology_dir: None) result = execute_on_runtime_pool( - run_id="run-1", - mode="suite", scenarios=scenarios, runtime=runtime, agent=SimpleNamespace(name="agent"), - artifact_dir=tmp_path / "artifacts", raw_dir=tmp_path / "artifacts" / "raw", - report_path=tmp_path / "artifacts" / "report.json", - metadata_path=tmp_path / "artifacts" / "metadata.json", - runtime_owner="platform", - teardown="skipped", - started_at=datetime(2026, 1, 1, tzinfo=UTC), - completed_at_factory=lambda: datetime(2026, 1, 1, 0, 1, tzinfo=UTC), - scenario_runner_cls=_FakeRunner, - evaluator_factory=_FakeEvaluator, - score_fault_episodes=score_fault_episodes, - diagnosis_callback_builder=lambda *_args: lambda _payload: {}, - worker_context_builder=lambda worker, topology_dir: WorkerExecutionContext( - topology_dir=topology_dir, - topology_id=f"topo-{worker.index}", - influxdb_bucket=worker.bucket, - ), - topology_metadata_loader=lambda _topology_dir: None, - create_run_report=create_run_report, - save_run_report=lambda _payload, _path: None, - save_run_metadata=lambda *_args, **_kwargs: None, - build_run_handle=lambda **kwargs: kwargs, - run_handle_adapter=lambda payload: payload, - artifact_manager=SimpleNamespace(), ) - summaries = created_reports[0]["scenario_summaries"] + summaries = result.scenarios assert [summary["scenario_id"] for summary in summaries] == ["scenario-1", "scenario-2"] assert [Path(summary["raw_result_path"]).exists() for summary in summaries] == [True, True] - - detailed = created_reports[0]["aggregate_report"]["detailed_results"] - assert {item["evaluator_id"] for item in detailed} == {1, 2} - assert created_reports[0]["aggregate_report"]["summary"]["evaluator_id"] == 3 - assert result["status"] == "completed" + assert {item.evaluator_id for item in result.evaluations} == {1, 2} + assert [summary["worker_id"] for summary in result.workers] == ["worker-1", "worker-2"] diff --git a/tests/test_session_reporting.py b/tests/test_session_reporting.py index 4d5c37f..977eb93 100644 --- a/tests/test_session_reporting.py +++ b/tests/test_session_reporting.py @@ -4,7 +4,10 @@ from pathlib import Path from types import SimpleNamespace -from netopsbench.platform.session.reporting import create_run_report, next_run_id +import pytest + +from netopsbench.platform.session.reporting import create_run_report, load_topology_metadata, next_run_id +from netopsbench.platform.topology.generator import generate_topology def test_next_run_id_uses_utc_timestamp_and_collision_suffix(tmp_path: Path): @@ -20,6 +23,25 @@ def test_next_run_id_uses_utc_timestamp_and_collision_suffix(tmp_path: Path): assert next_run_id(artifact_root, started_at=started_at) == "run-20260605T124040Z-03" +def test_session_runtime_loader_preserves_canonical_topology_schema(tmp_path: Path): + topology_dir = tmp_path / "topology" + generate_topology("xs", str(topology_dir)) + + metadata = load_topology_metadata(topology_dir) + + assert metadata["schema_version"] == "3" + assert isinstance(metadata["devices"], list) + assert {device["role"] for device in metadata["devices"]} == {"spine", "leaf", "client"} + + +def test_session_runtime_loader_requires_canonical_topology(tmp_path: Path): + missing = tmp_path / "missing-topology" + missing.mkdir() + + with pytest.raises(FileNotFoundError): + load_topology_metadata(missing) + + def test_create_run_report_preserves_topology_scale_and_agent_name(tmp_path: Path): runtime = SimpleNamespace(id="run-0001-runtime", scale="small") agent = SimpleNamespace(name="agent-x") diff --git a/tests/test_session_tracing.py b/tests/test_session_tracing.py index e2110e9..606a640 100644 --- a/tests/test_session_tracing.py +++ b/tests/test_session_tracing.py @@ -8,7 +8,8 @@ from netopsbench.agents.base import DiagnosticContext from netopsbench.agents.tracing import AgentTraceRecorder -from netopsbench.platform.session.tracing import TraceWriter, export_traces, load_trace_index +from netopsbench.platform.session.harbor_export import export_traces, load_trace_index +from netopsbench.platform.session.trace_store import TraceWriter def test_disabled_trace_recorder_preserves_api_without_collecting(): diff --git a/tests/test_toolkit_output_shape.py b/tests/test_toolkit_output_shape.py index 0a2f68e..208afda 100644 --- a/tests/test_toolkit_output_shape.py +++ b/tests/test_toolkit_output_shape.py @@ -1,29 +1,31 @@ from __future__ import annotations import subprocess +import tempfile +from netopsbench.platform.observability.influxdb import FluxQueryResult from netopsbench.platform.toolkit.toolkit import AgentToolkit +from netopsbench.platform.topology.generator import generate_topology -class _Response: - status_code = 200 - text = "" +def _toolkit() -> AgentToolkit: + with tempfile.TemporaryDirectory() as tmpdir: + metadata = generate_topology("xs", tmpdir)["metadata"] + return AgentToolkit(topology_metadata=metadata) - def __init__(self, text: str): - self.text = text +def test_toolkit_keeps_canonical_state_and_projects_public_topology(monkeypatch): + toolkit = _toolkit() -def _toolkit() -> AgentToolkit: - return AgentToolkit( - topology_metadata={ - "name": "dcn", - "devices": { - "spines": [], - "leafs": [{"name": "leaf1", "mgmt_ip": "172.20.20.15"}], - "clients": [], - }, - } - ) + assert toolkit.topology_metadata["schema_version"] == "3" + assert isinstance(toolkit.topology_metadata["devices"], list) + + public_topology = toolkit.get_topology() + assert public_topology.success is True + assert isinstance(public_topology.data["devices"], dict) + assert set(public_topology.data["devices"]) >= {"spines", "leafs", "clients"} + + assert not hasattr(toolkit, "get_all_bgp_status") def test_get_device_logs_excludes_raw_csv_by_default(monkeypatch): @@ -31,8 +33,8 @@ def test_get_device_logs_excludes_raw_csv_by_default(monkeypatch): csv_text = "_time,source,severity,_value\n2026-05-04T00:00:00Z,leaf1,notice,message-one\n" monkeypatch.setattr( - "netopsbench.platform.toolkit._core.device.log_ops.requests.post", - lambda *args, **kwargs: _Response(csv_text), + "netopsbench.platform.toolkit._core.device.log_ops.query_flux", + lambda *args, **kwargs: FluxQueryResult(status="ok", text=csv_text), ) result = toolkit.get_device_logs("leaf1", time_range_minutes=10) @@ -47,8 +49,8 @@ def test_get_device_logs_can_include_raw_csv(monkeypatch): csv_text = "_time,source,severity,_value\n2026-05-04T00:00:00Z,leaf1,notice,message-one\n" monkeypatch.setattr( - "netopsbench.platform.toolkit._core.device.log_ops.requests.post", - lambda *args, **kwargs: _Response(csv_text), + "netopsbench.platform.toolkit._core.device.log_ops.query_flux", + lambda *args, **kwargs: FluxQueryResult(status="ok", text=csv_text), ) result = toolkit.get_device_logs("leaf1", time_range_minutes=10, include_raw=True) diff --git a/tests/test_topology_context_propagation.py b/tests/test_topology_context_propagation.py index 083c7f0..83ff3dd 100644 --- a/tests/test_topology_context_propagation.py +++ b/tests/test_topology_context_propagation.py @@ -1,16 +1,13 @@ """Regression tests for explicit topology propagation across runtime helpers.""" -import json - from netopsbench.platform.scenario.executor import ScenarioExecutor from netopsbench.platform.session import context as runtime_agent_context +from netopsbench.platform.topology.generator import generate_topology def test_scenario_executor_passes_explicit_topology_to_fault_injector(monkeypatch, tmp_path): topology_dir = tmp_path / "generated_topology_xs" - topology_dir.mkdir() - metadata = {"name": "dcn-test", "devices": {"spines": [], "leafs": [], "clients": []}} - (topology_dir / "topology.json").write_text(json.dumps(metadata), encoding="utf-8") + metadata = generate_topology("xs", str(topology_dir), name="dcn-test")["metadata"] captured = {} @@ -29,10 +26,7 @@ def __init__(self, **kwargs): def test_build_toolkit_for_topology_uses_explicit_metadata(monkeypatch, tmp_path): topology_dir = tmp_path / "generated_topology_xs" - topology_dir.mkdir() - metadata = {"name": "dcn-test", "devices": {"spines": [], "leafs": [], "clients": []}} - (topology_dir / "topology.json").write_text(json.dumps(metadata), encoding="utf-8") - (topology_dir / "custom.clab.yaml").write_text("name: dcn-test\n", encoding="utf-8") + metadata = generate_topology("xs", str(topology_dir), name="dcn-test")["metadata"] captured = {} @@ -45,7 +39,6 @@ def __init__(self, **kwargs): runtime_agent_context._build_toolkit_for_topology(str(topology_dir)) assert captured["topology_metadata"] == metadata - assert captured["topology_file"].endswith("custom.clab.yaml") def test_scenario_executor_is_not_exported_from_platform_scenario_namespace(): diff --git a/tests/test_topology_fail_fast.py b/tests/test_topology_fail_fast.py index cc745c4..a93f27b 100644 --- a/tests/test_topology_fail_fast.py +++ b/tests/test_topology_fail_fast.py @@ -1,57 +1,39 @@ -"""Tests for fail-fast topology loading behavior.""" +"""Tests for explicit, fail-fast topology loading behavior.""" import pytest +from netopsbench.config import config from netopsbench.platform.faults.injector import FaultInjector from netopsbench.platform.toolkit.toolkit import AgentToolkit +from netopsbench.platform.topology.generator import generate_topology -def test_agent_toolkit_fails_fast_without_topology_metadata(monkeypatch, tmp_path): - monkeypatch.delenv("NETOPSBENCH_TOPOLOGY_DIR", raising=False) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(AgentToolkit, "_discover_topology_dir", lambda self, base_dir: str(tmp_path)) - - with pytest.raises(FileNotFoundError, match="topology.json"): - AgentToolkit() - - -def test_fault_injector_fails_fast_without_topology_metadata(monkeypatch, tmp_path): - monkeypatch.delenv("NETOPSBENCH_TOPOLOGY_DIR", raising=False) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(FaultInjector, "_discover_topology_dir", lambda self, base_dir: str(tmp_path)) - +def test_agent_toolkit_fails_fast_without_topology_metadata(tmp_path): with pytest.raises(FileNotFoundError, match="topology.json"): - FaultInjector() + AgentToolkit(topology_dir=tmp_path) -def test_agent_toolkit_ignores_deprecated_topology_flag(monkeypatch, tmp_path): - monkeypatch.delenv("NETOPSBENCH_TOPOLOGY_DIR", raising=False) - monkeypatch.setenv("NETOPSBENCH_ALLOW_TOPOLOGY_FALLBACK", "1") - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(AgentToolkit, "_discover_topology_dir", lambda self, base_dir: str(tmp_path)) - +def test_fault_injector_fails_fast_without_topology_metadata(tmp_path): with pytest.raises(FileNotFoundError, match="topology.json"): - AgentToolkit() - + FaultInjector(clab_dir=str(tmp_path)) -def test_fault_injector_ignores_deprecated_topology_flag(monkeypatch, tmp_path): - monkeypatch.delenv("NETOPSBENCH_TOPOLOGY_DIR", raising=False) - monkeypatch.setenv("NETOPSBENCH_ALLOW_TOPOLOGY_FALLBACK", "1") - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(FaultInjector, "_discover_topology_dir", lambda self, base_dir: str(tmp_path)) - with pytest.raises(FileNotFoundError, match="topology.json"): - FaultInjector() +def test_toolkit_uses_only_the_explicit_topology_directory(tmp_path): + selected_dir = tmp_path / "selected" + unrelated_dir = tmp_path / "newer" + generate_topology("xs", str(selected_dir), name="selected-lab") + generate_topology("small", str(unrelated_dir), name="unrelated-lab") + toolkit = AgentToolkit(topology_dir=selected_dir) -def test_agent_toolkit_discovery_considers_benchmark_generated_topologies(tmp_path): - base_dir = tmp_path - benchmark_dir = base_dir / "lab-topology" / "benchmarks" / "generated_topology_small" - benchmark_dir.mkdir(parents=True) - (benchmark_dir / "topology.json").write_text("{}", encoding="utf-8") + assert toolkit.manifest.name == "selected-lab" + assert toolkit.manifest.scale == "xs" - toolkit = AgentToolkit.__new__(AgentToolkit) - discovered = AgentToolkit._discover_topology_dir(toolkit, str(base_dir)) +def test_toolkit_does_not_discover_topology_from_current_directory(tmp_path, monkeypatch): + generate_topology("xs", str(tmp_path), name="cwd-lab") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(config, "topology_dir", None) - assert discovered == str(benchmark_dir) + with pytest.raises(ValueError, match="explicit topology_dir"): + AgentToolkit() diff --git a/tests/test_topology_generation_v3.py b/tests/test_topology_generation_v3.py new file mode 100644 index 0000000..13c6123 --- /dev/null +++ b/tests/test_topology_generation_v3.py @@ -0,0 +1,172 @@ +"""Focused contracts for unified topology planning and schema-v3 rendering.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from netopsbench.models.topology import DeviceRole, TopologyManifest +from netopsbench.platform.topology.clos_builder import build_clos_plan +from netopsbench.platform.topology.config import TopologyConfig, config_for_scale +from netopsbench.platform.topology.generator import generate_topology +from netopsbench.platform.topology.plan import FabricPlan +from netopsbench.platform.topology.topology_utils import ( + clab_container_name, + coerce_topology_manifest, + load_topology_manifest, +) + + +@pytest.mark.parametrize( + ("scale", "roles", "clients", "links"), + [ + ("xs", {"spine": 2, "leaf": 2}, 2, 6), + ("xlarge", {"spine": 16, "leaf": 128}, 128, 2176), + ("fat-tree-k8", {"core": 16, "agg": 32, "edge": 32}, 128, 384), + ("fat-tree-k12", {"core": 36, "agg": 72, "edge": 72}, 144, 1008), + ], +) +def test_renderer_persists_only_canonical_schema_v3_devices(tmp_path, scale, roles, clients, links): + result = generate_topology(scale, str(tmp_path / scale)) + persisted = json.loads(Path(result["metadata_file"]).read_text(encoding="utf-8")) + manifest = TopologyManifest.model_validate(persisted) + + assert persisted["schema_version"] == "3" + assert isinstance(persisted["devices"], list) + assert len(manifest.links) == links + assert all(link.mtu == 9232 for link in manifest.links) + assert all(endpoint.interface for link in manifest.links for endpoint in link.endpoints) + assert len(manifest.clients()) == clients + assert all(client.attached_switch for client in manifest.clients()) + assert result["metadata"] == manifest.model_dump(mode="json") + assert result["agent_topology"] == manifest.to_agent_topology() + + for role, count in roles.items(): + assert len(manifest.devices_by_role(role)) == count + for client in (entry for entry in persisted["devices"] if entry["role"] == "client"): + assert "leaf" not in client + assert "edge" not in client + + +def test_clos_builder_returns_complete_fabric_plan_without_writing(tmp_path): + output_dir = tmp_path / "not-rendered" + plan = build_clos_plan(TopologyConfig(scale_name="xs")) + + assert isinstance(plan, FabricPlan) + assert not output_dir.exists() + assert len(plan.device_plans) == 6 + spine1 = plan.device_plan("spine1") + leaf1 = plan.device_plan("leaf1") + assert spine1 is not None + assert spine1.required_ports == 2 + assert spine1.configdb_interface_cidrs["Ethernet0"] == ("10.1.1.1/30",) + assert spine1.bgp_neighbors[0].peer_ip == "10.1.1.2" + assert leaf1 is not None + assert leaf1.required_ports == 3 + assert leaf1.bgp_networks == ("192.168.101.0/30",) + assert plan.manifest.links[0].endpoints[0].interface == "eth1" + + +def test_rendered_clos_artifacts_keep_preseed_and_addressing_contract(tmp_path): + result = generate_topology("xlarge", str(tmp_path)) + manifest = TopologyManifest.model_validate(result["metadata"]) + rendered = yaml.safe_load(Path(result["yaml_file"]).read_text(encoding="utf-8")) + nodes = rendered["topology"]["nodes"] + links = rendered["topology"]["links"] + + assert len(nodes) == 16 + 128 + 128 + assert len(links) == (16 * 128) + 128 + assert manifest.routing.ecmp_hash_policy_by_role == {DeviceRole.SPINE: 1, DeviceRole.LEAF: 1} + assert nodes["spine16"]["mgmt-ipv4"] == "172.20.20.26" + assert nodes["leaf128"]["mgmt-ipv4"] == "172.20.20.154" + assert nodes["client128"]["mgmt-ipv4"] == "172.20.21.26" + assert links[0] == {"endpoints": ["spine1:eth1", "leaf1:eth1"], "mtu": 9232} + assert links[2047] == {"endpoints": ["spine16:eth128", "leaf128:eth16"], "mtu": 9232} + assert links[-1] == {"endpoints": ["leaf128:eth17", "client128:eth1"], "mtu": 9232} + + assert (tmp_path / "configs" / "sonic" / "start.sh").is_file() + assert (tmp_path / "configs" / "pingmesh").is_dir() + spine = json.loads((tmp_path / "configs" / "sonic" / "spine16" / "config_db.json").read_text()) + leaf = json.loads((tmp_path / "configs" / "sonic" / "leaf128" / "config_db.json").read_text()) + assert len([name for name in spine["PORT"]]) == 128 + assert len([name for name in leaf["PORT"]]) == 17 + assert "Ethernet508|10.16.128.1/30" in spine["INTERFACE"] + assert "Ethernet64|192.168.228.1/30" in leaf["INTERFACE"] + + +@pytest.mark.parametrize( + ("scale", "first_core", "first_agg", "first_edge", "switch_links"), + [ + ("fat-tree-k8", 8, 8, 8, 256), + ("fat-tree-k12", 12, 12, 8, 864), + ], +) +def test_rendered_fat_tree_artifacts_keep_ports_and_disjoint_switch_pools( + tmp_path, scale, first_core, first_agg, first_edge, switch_links +): + result = generate_topology(scale, str(tmp_path / scale)) + root = Path(result["metadata_file"]).parent + + def config(device: str) -> dict: + return json.loads((root / "configs" / "sonic" / device / "config_db.json").read_text()) + + core = config("core1") + agg = config("agg1") + edge = config("edge1") + assert len([key for key in core["INTERFACE"] if "|" not in key]) == first_core + assert len([key for key in agg["INTERFACE"] if "|" not in key]) == first_agg + assert len([key for key in edge["INTERFACE"] if "|" not in key]) == first_edge + + manifest = load_topology_manifest(root) + switch_fabric_links = [link for link in manifest.links if link.kind in {"core-agg", "agg-edge"}] + assert len(switch_fabric_links) == switch_links + + core_agg_networks = { + cidr.split("|", 1)[1] + for device in manifest.devices_by_role(DeviceRole.CORE) + for cidr in config(device.name)["INTERFACE"] + if "|" in cidr + } + agg_edge_networks = { + cidr.split("|", 1)[1] + for device in manifest.devices_by_role(DeviceRole.EDGE) + for cidr in config(device.name)["INTERFACE"] + if "|" in cidr and cidr.split("|", 1)[1].startswith("10.") + } + assert all(cidr.startswith("10.1.") for cidr in core_agg_networks) + assert all(cidr.startswith("10.2.") for cidr in agg_edge_networks) + + +def test_topology_helpers_require_v3_and_manifest_owns_role_queries(tmp_path): + generate_topology("fat-tree-k8", str(tmp_path)) + manifest = load_topology_manifest(tmp_path) + persisted = json.loads((tmp_path / "topology.json").read_text(encoding="utf-8")) + + assert persisted == manifest.model_dump(mode="json") + assert len(manifest.switches()) == 80 + expected = [clab_container_name(manifest.name, device.name) for device in manifest.devices] + assert "clab-dcn-agg1" in expected + assert len(expected) == 208 + assert manifest.clients()[0].attached_switch == "edge1" + + legacy = tmp_path / "legacy" + legacy.mkdir() + (legacy / "topology.json").write_text(json.dumps({"name": "legacy", "devices": {}}), encoding="utf-8") + with pytest.raises(ValueError, match="schema_version.*3"): + load_topology_manifest(legacy) + with pytest.raises(ValueError, match="schema_version.*3"): + coerce_topology_manifest({"name": "legacy", "devices": {}}) + + +def test_scale_config_factory_returns_fresh_values(tmp_path): + original = config_for_scale("xs") + separate = config_for_scale("xs") + + result = generate_topology("xs", str(tmp_path), name="custom-lab") + + assert result["metadata"]["name"] == "custom-lab" + assert original is not separate + assert original.name == separate.name == "dcn" diff --git a/tests/test_traffic_controller.py b/tests/test_traffic_controller.py new file mode 100644 index 0000000..b4b1f20 --- /dev/null +++ b/tests/test_traffic_controller.py @@ -0,0 +1,194 @@ +import subprocess + +import pytest + +from netopsbench.platform.traffic import controller as controller_mod +from netopsbench.platform.traffic.controller import TrafficController, TrafficFlow + + +def _flows() -> list[TrafficFlow]: + return [ + TrafficFlow(src="client1", dst="client3", dst_ip="192.168.103.2", dst_port=5201, protocol="udp"), + TrafficFlow(src="client1", dst="client4", dst_ip="192.168.104.2", dst_port=5202, protocol="tcp"), + TrafficFlow(src="client2", dst="client3", dst_ip="192.168.103.2", dst_port=5201, protocol="udp"), + TrafficFlow(src="client2", dst="client4", dst_ip="192.168.104.2", dst_port=5202, protocol="tcp"), + ] + + +def _controller() -> TrafficController: + return TrafficController( + { + "client1": "clab-test-client1", + "client2": "clab-test-client2", + "client3": "clab-test-client3", + "client4": "clab-test-client4", + } + ) + + +def test_start_matrix_batches_server_ensure_and_client_start_by_container(monkeypatch): + calls: list[list[str]] = [] + + def fake_safe_run(cmd, **kwargs): + calls.append([str(part) for part in cmd]) + return subprocess.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(controller_mod, "safe_run", fake_safe_run) + + controller = _controller() + flow_ids = controller.start_matrix(_flows()) + + command_texts = [" ".join(call) for call in calls] + server_calls = [text for text in command_texts if "iperf3 -s" in text] + client_calls = [text for text in command_texts if "iperf3 -c" in text] + + assert len(flow_ids) == 4 + assert len(controller.active_flows) == 4 + assert len(server_calls) == 2 + assert len(client_calls) == 2 + assert any("192.168.103.2" in text and "192.168.104.2" in text for text in client_calls) + + +def test_batched_server_ensure_fails_fast_and_verifies_listeners(monkeypatch): + calls: list[list[str]] = [] + + def fake_safe_run(cmd, **kwargs): + calls.append([str(part) for part in cmd]) + return subprocess.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(controller_mod, "safe_run", fake_safe_run) + + controller = _controller() + controller._ensure_iperf_servers_batch("clab-test-client3", {5201, 5202}) + + script = calls[0][-1] + assert script.startswith("set -e\n") + assert script.count("ss -lntH") == 2 + assert "required_ports='5201 5202'" in script + assert "for port in $required_ports" in script + assert "missing=0" in script + + +def test_batched_source_start_is_idempotent_for_safe_retry(monkeypatch): + calls: list[list[str]] = [] + + monkeypatch.setattr( + controller_mod, + "safe_run", + lambda cmd, **kwargs: calls.append([str(part) for part in cmd]) or subprocess.CompletedProcess(cmd, 0, "", ""), + ) + + _controller().start_matrix(_flows()) + + source_scripts = [call[-1] for call in calls if "iperf3 -c" in " ".join(call)] + assert source_scripts + assert all("flow_running" in script for script in source_scripts) + assert all("/tmp/netopsbench-traffic/" in script for script in source_scripts) + assert all("/proc/$pid/cmdline" in script for script in source_scripts) + assert all(" dict: + devices = [ + *[Device(name=f"spine{i}", role=DeviceRole.SPINE) for i in range(1, spines + 1)], + *[Device(name=f"leaf{i}", role=DeviceRole.LEAF) for i in range(1, leafs + 1)], + ] + for idx in range(1, (leafs * clients_per_leaf) + 1): + leaf_idx = ((idx - 1) // clients_per_leaf) + 1 + devices.append( + Device( + name=f"client{idx}", + role=DeviceRole.CLIENT, + data_ip=f"192.168.{100 + leaf_idx}.{(((idx - 1) % clients_per_leaf) * 4) + 2}", + attached_switch=f"leaf{leaf_idx}", + metadata={"rack": f"rack{leaf_idx}"}, + ) + ) + manifest = TopologyManifest( + topology_id=scale, + name=scale, + scale=scale, + family=family, + management=Management(network=f"clab-{scale}", ipv4_subnet="172.20.20.0/24"), + collector=Collector(ipv4="172.20.20.200"), + defaults=topology_models.TopologyDefaults(), + facts=topology_models.TopologyFacts( + num_spines=spines, + num_leafs=leafs, + clients_per_attached_switch=clients_per_leaf, + total_clients=leafs * clients_per_leaf, + total_switches=spines + leafs, + ), + routing=topology_models.RoutingMetadata( + ecmp_hash_policy_by_role={device.role: 1 for device in devices if device.role is not DeviceRole.CLIENT} + ), + devices=devices, + links=[], + ) + return manifest.model_dump(mode="json") -def test_intra_leaf_budget_counts_both_ingress_and_egress(monkeypatch): - monkeypatch.setattr(traffic_generator, "SWITCH_PPS_LIMIT", 300) - topology = { - "devices": { - "leafs": [{"name": "leaf1"}], - "spines": [], - "clients": [ - {"name": "client1", "data_ip": "192.168.1.2", "leaf": "leaf1"}, - {"name": "client2", "data_ip": "192.168.1.6", "leaf": "leaf1"}, - ], - } - } +def test_intra_leaf_budget_counts_both_ingress_and_egress(): + topology = _canonical_traffic_topology(scale="xs", spines=0, leafs=1, clients_per_leaf=2) + settings = TrafficSettings(switch_pps_limit=300) - config = traffic_generator.generate_traffic_config_from_topology(topology, "xs", "standard") + config = traffic_generator.generate_traffic_config_from_topology(topology, "xs", "standard", settings=settings) - assert config["stats"]["total_flows"] == 0 - assert config["stats"]["estimated_switch_pps"]["max_leaf_pps"] == 0.0 - assert traffic_generator.validate_traffic_config(config, "xs") is True + flow_pps = sum(traffic_generator.estimate_flow_pps(flow) for flow in config["flows"]) + assert config["stats"]["total_flows"] == 2 + assert config["stats"]["estimated_switch_pps"]["max_leaf_pps"] == pytest.approx(flow_pps * 2) + assert traffic_generator.validate_traffic_config(config, "xs", settings=settings) is True def test_large_standard_profile_respects_switch_budget(tmp_path): - topo_config = copy.deepcopy(TOPOLOGY_SCALES["large"]) - result = TopologyGenerator(config=topo_config, output_dir=str(tmp_path)).generate() + result = generate_topology("large", str(tmp_path)) config = traffic_generator.generate_traffic_config_from_topology( - result["metadata"], + load_topology_manifest(result["metadata_file"]).model_dump(mode="json"), "large", "standard", ) assert config["stats"]["total_flows"] > 0 - assert config["stats"]["estimated_switch_pps"]["max_leaf_pps"] <= traffic_generator.SWITCH_PPS_LIMIT - assert config["stats"]["estimated_switch_pps"]["max_spine_pps"] <= traffic_generator.SWITCH_PPS_LIMIT + assert config["stats"]["estimated_switch_pps"]["max_leaf_pps"] <= DEFAULT_SWITCH_PPS_LIMIT + assert config["stats"]["estimated_switch_pps"]["max_spine_pps"] <= DEFAULT_SWITCH_PPS_LIMIT assert traffic_generator.validate_traffic_config(config, "large") is True + + +def test_xlarge_standard_profile_respects_switch_budget(tmp_path): + result = generate_topology("xlarge", str(tmp_path)) + + config = traffic_generator.generate_traffic_config_from_topology( + load_topology_manifest(result["metadata_file"]).model_dump(mode="json"), + "xlarge", + "standard", + ) + + assert config["stats"]["total_flows"] > 0 + assert config["stats"]["estimated_switch_pps"]["max_leaf_pps"] <= DEFAULT_SWITCH_PPS_LIMIT + assert config["stats"]["estimated_switch_pps"]["max_spine_pps"] <= DEFAULT_SWITCH_PPS_LIMIT + assert traffic_generator.validate_traffic_config(config, "xlarge") is True + + +def test_fat_tree_profiles_are_bounded(tmp_path): + from netopsbench.platform.topology.generator import generate_topology + + k8_result = generate_topology("fat-tree-k8", str(tmp_path / "k8")) + k12_result = generate_topology("fat-tree-k12", str(tmp_path / "k12")) + k8_metadata = load_topology_manifest(k8_result["metadata_file"]).model_dump(mode="json") + k12_metadata = load_topology_manifest(k12_result["metadata_file"]).model_dump(mode="json") + + k8_config = traffic_generator.generate_traffic_config_from_topology(k8_metadata, "fat-tree-k8", "standard") + k12_config = traffic_generator.generate_traffic_config_from_topology(k12_metadata, "fat-tree-k12", "standard") + + assert k8_config["stats"]["total_flows"] > 0 + assert k12_config["stats"]["total_flows"] > 0 + assert ( + get_scale_profile("fat-tree-k12").traffic_max_pps_per_client + < get_scale_profile("fat-tree-k8").traffic_max_pps_per_client + ) + assert traffic_generator.validate_traffic_config(k8_config, "fat-tree-k8") is True + assert traffic_generator.validate_traffic_config(k12_config, "fat-tree-k12") is True + + +@pytest.mark.parametrize( + ("scale", "expected_flows", "expected_incoming"), + [ + ("xs", 2, 1), + ("small", 32, 4), + ("medium", 64, 4), + ("xlarge", 512, 4), + ("fat-tree-k8", 512, 4), + ("fat-tree-k12", 576, 4), + ], +) +def test_standard_matrix_balances_destination_listener_budget(tmp_path, scale, expected_flows, expected_incoming): + from netopsbench.platform.topology.generator import generate_topology + + result = generate_topology(scale, str(tmp_path / scale)) + topology = load_topology_manifest(result["metadata_file"]).model_dump(mode="json") + + config = traffic_generator.generate_traffic_config_from_topology(topology, scale, "standard") + + incoming = Counter(flow["dst"] for flow in config["flows"]) + listener_ports: dict[str, set[int]] = defaultdict(set) + source_protocols: dict[str, Counter] = defaultdict(Counter) + for flow in config["flows"]: + listener_ports[flow["dst"]].add(flow["dst_port"]) + source_protocols[flow["src"]][flow["protocol"]] += 1 + + assert len(config["flows"]) == expected_flows + assert set(incoming.values()) == {expected_incoming} + assert all(ports == set(range(5201, 5201 + expected_incoming)) for ports in listener_ports.values()) + if scale != "xs": + assert all(protocols == {"udp": 2, "tcp": 2} for protocols in source_protocols.values()) + assert config["stats"]["incoming_flows_per_client"] == dict(sorted(incoming.items())) + assert config["stats"]["min_incoming_flows"] == expected_incoming + assert config["stats"]["max_incoming_flows"] == expected_incoming + assert config["stats"]["required_listener_ports"] == list(range(5201, 5201 + expected_incoming)) + + +def test_large_standard_matrix_never_exceeds_four_destination_listeners(tmp_path): + from netopsbench.platform.topology.generator import generate_topology + + result = generate_topology("large", str(tmp_path / "large")) + topology = load_topology_manifest(result["metadata_file"]).model_dump(mode="json") + + config = traffic_generator.generate_traffic_config_from_topology(topology, "large", "standard") + incoming = Counter(flow["dst"] for flow in config["flows"]) + + assert config["stats"]["total_flows"] > 0 + assert max(incoming.values()) <= 4 + assert set(flow["dst_port"] for flow in config["flows"]) <= {5201, 5202, 5203, 5204} + + +@pytest.mark.parametrize("profile", ["light", "stress"]) +def test_traffic_generator_rejects_nonstandard_profiles(profile): + topology = _canonical_traffic_topology(scale="xs") + with pytest.raises(ValueError, match="Only the standard traffic profile is supported"): + traffic_generator.generate_traffic_config_from_topology(topology, "xs", profile) + + +def test_standard_matrix_is_deterministic(tmp_path): + from netopsbench.platform.topology.generator import generate_topology + + result = generate_topology("fat-tree-k8", str(tmp_path / "k8")) + topology = load_topology_manifest(result["metadata_file"]).model_dump(mode="json") + + first = traffic_generator.generate_traffic_config_from_topology(topology, "fat-tree-k8", "standard") + second = traffic_generator.generate_traffic_config_from_topology(topology, "fat-tree-k8", "standard") + + assert first == second + + +def test_traffic_generator_rejects_legacy_grouped_topology(): + with pytest.raises(ValueError, match="schema_version.*3"): + traffic_generator.generate_traffic_config_from_topology( + {"devices": {"clients": [], "leafs": [], "spines": []}}, + "xs", + "standard", + ) diff --git a/tests/test_wheel_smoke.py b/tests/test_wheel_smoke.py new file mode 100644 index 0000000..b6abc4b --- /dev/null +++ b/tests/test_wheel_smoke.py @@ -0,0 +1,80 @@ +"""Installed-wheel smoke coverage for packaged runtime resources.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import venv +import zipfile +from pathlib import Path + + +def test_installed_wheel_generates_topology_without_source_checkout(tmp_path): + repo = Path(__file__).resolve().parents[1] + wheel_dir = tmp_path / "wheel" + wheel_dir.mkdir() + subprocess.run( + [sys.executable, "-m", "pip", "wheel", ".", "--no-deps", "--wheel-dir", str(wheel_dir)], + cwd=repo, + check=True, + capture_output=True, + text=True, + ) + wheel = next(wheel_dir.glob("netopsbench-*.whl")) + + with zipfile.ZipFile(wheel) as archive: + names = set(archive.namelist()) + assert "netopsbench/platform/topology/sonic_start.sh" in names + assert "netopsbench/platform/topology/sonic_vs_base_config_db.json" in names + assert "netopsbench/platform/observability/assets/telegraf.conf.template" in names + assert "netopsbench/platform/observability/assets/grafana/provisioning/dashboards/default.yaml" in names + assert "netopsbench/platform/observability/assets/grafana/provisioning/datasources/default.yaml" in names + assert "netopsbench/platform/scenario/specs/fault_campaign.yaml" in names + assert not any(name.startswith("netopsbench/resources/") for name in names) + assert "netopsbench/platform/observability/assets/telegraf.conf" not in names + assert not any("__pycache__" in name or name.endswith(".pyc") for name in names) + + venv_dir = tmp_path / "venv" + venv.EnvBuilder(with_pip=True, system_site_packages=True).create(venv_dir) + python = venv_dir / "bin" / "python" + subprocess.run( + [str(python), "-m", "pip", "install", "--no-deps", "--force-reinstall", str(wheel)], + check=True, + capture_output=True, + text=True, + ) + + outside = tmp_path / "outside" + outside.mkdir() + script = f""" +from pathlib import Path +from importlib.resources import files +import netopsbench +from netopsbench.platform.observability.lifecycle import observability_asset_root +from netopsbench.platform.topology.generator import generate_topology + +source_root = Path({str(repo)!r}).resolve() +package_file = Path(netopsbench.__file__).resolve() +assert source_root not in package_file.parents, package_file +output = Path.cwd() / "generated" +result = generate_topology("xs", str(output), name="wheel-xs") +assert Path(result["metadata_file"]).is_file() +assert (output / "configs" / "sonic" / "start.sh").is_file() +assert (output / "configs" / "sonic" / "spine1" / "config_db.json").is_file() +assert (observability_asset_root() / "telegraf.conf.template").is_file() +assert files("netopsbench.platform.scenario").joinpath("specs", "fault_campaign.yaml").is_file() +print(package_file) +""" + env = os.environ.copy() + env.pop("PYTHONPATH", None) + completed = subprocess.run( + [str(python), "-c", script], + cwd=outside, + env=env, + check=True, + capture_output=True, + text=True, + ) + + assert str(repo) not in completed.stdout diff --git a/tests/test_worker_health.py b/tests/test_worker_health.py new file mode 100644 index 0000000..f37a6ab --- /dev/null +++ b/tests/test_worker_health.py @@ -0,0 +1,146 @@ +import subprocess +from pathlib import Path + +from netopsbench.models.runtime import RuntimeIdentity +from netopsbench.platform.runtime import health + + +def _canonical_topology_dict(*, scale: str, spines: int, leafs: int, clients: int) -> dict: + devices = [ + *[{"name": f"spine{i}", "role": "spine", "mgmt_ip": f"172.20.20.{10 + i}"} for i in range(1, spines + 1)], + *[{"name": f"leaf{i}", "role": "leaf", "mgmt_ip": f"172.20.20.{10 + spines + i}"} for i in range(1, leafs + 1)], + *[ + { + "name": f"client{i}", + "role": "client", + "attached_switch": f"leaf{((i - 1) % leafs) + 1}", + "data_ip": f"192.168.{100 + i}.2", + } + for i in range(1, clients + 1) + ], + ] + return { + "schema_version": "3", + "topology_id": scale, + "name": scale, + "scale": scale, + "family": "clos", + "management": {"network": f"clab-{scale}", "ipv4_subnet": "172.20.20.0/23"}, + "collector": {"ipv4": "172.20.20.200"}, + "defaults": {"link_mtu": 9232, "sonic_port_mtu": 9100}, + "facts": { + "num_spines": spines, + "num_leafs": leafs, + "clients_per_attached_switch": 1, + "total_clients": clients, + "total_switches": spines + leafs, + }, + "routing": {"ecmp_hash_policy_by_role": {"spine": 1, "leaf": 1}}, + "devices": devices, + "links": [], + } + + +def _runtime_identity(topology_dir: Path, name: str) -> RuntimeIdentity: + return RuntimeIdentity.create( + runtime_id=name, + worker_id="worker-1", + worker_index=1, + lab_name=name, + topology_dir=topology_dir, + mgmt_subnet="172.20.20.0/24", + mgmt_network=f"clab-mgmt-{name}", + bucket=f"network_data_{name}", + ) + + +def test_active_interface_coverage_flags_xlarge_spine_with_only_32_ports_up(): + topo = _canonical_topology_dict(scale="xlarge", spines=16, leafs=128, clients=128) + output = "\n".join(f"Ethernet{idx * 4} 1,2,3,4 100G 9100 N/A up up QSFP" for idx in range(32)) + + active = health._parse_active_interfaces(output) + assert len(active) == 32 + assert health._expected_active_interface_count(topo, "spine1") == 128 + assert health._expected_active_interface_count(topo, "leaf128") == 17 + + error = health._active_interface_coverage_error( + container="clab-xlarge-spine1", + device="spine1", + active_interfaces=active, + expected_count=128, + ) + assert error == "active interface coverage too low on clab-xlarge-spine1: active=32 expected>=128" + + +def test_active_interface_coverage_accepts_required_count(): + output = "\n".join(f"Ethernet{idx * 4} 1,2,3,4 100G 9100 N/A up up QSFP" for idx in range(128)) + + error = health._active_interface_coverage_error( + container="clab-xlarge-spine1", + device="spine1", + active_interfaces=health._parse_active_interfaces(output), + expected_count=128, + ) + assert error is None + + +def test_fat_tree_sparse_expected_active_interface_counts(): + import tempfile + + from netopsbench.platform.topology.generator import generate_topology + from netopsbench.platform.topology.topology_utils import load_topology_manifest + + with tempfile.TemporaryDirectory() as raw_dir: + generate_topology("fat-tree-k12", raw_dir) + topo = load_topology_manifest(raw_dir) + + assert health._expected_active_interface_count(topo, "core1") == 12 + assert health._expected_active_interface_count(topo, "agg1") == 12 + assert health._expected_active_interface_count(topo, "edge72") == 8 + + +def test_worker_health_retries_observability_until_collector_is_ready(tmp_path, monkeypatch): + from netopsbench.platform.observability import influxdb, validation + from netopsbench.platform.topology.generator import generate_topology + + generate_topology("xs", str(tmp_path), name="health-xs") + + monkeypatch.setattr( + health, + "safe_run", + lambda *args, **kwargs: subprocess.CompletedProcess(args[0], 0, stdout="telegraf-health-xs\n", stderr=""), + ) + monkeypatch.setattr(health, "_running_container_count", lambda _lab_name: 6) + + def fake_docker_exec(_container, *command, **_kwargs): + joined = " ".join(command) + if "show ip bgp summary" in joined: + output = "".join(f"10.0.0.{index} 4 65001 0 0 0 0 0 00:10:00 1\n" for index in range(1, 5)) + elif "show interfaces status" in joined: + output = "".join(f"Ethernet{index * 4} 1,2,3,4 100G 9100 N/A up up QSFP\n" for index in range(4)) + elif "ps aux" in joined: + output = "root 1 0.0 0.0 python3 -m netopsbench.platform.pingmesh.cli\n" + else: + output = "" + return subprocess.CompletedProcess(command, 0, stdout=output, stderr="") + + monkeypatch.setattr(health, "_docker_exec", fake_docker_exec) + attempts = [] + + def fake_check_observability(*_args, **_kwargs): + attempts.append(1) + return ["collector has not written its first sample"] if len(attempts) == 1 else [] + + monkeypatch.setattr(validation, "check_observability", fake_check_observability) + monkeypatch.setattr( + influxdb, "query_flux", lambda *_args, **_kwargs: type("Result", (), {"status": "ok", "text": ""})() + ) + + errors = health.check_worker_health( + _runtime_identity(tmp_path, "health-xs"), + health_retries=2, + health_delay=0, + ) + + assert errors == [] + assert len(attempts) == 2