From 3223e050a1123e2362c0c570667cb7715c4fd092 Mon Sep 17 00:00:00 2001 From: Jacob Taunton Date: Tue, 21 Apr 2026 16:10:39 -0700 Subject: [PATCH 1/3] feat(gateway): add local control plane mvp --- README.md | 81 ++ ax_cli/client.py | 75 ++ ax_cli/commands/auth.py | 6 + ax_cli/commands/gateway.py | 774 ++++++++++++++ ax_cli/commands/listen.py | 27 +- ax_cli/commands/messages.py | 149 ++- ax_cli/config.py | 30 + ax_cli/gateway.py | 1342 ++++++++++++++++++++++++ ax_cli/main.py | 2 + examples/codex_gateway/codex_bridge.py | 246 +++++ examples/gateway_probe/probe_bridge.py | 124 +++ tests/test_auth_commands.py | 27 + tests/test_channel.py | 41 +- tests/test_client.py | 117 +++ tests/test_codex_gateway_bridge.py | 20 + tests/test_config.py | 40 + tests/test_gateway_commands.py | 659 ++++++++++++ tests/test_gateway_probe_bridge.py | 52 + tests/test_messages.py | 112 +- 19 files changed, 3904 insertions(+), 20 deletions(-) create mode 100644 ax_cli/commands/gateway.py create mode 100644 ax_cli/gateway.py create mode 100644 examples/codex_gateway/codex_bridge.py create mode 100644 examples/gateway_probe/probe_bridge.py create mode 100644 tests/test_codex_gateway_bridge.py create mode 100644 tests/test_gateway_commands.py create mode 100644 tests/test_gateway_probe_bridge.py diff --git a/README.md b/README.md index 5655bc3..62be0a0 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,80 @@ axctl auth whoami --json The generated agent profile/config is what Claude Code Channel, headless MCP, MCP Jam, and long-running agents should use. +## Gateway MVP + +`ax gateway` is the first local control-plane runtime for ax-cli. It keeps the +bootstrap user PAT in one trusted local place, mints agent PATs on demand, and +supervises managed runtimes so the child process never needs the raw PAT or JWT. + +The first slice starts headless, but it now includes a live terminal operator +view: + +```bash +# 1. Store the Gateway bootstrap login +ax gateway login + +# 2. Register a managed echo bot +ax gateway agents add echo-bot --type echo + +# 3. Run the local Gateway supervisor +ax gateway run +``` + +In another shell or device: + +```bash +ax send --to echo-bot "ping" --no-wait +ax gateway status +ax gateway watch +ax gateway agents show echo-bot +``` + +`ax gateway status` now shows recent control-plane activity as well as managed +runtime state, and Gateway-authored replies carry control-plane metadata so +operator attribution is less ambiguous during dogfooding. + +`ax gateway watch` is the first dashboard-style operator surface: a live +terminal view with Gateway health, fleet counts, managed-agent roster, and +recent control-plane events. `ax gateway agents show ` gives the first +drill-in view for one managed runtime. + +Runtime support starts with: + +- `echo` — built-in ping/echo bot for proving the control plane. +- `inbox` — a connected inbox/queue identity that receives replies without + auto-responding. Useful for demos, operator testing, and sender/receiver + workflows. +- `exec` — Gateway-owned per-mention command execution. This is the bridge for + Hermes-style handlers without handing platform credentials to the child + process. + +Managed `exec` runtimes can now emit structured progress lines back to Gateway +while they are still working. The bridge prints lines prefixed with +`AX_GATEWAY_EVENT ` and Gateway turns them into control-plane activity, +`agent_processing`, and tool-call audit notifications. + +Example: connect the local Codex CLI as a managed runtime for `@codex`: + +```bash +ax gateway agents add codex \ + --type exec \ + --exec "python3 examples/codex_gateway/codex_bridge.py" \ + --workdir /absolute/path/to/ax-cli +``` + +Example: add a second connected sender identity and use it to message `@codex` +without creating an agent-to-agent reply loop: + +```bash +ax gateway agents add codex-gateway-inbox --type inbox +ax gateway agents send codex-gateway-inbox "pause for 8 seconds and narrate activity" --to codex +``` + +This is a compatibility-first Gateway: today it still uses agent PATs against +the existing platform APIs, but the Gateway owns those credentials centrally so +managed runtimes do not. + ## Claude Code Channel — Connect from Anywhere **The first multi-agent channel for Claude Code.** Send a message from your phone, Claude Code receives it in real-time, delegates work to specialist agents, and reports back. @@ -483,6 +557,10 @@ returned messages have actually been handled. | Command | Description | |---------|-------------| | `axctl login` | Set up or refresh the user login token without touching agent config | +| `ax gateway login` | Store the local Gateway bootstrap session | +| `ax gateway status` | Show Gateway daemon + managed runtime status | +| `ax gateway agents show NAME` | Drill into one managed agent | +| `ax gateway agents send NAME "msg" --to codex` | Send as a managed agent identity | | `ax auth whoami` | Current identity + profile + fingerprint | | `ax agents list` | List agents in the space | | `ax spaces list` | List spaces you belong to | @@ -496,6 +574,9 @@ returned messages have actually been handled. | Command | Description | |---------|-------------| | `ax events stream` | Raw SSE event stream | +| `ax gateway run` | Run the local Gateway supervisor | +| `ax gateway watch` | Live Gateway dashboard in the terminal | +| `ax gateway agents add NAME --type inbox` | Add a connected inbox-only managed agent | | `ax listen --exec "./bot"` | Listen for @mentions with handler | | `ax watch --mention` | Block until condition matches on SSE | diff --git a/ax_cli/client.py b/ax_cli/client.py index 6fe4c92..ccd7f84 100644 --- a/ax_cli/client.py +++ b/ax_cli/client.py @@ -436,6 +436,14 @@ def set_agent_processing_status( *, agent_name: str | None = None, space_id: str | None = None, + activity: str | None = None, + tool_name: str | None = None, + progress: dict | None = None, + detail: dict | None = None, + reason: str | None = None, + error_message: str | None = None, + retry_after_seconds: int | None = None, + parent_message_id: str | None = None, ) -> dict: """POST /api/v1/agents/processing-status. @@ -446,6 +454,19 @@ def set_agent_processing_status( body: dict = {"message_id": message_id, "status": status} if agent_name: body["agent_name"] = agent_name + optional_fields = { + "activity": activity, + "tool_name": tool_name, + "progress": progress, + "detail": detail, + "reason": reason, + "error_message": error_message, + "retry_after_seconds": retry_after_seconds, + "parent_message_id": parent_message_id, + } + for key, value in optional_fields.items(): + if value is not None: + body[key] = value headers = self._with_agent(self.agent_id) if space_id: headers["X-Space-Id"] = space_id @@ -453,6 +474,60 @@ def set_agent_processing_status( r.raise_for_status() return self._parse_json(r) + def record_tool_call( + self, + *, + tool_name: str, + tool_call_id: str, + space_id: str | None = None, + tool_action: str | None = None, + resource_uri: str | None = None, + arguments_hash: str | None = None, + kind: str | None = None, + arguments: dict | None = None, + initial_data: dict | None = None, + status: str = "success", + duration_ms: int | None = None, + agent_name: str | None = None, + agent_id: str | None = None, + message_id: str | None = None, + correlation_id: str | None = None, + ) -> dict: + """POST /api/v1/tool-calls. + + Records a tool-call audit event from an authenticated agent runtime. + The backend stores it durably and fans out progress/tool-call SSE so + the operator UI can show richer in-flight activity. + """ + body: dict = { + "tool_name": tool_name, + "tool_call_id": tool_call_id, + "status": status, + } + optional_fields = { + "space_id": space_id, + "tool_action": tool_action, + "resource_uri": resource_uri, + "arguments_hash": arguments_hash, + "kind": kind, + "arguments": arguments, + "initial_data": initial_data, + "duration_ms": duration_ms, + "agent_name": agent_name, + "agent_id": agent_id, + "message_id": message_id, + "correlation_id": correlation_id, + } + for key, value in optional_fields.items(): + if value is not None: + body[key] = value + headers = self._with_agent(agent_id) + if space_id: + headers["X-Space-Id"] = space_id + r = self._http.post("/api/v1/tool-calls", json=body, headers=headers) + r.raise_for_status() + return self._parse_json(r) + def upload_file(self, file_path: str, *, space_id: str | None = None) -> dict: """POST /api/v1/uploads — upload a local file. diff --git a/ax_cli/commands/auth.py b/ax_cli/commands/auth.py index aaae4ec..a40b38c 100644 --- a/ax_cli/commands/auth.py +++ b/ax_cli/commands/auth.py @@ -1,5 +1,6 @@ """ax auth — identity and token management.""" +import os from pathlib import Path import httpx @@ -192,6 +193,8 @@ def doctor( console.print(f" space_id = {effective.get('space_id')} ({effective.get('space_source')})") console.print(f" agent_name = {effective.get('agent_name')} ({effective.get('agent_name_source')})") console.print(f" agent_id = {effective.get('agent_id')} ({effective.get('agent_id_source')})") + if data.get("runtime_config"): + console.print(f" runtime_config = {data['runtime_config']}") if data.get("selected_env"): console.print(f" selected_env = {data['selected_env']}") if data.get("selected_profile"): @@ -235,6 +238,9 @@ def whoami(as_json: bool = JSON_OPTION): local = _local_config_dir() if local and (local / "config.toml").exists(): data["local_config"] = str(local / "config.toml") + runtime_config = os.environ.get("AX_CONFIG_FILE") + if runtime_config: + data["runtime_config"] = runtime_config if as_json: print_json(data) diff --git a/ax_cli/commands/gateway.py b/ax_cli/commands/gateway.py new file mode 100644 index 0000000..8d189e3 --- /dev/null +++ b/ax_cli/commands/gateway.py @@ -0,0 +1,774 @@ +"""ax gateway — local Gateway control plane.""" + +from __future__ import annotations + +import time +from datetime import datetime, timezone +from pathlib import Path + +import typer +from rich import box +from rich.columns import Columns +from rich.console import Group +from rich.live import Live +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +from ..client import AxClient +from ..commands import auth as auth_cmd +from ..commands.bootstrap import ( + _create_agent_in_space, + _find_agent_in_space, + _mint_agent_pat, + _polish_metadata, +) +from ..config import resolve_user_base_url, resolve_user_token +from ..gateway import ( + GatewayDaemon, + agent_token_path, + annotate_runtime_health, + daemon_status, + find_agent_entry, + gateway_dir, + load_gateway_registry, + load_gateway_session, + load_recent_gateway_activity, + record_gateway_activity, + remove_agent_entry, + save_gateway_registry, + save_gateway_session, + upsert_agent_entry, +) +from ..output import JSON_OPTION, console, err_console, print_json, print_table + +app = typer.Typer(name="gateway", help="Run the local Gateway control plane", no_args_is_help=True) +agents_app = typer.Typer(name="agents", help="Manage Gateway-controlled agents", no_args_is_help=True) +app.add_typer(agents_app, name="agents") + +_STATE_STYLES = { + "running": "green", + "starting": "cyan", + "reconnecting": "yellow", + "stale": "yellow", + "error": "red", + "stopped": "dim", +} +_STATE_ORDER = { + "running": 0, + "starting": 1, + "reconnecting": 2, + "stale": 3, + "error": 4, + "stopped": 5, +} + + +def _resolve_gateway_login_token(explicit_token: str | None) -> str: + if explicit_token and explicit_token.strip(): + return auth_cmd._resolve_login_token(explicit_token) + existing = resolve_user_token() + if existing: + err_console.print("[cyan]Using existing axctl user login for Gateway bootstrap.[/cyan]") + return existing + return auth_cmd._resolve_login_token(None) + + +def _load_gateway_user_client() -> AxClient: + session = load_gateway_session() + if not session: + err_console.print("[red]Gateway is not logged in.[/red] Run `ax gateway login` first.") + raise typer.Exit(1) + token = str(session.get("token") or "") + if not token: + err_console.print("[red]Gateway session is missing its bootstrap token.[/red]") + raise typer.Exit(1) + if not token.startswith("axp_u_"): + err_console.print("[red]Gateway bootstrap currently requires a user PAT (axp_u_).[/red]") + raise typer.Exit(1) + return AxClient(base_url=str(session.get("base_url") or auth_cmd.DEFAULT_LOGIN_BASE_URL), token=token) + + +def _load_gateway_session_or_exit() -> dict: + session = load_gateway_session() + if not session: + err_console.print("[red]Gateway is not logged in.[/red] Run `ax gateway login` first.") + raise typer.Exit(1) + return session + + +def _save_agent_token(name: str, token: str) -> Path: + token_path = agent_token_path(name) + token_path.write_text(token.strip() + "\n") + token_path.chmod(0o600) + return token_path + + +def _load_managed_agent_or_exit(name: str) -> dict: + registry = load_gateway_registry() + entry = find_agent_entry(registry, name) + if not entry: + err_console.print(f"[red]Managed agent not found:[/red] {name}") + raise typer.Exit(1) + return entry + + +def _load_managed_agent_client(entry: dict) -> AxClient: + token_file = Path(str(entry.get("token_file") or "")).expanduser() + if not token_file.exists(): + err_console.print(f"[red]Managed agent token is missing:[/red] {token_file}") + raise typer.Exit(1) + token = token_file.read_text().strip() + if not token: + err_console.print(f"[red]Managed agent token file is empty:[/red] {token_file}") + raise typer.Exit(1) + return AxClient( + base_url=str(entry.get("base_url") or ""), + token=token, + agent_name=str(entry.get("name") or ""), + agent_id=str(entry.get("agent_id") or "") or None, + ) + + +def _status_payload(*, activity_limit: int = 10) -> dict: + daemon = daemon_status() + session = load_gateway_session() + registry = daemon["registry"] + agents = [annotate_runtime_health(agent) for agent in registry.get("agents", [])] + running_agents = [a for a in agents if str(a.get("effective_state")) == "running"] + connected_agents = [a for a in agents if bool(a.get("connected"))] + stale_agents = [a for a in agents if str(a.get("effective_state")) == "stale"] + errored_agents = [a for a in agents if str(a.get("effective_state")) == "error"] + gateway = dict(registry.get("gateway", {})) + if not daemon["running"]: + gateway["effective_state"] = "stopped" + gateway["pid"] = None + return { + "gateway_dir": str(gateway_dir()), + "connected": bool(session), + "base_url": session.get("base_url") if session else None, + "space_id": session.get("space_id") if session else None, + "user": session.get("username") if session else None, + "daemon": { + "running": daemon["running"], + "pid": daemon["pid"], + }, + "gateway": gateway, + "agents": agents, + "recent_activity": load_recent_gateway_activity(limit=activity_limit), + "summary": { + "managed_agents": len(agents), + "running_agents": len(running_agents), + "connected_agents": len(connected_agents), + "stale_agents": len(stale_agents), + "errored_agents": len(errored_agents), + }, + } + + +def _parse_iso8601(value: object) -> datetime | None: + if not value or not isinstance(value, str): + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _age_seconds(value: object) -> int | None: + parsed = _parse_iso8601(value) + if parsed is None: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return max(0, int((datetime.now(timezone.utc) - parsed.astimezone(timezone.utc)).total_seconds())) + + +def _format_age(seconds: object) -> str: + if seconds is None: + return "-" + try: + total = int(seconds) + except (TypeError, ValueError): + return "-" + if total < 60: + return f"{total}s" + minutes, seconds = divmod(total, 60) + if minutes < 60: + return f"{minutes}m {seconds:02d}s" + hours, minutes = divmod(minutes, 60) + if hours < 24: + return f"{hours}h {minutes:02d}m" + days, hours = divmod(hours, 24) + return f"{days}d {hours:02d}h" + + +def _format_timestamp(value: object) -> str: + return _format_age(_age_seconds(value)) + + +def _state_text(state: object) -> Text: + label = str(state or "unknown").lower() + style = _STATE_STYLES.get(label, "white") + return Text(f"● {label}", style=style) + + +def _metric_panel(label: str, value: object, *, tone: str = "cyan", subtitle: str | None = None) -> Panel: + body = Text() + body.append(str(value), style=f"bold {tone}") + body.append(f"\n{label}", style="dim") + if subtitle: + body.append(f"\n{subtitle}", style="dim") + return Panel(body, border_style=tone, padding=(1, 2)) + + +def _sorted_agents(agents: list[dict]) -> list[dict]: + return sorted( + agents, + key=lambda agent: ( + _STATE_ORDER.get(str(agent.get("effective_state") or "").lower(), 99), + str(agent.get("name") or "").lower(), + ), + ) + + +def _render_gateway_overview(payload: dict) -> Panel: + gateway = payload.get("gateway") or {} + grid = Table.grid(expand=True, padding=(0, 2)) + grid.add_column(style="bold") + grid.add_column(ratio=2) + grid.add_column(style="bold") + grid.add_column(ratio=2) + grid.add_row("Gateway", str(gateway.get("gateway_id") or "-")[:8], "Daemon", "running" if payload["daemon"]["running"] else "stopped") + grid.add_row("User", str(payload.get("user") or "-"), "Base URL", str(payload.get("base_url") or "-")) + grid.add_row("Space", str(payload.get("space_id") or "-"), "PID", str(payload["daemon"].get("pid") or "-")) + grid.add_row( + "Session", + "connected" if payload.get("connected") else "disconnected", + "Last Reconcile", + _format_timestamp(gateway.get("last_reconcile_at")), + ) + return Panel(grid, title="Gateway Overview", border_style="cyan") + + +def _render_agent_table(agents: list[dict]) -> Table: + table = Table(expand=True, box=box.SIMPLE_HEAVY) + table.add_column("Agent", style="bold") + table.add_column("Type") + table.add_column("State") + table.add_column("Phase") + table.add_column("Queue", justify="right") + table.add_column("Seen", justify="right") + table.add_column("Processed", justify="right") + table.add_column("Activity", overflow="fold") + if not agents: + table.add_row("No managed agents", "-", Text("● stopped", style="dim"), "-", "0", "-", "0", "-") + return table + for agent in _sorted_agents(agents): + activity = str(agent.get("current_activity") or agent.get("current_tool") or agent.get("last_reply_preview") or "-") + table.add_row( + f"@{agent.get('name')}", + str(agent.get("runtime_type") or "-"), + _state_text(agent.get("effective_state")), + str(agent.get("current_status") or "-"), + str(agent.get("backlog_depth") or 0), + _format_age(agent.get("last_seen_age_seconds")), + str(agent.get("processed_count") or 0), + activity, + ) + return table + + +def _render_activity_table(activity: list[dict]) -> Table: + table = Table(expand=True, box=box.SIMPLE_HEAVY) + table.add_column("When", justify="right", no_wrap=True) + table.add_column("Event", no_wrap=True) + table.add_column("Agent", no_wrap=True) + table.add_column("Detail", overflow="fold") + if not activity: + table.add_row("-", "idle", "-", "No activity yet") + return table + for item in activity: + detail = ( + item.get("activity_message") + or item.get("reply_preview") + or item.get("tool_name") + or item.get("error") + or item.get("message_id") + or "-" + ) + agent_name = item.get("agent_name") + table.add_row( + _format_timestamp(item.get("ts")), + str(item.get("event") or "-"), + f"@{agent_name}" if agent_name else "-", + str(detail), + ) + return table + + +def _render_gateway_dashboard(payload: dict) -> Group: + agents = payload.get("agents", []) + summary = payload.get("summary", {}) + queue_depth = sum(int(agent.get("backlog_depth") or 0) for agent in agents) + metrics = Columns( + [ + _metric_panel("managed agents", summary.get("managed_agents", 0), tone="cyan"), + _metric_panel("connected", summary.get("connected_agents", 0), tone="green"), + _metric_panel("stale", summary.get("stale_agents", 0), tone="yellow"), + _metric_panel("errors", summary.get("errored_agents", 0), tone="red"), + _metric_panel("queue depth", queue_depth, tone="blue"), + ], + expand=True, + equal=True, + ) + return Group( + _render_gateway_overview(payload), + metrics, + Panel(_render_agent_table(agents), title="Managed Agents", border_style="green"), + Panel(_render_activity_table(payload.get("recent_activity", [])), title="Recent Activity", border_style="magenta"), + ) + + +def _render_agent_detail(entry: dict, *, activity: list[dict]) -> Group: + overview = Table.grid(expand=True, padding=(0, 2)) + overview.add_column(style="bold") + overview.add_column(ratio=2) + overview.add_column(style="bold") + overview.add_column(ratio=2) + overview.add_row("Agent", f"@{entry.get('name')}", "Runtime", str(entry.get("runtime_type") or "-")) + overview.add_row("Desired", str(entry.get("desired_state") or "-"), "Effective", str(entry.get("effective_state") or "-")) + overview.add_row("Connected", "yes" if entry.get("connected") else "no", "Queue", str(entry.get("backlog_depth") or 0)) + overview.add_row("Seen", _format_age(entry.get("last_seen_age_seconds")), "Reconnect", _format_age(entry.get("reconnect_backoff_seconds"))) + overview.add_row("Processed", str(entry.get("processed_count") or 0), "Dropped", str(entry.get("dropped_count") or 0)) + overview.add_row("Last Work", _format_timestamp(entry.get("last_work_received_at")), "Completed", _format_timestamp(entry.get("last_work_completed_at"))) + overview.add_row("Phase", str(entry.get("current_status") or "-"), "Activity", str(entry.get("current_activity") or "-")) + overview.add_row("Tool", str(entry.get("current_tool") or "-"), "Transport", str(entry.get("transport") or "-")) + overview.add_row("Cred Source", str(entry.get("credential_source") or "-"), "Space", str(entry.get("space_id") or "-")) + overview.add_row("Agent ID", str(entry.get("agent_id") or "-"), "Last Reply", str(entry.get("last_reply_preview") or "-")) + overview.add_row("Last Error", str(entry.get("last_error") or "-"), "", "") + + paths = Table.grid(expand=True, padding=(0, 2)) + paths.add_column(style="bold") + paths.add_column(ratio=3) + paths.add_row("Token File", str(entry.get("token_file") or "-")) + paths.add_row("Workdir", str(entry.get("workdir") or "-")) + paths.add_row("Exec", str(entry.get("exec_command") or "-")) + paths.add_row("Added", _format_timestamp(entry.get("added_at"))) + + return Group( + Panel(overview, title=f"Managed Agent · @{entry.get('name')}", border_style="cyan"), + Panel(paths, title="Runtime Details", border_style="blue"), + Panel(_render_activity_table(activity), title="Recent Agent Activity", border_style="magenta"), + ) + + +@app.command("login") +def login( + token: str = typer.Option(None, "--token", "-t", help="User PAT (prompted or reused from axctl login when omitted)"), + base_url: str = typer.Option(None, "--url", "-u", help="API base URL (defaults to existing axctl login or paxai.app)"), + space_id: str = typer.Option(None, "--space-id", "-s", help="Optional default space for managed agents"), + as_json: bool = JSON_OPTION, +): + """Store the Gateway bootstrap session. + + The Gateway keeps the user PAT centrally and uses it to mint agent PATs for + managed runtimes. Managed runtimes themselves never receive the PAT or JWT. + """ + resolved_token = _resolve_gateway_login_token(token) + if not resolved_token.startswith("axp_u_"): + err_console.print("[red]Gateway bootstrap requires a user PAT (axp_u_).[/red]") + raise typer.Exit(1) + resolved_base_url = base_url or resolve_user_base_url() or auth_cmd.DEFAULT_LOGIN_BASE_URL + + err_console.print(f"[cyan]Verifying Gateway login against {resolved_base_url}...[/cyan]") + from ..token_cache import TokenExchanger + + try: + exchanger = TokenExchanger(resolved_base_url, resolved_token) + exchanger.get_token( + "user_access", + scope="messages tasks context agents spaces search", + force_refresh=True, + ) + client = AxClient(base_url=resolved_base_url, token=resolved_token) + me = client.whoami() + except Exception as exc: + err_console.print(f"[red]Gateway login failed:[/red] {exc}") + raise typer.Exit(1) + + selected_space = space_id + if not selected_space: + try: + spaces = client.list_spaces() + space_list = spaces.get("spaces", spaces) if isinstance(spaces, dict) else spaces + selected = auth_cmd._select_login_space([s for s in space_list if isinstance(s, dict)]) + if selected: + selected_space = auth_cmd._candidate_space_id(selected) + except Exception: + selected_space = None + + payload = { + "token": resolved_token, + "base_url": resolved_base_url, + "principal_type": "user", + "space_id": selected_space, + "username": me.get("username"), + "email": me.get("email"), + "saved_at": None, + } + path = save_gateway_session(payload) + registry = load_gateway_registry() + registry.setdefault("gateway", {}) + registry["gateway"]["session_connected"] = True + save_gateway_registry(registry) + record_gateway_activity("gateway_login", username=me.get("username"), base_url=resolved_base_url, space_id=selected_space) + + result = { + "session_path": str(path), + "base_url": resolved_base_url, + "space_id": selected_space, + "username": me.get("username"), + "email": me.get("email"), + } + if as_json: + print_json(result) + else: + err_console.print(f"[green]Gateway login saved:[/green] {path}") + for key, value in result.items(): + err_console.print(f" {key} = {value}") + + +@app.command("status") +def status(as_json: bool = JSON_OPTION): + """Show Gateway status, daemon state, and managed runtimes.""" + payload = _status_payload() + if as_json: + print_json(payload) + return + + err_console.print("[bold]ax gateway status[/bold]") + err_console.print(f" gateway_dir = {payload['gateway_dir']}") + err_console.print(f" connected = {payload['connected']}") + err_console.print(f" daemon = {'running' if payload['daemon']['running'] else 'stopped'}") + if payload["daemon"]["pid"]: + err_console.print(f" pid = {payload['daemon']['pid']}") + err_console.print(f" base_url = {payload['base_url']}") + err_console.print(f" space_id = {payload['space_id']}") + err_console.print(f" user = {payload['user']}") + err_console.print(f" agents = {payload['summary']['managed_agents']}") + err_console.print(f" connected = {payload['summary']['connected_agents']}") + if payload["agents"]: + print_table( + ["Agent", "Type", "Desired", "Effective", "Phase", "Seen", "Backlog", "Activity", "Last Error"], + payload["agents"], + keys=[ + "name", + "runtime_type", + "desired_state", + "effective_state", + "current_status", + "last_seen_age_seconds", + "backlog_depth", + "current_activity", + "last_error", + ], + ) + if payload["recent_activity"]: + print_table( + ["Time", "Event", "Agent", "Message", "Preview"], + payload["recent_activity"], + keys=["ts", "event", "agent_name", "message_id", "reply_preview"], + ) + + +@app.command("watch") +def watch( + interval: float = typer.Option(2.0, "--interval", "-n", help="Dashboard refresh interval in seconds"), + activity_limit: int = typer.Option(8, "--activity-limit", help="Number of recent events to display"), + once: bool = typer.Option(False, "--once", help="Render one dashboard frame and exit"), +): + """Watch the Gateway in a live terminal dashboard.""" + + def render_dashboard() -> Group: + return _render_gateway_dashboard(_status_payload(activity_limit=activity_limit)) + + if once: + console.print(render_dashboard()) + return + + try: + with Live(render_dashboard(), console=console, screen=True, auto_refresh=False) as live: + while True: + live.update(render_dashboard(), refresh=True) + time.sleep(interval) + except KeyboardInterrupt: + err_console.print("[yellow]Gateway watch stopped.[/yellow]") + + +@app.command("run") +def run( + poll_interval: float = typer.Option(1.0, "--poll-interval", help="Registry reconcile interval in seconds"), + once: bool = typer.Option(False, "--once", help="Run one reconcile pass and exit"), +): + """Run the foreground Gateway supervisor.""" + _load_gateway_session_or_exit() + err_console.print("[bold]ax gateway[/bold] — local control plane") + err_console.print(f" state_dir = {gateway_dir()}") + err_console.print(f" interval = {poll_interval}s") + err_console.print(f" mode = {'single-pass' if once else 'foreground'}") + daemon = GatewayDaemon(logger=lambda msg: err_console.print(f"[dim]{msg}[/dim]"), poll_interval=poll_interval) + try: + daemon.run(once=once) + except RuntimeError as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) + except KeyboardInterrupt: + daemon.stop() + err_console.print("[yellow]Gateway stopped.[/yellow]") + + +@agents_app.command("add") +def add_agent( + name: str = typer.Argument(..., help="Managed agent name"), + runtime_type: str = typer.Option("echo", "--type", help="Runtime type: echo | exec | inbox"), + exec_cmd: str = typer.Option(None, "--exec", help="Per-mention command for exec runtimes"), + workdir: str = typer.Option(None, "--workdir", help="Working directory for exec runtimes"), + space_id: str = typer.Option(None, "--space-id", help="Target space (defaults to gateway session)"), + audience: str = typer.Option("both", "--audience", help="Minted PAT audience"), + description: str = typer.Option(None, "--description", help="Create/update description"), + model: str = typer.Option(None, "--model", help="Create/update model"), + start: bool = typer.Option(True, "--start/--no-start", help="Desired running state after registration"), + as_json: bool = JSON_OPTION, +): + """Register a managed agent and mint a Gateway-owned PAT for it.""" + runtime_type = runtime_type.lower().strip() + if runtime_type not in {"echo", "exec", "command", "inbox"}: + err_console.print("[red]Unsupported runtime type.[/red] Use echo, exec, or inbox.") + raise typer.Exit(1) + if runtime_type in {"exec", "command"} and not exec_cmd: + err_console.print("[red]Exec runtimes require --exec.[/red]") + raise typer.Exit(1) + if runtime_type in {"echo", "inbox"} and exec_cmd: + err_console.print("[red]Echo and inbox runtimes do not accept --exec.[/red]") + raise typer.Exit(1) + + session = _load_gateway_session_or_exit() + selected_space = space_id or session.get("space_id") + if not selected_space: + err_console.print("[red]No space selected.[/red] Use --space-id or re-run `ax gateway login` with one.") + raise typer.Exit(1) + + client = _load_gateway_user_client() + existing = _find_agent_in_space(client, name, selected_space) + if existing: + agent = existing + if description or model: + client.update_agent(name, **{k: v for k, v in {"description": description, "model": model}.items() if v}) + else: + agent = _create_agent_in_space( + client, + name=name, + space_id=selected_space, + description=description, + model=model, + ) + _polish_metadata(client, name=name, bio=None, specialization=None, system_prompt=None) + + agent_id = str(agent.get("id") or agent.get("agent_id") or "") + token, pat_source = _mint_agent_pat( + client, + agent_id=agent_id, + agent_name=name, + audience=audience, + expires_in_days=90, + pat_name=f"gateway-{name}", + space_id=selected_space, + ) + token_file = _save_agent_token(name, token) + + registry = load_gateway_registry() + entry = upsert_agent_entry( + registry, + { + "name": name, + "agent_id": agent_id, + "space_id": selected_space, + "base_url": session["base_url"], + "runtime_type": "exec" if runtime_type == "command" else runtime_type, + "exec_command": exec_cmd, + "workdir": workdir, + "token_file": str(token_file), + "desired_state": "running" if start else "stopped", + "effective_state": "stopped", + "transport": "gateway", + "credential_source": "gateway", + "last_error": None, + "backlog_depth": 0, + "processed_count": 0, + "dropped_count": 0, + "pat_source": pat_source, + "added_at": __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat(), + }, + ) + save_gateway_registry(registry) + record_gateway_activity( + "managed_agent_added", + entry=entry, + space_id=selected_space, + token_file=str(token_file), + ) + + if as_json: + print_json(entry) + else: + err_console.print(f"[green]Managed agent ready:[/green] @{name}") + err_console.print(f" runtime_type = {entry['runtime_type']}") + err_console.print(f" desired_state = {entry['desired_state']}") + err_console.print(f" token_file = {token_file}") + + +@agents_app.command("list") +def list_agents(as_json: bool = JSON_OPTION): + """List Gateway-managed agents.""" + agents = _status_payload()["agents"] + if as_json: + print_json({"agents": agents, "count": len(agents)}) + return + print_table( + ["Agent", "Type", "Desired", "Effective", "Space"], + agents, + keys=["name", "runtime_type", "desired_state", "effective_state", "space_id"], + ) + + +@agents_app.command("show") +def show_agent( + name: str = typer.Argument(..., help="Managed agent name"), + activity_limit: int = typer.Option(12, "--activity-limit", help="Number of recent agent events to display"), + as_json: bool = JSON_OPTION, +): + """Show one managed agent in detail.""" + payload = _status_payload(activity_limit=activity_limit) + entry = next((agent for agent in payload["agents"] if str(agent.get("name") or "").lower() == name.lower()), None) + if not entry: + err_console.print(f"[red]Managed agent not found:[/red] {name}") + raise typer.Exit(1) + activity = load_recent_gateway_activity(limit=activity_limit, agent_name=name) + result = { + "gateway": { + "connected": payload["connected"], + "base_url": payload["base_url"], + "space_id": payload["space_id"], + "daemon": payload["daemon"], + }, + "agent": entry, + "recent_activity": activity, + } + if as_json: + print_json(result) + return + console.print(_render_agent_detail(entry, activity=activity)) + + +@agents_app.command("send") +def send_as_agent( + name: str = typer.Argument(..., help="Managed agent name to send as"), + content: str = typer.Argument(..., help="Message content"), + to: str = typer.Option(None, "--to", help="Prepend a mention like @codex automatically"), + parent_id: str = typer.Option(None, "--parent-id", help="Reply inside an existing thread"), + as_json: bool = JSON_OPTION, +): + """Send a message as a Gateway-managed agent.""" + entry = _load_managed_agent_or_exit(name) + client = _load_managed_agent_client(entry) + space_id = str(entry.get("space_id") or "") + if not space_id: + err_console.print(f"[red]Managed agent is missing a space id:[/red] @{name}") + raise typer.Exit(1) + + message_content = content.strip() + mention = str(to or "").strip().lstrip("@") + if mention: + prefix = f"@{mention}" + if not message_content.startswith(prefix): + message_content = f"{prefix} {message_content}".strip() + + metadata = { + "control_plane": "gateway", + "gateway": { + "managed": True, + "agent_name": entry.get("name"), + "agent_id": entry.get("agent_id"), + "runtime_type": entry.get("runtime_type"), + "transport": entry.get("transport", "gateway"), + "credential_source": entry.get("credential_source", "gateway"), + "sent_via": "gateway_cli", + }, + } + result = client.send_message( + space_id, + message_content, + agent_id=str(entry.get("agent_id") or "") or None, + parent_id=parent_id or None, + metadata=metadata, + ) + payload = result.get("message", result) if isinstance(result, dict) else result + if isinstance(payload, dict): + record_gateway_activity( + "manual_message_sent", + entry=entry, + message_id=payload.get("id"), + reply_preview=message_content[:120] or None, + ) + if as_json: + print_json({"agent": entry.get("name"), "message": payload, "content": message_content}) + return + err_console.print(f"[green]Sent as managed agent:[/green] @{entry.get('name')}") + if isinstance(payload, dict) and payload.get("id"): + err_console.print(f" id = {payload['id']}") + err_console.print(f" content = {message_content}") + + +@agents_app.command("start") +def start_agent(name: str = typer.Argument(..., help="Managed agent name")): + """Set a managed agent's desired state to running.""" + registry = load_gateway_registry() + entry = find_agent_entry(registry, name) + if not entry: + err_console.print(f"[red]Managed agent not found:[/red] {name}") + raise typer.Exit(1) + entry["desired_state"] = "running" + save_gateway_registry(registry) + record_gateway_activity("managed_agent_desired_running", entry=entry) + err_console.print(f"[green]Desired state set to running:[/green] @{name}") + + +@agents_app.command("stop") +def stop_agent(name: str = typer.Argument(..., help="Managed agent name")): + """Set a managed agent's desired state to stopped.""" + registry = load_gateway_registry() + entry = find_agent_entry(registry, name) + if not entry: + err_console.print(f"[red]Managed agent not found:[/red] {name}") + raise typer.Exit(1) + entry["desired_state"] = "stopped" + save_gateway_registry(registry) + record_gateway_activity("managed_agent_desired_stopped", entry=entry) + err_console.print(f"[green]Desired state set to stopped:[/green] @{name}") + + +@agents_app.command("remove") +def remove_agent(name: str = typer.Argument(..., help="Managed agent name")): + """Remove a managed agent from local Gateway control.""" + registry = load_gateway_registry() + entry = remove_agent_entry(registry, name) + if not entry: + err_console.print(f"[red]Managed agent not found:[/red] {name}") + raise typer.Exit(1) + save_gateway_registry(registry) + token_file = Path(str(entry.get("token_file") or "")) + if token_file.exists(): + token_file.unlink() + record_gateway_activity("managed_agent_removed", entry=entry) + err_console.print(f"[green]Removed managed agent:[/green] @{name}") diff --git a/ax_cli/commands/listen.py b/ax_cli/commands/listen.py index debb8a6..360e9c6 100644 --- a/ax_cli/commands/listen.py +++ b/ax_cli/commands/listen.py @@ -75,6 +75,14 @@ def _message_sender_identity(data: dict) -> tuple[str, str]: ) +def _message_sender_type(data: dict) -> str: + """Return the author type from an SSE payload when available.""" + author = data.get("author") + if isinstance(author, dict): + return str(author.get("type") or "") + return str(data.get("sender_type") or "") + + def _is_self_authored(data: dict, agent_name: str, agent_id: str | None) -> bool: """Return True when an SSE payload was authored by this listener's agent.""" sender, sender_id = _message_sender_identity(data) @@ -124,11 +132,6 @@ def _should_respond( if _is_self_authored(data, agent_name, agent_id): return False - parent_id = str(data.get("parent_id") or "") - conversation_id = str(data.get("conversation_id") or "") - if reply_anchor_ids and (parent_id in reply_anchor_ids or conversation_id in reply_anchor_ids): - return True - # Primary path: trust the backend's authoritative mentions list. # An empty list is MEANINGFUL — it means "no active mentions for # this message," which covers the kill-switch filter case where @@ -137,16 +140,30 @@ def _should_respond( mentions = data.get("mentions") if mentions is not None and isinstance(mentions, list): agent_name_lower = agent_name.lower() + sender_type = _message_sender_type(data).lower() for m in mentions: handle = "" + source = "" if isinstance(m, str): handle = m elif isinstance(m, dict): handle = m.get("agent_name") or m.get("handle") or m.get("name") or "" + source = str(m.get("source") or "") + if sender_type == "agent" and source in {"thread_parent", "reply_target"}: + continue if handle.lower().lstrip("@").strip() == agent_name_lower: return True + parent_id = str(data.get("parent_id") or "") + conversation_id = str(data.get("conversation_id") or "") + if reply_anchor_ids and (parent_id in reply_anchor_ids or conversation_id in reply_anchor_ids): + return _message_sender_type(data).lower() != "agent" return False + parent_id = str(data.get("parent_id") or "") + conversation_id = str(data.get("conversation_id") or "") + if reply_anchor_ids and (parent_id in reply_anchor_ids or conversation_id in reply_anchor_ids): + return _message_sender_type(data).lower() != "agent" + # Fallback: `mentions` field absent entirely (legacy / non-standard # event shape). Use content regex so we don't silently stop reacting # to payloads that predate the current mentions contract. diff --git a/ax_cli/commands/messages.py b/ax_cli/commands/messages.py index 74386e6..0da9578 100644 --- a/ax_cli/commands/messages.py +++ b/ax_cli/commands/messages.py @@ -34,12 +34,63 @@ def _processing_status_from_event(message_id: str, event_type: str | None, data: status = str(data.get("status") or "").strip() if not status: return None - return { + event = { "message_id": event_message_id, "status": status, "agent_id": data.get("agent_id"), "agent_name": data.get("agent_name"), } + for field in ( + "activity", + "tool_name", + "progress", + "detail", + "reason", + "error_message", + "retry_after_seconds", + "parent_message_id", + ): + if data.get(field) is not None: + event[field] = data.get(field) + return event + + +def _processing_status_text(status_event: dict, *, wait_label: str = "reply") -> str: + """Render tooling-side delivery/progress in human-friendly language.""" + status = str(status_event.get("status") or "").strip().lower() + agent_name = str(status_event.get("agent_name") or wait_label).strip().lstrip("@") + target = f"@{agent_name}" if agent_name else wait_label + activity = str(status_event.get("activity") or "").strip() + tool_name = str(status_event.get("tool_name") or "").strip() + + if status == "accepted": + base = f"tooling: {target} acknowledged the message" + elif status in {"started", "claimed", "forwarded"}: + base = f"tooling: {target} picked up the message" + elif status in {"queued", "queued locally"}: + base = f"tooling: {target} queued the message" + elif status in {"working", "processing"}: + base = f"tooling: {target} is working" + elif status == "thinking": + base = f"tooling: {target} is thinking" + elif status in {"tool_use", "tool_call"}: + base = f"tooling: {target} is using tools" + elif status == "tool_complete": + base = f"tooling: {target} finished a tool step" + elif status == "streaming": + base = f"tooling: {target} is streaming a reply" + elif status == "completed": + base = f"tooling: {target} finished processing" + elif status == "error": + base = f"tooling: {target} hit an error" + else: + base = f"tooling: {target} status={status}" + + if activity: + return f"{base} — {activity}" + if tool_name: + return f"{base} — {tool_name}" + return base class _ProcessingStatusWatcher: @@ -52,6 +103,8 @@ def __init__(self, client, *, space_id: str, timeout: int) -> None: self.message_id: str | None = None self.events: list[dict] = [] self._queue: queue.Queue[dict] = queue.Queue() + self._pending: list[dict] = [] + self._lock = threading.Lock() self._ready = threading.Event() self._stop = threading.Event() self._thread: threading.Thread | None = None @@ -64,7 +117,12 @@ def wait_ready(self, timeout: float = 1.5) -> bool: return self._ready.wait(timeout) def set_message_id(self, message_id: str) -> None: - self.message_id = message_id + with self._lock: + self.message_id = message_id + queued = [event for event in self._pending if event.get("message_id") == message_id] + self._pending = [event for event in self._pending if event.get("message_id") != message_id] + for event in queued: + self._queue.put(event) def close(self) -> None: self._stop.set() @@ -79,6 +137,17 @@ def drain(self) -> list[dict]: self.events.append(item) drained.append(item) + def _accept_status_event(self, status_event: dict) -> None: + with self._lock: + message_id = self.message_id + if not message_id: + self._pending.append(status_event) + if len(self._pending) > 100: + self._pending = self._pending[-100:] + return + if status_event.get("message_id") == message_id: + self._queue.put(status_event) + def _run(self) -> None: while not self._stop.is_set() and time.time() < self.deadline: try: @@ -90,12 +159,14 @@ def _run(self) -> None: for event_type, data in _iter_sse(response): if self._stop.is_set() or time.time() >= self.deadline: return - message_id = self.message_id - if not message_id: + if event_type != "agent_processing" or not isinstance(data, dict): continue - status = _processing_status_from_event(message_id, event_type, data) + event_message_id = str(data.get("message_id") or data.get("source_message_id") or "") + if not event_message_id: + continue + status = _processing_status_from_event(event_message_id, event_type, data) if status: - self._queue.put(status) + self._accept_status_event(status) except httpx.ReadTimeout: continue except (httpx.HTTPError, RuntimeError, AttributeError): @@ -146,7 +217,7 @@ def _wait_for_reply_polling( ) -> dict | None: """Poll for a reply as a fallback when SSE is unavailable.""" last_remaining = None - announced_processing: set[tuple[str | None, str]] = set() + announced_processing: set[tuple[str | None, str, str, str]] = set() while time.time() < deadline: remaining = int(deadline - time.time()) @@ -155,10 +226,15 @@ def _wait_for_reply_polling( for status_event in processing_watcher.drain(): status = str(status_event.get("status") or "") agent_name = status_event.get("agent_name") or wait_label - key = (status_event.get("agent_id"), status) + key = ( + status_event.get("agent_id"), + status, + str(status_event.get("activity") or ""), + str(status_event.get("tool_name") or ""), + ) if status and key not in announced_processing: console.print(" " * 60, end="\r") - console.print(f" [cyan]@{str(agent_name).lstrip('@')} is {status}[/cyan]") + console.print(f" [cyan]{_processing_status_text(status_event, wait_label=agent_name)}[/cyan]") announced_processing.add(key) try: @@ -243,6 +319,46 @@ def _starts_with_mention(content: str, mention: str) -> bool: return content.lstrip().lower().startswith(mention.lower()) +def _sender_label(message: dict) -> str | None: + display_name = str(message.get("display_name") or "").strip() + sender_type = str(message.get("sender_type") or "").strip() + if display_name: + if sender_type == "agent": + return f"@{display_name.lstrip('@')}" + return display_name + if sender_type: + return sender_type + return None + + +def _gateway_reply_note(message: dict) -> str | None: + metadata = message.get("metadata") + if not isinstance(metadata, dict) or metadata.get("control_plane") != "gateway": + return None + gateway = metadata.get("gateway") + if not isinstance(gateway, dict): + return None + + parts = ["via Gateway"] + gateway_id = str(gateway.get("gateway_id") or "").strip() + if gateway_id: + parts[0] = f"{parts[0]} {gateway_id[:8]}" + + agent_name = str(gateway.get("agent_name") or "").strip() + if agent_name: + parts.append(f"agent=@{agent_name.lstrip('@')}") + + runtime_type = str(gateway.get("runtime_type") or "").strip() + if runtime_type: + parts.append(f"runtime={runtime_type}") + + transport = str(gateway.get("transport") or "").strip() + if transport: + parts.append(f"transport={transport}") + + return " · ".join(parts) + + def _attachment_ref( *, attachment_id: str, @@ -498,10 +614,18 @@ def send( if as_json: print_json(data) else: - console.print(f"[green]Sent.[/green] id={msg_id}") + sent_line = f"[green]Sent.[/green] id={msg_id}" + sender = _sender_label(msg) + if sender: + sent_line += f" as {sender}" + console.print(sent_line) return - console.print(f"[green]Sent.[/green] id={msg_id}") + sent_line = f"[green]Sent.[/green] id={msg_id}" + sender = _sender_label(msg) + if sender: + sent_line += f" as {sender}" + console.print(sent_line) wait_label = _target_mention("aX") if ask_ax else (_target_mention(to) if to else "reply") reply = _wait_for_reply( client, @@ -519,6 +643,9 @@ def send( print_json({"sent": data, "reply": reply, "processing_statuses": processing_statuses}) else: console.print(f"\n[bold cyan]aX:[/bold cyan] {reply.get('content', '')}") + gateway_note = _gateway_reply_note(reply) + if gateway_note: + console.print(f"[dim]{gateway_note}[/dim]") else: if as_json: print_json( diff --git a/ax_cli/config.py b/ax_cli/config.py index 0ee9e86..e9f6a72 100644 --- a/ax_cli/config.py +++ b/ax_cli/config.py @@ -400,6 +400,9 @@ def apply_cfg(cfg: dict, source: str) -> None: selected_user_env = normalized_env or _resolve_user_env() user_cfg = _load_user_config(selected_user_env) user_path = _user_config_path(selected_user_env) + explicit_cfg_env = os.environ.get("AX_CONFIG_FILE") + explicit_cfg_path = Path(explicit_cfg_env).expanduser() if explicit_cfg_env else None + explicit_cfg = _load_runtime_config_file(explicit_cfg_env) local_dir = _local_config_dir() local_path = (local_dir / "config.toml") if local_dir else None @@ -533,6 +536,32 @@ def apply_cfg(cfg: dict, source: str) -> None: effective["principal_type"] = "agent" field_sources["principal_type"] = "local_config" + if explicit_cfg_path: + sources.append( + _source_record( + "runtime_config", + path=explicit_cfg_path, + exists=explicit_cfg_path.exists(), + used=bool(explicit_cfg), + keys=list(explicit_cfg.keys()) if explicit_cfg else None, + ) + ) + if explicit_cfg: + runtime_source = f"runtime_config:{explicit_cfg_path}" + apply_cfg(explicit_cfg, runtime_source) + if "principal_type" not in explicit_cfg and _has_agent_identity(explicit_cfg): + effective["principal_type"] = "agent" + field_sources["principal_type"] = runtime_source + else: + sources.append( + _source_record( + "runtime_config", + path=None, + exists=False, + used=False, + ) + ) + used_env_keys: list[str] = [] if not normalized_env: env_overrides = { @@ -591,6 +620,7 @@ def apply_cfg(cfg: dict, source: str) -> None: "ok": not problems, "selected_env": normalized_env or selected_user_env, "selected_profile": selected_profile_name, + "runtime_config": str(explicit_cfg_path) if explicit_cfg_path else None, "effective": { "auth_source": field_sources.get("token"), "token_kind": token_kind, diff --git a/ax_cli/gateway.py b/ax_cli/gateway.py new file mode 100644 index 0000000..abff838 --- /dev/null +++ b/ax_cli/gateway.py @@ -0,0 +1,1342 @@ +"""Local Gateway runtime and state management. + +The Gateway is a local control-plane daemon that owns bootstrap and agent +credentials, supervises managed runtimes, and keeps lightweight desired vs +effective state in a registry file. The first slice intentionally uses +filesystem state plus a foreground daemon so it can ship quickly without +introducing a second backend. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import queue +import re +import shlex +import subprocess +import threading +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +import httpx + +from .client import AxClient +from .commands.listen import ( + _is_self_authored, + _iter_sse, + _remember_reply_anchor, + _should_respond, + _strip_mention, +) +from .config import _global_config_dir + +RuntimeLogger = Callable[[str], None] + +REPLY_ANCHOR_MAX = 500 +SEEN_IDS_MAX = 500 +DEFAULT_QUEUE_SIZE = 50 +DEFAULT_ACTIVITY_LIMIT = 10 +DEFAULT_HANDLER_TIMEOUT_SECONDS = 900 +SSE_IDLE_TIMEOUT_SECONDS = 45.0 +RUNTIME_STALE_AFTER_SECONDS = 75.0 +GATEWAY_EVENT_PREFIX = "AX_GATEWAY_EVENT " +ENV_DENYLIST = { + "AX_AGENT_ID", + "AX_AGENT_NAME", + "AX_BASE_URL", + "AX_CONFIG_FILE", + "AX_ENV", + "AX_SPACE_ID", + "AX_TOKEN", + "AX_TOKEN_FILE", + "AX_USER_BASE_URL", + "AX_USER_ENV", + "AX_USER_TOKEN", +} +_ACTIVITY_LOCK = threading.Lock() +_GATEWAY_PROCESS_RE = re.compile(r"(?:^|\s)(?:uv\s+run\s+ax\s+gateway\s+run|.+?/ax\s+gateway\s+run)(?:\s|$)") + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _parse_iso8601(value: object) -> datetime | None: + if not value or not isinstance(value, str): + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _age_seconds(value: object, *, now: datetime | None = None) -> int | None: + parsed = _parse_iso8601(value) + if parsed is None: + return None + current = now or datetime.now(timezone.utc) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + delta = current - parsed.astimezone(timezone.utc) + return max(0, int(delta.total_seconds())) + + +def annotate_runtime_health(snapshot: dict[str, Any], *, now: datetime | None = None) -> dict[str, Any]: + enriched = dict(snapshot) + last_seen_age = _age_seconds(enriched.get("last_seen_at"), now=now) + last_error_age = _age_seconds(enriched.get("last_listener_error_at"), now=now) + if last_seen_age is not None: + enriched["last_seen_age_seconds"] = last_seen_age + if last_error_age is not None: + enriched["last_listener_error_age_seconds"] = last_error_age + + state = str(enriched.get("effective_state") or "stopped").lower() + connected = False + if state == "running": + if last_seen_age is None or last_seen_age > RUNTIME_STALE_AFTER_SECONDS: + state = "stale" + else: + connected = True + enriched["effective_state"] = state + enriched["connected"] = connected + return enriched + + +def gateway_dir() -> Path: + path = _global_config_dir() / "gateway" + path.mkdir(parents=True, exist_ok=True) + path.chmod(0o700) + return path + + +def gateway_agents_dir() -> Path: + path = gateway_dir() / "agents" + path.mkdir(parents=True, exist_ok=True) + path.chmod(0o700) + return path + + +def session_path() -> Path: + return gateway_dir() / "session.json" + + +def registry_path() -> Path: + return gateway_dir() / "registry.json" + + +def pid_path() -> Path: + return gateway_dir() / "gateway.pid" + + +def activity_log_path() -> Path: + return gateway_dir() / "activity.jsonl" + + +def agent_dir(name: str) -> Path: + path = gateway_agents_dir() / name + path.mkdir(parents=True, exist_ok=True) + path.chmod(0o700) + return path + + +def agent_token_path(name: str) -> Path: + return agent_dir(name) / "token" + + +def _default_registry() -> dict[str, Any]: + return { + "version": 1, + "gateway": { + "gateway_id": str(uuid.uuid4()), + "desired_state": "stopped", + "effective_state": "stopped", + "session_connected": False, + "pid": None, + "last_started_at": None, + "last_reconcile_at": None, + }, + "agents": [], + } + + +def _write_json(path: Path, payload: dict[str, Any], *, mode: int = 0o600) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + tmp.chmod(mode) + tmp.replace(path) + path.chmod(mode) + + +def _read_json(path: Path, *, default: dict[str, Any]) -> dict[str, Any]: + if not path.exists(): + return copy.deepcopy(default) + return json.loads(path.read_text()) + + +def load_gateway_session() -> dict[str, Any]: + return _read_json(session_path(), default={}) + + +def save_gateway_session(data: dict[str, Any]) -> Path: + payload = dict(data) + payload.setdefault("saved_at", _now_iso()) + _write_json(session_path(), payload) + return session_path() + + +def load_gateway_registry() -> dict[str, Any]: + registry = _read_json(registry_path(), default=_default_registry()) + registry.setdefault("version", 1) + registry.setdefault("gateway", {}) + registry.setdefault("agents", []) + gateway = registry["gateway"] + gateway.setdefault("gateway_id", str(uuid.uuid4())) + gateway.setdefault("desired_state", "stopped") + gateway.setdefault("effective_state", "stopped") + gateway.setdefault("session_connected", False) + gateway.setdefault("pid", None) + gateway.setdefault("last_started_at", None) + gateway.setdefault("last_reconcile_at", None) + return registry + + +def save_gateway_registry(registry: dict[str, Any]) -> Path: + _write_json(registry_path(), registry) + return registry_path() + + +def _pid_alive(pid: int | None) -> bool: + if not pid: + return False + try: + os.kill(pid, 0) + return True + except OSError: + return False + + +def daemon_status() -> dict[str, Any]: + pid = None + if pid_path().exists(): + try: + pid = int(pid_path().read_text().strip()) + except ValueError: + pid = None + registry = load_gateway_registry() + return { + "pid": pid, + "running": _pid_alive(pid), + "registry_path": str(registry_path()), + "session_path": str(session_path()), + "registry": registry, + } + + +def _scan_gateway_process_pids() -> list[int]: + """Best-effort fallback for live daemons that predate the pid file.""" + current_pid = os.getpid() + parent_pid = os.getppid() + try: + output = subprocess.check_output( + ["ps", "-axo", "pid=,command="], + text=True, + stderr=subprocess.DEVNULL, + ) + except Exception: + return [] + + pids: list[int] = [] + for raw_line in output.splitlines(): + line = raw_line.strip() + if not line: + continue + pid_text, _, command = line.partition(" ") + try: + pid = int(pid_text) + except ValueError: + continue + if pid in {current_pid, parent_pid} or not _pid_alive(pid): + continue + command = command.strip() + if command and _GATEWAY_PROCESS_RE.search(command): + pids.append(pid) + return sorted(set(pids)) + + +def active_gateway_pids() -> list[int]: + """Return all known live Gateway daemon PIDs except the current process.""" + status = daemon_status() + pids: list[int] = [] + pid = status.get("pid") + if isinstance(pid, int) and status.get("running") and pid != os.getpid(): + pids.append(pid) + pids.extend(_scan_gateway_process_pids()) + return sorted(set(pids)) + + +def active_gateway_pid() -> int | None: + """Return the PID of a live Gateway daemon, if one is already running.""" + pids = active_gateway_pids() + return pids[0] if pids else None + + +def write_gateway_pid(pid: int) -> None: + pid_path().write_text(f"{pid}\n") + pid_path().chmod(0o600) + + +def clear_gateway_pid(pid: int | None = None) -> None: + if not pid_path().exists(): + return + if pid is not None: + try: + existing_pid = int(pid_path().read_text().strip()) + except ValueError: + existing_pid = None + if existing_pid not in {None, pid}: + return + pid_path().unlink() + + +def record_gateway_activity( + event: str, + *, + entry: dict[str, Any] | None = None, + **fields: Any, +) -> dict[str, Any]: + record: dict[str, Any] = { + "ts": _now_iso(), + "event": event, + } + registry = load_gateway_registry() + gateway = registry.get("gateway", {}) + if gateway.get("gateway_id"): + record["gateway_id"] = gateway["gateway_id"] + if entry: + record.update( + { + "agent_name": entry.get("name"), + "agent_id": entry.get("agent_id"), + "runtime_type": entry.get("runtime_type"), + "transport": entry.get("transport", "gateway"), + "credential_source": entry.get("credential_source", "gateway"), + } + ) + for key, value in fields.items(): + if value is not None: + record[key] = value + + path = activity_log_path() + path.parent.mkdir(parents=True, exist_ok=True) + with _ACTIVITY_LOCK: + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + path.chmod(0o600) + return record + + +def load_recent_gateway_activity( + limit: int = DEFAULT_ACTIVITY_LIMIT, + *, + agent_name: str | None = None, +) -> list[dict[str, Any]]: + path = activity_log_path() + if not path.exists(): + return [] + try: + lines = path.read_text().splitlines() + except OSError: + return [] + if limit <= 0: + return [] + agent_filter = agent_name.strip().lower() if agent_name else None + items: list[dict[str, Any]] = [] + for line in reversed(lines): + if not line.strip(): + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(payload, dict): + continue + if agent_filter and str(payload.get("agent_name") or "").lower() != agent_filter: + continue + items.append(payload) + if len(items) >= limit: + break + items.reverse() + return items + + +def find_agent_entry(registry: dict[str, Any], name: str) -> dict[str, Any] | None: + for entry in registry.get("agents", []): + if str(entry.get("name", "")).lower() == name.lower(): + return entry + return None + + +def upsert_agent_entry(registry: dict[str, Any], agent: dict[str, Any]) -> dict[str, Any]: + agents = registry.setdefault("agents", []) + for idx, existing in enumerate(agents): + if str(existing.get("name", "")).lower() == str(agent.get("name", "")).lower(): + merged = dict(existing) + merged.update(agent) + agents[idx] = merged + return merged + agents.append(agent) + return agent + + +def remove_agent_entry(registry: dict[str, Any], name: str) -> dict[str, Any] | None: + agents = registry.setdefault("agents", []) + for idx, entry in enumerate(agents): + if str(entry.get("name", "")).lower() == name.lower(): + return agents.pop(idx) + return None + + +def sanitize_exec_env(prompt: str, entry: dict[str, Any]) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if k not in ENV_DENYLIST} + env["AX_GATEWAY_AGENT_ID"] = str(entry.get("agent_id") or "") + env["AX_GATEWAY_AGENT_NAME"] = str(entry.get("name") or "") + env["AX_GATEWAY_RUNTIME_TYPE"] = str(entry.get("runtime_type") or "") + env["AX_MENTION_CONTENT"] = prompt + return env + + +def _parse_gateway_exec_event(raw_line: str) -> dict[str, Any] | None: + line = raw_line.strip() + if not line.startswith(GATEWAY_EVENT_PREFIX): + return None + payload = line[len(GATEWAY_EVENT_PREFIX) :].strip() + if not payload: + return None + try: + data = json.loads(payload) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + +def _hash_tool_arguments(arguments: dict[str, Any] | None) -> str | None: + if not arguments: + return None + encoded = json.dumps(arguments, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _run_exec_handler( + command: str, + prompt: str, + entry: dict[str, Any], + *, + message_id: str | None = None, + space_id: str | None = None, + on_event: Callable[[dict[str, Any]], None] | None = None, +) -> str: + argv = [*shlex.split(command), prompt] + env = sanitize_exec_env(prompt, entry) + if message_id: + env["AX_GATEWAY_MESSAGE_ID"] = message_id + if space_id: + env["AX_GATEWAY_SPACE_ID"] = space_id + try: + process = subprocess.Popen( + argv, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + cwd=entry.get("workdir") or None, + env=env, + ) + except FileNotFoundError: + return f"(handler not found: {argv[0]})" + + stdout_lines: list[str] = [] + stderr_lines: list[str] = [] + + def _consume_stdout() -> None: + if process.stdout is None: + return + for raw in process.stdout: + event = _parse_gateway_exec_event(raw) + if event is not None: + if on_event is not None: + try: + on_event(event) + except Exception: + pass + continue + stdout_lines.append(raw) + + def _consume_stderr() -> None: + if process.stderr is None: + return + for raw in process.stderr: + stderr_lines.append(raw) + + stdout_thread = threading.Thread(target=_consume_stdout, daemon=True, name=f"gw-exec-stdout-{entry.get('name')}") + stderr_thread = threading.Thread(target=_consume_stderr, daemon=True, name=f"gw-exec-stderr-{entry.get('name')}") + stdout_thread.start() + stderr_thread.start() + + timed_out = False + try: + process.wait(timeout=DEFAULT_HANDLER_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + finally: + stdout_thread.join(timeout=1.0) + stderr_thread.join(timeout=1.0) + if process.stdout is not None: + process.stdout.close() + if process.stderr is not None: + process.stderr.close() + + if timed_out: + return f"(handler timed out after {DEFAULT_HANDLER_TIMEOUT_SECONDS}s)" + + output = "".join(stdout_lines).strip() + stderr = "".join(stderr_lines).strip() + if process.returncode != 0 and stderr: + output = f"{output}\n(stderr: {stderr[:400]})".strip() + return output or "(no output)" + + +def _echo_handler(prompt: str, _entry: dict[str, Any]) -> str: + return f"Echo: {prompt}" + + +def _is_passive_runtime(runtime_type: object) -> bool: + return str(runtime_type or "").lower() in {"inbox", "passive", "monitor"} + + +class ManagedAgentRuntime: + """Listener + worker pair for one managed agent.""" + + def __init__( + self, + entry: dict[str, Any], + *, + client_factory: Callable[..., Any] = AxClient, + logger: RuntimeLogger | None = None, + ) -> None: + self.entry = dict(entry) + self.client_factory = client_factory + self.logger = logger or (lambda _msg: None) + self.stop_event = threading.Event() + self._listener_thread: threading.Thread | None = None + self._worker_thread: threading.Thread | None = None + self._queue: queue.Queue = queue.Queue(maxsize=int(entry.get("queue_size") or DEFAULT_QUEUE_SIZE)) + self._reply_anchor_ids: set[str] = set() + self._seen_ids: set[str] = set() + self._completed_seen_ids: set[str] = set() + self._state_lock = threading.Lock() + self._stream_client = None + self._send_client = None + self._stream_response = None + self._state: dict[str, Any] = { + "effective_state": "stopped", + "backlog_depth": 0, + "dropped_count": 0, + "processed_count": 0, + "current_status": None, + "current_activity": None, + "current_tool": None, + "current_tool_call_id": None, + "last_error": None, + "last_connected_at": None, + "last_listener_error_at": None, + "last_started_at": None, + "last_seen_at": None, + "last_work_received_at": None, + "last_work_completed_at": None, + "last_received_message_id": None, + "last_reply_message_id": None, + "last_reply_preview": None, + "reconnect_backoff_seconds": 0, + } + + @property + def name(self) -> str: + return str(self.entry.get("name") or "") + + @property + def agent_id(self) -> str | None: + value = self.entry.get("agent_id") + return str(value) if value else None + + @property + def base_url(self) -> str: + return str(self.entry.get("base_url") or "") + + @property + def space_id(self) -> str: + return str(self.entry.get("space_id") or "") + + @property + def token_file(self) -> Path: + return Path(str(self.entry.get("token_file") or "")).expanduser() + + def _log(self, message: str) -> None: + self.logger(f"{self.name}: {message}") + + def _token(self) -> str: + return self.token_file.read_text().strip() + + def _new_client(self): + return self.client_factory( + base_url=self.base_url, + token=self._token(), + agent_name=self.name, + agent_id=self.agent_id, + ) + + def _update_state(self, **fields: Any) -> None: + with self._state_lock: + self._state.update(fields) + + def _bump(self, field: str, amount: int = 1) -> None: + with self._state_lock: + self._state[field] = int(self._state.get(field) or 0) + amount + + def _mark_completed_seen(self, message_id: str) -> None: + if not message_id: + return + with self._state_lock: + self._completed_seen_ids.add(message_id) + + def _consume_completed_seen(self, message_id: str) -> bool: + if not message_id: + return False + with self._state_lock: + seen = message_id in self._completed_seen_ids + if seen: + self._completed_seen_ids.discard(message_id) + return seen + + def snapshot(self) -> dict[str, Any]: + with self._state_lock: + return annotate_runtime_health(dict(self._state)) + + def start(self) -> None: + if self._listener_thread and self._listener_thread.is_alive(): + return + self.stop_event.clear() + self._queue = queue.Queue(maxsize=int(self.entry.get("queue_size") or DEFAULT_QUEUE_SIZE)) + self._reply_anchor_ids = set() + self._seen_ids = set() + self._completed_seen_ids = set() + self._update_state( + effective_state="starting", + backlog_depth=0, + current_status=None, + current_activity=None, + current_tool=None, + current_tool_call_id=None, + last_error=None, + last_listener_error_at=None, + last_started_at=_now_iso(), + reconnect_backoff_seconds=0, + ) + self._worker_thread = None + if not _is_passive_runtime(self.entry.get("runtime_type")): + self._worker_thread = threading.Thread( + target=self._worker_loop, + daemon=True, + name=f"gw-worker-{self.name}", + ) + self._listener_thread = threading.Thread( + target=self._listener_loop, + daemon=True, + name=f"gw-listener-{self.name}", + ) + if self._worker_thread is not None: + self._worker_thread.start() + self._listener_thread.start() + record_gateway_activity("runtime_started", entry=self.entry) + self._log("started") + + def stop(self, timeout: float = 5.0) -> None: + self.stop_event.set() + try: + self._queue.put_nowait(None) + except queue.Full: + pass + if self._stream_response is not None: + try: + self._stream_response.close() + except Exception: + pass + for thread in (self._listener_thread, self._worker_thread): + if thread and thread.is_alive(): + thread.join(timeout=timeout) + for client in (self._stream_client, self._send_client): + if client is not None: + try: + client.close() + except Exception: + pass + self._stream_client = None + self._send_client = None + self._stream_response = None + self._update_state( + effective_state="stopped", + backlog_depth=0, + current_status=None, + current_activity=None, + current_tool=None, + current_tool_call_id=None, + ) + record_gateway_activity("runtime_stopped", entry=self.entry) + self._log("stopped") + + def _publish_processing_status( + self, + message_id: str, + status: str, + *, + activity: str | None = None, + tool_name: str | None = None, + progress: dict[str, Any] | None = None, + detail: dict[str, Any] | None = None, + reason: str | None = None, + error_message: str | None = None, + retry_after_seconds: int | None = None, + parent_message_id: str | None = None, + ) -> None: + if not self._send_client: + return + try: + self._send_client.set_agent_processing_status( + message_id, + status, + agent_name=self.name, + space_id=self.space_id, + activity=activity, + tool_name=tool_name, + progress=progress, + detail=detail, + reason=reason, + error_message=error_message, + retry_after_seconds=retry_after_seconds, + parent_message_id=parent_message_id, + ) + except Exception: + pass + + @staticmethod + def _processing_status_metadata(event: dict[str, Any]) -> dict[str, Any]: + progress = event.get("progress") if isinstance(event.get("progress"), dict) else None + detail = event.get("detail") if isinstance(event.get("detail"), dict) else None + if detail is None and isinstance(event.get("initial_data"), dict): + detail = event.get("initial_data") + reason = str(event.get("reason") or "").strip() or None + error_message = str(event.get("error_message") or "").strip() or None + parent_message_id = str(event.get("parent_message_id") or "").strip() or None + + retry_after_seconds = None + retry_after_raw = event.get("retry_after_seconds") + if retry_after_raw is not None: + try: + retry_after_seconds = int(retry_after_raw) + except (TypeError, ValueError): + retry_after_seconds = None + + return { + "progress": progress, + "detail": detail, + "reason": reason, + "error_message": error_message, + "retry_after_seconds": retry_after_seconds, + "parent_message_id": parent_message_id, + } + + def _record_tool_call(self, *, message_id: str, event: dict[str, Any]) -> None: + if not self._send_client: + return + tool_name = str(event.get("tool_name") or event.get("tool") or "").strip() + if not tool_name: + return + tool_call_id = str(event.get("tool_call_id") or uuid.uuid4()) + arguments = event.get("arguments") if isinstance(event.get("arguments"), dict) else None + initial_data = event.get("initial_data") if isinstance(event.get("initial_data"), dict) else None + duration_raw = event.get("duration_ms") + try: + duration_ms = int(duration_raw) if duration_raw is not None else None + except (TypeError, ValueError): + duration_ms = None + try: + self._send_client.record_tool_call( + tool_name=tool_name, + tool_call_id=tool_call_id, + space_id=self.space_id, + tool_action=str(event.get("tool_action") or event.get("tool_action_name") or event.get("command") or "") or None, + resource_uri=str(event.get("resource_uri") or "ui://gateway/tool-call"), + arguments_hash=_hash_tool_arguments(arguments), + kind=str(event.get("kind_name") or event.get("result_kind") or "gateway_runtime"), + arguments=arguments, + initial_data=initial_data, + status=str(event.get("status") or "success"), + duration_ms=duration_ms, + agent_name=self.name, + agent_id=self.agent_id, + message_id=message_id, + correlation_id=str(event.get("correlation_id") or message_id), + ) + record_gateway_activity( + "tool_call_recorded", + entry=self.entry, + message_id=message_id, + tool_name=tool_name, + tool_call_id=tool_call_id, + ) + except Exception as exc: + record_gateway_activity( + "tool_call_record_failed", + entry=self.entry, + message_id=message_id, + tool_name=tool_name, + tool_call_id=tool_call_id, + error=str(exc)[:400], + ) + + def _handle_exec_event(self, event: dict[str, Any], *, message_id: str) -> None: + kind = str(event.get("kind") or event.get("type") or "").strip().lower() + if not kind: + return + if kind == "status": + status = str(event.get("status") or "processing").strip() + if status == "completed": + self._mark_completed_seen(message_id) + activity = str(event.get("message") or event.get("activity") or "").strip() or None + tool_name = str(event.get("tool") or event.get("tool_name") or "").strip() or None + metadata = self._processing_status_metadata(event) + updates: dict[str, Any] = {} + updates["current_status"] = status + if activity is not None: + updates["current_activity"] = activity[:240] + if tool_name is not None: + updates["current_tool"] = tool_name[:120] + if status == "completed": + updates["current_status"] = None + updates.setdefault("current_activity", None) + updates.setdefault("current_tool", None) + updates["current_tool_call_id"] = None + if updates: + self._update_state(**updates) + if message_id: + self._publish_processing_status( + message_id, + status, + activity=activity, + tool_name=tool_name, + **metadata, + ) + record_gateway_activity( + "runtime_status", + entry=self.entry, + message_id=message_id, + status=status, + activity_message=activity, + tool_name=tool_name, + ) + return + + if kind == "tool_start": + tool_name = str(event.get("tool_name") or event.get("tool") or "tool").strip() + tool_call_id = str(event.get("tool_call_id") or uuid.uuid4()) + activity = str(event.get("message") or f"Using {tool_name}").strip() + status = str(event.get("status") or "tool_call").strip() + metadata = self._processing_status_metadata(event) + self._update_state( + current_status=status, + current_activity=activity[:240], + current_tool=tool_name[:120] or None, + current_tool_call_id=tool_call_id, + ) + if message_id: + self._publish_processing_status( + message_id, + status, + activity=activity, + tool_name=tool_name or None, + **metadata, + ) + record_gateway_activity( + "tool_started", + entry=self.entry, + message_id=message_id, + tool_name=tool_name, + tool_call_id=tool_call_id, + tool_action=str(event.get("tool_action") or event.get("command") or "") or None, + ) + return + + if kind == "tool_result": + tool_name = str(event.get("tool_name") or event.get("tool") or "tool").strip() + tool_call_id = str(event.get("tool_call_id") or uuid.uuid4()) + status = str(event.get("status") or "success").strip() + metadata = self._processing_status_metadata(event) + self._record_tool_call(message_id=message_id, event=event) + step_status = "tool_complete" if status.lower() in {"success", "completed", "ok", "tool_complete"} else "error" + self._update_state( + current_status=None if step_status == "tool_complete" else step_status, + current_activity=None, + current_tool=None, + current_tool_call_id=None, + ) + if message_id: + self._publish_processing_status( + message_id, + step_status, + tool_name=tool_name or None, + detail=metadata["detail"], + reason=metadata["reason"] or (None if step_status == "tool_complete" else status), + error_message=metadata["error_message"], + retry_after_seconds=metadata["retry_after_seconds"], + parent_message_id=metadata["parent_message_id"], + ) + record_gateway_activity( + "tool_finished", + entry=self.entry, + message_id=message_id, + tool_name=tool_name, + tool_call_id=tool_call_id, + status=status, + ) + return + + if kind == "activity": + activity = str(event.get("message") or event.get("activity") or "").strip() + if activity: + self._update_state(current_activity=activity[:240]) + record_gateway_activity( + "runtime_activity", + entry=self.entry, + message_id=message_id, + activity_message=activity or None, + ) + + def _handle_prompt(self, prompt: str, *, message_id: str) -> str: + runtime_type = str(self.entry.get("runtime_type") or "echo").lower() + if runtime_type == "echo": + return _echo_handler(prompt, self.entry) + if runtime_type in {"inbox", "passive", "monitor"}: + return "" + if runtime_type in {"exec", "command"}: + command = str(self.entry.get("exec_command") or "").strip() + if not command: + raise ValueError("exec runtime requires exec_command") + return _run_exec_handler( + command, + prompt, + self.entry, + message_id=message_id or None, + space_id=self.space_id, + on_event=lambda event: self._handle_exec_event(event, message_id=message_id), + ) + raise ValueError(f"Unsupported runtime_type: {runtime_type}") + + def _gateway_message_metadata(self, parent_message_id: str | None = None) -> dict[str, Any]: + registry = load_gateway_registry() + gateway = registry.get("gateway", {}) + metadata: dict[str, Any] = { + "control_plane": "gateway", + "gateway": { + "managed": True, + "gateway_id": gateway.get("gateway_id"), + "agent_name": self.name, + "agent_id": self.agent_id, + "runtime_type": self.entry.get("runtime_type"), + "transport": self.entry.get("transport", "gateway"), + "credential_source": self.entry.get("credential_source", "gateway"), + }, + } + if parent_message_id: + metadata["gateway"]["parent_message_id"] = parent_message_id + return metadata + + def _worker_loop(self) -> None: + while not self.stop_event.is_set(): + try: + data = self._queue.get(timeout=0.5) + except queue.Empty: + continue + if data is None: + break + + message_id = str(data.get("id") or "") + prompt = _strip_mention(str(data.get("content") or ""), self.name) + self._update_state(backlog_depth=self._queue.qsize()) + if not prompt: + self._queue.task_done() + continue + + if message_id: + runtime_type = str(self.entry.get("runtime_type") or "echo").lower() + start_status = "processing" + start_activity = "Preparing response" + if runtime_type == "echo": + start_activity = "Composing echo reply" + elif runtime_type in {"exec", "command"}: + start_activity = "Preparing runtime" + if runtime_type in {"echo", "exec", "command"}: + self._update_state(current_status=start_status, current_activity=start_activity[:240]) + self._publish_processing_status(message_id, start_status, activity=start_activity) + record_gateway_activity( + "runtime_status", + entry=self.entry, + message_id=message_id, + status=start_status, + activity_message=start_activity, + ) + try: + response_text = self._handle_prompt(prompt, message_id=message_id) + if response_text and self._send_client: + result = self._send_client.send_message( + self.space_id, + response_text, + agent_id=self.agent_id, + parent_id=message_id or None, + metadata=self._gateway_message_metadata(message_id or None), + ) + message = result.get("message", result) if isinstance(result, dict) else {} + _remember_reply_anchor(self._reply_anchor_ids, message.get("id")) + reply_id = message.get("id") + preview = response_text.strip().replace("\n", " ") + if len(preview) > 120: + preview = preview[:117] + "..." + self._update_state(last_reply_message_id=reply_id, last_reply_preview=preview or None) + record_gateway_activity( + "reply_sent", + entry=self.entry, + message_id=message_id or None, + reply_message_id=reply_id, + reply_preview=preview or None, + ) + runtime_type = str(self.entry.get("runtime_type") or "echo").lower() + bridge_already_closed = runtime_type in {"exec", "command"} and self._consume_completed_seen(message_id) + if message_id and not bridge_already_closed: + self._publish_processing_status(message_id, "completed") + self._bump("processed_count") + self._update_state( + current_status=None, + current_activity=None, + current_tool=None, + current_tool_call_id=None, + last_error=None, + last_work_completed_at=_now_iso(), + backlog_depth=self._queue.qsize(), + ) + except Exception as exc: + self._update_state( + current_status="error", + current_activity=None, + current_tool=None, + current_tool_call_id=None, + last_error=str(exc)[:400], + backlog_depth=self._queue.qsize(), + ) + if message_id: + self._publish_processing_status( + message_id, + "error", + error_message=str(exc)[:400], + ) + record_gateway_activity( + "runtime_error", + entry=self.entry, + message_id=message_id or None, + error=str(exc)[:400], + ) + self._log(f"worker error: {exc}") + finally: + self._queue.task_done() + + def _listener_loop(self) -> None: + backoff = 1.0 + while not self.stop_event.is_set(): + try: + self._stream_client = self._new_client() + self._send_client = self._new_client() + timeout = httpx.Timeout( + connect=10.0, + read=SSE_IDLE_TIMEOUT_SECONDS, + write=10.0, + pool=10.0, + ) + reconnected = backoff > 1.0 + with self._stream_client.connect_sse(space_id=self.space_id, timeout=timeout) as response: + self._stream_response = response + if response.status_code != 200: + raise ConnectionError(f"SSE failed: {response.status_code}") + self._update_state( + effective_state="running", + current_status=None, + last_error=None, + last_connected_at=_now_iso(), + last_listener_error_at=None, + last_seen_at=_now_iso(), + reconnect_backoff_seconds=0, + ) + record_gateway_activity("listener_connected", entry=self.entry, reconnected=reconnected) + backoff = 1.0 + for event_type, data in _iter_sse(response): + if self.stop_event.is_set(): + break + if event_type in {"bootstrap", "heartbeat", "ping", "identity_bootstrap", "connected"}: + self._update_state(last_seen_at=_now_iso()) + continue + if event_type not in {"message", "mention"} or not isinstance(data, dict): + continue + message_id = str(data.get("id") or "") + if not message_id or message_id in self._seen_ids: + continue + if _is_self_authored(data, self.name, self.agent_id): + _remember_reply_anchor(self._reply_anchor_ids, message_id) + self._seen_ids.add(message_id) + continue + if not _should_respond( + data, + self.name, + self.agent_id, + reply_anchor_ids=self._reply_anchor_ids, + ): + continue + + self._seen_ids.add(message_id) + if len(self._seen_ids) > SEEN_IDS_MAX: + self._seen_ids = set(list(self._seen_ids)[-SEEN_IDS_MAX // 2 :]) + _remember_reply_anchor(self._reply_anchor_ids, message_id) + self._update_state( + last_seen_at=_now_iso(), + last_work_received_at=_now_iso(), + last_received_message_id=message_id, + ) + record_gateway_activity("message_received", entry=self.entry, message_id=message_id) + try: + self._queue.put_nowait(data) + backlog_depth = self._queue.qsize() + runtime_type = str(self.entry.get("runtime_type") or "").lower() + pickup_status = "queued" if _is_passive_runtime(runtime_type) else "started" + accepted_activity = "Queued in Gateway" + if not _is_passive_runtime(runtime_type): + accepted_activity = "Picked up by Gateway" + if backlog_depth > 1: + if _is_passive_runtime(runtime_type): + accepted_activity = f"Queued in Gateway ({backlog_depth} pending)" + else: + accepted_activity = f"Picked up by Gateway ({backlog_depth} pending)" + self._update_state( + backlog_depth=backlog_depth, + current_status=pickup_status, + current_activity=accepted_activity[:240], + ) + self._publish_processing_status( + message_id, + pickup_status, + activity=accepted_activity, + detail={ + "backlog_depth": backlog_depth, + "pickup_state": "queued" if _is_passive_runtime(runtime_type) else "claimed", + }, + ) + if _is_passive_runtime(self.entry.get("runtime_type")): + record_gateway_activity( + "message_queued", + entry=self.entry, + message_id=message_id, + backlog_depth=backlog_depth, + ) + else: + record_gateway_activity( + "message_claimed", + entry=self.entry, + message_id=message_id, + backlog_depth=backlog_depth, + ) + except queue.Full: + self._bump("dropped_count") + self._update_state(last_error="queue full", backlog_depth=self._queue.qsize()) + self._publish_processing_status( + message_id, + "error", + reason="queue_full", + error_message="Gateway queue full", + ) + record_gateway_activity( + "message_dropped", + entry=self.entry, + message_id=message_id, + error="queue full", + ) + self._log("queue full; dropped message") + except Exception as exc: + if self.stop_event.is_set(): + break + error_text = str(exc)[:400] + event_name = "listener_error" + if isinstance(exc, httpx.ReadTimeout): + error_text = f"idle timeout after {int(SSE_IDLE_TIMEOUT_SECONDS)}s without SSE heartbeat" + event_name = "listener_timeout" + self._update_state( + effective_state="reconnecting", + last_error=error_text, + last_listener_error_at=_now_iso(), + reconnect_backoff_seconds=int(backoff), + ) + record_gateway_activity(event_name, entry=self.entry, error=error_text, reconnect_in_seconds=int(backoff)) + self._log(f"listener error: {error_text}") + time.sleep(backoff) + backoff = min(backoff * 2, 30.0) + finally: + self._stream_response = None + if self._stream_client is not None: + try: + self._stream_client.close() + except Exception: + pass + self._stream_client = None + self._update_state( + effective_state="stopped", + backlog_depth=self._queue.qsize(), + current_status=None, + current_activity=None, + current_tool=None, + current_tool_call_id=None, + ) + + +class GatewayDaemon: + """Foreground Gateway supervisor.""" + + def __init__( + self, + *, + client_factory: Callable[..., Any] = AxClient, + logger: RuntimeLogger | None = None, + poll_interval: float = 1.0, + ) -> None: + self.client_factory = client_factory + self.logger = logger or (lambda _msg: None) + self.poll_interval = poll_interval + self._runtimes: dict[str, ManagedAgentRuntime] = {} + self._stop = threading.Event() + + def _log(self, message: str) -> None: + self.logger(message) + + def stop(self) -> None: + self._stop.set() + + def _reconcile_runtime(self, entry: dict[str, Any]) -> None: + name = str(entry.get("name") or "") + desired_state = str(entry.get("desired_state") or "stopped").lower() + runtime = self._runtimes.get(name) + if desired_state == "running": + if runtime is None: + runtime = ManagedAgentRuntime(entry, client_factory=self.client_factory, logger=self.logger) + self._runtimes[name] = runtime + runtime.start() + else: + runtime.entry.update(entry) + runtime.start() + else: + if runtime is not None: + runtime.stop() + self._runtimes.pop(name, None) + + def _reconcile_registry(self, registry: dict[str, Any], session: dict[str, Any]) -> dict[str, Any]: + agents = registry.setdefault("agents", []) + agent_names = {str(entry.get("name") or "") for entry in agents} + for name, runtime in list(self._runtimes.items()): + if name not in agent_names: + runtime.stop() + self._runtimes.pop(name, None) + + for entry in agents: + entry.setdefault("transport", "gateway") + entry.setdefault("credential_source", "gateway") + entry.setdefault("runtime_type", "echo") + entry.setdefault("desired_state", "stopped") + self._reconcile_runtime(entry) + runtime = self._runtimes.get(str(entry.get("name") or "")) + snapshot = runtime.snapshot() if runtime is not None else annotate_runtime_health({"effective_state": "stopped"}) + entry.update(snapshot) + + gateway = registry.setdefault("gateway", {}) + gateway.update( + { + "desired_state": "running", + "effective_state": "running" if session else "stopped", + "session_connected": bool(session), + "pid": os.getpid(), + "last_started_at": gateway.get("last_started_at") or _now_iso(), + "last_reconcile_at": _now_iso(), + } + ) + return registry + + def run(self, *, once: bool = False) -> None: + session = load_gateway_session() + if not session: + raise RuntimeError("Gateway login required. Run `ax gateway login` first.") + + existing_pids = active_gateway_pids() + if existing_pids: + existing_pid = existing_pids[0] + record_gateway_activity( + "gateway_start_blocked", + pid=os.getpid(), + existing_pid=existing_pid, + existing_pids=existing_pids, + ) + raise RuntimeError(f"Gateway already running (pid {existing_pid}).") + + write_gateway_pid(os.getpid()) + registry = load_gateway_registry() + registry.setdefault("gateway", {}) + registry["gateway"]["last_started_at"] = registry["gateway"].get("last_started_at") or _now_iso() + record_gateway_activity("gateway_started", pid=os.getpid()) + try: + while not self._stop.is_set(): + registry = load_gateway_registry() + registry = self._reconcile_registry(registry, session) + save_gateway_registry(registry) + if once: + break + time.sleep(self.poll_interval) + finally: + for runtime in list(self._runtimes.values()): + runtime.stop() + final_registry = load_gateway_registry() + final_gateway = final_registry.setdefault("gateway", {}) + final_gateway.update( + { + "desired_state": final_gateway.get("desired_state") or "stopped", + "effective_state": "stopped", + "session_connected": bool(session), + "pid": None, + "last_reconcile_at": _now_iso(), + } + ) + for entry in final_registry.get("agents", []): + name = str(entry.get("name") or "") + entry.update({"effective_state": "stopped", "backlog_depth": 0}) + runtime = self._runtimes.get(name) + if runtime is not None: + entry.update(runtime.snapshot()) + save_gateway_registry(final_registry) + record_gateway_activity("gateway_stopped") + clear_gateway_pid(os.getpid()) diff --git a/ax_cli/main.py b/ax_cli/main.py index a7fe3c3..8bb31dc 100644 --- a/ax_cli/main.py +++ b/ax_cli/main.py @@ -16,6 +16,7 @@ context, credentials, events, + gateway, handoff, keys, listen, @@ -42,6 +43,7 @@ app.add_typer(tasks.app, name="tasks") app.add_typer(events.app, name="events") app.add_typer(listen.app, name="listen") +app.add_typer(gateway.app, name="gateway") app.add_typer(context.app, name="context") app.add_typer(watch.app, name="watch") app.add_typer(upload.app, name="upload") diff --git a/examples/codex_gateway/codex_bridge.py b/examples/codex_gateway/codex_bridge.py new file mode 100644 index 0000000..2f51c9e --- /dev/null +++ b/examples/codex_gateway/codex_bridge.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""codex_bridge.py — Gateway-managed bridge for the Codex CLI. + +This bridge is designed for `ax gateway agents add ... --type exec`. +It converts `codex exec --json` events into lightweight Gateway progress +events so the Gateway can publish agent-processing and tool-call activity +back to aX while Codex is still working. + +Usage example: + + ax gateway agents add codex \ + --type exec \ + --exec "python3 examples/codex_gateway/codex_bridge.py" \ + --workdir /absolute/path/to/repo +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import time +import uuid +from typing import Any + +EVENT_PREFIX = "AX_GATEWAY_EVENT " +DEFAULT_MODEL = os.environ.get("CODEX_GATEWAY_MODEL", "gpt-5.4") +DEFAULT_SANDBOX = os.environ.get("CODEX_GATEWAY_SANDBOX", "workspace-write") +MAX_SLEEP_SECONDS = 300 +SLEEP_RE = re.compile(r"\b(?:sleep|pause|wait)\s+(?:for\s+)?(\d+)\s*(?:seconds?|secs?|s)\b", re.IGNORECASE) +TIMER_RE = re.compile(r"\b(\d+)\s*(?:seconds?|secs?|s)\s+(?:timer|countdown)\b", re.IGNORECASE) +TIMER_FOR_RE = re.compile(r"\b(?:timer|countdown)\s+(?:for\s+)?(\d+)\s*(?:seconds?|secs?|s)\b", re.IGNORECASE) + + +def emit_event(payload: dict[str, Any]) -> None: + print(f"{EVENT_PREFIX}{json.dumps(payload, sort_keys=True)}", flush=True) + + +def _read_prompt() -> str: + if len(sys.argv) > 1 and sys.argv[-1] != "-": + return sys.argv[-1] + env_prompt = os.environ.get("AX_MENTION_CONTENT", "").strip() + if env_prompt: + return env_prompt + stdin_text = sys.stdin.read().strip() + return stdin_text + + +def _sleep_demo_seconds(prompt: str) -> int | None: + for regex in (SLEEP_RE, TIMER_RE, TIMER_FOR_RE): + match = regex.search(prompt) + if not match: + continue + seconds = int(match.group(1)) + if 0 < seconds <= MAX_SLEEP_SECONDS: + return seconds + return None + + +def _run_sleep_demo(seconds: int) -> int: + tool_call_id = f"sleep-{uuid.uuid4()}" + start = time.monotonic() + emit_event({"kind": "status", "status": "thinking", "message": f"Planning sleep for {seconds}s"}) + emit_event({"kind": "status", "status": "processing", "message": f"Sleeping for {seconds}s"}) + emit_event( + { + "kind": "tool_start", + "tool_name": "sleep", + "tool_action": "sleep", + "status": "tool_call", + "tool_call_id": tool_call_id, + "arguments": {"seconds": seconds}, + "message": f"Sleeping for {seconds}s", + } + ) + deadline = time.monotonic() + seconds + while True: + remaining = max(0, int(round(deadline - time.monotonic()))) + if remaining <= 0: + break + emit_event( + { + "kind": "activity", + "activity": f"Sleeping... {remaining}s remaining", + } + ) + time.sleep(min(5, remaining)) + emit_event( + { + "kind": "tool_result", + "tool_name": "sleep", + "tool_action": "sleep", + "tool_call_id": tool_call_id, + "arguments": {"seconds": seconds}, + "initial_data": {"slept_seconds": seconds}, + "status": "tool_complete", + "duration_ms": int((time.monotonic() - start) * 1000), + } + ) + emit_event({"kind": "status", "status": "completed"}) + print(f"Paused for {seconds} seconds and I am back.") + return 0 + + +def _codex_command(prompt: str) -> list[str]: + workdir = os.environ.get("CODEX_GATEWAY_WORKDIR", os.getcwd()) + system_prompt = os.environ.get( + "CODEX_GATEWAY_SYSTEM_PROMPT", + "You are Codex running as a Gateway-managed aX agent. " + "Be concise and helpful. Keep replies under 2000 characters unless the task truly needs more detail. " + "When useful, inspect the local workspace and use tools.", + ).strip() + full_prompt = f"{system_prompt}\n\nUser message:\n{prompt.strip()}" + return [ + "codex", + "exec", + "--json", + "--color", + "never", + "--skip-git-repo-check", + "--sandbox", + DEFAULT_SANDBOX, + "--model", + DEFAULT_MODEL, + "--cd", + workdir, + full_prompt, + ] + + +def _tool_event_payload(item: dict[str, Any], *, phase: str) -> tuple[str, dict[str, Any]]: + item_type = str(item.get("type") or "tool") + item_id = str(item.get("id") or uuid.uuid4()) + if item_type == "command_execution": + command = str(item.get("command") or "").strip() + arguments = {"command": command} if command else None + initial_data: dict[str, Any] = {} + if item.get("aggregated_output"): + initial_data["output"] = str(item.get("aggregated_output"))[:4000] + if item.get("exit_code") is not None: + initial_data["exit_code"] = item.get("exit_code") + payload = { + "tool_name": "shell", + "tool_action": command or "command_execution", + "tool_call_id": item_id, + "arguments": arguments, + "initial_data": initial_data or None, + "message": f"Running command: {command}" if command else "Running command", + "status": ( + "tool_call" + if phase == "start" + else ("tool_complete" if int(item.get("exit_code") or 0) == 0 else "error") + ), + } + return item_type, payload + payload = { + "tool_name": item_type, + "tool_action": str(item.get("title") or item_type), + "tool_call_id": item_id, + "initial_data": {"item": item}, + "message": f"Using {item_type}", + "status": "tool_call" if phase == "start" else "tool_complete", + } + return item_type, payload + + +def _run_codex(prompt: str) -> int: + cmd = _codex_command(prompt) + final_text = "" + stderr_lines: list[str] = [] + + emit_event({"kind": "status", "status": "thinking", "message": "Starting Codex"}) + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + assert process.stdout is not None + assert process.stderr is not None + + for raw in process.stdout: + line = raw.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(payload, dict): + continue + event_type = str(payload.get("type") or "") + item = payload.get("item") if isinstance(payload.get("item"), dict) else None + if event_type == "item.started" and item is not None and str(item.get("type") or "") != "agent_message": + _, tool_payload = _tool_event_payload(item, phase="start") + emit_event({"kind": "tool_start", **tool_payload}) + continue + if event_type == "item.completed" and item is not None: + item_type = str(item.get("type") or "") + if item_type == "agent_message": + text = str(item.get("text") or "").strip() + if text: + final_text = text + continue + _, tool_payload = _tool_event_payload(item, phase="result") + emit_event({"kind": "tool_result", **tool_payload}) + continue + if event_type == "turn.completed": + emit_event({"kind": "status", "status": "completed"}) + + stderr_lines.extend(process.stderr.readlines()) + return_code = process.wait() + stderr_text = "\n".join(line.strip() for line in stderr_lines if line.strip()) + if return_code != 0: + if stderr_text: + print(f"Codex bridge failed:\n{stderr_text[:2000]}") + else: + print(f"Codex bridge failed with exit code {return_code}.") + return return_code + + if final_text: + print(final_text) + else: + print("Codex finished without a final reply.") + return 0 + + +def main() -> int: + prompt = _read_prompt() + if not prompt: + print("(no mention content received)", file=sys.stderr) + return 1 + + sleep_seconds = _sleep_demo_seconds(prompt) + if sleep_seconds is not None: + return _run_sleep_demo(sleep_seconds) + + return _run_codex(prompt) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/gateway_probe/probe_bridge.py b/examples/gateway_probe/probe_bridge.py new file mode 100644 index 0000000..cd24e31 --- /dev/null +++ b/examples/gateway_probe/probe_bridge.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""probe_bridge.py — deterministic Gateway-managed probe runtime. + +This bridge is intentionally boring. It does not call Codex or any external +tooling. Instead, it emits a fixed sequence of Gateway status/tool/activity +events so we can test the message monitor with a predictable trace. + +Usage example: + + ax gateway agents add gateway-probe \ + --type exec \ + --exec "python3 examples/gateway_probe/probe_bridge.py" \ + --workdir /absolute/path/to/repo + +Example prompts: + + @gateway-probe probe + @gateway-probe probe 6 + @gateway-probe run a 10 second probe +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import time +import uuid +from typing import Any + +EVENT_PREFIX = "AX_GATEWAY_EVENT " +DEFAULT_SECONDS = 6 +MAX_SECONDS = 60 +SECONDS_RE = re.compile(r"\b(\d+)\s*(?:seconds?|secs?|s)?\b", re.IGNORECASE) +PROBE_RE = re.compile(r"\bprobe\b", re.IGNORECASE) + + +def emit_event(payload: dict[str, Any]) -> None: + print(f"{EVENT_PREFIX}{json.dumps(payload, sort_keys=True)}", flush=True) + + +def _read_prompt() -> str: + if len(sys.argv) > 1 and sys.argv[-1] != "-": + return sys.argv[-1] + env_prompt = os.environ.get("AX_MENTION_CONTENT", "").strip() + if env_prompt: + return env_prompt + return sys.stdin.read().strip() + + +def _probe_seconds(prompt: str) -> int: + match = SECONDS_RE.search(prompt) + if not match: + return DEFAULT_SECONDS + seconds = int(match.group(1)) + if seconds < 1: + return 1 + return min(MAX_SECONDS, seconds) + + +def _is_probe_prompt(prompt: str) -> bool: + if not prompt.strip(): + return True + return bool(PROBE_RE.search(prompt)) + + +def _run_probe(seconds: int) -> int: + tool_call_id = f"probe-sleep-{uuid.uuid4()}" + start = time.monotonic() + emit_event({"kind": "status", "status": "started", "message": "Probe accepted"}) + emit_event({"kind": "status", "status": "thinking", "message": f"Probe planning {seconds}s run"}) + emit_event({"kind": "status", "status": "processing", "message": f"Probe sleeping for {seconds}s"}) + emit_event( + { + "kind": "tool_start", + "tool_name": "probe_sleep", + "tool_action": "sleep", + "tool_call_id": tool_call_id, + "status": "tool_call", + "arguments": {"seconds": seconds}, + "message": f"Probe sleeping for {seconds}s", + } + ) + + for remaining in range(seconds, 0, -1): + emit_event( + { + "kind": "activity", + "activity": f"Probe tick {seconds - remaining + 1}/{seconds} ({remaining}s left)", + } + ) + time.sleep(1) + + emit_event( + { + "kind": "tool_result", + "tool_name": "probe_sleep", + "tool_action": "sleep", + "tool_call_id": tool_call_id, + "arguments": {"seconds": seconds}, + "initial_data": {"slept_seconds": seconds, "probe": True}, + "status": "tool_complete", + "duration_ms": int((time.monotonic() - start) * 1000), + "message": "Probe sleep finished", + } + ) + emit_event({"kind": "status", "status": "completed", "message": "Probe complete"}) + print(f"PROBE_OK seconds={seconds}") + return 0 + + +def main() -> int: + prompt = _read_prompt() + if not _is_probe_prompt(prompt): + print( + "PROBE_ERROR unsupported prompt. Use 'probe' optionally followed by a duration, for example 'probe 6'." + ) + return 1 + return _run_probe(_probe_seconds(prompt)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_auth_commands.py b/tests/test_auth_commands.py index 3dae0f4..6f1110b 100644 --- a/tests/test_auth_commands.py +++ b/tests/test_auth_commands.py @@ -232,6 +232,7 @@ def test_auth_doctor_json_outputs_diagnostics(monkeypatch): "ok": True, "selected_env": env_name, "selected_profile": None, + "runtime_config": "/tmp/codex/.ax/config.toml", "effective": { "auth_source": "user_login:dev", "token_kind": "user_pat", @@ -267,3 +268,29 @@ def test_auth_doctor_json_outputs_diagnostics(monkeypatch): assert payload["details"] == [] assert payload["effective"]["auth_source"] == "user_login:dev" assert payload["effective"]["space_id"] == "space-1" + + +def test_auth_whoami_reports_runtime_config(monkeypatch, tmp_path): + runtime_config = tmp_path / "runtime-config.toml" + runtime_config.write_text("") + monkeypatch.setenv("AX_CONFIG_FILE", str(runtime_config)) + + class FakeClient: + def whoami(self): + return { + "id": "user-1", + "bound_agent": { + "default_space_id": "space-1", + }, + } + + monkeypatch.setattr(auth, "get_client", lambda: FakeClient()) + monkeypatch.setattr(auth, "resolve_agent_name", lambda *, client: "codex") + monkeypatch.setattr(auth, "_local_config_dir", lambda: None) + + result = runner.invoke(app, ["auth", "whoami", "--json"]) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["runtime_config"] == str(runtime_config) + assert payload["resolved_agent"] == "codex" diff --git a/tests/test_channel.py b/tests/test_channel.py index 0325101..d84fa91 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -538,7 +538,7 @@ def test_listener_treats_parent_reply_as_delivery_signal(): "id": "reply-1", "content": "I looked at this", "parent_id": "agent-message-1", - "author": {"id": "other-agent", "name": "demo-agent", "type": "agent"}, + "author": {"id": "user-1", "name": "Jacob", "type": "user"}, "mentions": [], } @@ -551,13 +551,52 @@ def test_listener_treats_conversation_reply_as_delivery_signal(): "id": "reply-1", "content": "I looked at this", "conversation_id": "agent-message-1", + "author": {"id": "user-1", "name": "Jacob", "type": "user"}, + "mentions": [], + } + + assert _should_respond(data, "peer-agent", "agent-123", reply_anchor_ids=anchors) is True + + +def test_listener_does_not_auto_reply_to_other_agent_thread_reply_without_mention(): + anchors = {"agent-message-1"} + data = { + "id": "reply-1", + "content": "I looked at this", + "parent_id": "agent-message-1", "author": {"id": "other-agent", "name": "demo-agent", "type": "agent"}, "mentions": [], } + assert _should_respond(data, "peer-agent", "agent-123", reply_anchor_ids=anchors) is False + + +def test_listener_still_replies_to_other_agent_thread_reply_when_explicitly_mentioned(): + anchors = {"agent-message-1"} + data = { + "id": "reply-1", + "content": "@peer-agent I looked at this", + "parent_id": "agent-message-1", + "author": {"id": "other-agent", "name": "demo-agent", "type": "agent"}, + "mentions": ["peer-agent"], + } + assert _should_respond(data, "peer-agent", "agent-123", reply_anchor_ids=anchors) is True +def test_listener_ignores_thread_parent_mentions_from_other_agents(): + anchors = {"agent-message-1"} + data = { + "id": "reply-1", + "content": "continuing the thread", + "parent_id": "agent-message-1", + "sender_type": "agent", + "mentions": [{"agent_name": "peer-agent", "source": "thread_parent"}], + } + + assert _should_respond(data, "peer-agent", "agent-123", reply_anchor_ids=anchors) is False + + def test_listener_tracks_self_authored_messages_without_responding(): anchors: set[str] = set() data = { diff --git a/tests/test_client.py b/tests/test_client.py index 13505c6..fdbd712 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -217,6 +217,123 @@ def test_mark_all_messages_read_calls_backend_endpoint(): assert client._http.post.call_args.args[0] == "/api/v1/messages/mark-all-read" +def test_record_tool_call_posts_audit_payload(): + client = AxClient("https://example.com", "legacy-token", agent_id="agent-123", agent_name="codex") + response = httpx.Response( + 202, + json={"ok": True, "tool_call_id": "tool-1"}, + request=httpx.Request("POST", "https://example.com/api/v1/tool-calls"), + ) + client._http.post = MagicMock(return_value=response) + + result = client.record_tool_call( + tool_name="shell", + tool_call_id="tool-1", + space_id="space-123", + tool_action="wc -c README.md", + arguments={"command": "wc -c README.md"}, + initial_data={"output": "28358 README.md"}, + status="success", + message_id="msg-1", + correlation_id="msg-1", + ) + + assert result["tool_call_id"] == "tool-1" + assert client._http.post.call_args.args[0] == "/api/v1/tool-calls" + assert client._http.post.call_args.kwargs["json"] == { + "tool_name": "shell", + "tool_call_id": "tool-1", + "status": "success", + "space_id": "space-123", + "tool_action": "wc -c README.md", + "arguments": {"command": "wc -c README.md"}, + "initial_data": {"output": "28358 README.md"}, + "message_id": "msg-1", + "correlation_id": "msg-1", + } + + +def test_set_agent_processing_status_includes_optional_fields(): + client = AxClient("https://example.com", "legacy-token", agent_id="agent-123", agent_name="codex") + response = httpx.Response( + 200, + json={"ok": True, "event": "agent_processing", "status": "processing"}, + request=httpx.Request("POST", "https://example.com/api/v1/agents/processing-status"), + ) + client._http.post = MagicMock(return_value=response) + + result = client.set_agent_processing_status( + "msg-1", + "processing", + agent_name="codex", + space_id="space-123", + activity="Running command", + tool_name="shell", + progress={"current": 1, "total": 3, "unit": "steps"}, + detail={"command": "pwd"}, + reason="gateway_runtime", + error_message=None, + retry_after_seconds=5, + parent_message_id="parent-1", + ) + + assert result["status"] == "processing" + assert client._http.post.call_args.args[0] == "/api/v1/agents/processing-status" + assert client._http.post.call_args.kwargs["json"] == { + "message_id": "msg-1", + "status": "processing", + "agent_name": "codex", + "activity": "Running command", + "tool_name": "shell", + "progress": {"current": 1, "total": 3, "unit": "steps"}, + "detail": {"command": "pwd"}, + "reason": "gateway_runtime", + "retry_after_seconds": 5, + "parent_message_id": "parent-1", + } + + +def test_set_agent_processing_status_posts_rich_payload(): + client = AxClient("https://example.com", "legacy-token", agent_id="agent-123", agent_name="codex") + response = httpx.Response( + 202, + json={"ok": True}, + request=httpx.Request("POST", "https://example.com/api/v1/agents/processing-status"), + ) + client._http.post = MagicMock(return_value=response) + + result = client.set_agent_processing_status( + "msg-1", + "tool_call", + agent_name="codex", + space_id="space-123", + activity="Running tests", + tool_name="shell", + progress={"current": 1, "total": 3, "unit": "steps"}, + detail={"command": "pytest tests/test_gateway_commands.py"}, + reason="tool started", + error_message="", + retry_after_seconds=5, + parent_message_id="parent-1", + ) + + assert result["ok"] is True + assert client._http.post.call_args.args[0] == "/api/v1/agents/processing-status" + assert client._http.post.call_args.kwargs["json"] == { + "message_id": "msg-1", + "status": "tool_call", + "agent_name": "codex", + "activity": "Running tests", + "tool_name": "shell", + "progress": {"current": 1, "total": 3, "unit": "steps"}, + "detail": {"command": "pytest tests/test_gateway_commands.py"}, + "reason": "tool started", + "error_message": "", + "retry_after_seconds": 5, + "parent_message_id": "parent-1", + } + + def test_list_tasks_passes_explicit_space_id(): client = AxClient("https://example.com", "legacy-token") response = httpx.Response( diff --git a/tests/test_codex_gateway_bridge.py b/tests/test_codex_gateway_bridge.py new file mode 100644 index 0000000..7c39edc --- /dev/null +++ b/tests/test_codex_gateway_bridge.py @@ -0,0 +1,20 @@ +import importlib.util +from pathlib import Path + + +def _load_bridge_module(): + bridge_path = Path(__file__).resolve().parents[1] / "examples" / "codex_gateway" / "codex_bridge.py" + spec = importlib.util.spec_from_file_location("codex_gateway_bridge", bridge_path) + module = importlib.util.module_from_spec(spec) + assert spec is not None and spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_sleep_demo_matches_timer_phrases(): + bridge = _load_bridge_module() + + assert bridge._sleep_demo_seconds("pause for 30 seconds") == 30 + assert bridge._sleep_demo_seconds("do a 30 second timer") == 30 + assert bridge._sleep_demo_seconds("start a 12 second countdown") == 12 + assert bridge._sleep_demo_seconds("timer for 9 seconds") == 9 diff --git a/tests/test_config.py b/tests/test_config.py index fed88dd..4fb34a6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -356,6 +356,46 @@ def test_explicit_env_vars_report_environment_sources(self, tmp_path, monkeypatc assert diagnostic["effective"]["host"] == "env.paxai.app" assert diagnostic["effective"]["principal_intent"] == "agent" + def test_explicit_runtime_config_reports_runtime_source(self, tmp_path, monkeypatch): + global_dir = tmp_path / "global" + global_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(global_dir)) + + local_ax = tmp_path / ".ax" + local_ax.mkdir() + (local_ax / "config.toml").write_text( + 'base_url = "https://next.paxai.app"\n' + 'agent_name = "night_owl"\n' + 'agent_id = "agent-night-owl"\n' + 'space_id = "night-space"\n' + ) + runtime_dir = tmp_path / "runtime" + runtime_dir.mkdir() + token_file = runtime_dir / "codex.pat" + token_file.write_text("axp_a_codex.secret") + runtime_config = runtime_dir / "config.toml" + runtime_config.write_text( + f'token_file = "{token_file.name}"\n' + 'base_url = "https://paxai.app"\n' + 'agent_name = "codex"\n' + 'agent_id = "agent-codex"\n' + 'space_id = "codex-space"\n' + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AX_CONFIG_FILE", str(runtime_config)) + + diagnostic = diagnose_auth_config() + + assert diagnostic["ok"] is True + assert diagnostic["runtime_config"] == str(runtime_config) + assert diagnostic["effective"]["auth_source"] == f"runtime_config:{runtime_config}" + assert diagnostic["effective"]["base_url_source"] == f"runtime_config:{runtime_config}" + assert diagnostic["effective"]["agent_name"] == "codex" + assert diagnostic["effective"]["space_id"] == "codex-space" + runtime_source = next(source for source in diagnostic["sources"] if source["name"] == "runtime_config") + assert runtime_source["used"] is True + assert runtime_source["path"] == str(runtime_config) + def test_unsafe_local_config_reports_ignored_reason_and_uses_profile(self, tmp_path, monkeypatch): global_dir = tmp_path / "global" global_dir.mkdir() diff --git a/tests/test_gateway_commands.py b/tests/test_gateway_commands.py new file mode 100644 index 0000000..868bd87 --- /dev/null +++ b/tests/test_gateway_commands.py @@ -0,0 +1,659 @@ +import json +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import httpx +from typer.testing import CliRunner + +from ax_cli import gateway as gateway_core +from ax_cli.commands import gateway as gateway_cmd +from ax_cli.main import app + +runner = CliRunner() + + +class _FakeTokenExchanger: + def __init__(self, base_url, token): + self.base_url = base_url + self.token = token + + def get_token(self, *args, **kwargs): + return "jwt-test" + + +class _FakeLoginClient: + def __init__(self, *args, **kwargs): + self.base_url = kwargs["base_url"] + self.token = kwargs["token"] + + def whoami(self): + return {"username": "madtank", "email": "madtank@example.com"} + + def list_spaces(self): + return {"spaces": [{"id": "space-1", "name": "Workspace", "is_default": True}]} + + +class _FakeUserClient: + def update_agent(self, *args, **kwargs): + return {"ok": True} + + +class _FakeSseResponse: + status_code = 200 + + def __init__(self, payload): + self.payload = payload + self.closed = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def close(self): + self.closed = True + + def iter_lines(self): + yield "event: connected" + yield "data: {}" + yield "" + yield "event: message" + yield f"data: {json.dumps(self.payload)}" + yield "" + + +class _SharedRuntimeClient: + def __init__(self, payload): + self.payload = payload + self.sent = [] + self.processing = [] + self.tool_calls = [] + self.connect_calls = 0 + + def connect_sse(self, *, space_id, timeout=None): + self.connect_calls += 1 + if self.connect_calls > 1: + raise ConnectionError("test done") + return _FakeSseResponse(self.payload) + + def send_message(self, space_id, content, *, agent_id=None, parent_id=None, **kwargs): + self.sent.append( + { + "space_id": space_id, + "content": content, + "agent_id": agent_id, + "parent_id": parent_id, + "metadata": kwargs.get("metadata"), + } + ) + return {"message": {"id": "reply-1"}} + + def set_agent_processing_status(self, message_id, status, *, agent_name=None, space_id=None, **kwargs): + payload = { + "message_id": message_id, + "status": status, + "agent_name": agent_name, + "space_id": space_id, + } + payload.update(kwargs) + self.processing.append(payload) + return {"ok": True} + + def record_tool_call(self, **payload): + self.tool_calls.append(payload) + return {"ok": True, "tool_call_id": payload["tool_call_id"]} + + def close(self): + return None + + +class _FakeManagedSendClient: + def __init__(self, *args, **kwargs): + self.base_url = kwargs["base_url"] + self.token = kwargs["token"] + self.agent_name = kwargs.get("agent_name") + self.agent_id = kwargs.get("agent_id") + + def send_message(self, space_id, content, *, agent_id=None, parent_id=None, metadata=None): + return { + "message": { + "id": "msg-sent-1", + "space_id": space_id, + "content": content, + "agent_id": agent_id, + "parent_id": parent_id, + "metadata": metadata, + } + } + + +def test_gateway_login_saves_gateway_session(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + monkeypatch.setattr("ax_cli.token_cache.TokenExchanger", _FakeTokenExchanger) + monkeypatch.setattr(gateway_cmd, "AxClient", _FakeLoginClient) + + result = runner.invoke( + app, + ["gateway", "login", "--token", "axp_u_test.token", "--url", "https://paxai.app", "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["base_url"] == "https://paxai.app" + assert payload["space_id"] == "space-1" + session = gateway_core.load_gateway_session() + assert session["token"] == "axp_u_test.token" + assert session["base_url"] == "https://paxai.app" + assert not (config_dir / "user.toml").exists() + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] == "gateway_login" + assert recent[-1]["username"] == "madtank" + + +def test_gateway_run_refuses_second_live_daemon(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "madtank", + } + ) + gateway_core.write_gateway_pid(4242) + monkeypatch.setattr(gateway_core, "_pid_alive", lambda pid: pid == 4242) + + result = runner.invoke(app, ["gateway", "run", "--once"]) + + assert result.exit_code == 1, result.output + assert "Gateway already running (pid 4242)." in result.output + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] == "gateway_start_blocked" + assert recent[-1]["existing_pid"] == 4242 + + +def test_gateway_run_refuses_process_table_daemon_when_pid_file_missing(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "madtank", + } + ) + monkeypatch.setattr(gateway_core, "_scan_gateway_process_pids", lambda: [5514]) + + result = runner.invoke(app, ["gateway", "run", "--once"]) + + assert result.exit_code == 1, result.output + assert "Gateway already running (pid 5514)." in result.output + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] == "gateway_start_blocked" + assert recent[-1]["existing_pids"] == [5514] + + +def test_clear_gateway_pid_keeps_newer_owner(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.write_gateway_pid(22179) + + gateway_core.clear_gateway_pid(5514) + + assert gateway_core.pid_path().exists() + assert gateway_core.pid_path().read_text().strip() == "22179" + + +def test_scan_gateway_process_pids_ignores_current_parent_wrapper(monkeypatch): + monkeypatch.setattr(gateway_core.os, "getpid", lambda: 22179) + monkeypatch.setattr(gateway_core.os, "getppid", lambda: 22178) + monkeypatch.setattr(gateway_core, "_pid_alive", lambda pid: True) + monkeypatch.setattr( + gateway_core.subprocess, + "check_output", + lambda *args, **kwargs: "\n".join( + [ + "22178 uv run ax gateway run", + "22179 /Users/jacob/claude_home/ax-cli/.venv/bin/python3 /Users/jacob/claude_home/ax-cli/.venv/bin/ax gateway run", + "5514 /Users/jacob/claude_home/ax-cli/.venv/bin/python3 /Users/jacob/claude_home/ax-cli/.venv/bin/ax gateway run", + ] + ), + ) + + assert gateway_core._scan_gateway_process_pids() == [5514] + + +def test_gateway_agents_add_mints_token_and_writes_registry(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "madtank", + } + ) + monkeypatch.setattr(gateway_cmd, "_load_gateway_user_client", lambda: _FakeUserClient()) + monkeypatch.setattr(gateway_cmd, "_find_agent_in_space", lambda *args, **kwargs: None) + monkeypatch.setattr( + gateway_cmd, + "_create_agent_in_space", + lambda *args, **kwargs: {"id": "agent-1", "name": "echo-bot"}, + ) + monkeypatch.setattr(gateway_cmd, "_polish_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_mint_agent_pat", lambda *args, **kwargs: ("axp_a_agent.secret", "mgmt")) + + result = runner.invoke(app, ["gateway", "agents", "add", "echo-bot", "--type", "echo", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["name"] == "echo-bot" + assert payload["runtime_type"] == "echo" + assert payload["desired_state"] == "running" + assert payload["credential_source"] == "gateway" + assert payload["transport"] == "gateway" + registry = gateway_core.load_gateway_registry() + assert registry["agents"][0]["name"] == "echo-bot" + token_file = Path(registry["agents"][0]["token_file"]) + assert token_file.exists() + assert token_file.read_text().strip() == "axp_a_agent.secret" + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] == "managed_agent_added" + assert recent[-1]["agent_name"] == "echo-bot" + + +def test_sanitize_exec_env_strips_ax_credentials(monkeypatch): + monkeypatch.setenv("AX_TOKEN", "secret-token") + monkeypatch.setenv("AX_USER_TOKEN", "secret-user") + monkeypatch.setenv("AX_BASE_URL", "https://paxai.app") + monkeypatch.setenv("AX_AGENT_NAME", "orion") + monkeypatch.setenv("OPENAI_API_KEY", "keep-me") + + env = gateway_core.sanitize_exec_env("hello", {"name": "echo-bot", "agent_id": "agent-1", "runtime_type": "exec"}) + + assert "AX_TOKEN" not in env + assert "AX_USER_TOKEN" not in env + assert "AX_BASE_URL" not in env + assert "AX_AGENT_NAME" not in env + assert env["AX_MENTION_CONTENT"] == "hello" + assert env["AX_GATEWAY_AGENT_NAME"] == "echo-bot" + assert env["OPENAI_API_KEY"] == "keep-me" + + +def test_managed_echo_runtime_processes_message(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + token_file = tmp_path / "token" + token_file.write_text("axp_a_agent.secret") + payload = { + "id": "msg-1", + "content": "@echo-bot ping", + "author": {"id": "user-1", "name": "madtank", "type": "user"}, + "mentions": ["echo-bot"], + } + shared = _SharedRuntimeClient(payload) + + runtime = gateway_core.ManagedAgentRuntime( + { + "name": "echo-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "echo", + "token_file": str(token_file), + }, + client_factory=lambda **kwargs: shared, + ) + + runtime.start() + deadline = time.time() + 2.0 + while time.time() < deadline and not shared.sent: + time.sleep(0.05) + runtime.stop() + + assert shared.sent, "echo runtime should have replied" + assert shared.sent[0]["content"] == "Echo: ping" + assert shared.sent[0]["parent_id"] == "msg-1" + assert shared.sent[0]["agent_id"] == "agent-1" + assert shared.sent[0]["metadata"]["control_plane"] == "gateway" + assert shared.sent[0]["metadata"]["gateway"]["managed"] is True + assert shared.sent[0]["metadata"]["gateway"]["agent_name"] == "echo-bot" + assert [row["status"] for row in shared.processing] == ["started", "processing", "completed"] + assert shared.processing[0]["activity"] == "Picked up by Gateway" + assert shared.processing[0]["detail"] == {"backlog_depth": 1, "pickup_state": "claimed"} + assert shared.processing[1]["activity"] == "Composing echo reply" + recent = gateway_core.load_recent_gateway_activity() + event_names = [row["event"] for row in recent] + assert "message_received" in event_names + assert "message_claimed" in event_names + assert "reply_sent" in event_names + + +def test_managed_exec_runtime_parses_gateway_progress_events(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + token_file = tmp_path / "token" + token_file.write_text("axp_a_agent.secret") + script = tmp_path / "bridge.py" + script.write_text( + """ +import json +import sys + +prefix = "AX_GATEWAY_EVENT " +print(prefix + json.dumps({"kind": "status", "status": "working", "message": "warming up"}), flush=True) +print(prefix + json.dumps({"kind": "status", "status": "working", "message": "warming up", "progress": {"current": 1, "total": 3, "unit": "steps"}}), flush=True) +print(prefix + json.dumps({"kind": "tool_start", "tool_name": "sleep", "tool_call_id": "tool-1", "arguments": {"seconds": 1}}), flush=True) +print(prefix + json.dumps({"kind": "tool_result", "tool_name": "sleep", "tool_call_id": "tool-1", "arguments": {"seconds": 1}, "initial_data": {"slept_seconds": 1}, "status": "success"}), flush=True) +print("done", flush=True) +""".strip() + ) + payload = { + "id": "msg-1", + "content": "@exec-bot pause 1s", + "author": {"id": "user-1", "name": "madtank", "type": "user"}, + "mentions": ["exec-bot"], + } + shared = _SharedRuntimeClient(payload) + + runtime = gateway_core.ManagedAgentRuntime( + { + "name": "exec-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "exec", + "exec_command": f"{sys.executable} {script}", + "token_file": str(token_file), + }, + client_factory=lambda **kwargs: shared, + ) + + runtime.start() + deadline = time.time() + 3.0 + while time.time() < deadline and not shared.sent: + time.sleep(0.05) + snapshot = runtime.snapshot() + runtime.stop() + + assert shared.sent, "exec runtime should have replied" + assert shared.sent[0]["content"] == "done" + assert [row["status"] for row in shared.processing] == [ + "started", + "processing", + "working", + "working", + "tool_call", + "tool_complete", + "completed", + ] + assert shared.processing[0]["activity"] == "Picked up by Gateway" + assert shared.processing[0]["detail"] == {"backlog_depth": 1, "pickup_state": "claimed"} + assert shared.processing[1]["activity"] == "Preparing runtime" + assert shared.processing[2]["activity"] == "warming up" + assert shared.processing[3]["activity"] == "warming up" + assert shared.processing[3]["progress"] == {"current": 1, "total": 3, "unit": "steps"} + assert shared.processing[4]["tool_name"] == "sleep" + assert shared.processing[4]["activity"] == "Using sleep" + assert shared.processing[5]["tool_name"] == "sleep" + assert shared.processing[5]["detail"] == {"slept_seconds": 1} + assert shared.tool_calls + assert shared.tool_calls[0]["tool_name"] == "sleep" + assert shared.tool_calls[0]["message_id"] == "msg-1" + assert snapshot["current_activity"] in {None, "warming up"} + recent = gateway_core.load_recent_gateway_activity() + events = [row["event"] for row in recent] + assert "message_claimed" in events + assert "tool_started" in events + assert "tool_finished" in events + + +def test_managed_inbox_runtime_queues_message_without_reply(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + token_file = tmp_path / "token" + token_file.write_text("axp_a_agent.secret") + payload = { + "id": "msg-1", + "content": "@inbox-bot hello there", + "author": {"id": "user-1", "name": "madtank", "type": "user"}, + "mentions": ["inbox-bot"], + } + shared = _SharedRuntimeClient(payload) + + runtime = gateway_core.ManagedAgentRuntime( + { + "name": "inbox-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "inbox", + "token_file": str(token_file), + }, + client_factory=lambda **kwargs: shared, + ) + + runtime.start() + deadline = time.time() + 2.0 + snapshot = runtime.snapshot() + while time.time() < deadline and snapshot["backlog_depth"] < 1: + time.sleep(0.05) + snapshot = runtime.snapshot() + runtime.stop() + + assert not shared.sent + assert snapshot["backlog_depth"] >= 1 + assert [row["status"] for row in shared.processing] == ["queued"] + assert shared.processing[0]["activity"] == "Queued in Gateway" + assert shared.processing[0]["detail"] == {"backlog_depth": 1, "pickup_state": "queued"} + recent = gateway_core.load_recent_gateway_activity() + events = [row["event"] for row in recent] + assert "message_received" in events + assert "message_queued" in events + + +def test_annotate_runtime_health_marks_stale_after_missed_heartbeat(): + old_seen = (datetime.now(timezone.utc) - timedelta(seconds=gateway_core.RUNTIME_STALE_AFTER_SECONDS + 5)).isoformat() + + snapshot = gateway_core.annotate_runtime_health( + { + "effective_state": "running", + "last_seen_at": old_seen, + } + ) + + assert snapshot["effective_state"] == "stale" + assert snapshot["connected"] is False + assert snapshot["last_seen_age_seconds"] >= gateway_core.RUNTIME_STALE_AFTER_SECONDS + + +def test_listener_timeout_enters_reconnecting_state(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + token_file = tmp_path / "token" + token_file.write_text("axp_a_agent.secret") + + class _TimeoutRuntimeClient: + def __init__(self): + self.timeout = None + + def connect_sse(self, *, space_id, timeout=None): + self.timeout = timeout + raise httpx.ReadTimeout("boom", request=httpx.Request("GET", "https://paxai.app/api/v1/sse/messages")) + + def close(self): + return None + + shared = _TimeoutRuntimeClient() + runtime = gateway_core.ManagedAgentRuntime( + { + "name": "echo-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "echo", + "token_file": str(token_file), + }, + client_factory=lambda **kwargs: shared, + ) + + runtime.start() + deadline = time.time() + 1.0 + snapshot = runtime.snapshot() + while time.time() < deadline and snapshot["effective_state"] != "reconnecting": + time.sleep(0.05) + snapshot = runtime.snapshot() + runtime.stop() + + assert shared.timeout is not None + assert shared.timeout.read == gateway_core.SSE_IDLE_TIMEOUT_SECONDS + assert snapshot["effective_state"] == "reconnecting" + assert snapshot["last_error"] == "idle timeout after 45s without SSE heartbeat" + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] in {"runtime_stopped", "listener_timeout"} + assert any(row["event"] == "listener_timeout" for row in recent) + + +def test_gateway_watch_once_renders_dashboard(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + registry = gateway_core.load_gateway_registry() + registry["gateway"].update( + { + "gateway_id": "gw-12345678", + "desired_state": "running", + "effective_state": "running", + "last_reconcile_at": datetime.now(timezone.utc).isoformat(), + } + ) + registry["agents"] = [ + { + "name": "echo-bot", + "runtime_type": "echo", + "desired_state": "running", + "effective_state": "running", + "backlog_depth": 2, + "processed_count": 7, + "last_seen_at": datetime.now(timezone.utc).isoformat(), + "last_reply_preview": "Echo: ping", + } + ] + gateway_core.save_gateway_registry(registry) + gateway_core.record_gateway_activity("message_received", entry=registry["agents"][0], message_id="msg-1") + + result = runner.invoke(app, ["gateway", "watch", "--once"]) + + assert result.exit_code == 0, result.output + assert "Gateway Overview" in result.output + assert "Managed Agents" in result.output + assert "@echo-bot" in result.output + assert "Recent Activity" in result.output + + +def test_gateway_agents_show_json_filters_activity(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "echo-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "runtime_type": "echo", + "desired_state": "running", + "effective_state": "running", + "last_seen_at": datetime.now(timezone.utc).isoformat(), + "last_reply_preview": "Echo: ping", + "token_file": "/tmp/echo-token", + }, + { + "name": "other-bot", + "agent_id": "agent-2", + "space_id": "space-1", + "runtime_type": "exec", + "desired_state": "running", + "effective_state": "running", + "last_seen_at": datetime.now(timezone.utc).isoformat(), + "token_file": "/tmp/other-token", + }, + ] + gateway_core.save_gateway_registry(registry) + gateway_core.record_gateway_activity("reply_sent", entry=registry["agents"][0], reply_preview="Echo: ping") + gateway_core.record_gateway_activity("reply_sent", entry=registry["agents"][1], reply_preview="Other reply") + + result = runner.invoke(app, ["gateway", "agents", "show", "echo-bot", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["agent"]["name"] == "echo-bot" + assert payload["recent_activity"] + assert all(row["agent_name"] == "echo-bot" for row in payload["recent_activity"]) + + +def test_gateway_agents_send_uses_managed_identity(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + token_file = tmp_path / "sender.token" + token_file.write_text("axp_a_agent.secret") + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "sender-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "inbox", + "desired_state": "running", + "effective_state": "running", + "token_file": str(token_file), + "transport": "gateway", + "credential_source": "gateway", + } + ] + gateway_core.save_gateway_registry(registry) + monkeypatch.setattr(gateway_cmd, "AxClient", _FakeManagedSendClient) + + result = runner.invoke(app, ["gateway", "agents", "send", "sender-bot", "hello there", "--to", "codex", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["agent"] == "sender-bot" + assert payload["content"] == "@codex hello there" + assert payload["message"]["metadata"]["gateway"]["sent_via"] == "gateway_cli" + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] == "manual_message_sent" diff --git a/tests/test_gateway_probe_bridge.py b/tests/test_gateway_probe_bridge.py new file mode 100644 index 0000000..7438e02 --- /dev/null +++ b/tests/test_gateway_probe_bridge.py @@ -0,0 +1,52 @@ +import importlib.util +from pathlib import Path + + +def _load_probe_module(): + probe_path = Path(__file__).resolve().parents[1] / "examples" / "gateway_probe" / "probe_bridge.py" + spec = importlib.util.spec_from_file_location("gateway_probe_bridge", probe_path) + module = importlib.util.module_from_spec(spec) + assert spec is not None and spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_probe_seconds_defaults_and_caps(): + probe = _load_probe_module() + + assert probe._probe_seconds("probe") == probe.DEFAULT_SECONDS + assert probe._probe_seconds("probe 3") == 3 + assert probe._probe_seconds("run a 999 second probe") == probe.MAX_SECONDS + + +def test_run_probe_emits_predictable_sequence(monkeypatch, capsys): + probe = _load_probe_module() + events = [] + + monkeypatch.setattr(probe, "emit_event", events.append) + monkeypatch.setattr(probe.uuid, "uuid4", lambda: "fixed-id") + monkeypatch.setattr(probe.time, "sleep", lambda _: None) + + assert probe._run_probe(3) == 0 + + out = capsys.readouterr().out.strip() + assert out == "PROBE_OK seconds=3" + assert [event["kind"] for event in events] == [ + "status", + "status", + "status", + "tool_start", + "activity", + "activity", + "activity", + "tool_result", + "status", + ] + assert events[0]["status"] == "started" + assert events[1]["status"] == "thinking" + assert events[2]["status"] == "processing" + assert events[3]["tool_name"] == "probe_sleep" + assert events[4]["activity"] == "Probe tick 1/3 (3s left)" + assert events[6]["activity"] == "Probe tick 3/3 (1s left)" + assert events[7]["status"] == "tool_complete" + assert events[8]["status"] == "completed" diff --git a/tests/test_messages.py b/tests/test_messages.py index 9e156f4..bbc6398 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -3,7 +3,11 @@ from typer.testing import CliRunner -from ax_cli.commands.messages import _processing_status_from_event +from ax_cli.commands.messages import ( + _processing_status_from_event, + _processing_status_text, + _ProcessingStatusWatcher, +) from ax_cli.main import app runner = CliRunner() @@ -417,28 +421,130 @@ def fake_wait(client, message_id, timeout=60, wait_label="reply", **kwargs): assert calls["wait"]["processing_watcher"] is not None +def test_send_prints_sender_identity_in_human_output(monkeypatch): + class FakeClient: + _base_headers = {} + + def send_message(self, space_id, content, *, channel="main", parent_id=None, attachments=None): + return { + "message": { + "id": "msg-1", + "display_name": "codex", + "sender_type": "agent", + } + } + + monkeypatch.setattr("ax_cli.commands.messages.get_client", lambda: FakeClient()) + monkeypatch.setattr("ax_cli.commands.messages.resolve_space_id", lambda client, explicit=None: "space-1") + monkeypatch.setattr("ax_cli.commands.messages.resolve_agent_name", lambda client=None: "codex") + + result = runner.invoke(app, ["send", "checkpoint", "--no-wait"]) + + assert result.exit_code == 0, result.output + output = _strip_ansi(result.output) + assert "Sent. id=msg-1 as @codex" in output + + +def test_send_prints_gateway_reply_note_in_human_output(monkeypatch): + class FakeClient: + _base_headers = {} + + def send_message(self, space_id, content, *, channel="main", parent_id=None, attachments=None): + return { + "message": { + "id": "msg-1", + "display_name": "codex", + "sender_type": "agent", + } + } + + def fake_wait(client, message_id, timeout=60, wait_label="reply", **kwargs): + return { + "id": "reply-1", + "content": "ack", + "metadata": { + "control_plane": "gateway", + "gateway": { + "gateway_id": "12345678-90ab-cdef-1234-567890abcdef", + "agent_name": "echo-bot", + "runtime_type": "echo", + "transport": "gateway", + }, + }, + } + + monkeypatch.setattr("ax_cli.commands.messages.get_client", lambda: FakeClient()) + monkeypatch.setattr("ax_cli.commands.messages.resolve_space_id", lambda client, explicit=None: "space-1") + monkeypatch.setattr("ax_cli.commands.messages.resolve_agent_name", lambda client=None: "codex") + monkeypatch.setattr("ax_cli.commands.messages._wait_for_reply", fake_wait) + + result = runner.invoke(app, ["send", "checkpoint"]) + + assert result.exit_code == 0, result.output + output = _strip_ansi(result.output) + assert "Sent. id=msg-1 as @codex" in output + assert "aX: ack" in output + assert "via Gateway 12345678" in output + assert "agent=@echo-bot" in output + + def test_processing_status_from_event_matches_message(): event = _processing_status_from_event( "msg-1", "agent_processing", { "message_id": "msg-1", - "status": "working", + "status": "processing", "agent_id": "agent-1", "agent_name": "orion", + "activity": "Running command", + "tool_name": "shell", }, ) assert event == { "message_id": "msg-1", - "status": "working", + "status": "processing", "agent_id": "agent-1", "agent_name": "orion", + "activity": "Running command", + "tool_name": "shell", } assert _processing_status_from_event("msg-2", "agent_processing", {"message_id": "msg-1"}) is None assert _processing_status_from_event("msg-1", "message", {"message_id": "msg-1"}) is None +def test_processing_status_watcher_buffers_fast_tooling_receipt_until_message_id_known(): + watcher = _ProcessingStatusWatcher(client=None, space_id="space-1", timeout=5) + status_event = { + "message_id": "msg-1", + "status": "accepted", + "agent_id": "agent-1", + "agent_name": "orion", + "activity": "Queued in Gateway", + } + + watcher._accept_status_event(status_event) + + assert watcher.drain() == [] + + watcher.set_message_id("msg-1") + + assert watcher.drain() == [status_event] + + +def test_processing_status_text_marks_tooling_as_the_source(): + text = _processing_status_text({"status": "accepted", "agent_name": "orion", "activity": "Queued in Gateway"}) + + assert text == "tooling: @orion acknowledged the message — Queued in Gateway" + + +def test_processing_status_text_highlights_gateway_pickup(): + text = _processing_status_text({"status": "started", "agent_name": "orion", "activity": "Picked up by Gateway"}) + + assert text == "tooling: @orion picked up the message — Picked up by Gateway" + + def test_messages_edit_and_delete_resolve_short_id_prefix(monkeypatch): message_id = "12345678-90ab-cdef-1234-567890abcdef" calls = {} From b6666fda45d80e68b02bee497f9b8e58becc8202 Mon Sep 17 00:00:00 2001 From: Jacob Taunton Date: Thu, 23 Apr 2026 08:36:34 -0700 Subject: [PATCH 2/3] Build Gateway control-plane MVP --- README.md | 76 +- ax_cli/commands/gateway.py | 3526 ++++++++++++++++-- ax_cli/gateway.py | 2055 +++++++++- ax_cli/gateway_runtime_types.py | 313 ++ examples/gateway_ollama/ollama_bridge.py | 114 + skills/SKILL.md | 16 + skills/gateway-agent-setup/SKILL.md | 159 + specs/CONNECTED-ASSET-GOVERNANCE-001/spec.md | 759 ++++ specs/GATEWAY-ASSET-TAXONOMY-001/spec.md | 586 +++ specs/GATEWAY-CONNECTIVITY-001/mockups.md | 299 ++ specs/GATEWAY-CONNECTIVITY-001/spec.md | 1050 ++++++ specs/GATEWAY-IDENTITY-SPACE-001/spec.md | 659 ++++ specs/README.md | 4 + tests/test_gateway_commands.py | 1658 +++++++- 14 files changed, 10940 insertions(+), 334 deletions(-) create mode 100644 ax_cli/gateway_runtime_types.py create mode 100644 examples/gateway_ollama/ollama_bridge.py create mode 100644 skills/gateway-agent-setup/SKILL.md create mode 100644 specs/CONNECTED-ASSET-GOVERNANCE-001/spec.md create mode 100644 specs/GATEWAY-ASSET-TAXONOMY-001/spec.md create mode 100644 specs/GATEWAY-CONNECTIVITY-001/mockups.md create mode 100644 specs/GATEWAY-CONNECTIVITY-001/spec.md create mode 100644 specs/GATEWAY-IDENTITY-SPACE-001/spec.md diff --git a/README.md b/README.md index 62be0a0..2aa6f35 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ In another shell or device: ax send --to echo-bot "ping" --no-wait ax gateway status ax gateway watch +ax gateway ui ax gateway agents show echo-bot ``` @@ -105,20 +106,39 @@ ax gateway agents show echo-bot runtime state, and Gateway-authored replies carry control-plane metadata so operator attribution is less ambiguous during dogfooding. -`ax gateway watch` is the first dashboard-style operator surface: a live -terminal view with Gateway health, fleet counts, managed-agent roster, and -recent control-plane events. `ax gateway agents show ` gives the first -drill-in view for one managed runtime. - -Runtime support starts with: +Gateway setup should be treated as an agent-operable workflow, not a browser- +only wizard. The repo skill for that flow is +[`skills/gateway-agent-setup/SKILL.md`](skills/gateway-agent-setup/SKILL.md), +which wraps add/update/doctor/approval work on top of the local Gateway. -- `echo` — built-in ping/echo bot for proving the control plane. -- `inbox` — a connected inbox/queue identity that receives replies without - auto-responding. Useful for demos, operator testing, and sender/receiver - workflows. -- `exec` — Gateway-owned per-mention command execution. This is the bridge for - Hermes-style handlers without handing platform credentials to the child - process. +`ax gateway watch` is the first dashboard-style operator surface: a live +terminal view with Gateway health, alerts, fleet counts, managed-agent roster, +and recent control-plane events. `ax gateway agents show ` gives the +first drill-in view for one managed runtime, and `ax gateway agents test ` +sends a Gateway-authored smoke test to that managed agent. + +`ax gateway ui` serves the same local Gateway state over a small local web +dashboard on `127.0.0.1` by default. It now speaks the same local Gateway +contract as the CLI for status, agent drill-in, add/start/stop/remove, and +managed test sends, so the browser UI and terminal UI stay aligned to one +control-plane source of truth while the product shape is still forming. + +`ax gateway templates` exposes the same user-facing agent catalog the web UI +uses, including what each agent type needs and what kind of delivery, liveness, +activity, and tool telemetry the operator should expect. + +The primary agent types start with: + +- `Echo (Test)` — built-in ping/echo bot for proving the control plane. +- `Ollama` — a local-model runtime managed through Gateway. +- `Hermes` — a local Hermes bridge with rich activity and tool telemetry. +- `Claude Code Channel` — planned managed channel adapter. + +The lower-level runtime backends still exist under `ax gateway runtime-types` +for advanced/debug use, but they are not the main operator-facing choices. +Advanced users can still override launch commands and working directories from +the CLI or the UI's advanced launch section when they are building a custom +bridge. Managed `exec` runtimes can now emit structured progress lines back to Gateway while they are still working. The bridge prints lines prefixed with @@ -142,6 +162,29 @@ ax gateway agents add codex-gateway-inbox --type inbox ax gateway agents send codex-gateway-inbox "pause for 8 seconds and narrate activity" --to codex ``` +Example: update a managed runtime without recreating its identity: + +```bash +ax gateway agents update northstar --template hermes +ax gateway agents doctor northstar +``` + +Gateway test sends now default to an agent-authored path using a passive +Gateway-managed sender identity. For diagnostics, you can still force a +user-authored test explicitly: + +```bash +ax gateway agents test northstar +ax gateway agents test northstar --author user +``` + +For alert-style or scheduled custom payloads, use the normal send path instead +of the test button: + +```bash +ax gateway agents send switchboard-12d6eafd "Cron job: nightly sync finished" --to northstar +``` + This is a compatibility-first Gateway: today it still uses agent PATs against the existing platform APIs, but the Gateway owns those credentials centrally so managed runtimes do not. @@ -559,6 +602,10 @@ returned messages have actually been handled. | `axctl login` | Set up or refresh the user login token without touching agent config | | `ax gateway login` | Store the local Gateway bootstrap session | | `ax gateway status` | Show Gateway daemon + managed runtime status | +| `ax gateway agents test NAME` | Send a Gateway-authored smoke test to one managed agent | +| `ax gateway templates` | List the main Gateway agent types users can add | +| `ax gateway runtime-types` | List advanced/internal runtime backends | +| `ax gateway ui` | Serve the local Gateway web dashboard | | `ax gateway agents show NAME` | Drill into one managed agent | | `ax gateway agents send NAME "msg" --to codex` | Send as a managed agent identity | | `ax auth whoami` | Current identity + profile + fingerprint | @@ -576,7 +623,8 @@ returned messages have actually been handled. | `ax events stream` | Raw SSE event stream | | `ax gateway run` | Run the local Gateway supervisor | | `ax gateway watch` | Live Gateway dashboard in the terminal | -| `ax gateway agents add NAME --type inbox` | Add a connected inbox-only managed agent | +| `ax gateway ui --port 8765` | Local browser dashboard over Gateway state | +| `ax gateway agents add NAME --template hermes` | Add a Hermes-managed agent using the default bridge | | `ax listen --exec "./bot"` | Listen for @mentions with handler | | `ax watch --mention` | Block until condition matches on SSE | diff --git a/ax_cli/commands/gateway.py b/ax_cli/commands/gateway.py index 8d189e3..5131481 100644 --- a/ax_cli/commands/gateway.py +++ b/ax_cli/commands/gateway.py @@ -2,9 +2,20 @@ from __future__ import annotations +import json +import os +import shutil +import signal +import socket +import subprocess +import sys import time +import webbrowser from datetime import datetime, timezone +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from urllib.parse import unquote, urlparse import typer from rich import box @@ -15,6 +26,7 @@ from rich.table import Table from rich.text import Text +from .. import gateway as gateway_core from ..client import AxClient from ..commands import auth as auth_cmd from ..commands.bootstrap import ( @@ -26,25 +38,52 @@ from ..config import resolve_user_base_url, resolve_user_token from ..gateway import ( GatewayDaemon, + active_gateway_pid, + active_gateway_pids, + active_gateway_ui_pid, + active_gateway_ui_pids, + agent_dir, agent_token_path, annotate_runtime_health, + approve_gateway_approval, + clear_gateway_ui_state, + daemon_log_path, daemon_status, + deny_gateway_approval, + ensure_gateway_identity_binding, + ensure_local_asset_binding, find_agent_entry, gateway_dir, + get_gateway_approval, + hermes_setup_status, + infer_asset_descriptor, + list_gateway_approvals, load_gateway_registry, load_gateway_session, load_recent_gateway_activity, + ollama_setup_status, record_gateway_activity, remove_agent_entry, save_gateway_registry, save_gateway_session, + ui_log_path, + ui_status, upsert_agent_entry, + write_gateway_ui_state, +) +from ..gateway_runtime_types import ( + agent_template_definition, + agent_template_list, + runtime_type_definition, + runtime_type_list, ) from ..output import JSON_OPTION, console, err_console, print_json, print_table app = typer.Typer(name="gateway", help="Run the local Gateway control plane", no_args_is_help=True) agents_app = typer.Typer(name="agents", help="Manage Gateway-controlled agents", no_args_is_help=True) +approvals_app = typer.Typer(name="approvals", help="Review and decide Gateway approval requests", no_args_is_help=True) app.add_typer(agents_app, name="agents") +app.add_typer(approvals_app, name="approvals") _STATE_STYLES = { "running": "green", @@ -54,14 +93,32 @@ "error": "red", "stopped": "dim", } -_STATE_ORDER = { - "running": 0, - "starting": 1, - "reconnecting": 2, - "stale": 3, - "error": 4, - "stopped": 5, +_PRESENCE_STYLES = { + "IDLE": "green", + "QUEUED": "cyan", + "WORKING": "green", + "BLOCKED": "yellow", + "STALE": "yellow", + "OFFLINE": "dim", + "ERROR": "red", +} +_CONFIDENCE_STYLES = { + "HIGH": "green", + "MEDIUM": "cyan", + "LOW": "yellow", + "BLOCKED": "red", } +_PRESENCE_ORDER = { + "ERROR": 0, + "BLOCKED": 1, + "WORKING": 2, + "QUEUED": 3, + "STALE": 4, + "OFFLINE": 5, + "IDLE": 6, +} + +_UNSET = object() def _resolve_gateway_login_token(explicit_token: str | None) -> str: @@ -130,39 +187,939 @@ def _load_managed_agent_client(entry: dict) -> AxClient: ) +def _normalize_runtime_type(runtime_type: str) -> str: + try: + return str(runtime_type_definition(runtime_type)["id"]) + except KeyError as exc: + raise ValueError("Unsupported runtime type. Use echo, exec, or inbox.") from exc + + +def _validate_runtime_registration(runtime_type: str, exec_cmd: str | None) -> None: + definition = runtime_type_definition(runtime_type) + required = set(definition.get("requires") or []) + if "exec_command" in required and not exec_cmd: + raise ValueError("Exec runtimes require --exec.") + if "exec_command" not in required and exec_cmd: + raise ValueError("Echo and inbox runtimes do not accept --exec.") + + +def _register_managed_agent( + *, + name: str, + runtime_type: str | None = None, + template_id: str | None = None, + exec_cmd: str | None = None, + workdir: str | None = None, + ollama_model: str | None = None, + space_id: str | None = None, + audience: str = "both", + description: str | None = None, + model: str | None = None, + start: bool = True, +) -> dict: + name = name.strip() + if not name: + raise ValueError("Managed agent name is required.") + template = None + if template_id: + try: + template = agent_template_definition(template_id) + except KeyError as exc: + raise ValueError(f"Unknown template: {template_id}") from exc + if not bool(template.get("launchable", True)): + raise ValueError(f"Template {template['label']} is not launchable yet.") + defaults = template.get("defaults") or {} + runtime_type = runtime_type or str(defaults.get("runtime_type") or "") + exec_cmd = exec_cmd or (str(defaults.get("exec_command") or "").strip() or None) + workdir = workdir or (str(defaults.get("workdir") or "").strip() or None) + runtime_type = runtime_type or "echo" + runtime_type = _normalize_runtime_type(runtime_type) + normalized_ollama_model = str(ollama_model or "").strip() or None + template_effective_id = str(template.get("id") if template else "").strip().lower() + if normalized_ollama_model and template_effective_id != "ollama": + raise ValueError("--ollama-model is only supported with the Ollama template.") + if template_effective_id == "ollama" and not normalized_ollama_model: + normalized_ollama_model = str(ollama_setup_status().get("recommended_model") or "").strip() or None + _validate_runtime_registration(runtime_type, exec_cmd) + + session = _load_gateway_session_or_exit() + selected_space = space_id or session.get("space_id") + if not selected_space: + raise ValueError("No space selected. Use --space-id or re-run `ax gateway login` with one.") + + client = _load_gateway_user_client() + existing = _find_agent_in_space(client, name, selected_space) + if existing: + agent = existing + if description or model: + client.update_agent(name, **{k: v for k, v in {"description": description, "model": model}.items() if v}) + else: + agent = _create_agent_in_space( + client, + name=name, + space_id=selected_space, + description=description, + model=model, + ) + _polish_metadata(client, name=name, bio=None, specialization=None, system_prompt=None) + + agent_id = str(agent.get("id") or agent.get("agent_id") or "") + token, pat_source = _mint_agent_pat( + client, + agent_id=agent_id, + agent_name=name, + audience=audience, + expires_in_days=90, + pat_name=f"gateway-{name}", + space_id=selected_space, + ) + token_file = _save_agent_token(name, token) + + registry = load_gateway_registry() + entry = upsert_agent_entry( + registry, + { + "name": name, + "template_id": template.get("id") if template else None, + "template_label": template.get("label") if template else None, + "agent_id": agent_id, + "space_id": selected_space, + "base_url": session["base_url"], + "runtime_type": runtime_type, + "exec_command": exec_cmd, + "workdir": workdir, + "ollama_model": normalized_ollama_model, + "token_file": str(token_file), + "desired_state": "running" if start else "stopped", + "effective_state": "stopped", + "transport": "gateway", + "credential_source": "gateway", + "last_error": None, + "backlog_depth": 0, + "processed_count": 0, + "dropped_count": 0, + "pat_source": pat_source, + "added_at": datetime.now(timezone.utc).isoformat(), + }, + ) + ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) + ensure_gateway_identity_binding(registry, entry, session=session, created_via="cli") + hermes_status = hermes_setup_status(entry) + if not hermes_status.get("ready", True): + entry["effective_state"] = "error" + entry["last_error"] = str(hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete.") + entry["current_activity"] = str(hermes_status.get("summary") or "Hermes setup is incomplete.") + elif hermes_status.get("resolved_path"): + entry["hermes_repo_path"] = str(hermes_status["resolved_path"]) + save_gateway_registry(registry) + record_gateway_activity( + "managed_agent_added", + entry=entry, + space_id=selected_space, + token_file=str(token_file), + ) + return annotate_runtime_health(entry, registry=registry) + + +def _update_managed_agent( + *, + name: str, + template_id: str | None = None, + runtime_type: str | None = None, + exec_cmd: str | object = _UNSET, + workdir: str | object = _UNSET, + ollama_model: str | object = _UNSET, + description: str | None = None, + model: str | None = None, + desired_state: str | None = None, +) -> dict: + name = name.strip() + if not name: + raise ValueError("Managed agent name is required.") + + registry = load_gateway_registry() + entry = find_agent_entry(registry, name) + if not entry: + raise LookupError(f"Managed agent not found: {name}") + + template = None + if template_id: + try: + template = agent_template_definition(template_id) + except KeyError as exc: + raise ValueError(f"Unknown template: {template_id}") from exc + if not bool(template.get("launchable", True)): + raise ValueError(f"Template {template['label']} is not launchable yet.") + + runtime_candidate = runtime_type or (template.get("defaults") or {}).get("runtime_type") if template else runtime_type + runtime_effective = str(runtime_candidate or entry.get("runtime_type") or "echo") + runtime_effective = _normalize_runtime_type(runtime_effective) + template_effective_id = str(template.get("id") if template else entry.get("template_id") or "").strip().lower() + + if template: + defaults = template.get("defaults") or {} + exec_effective = ( + str(exec_cmd).strip() or None + if exec_cmd is not _UNSET + else (str(defaults.get("exec_command") or "").strip() or None) + ) + workdir_effective = ( + str(workdir).strip() or None + if workdir is not _UNSET + else (str(defaults.get("workdir") or "").strip() or None) + ) + else: + exec_effective = ( + str(entry.get("exec_command") or "").strip() or None + if exec_cmd is _UNSET + else (str(exec_cmd).strip() or None) + ) + workdir_effective = ( + str(entry.get("workdir") or "").strip() or None + if workdir is _UNSET + else (str(workdir).strip() or None) + ) + + if ollama_model is _UNSET: + ollama_model_effective = str(entry.get("ollama_model") or "").strip() or None + else: + ollama_model_effective = str(ollama_model).strip() or None + if ollama_model_effective and template_effective_id != "ollama": + raise ValueError("--ollama-model is only supported with the Ollama template.") + if template_effective_id == "ollama" and ollama_model is _UNSET and not ollama_model_effective: + ollama_model_effective = str(ollama_setup_status().get("recommended_model") or "").strip() or None + + _validate_runtime_registration(runtime_effective, exec_effective) + + if desired_state is not None: + normalized_desired = desired_state.lower().strip() + if normalized_desired not in {"running", "stopped"}: + raise ValueError("Desired state must be running or stopped.") + entry["desired_state"] = normalized_desired + + session = _load_gateway_session_or_exit() + if description or model: + client = _load_gateway_user_client() + client.update_agent(name, **{k: v for k, v in {"description": description, "model": model}.items() if v}) + + if template: + entry["template_id"] = template.get("id") + entry["template_label"] = template.get("label") + entry["runtime_type"] = runtime_effective + entry["exec_command"] = exec_effective + entry["workdir"] = workdir_effective + if template_effective_id == "ollama": + entry["ollama_model"] = ollama_model_effective + else: + entry.pop("ollama_model", None) + entry["updated_at"] = datetime.now(timezone.utc).isoformat() + entry.setdefault("transport", "gateway") + entry.setdefault("credential_source", "gateway") + + if template and template.get("id") != "hermes": + entry.pop("hermes_repo_path", None) + + ensure_gateway_identity_binding(registry, entry, session=session) + hermes_status = hermes_setup_status(entry) + if not hermes_status.get("ready", True): + entry["effective_state"] = "error" + entry["last_error"] = str(hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete.") + entry["current_activity"] = str(hermes_status.get("summary") or "Hermes setup is incomplete.") + elif hermes_status.get("resolved_path"): + entry["hermes_repo_path"] = str(hermes_status["resolved_path"]) + + save_gateway_registry(registry) + record_gateway_activity( + "managed_agent_updated", + entry=entry, + template_id=entry.get("template_id"), + runtime_type=runtime_effective, + workdir=workdir_effective, + exec_command=exec_effective, + desired_state=entry.get("desired_state"), + ) + return annotate_runtime_health(entry, registry=registry) + + +def _set_managed_agent_desired_state(name: str, desired_state: str) -> dict: + desired_state = desired_state.lower().strip() + if desired_state not in {"running", "stopped"}: + raise ValueError("Desired state must be running or stopped.") + registry = load_gateway_registry() + entry = find_agent_entry(registry, name) + if not entry: + raise LookupError(f"Managed agent not found: {name}") + entry["desired_state"] = desired_state + save_gateway_registry(registry) + event = "managed_agent_desired_running" if desired_state == "running" else "managed_agent_desired_stopped" + record_gateway_activity(event, entry=entry) + return annotate_runtime_health(entry, registry=registry) + + +def _remove_managed_agent(name: str) -> dict: + registry = load_gateway_registry() + entry = remove_agent_entry(registry, name) + if not entry: + raise LookupError(f"Managed agent not found: {name}") + save_gateway_registry(registry) + token_file = Path(str(entry.get("token_file") or "")) + if token_file.exists(): + token_file.unlink() + record_gateway_activity("managed_agent_removed", entry=entry) + return entry + + +def _identity_space_send_guard(entry: dict, *, explicit_space_id: str | None = None) -> dict: + registry = load_gateway_registry() + stored = find_agent_entry(registry, str(entry.get("name") or "")) or entry + ensure_gateway_identity_binding(registry, stored, session=load_gateway_session()) + snapshot = annotate_runtime_health(stored, registry=registry, explicit_space_id=explicit_space_id) + save_gateway_registry(registry) + if str(snapshot.get("confidence") or "").upper() == "BLOCKED": + reason = str(snapshot.get("confidence_reason") or "blocked") + detail = str(snapshot.get("confidence_detail") or "Gateway blocked this action.") + raise ValueError(f"{detail} ({reason})") + return snapshot + + +def _send_from_managed_agent( + *, + name: str, + content: str, + to: str | None = None, + parent_id: str | None = None, + sent_via: str = "gateway_cli", + metadata_extra: dict[str, object] | None = None, +) -> dict: + if not content.strip(): + raise ValueError("Message content is required.") + entry = _load_managed_agent_or_exit(name) + snapshot = _identity_space_send_guard(entry) + client = _load_managed_agent_client(entry) + space_id = str(snapshot.get("active_space_id") or entry.get("space_id") or "") + if not space_id: + raise ValueError(f"Managed agent is missing a space id: @{name}") + + message_content = content.strip() + mention = str(to or "").strip().lstrip("@") + if mention: + prefix = f"@{mention}" + if not message_content.startswith(prefix): + message_content = f"{prefix} {message_content}".strip() + + metadata = { + "control_plane": "gateway", + "gateway": { + "managed": True, + "agent_name": entry.get("name"), + "agent_id": entry.get("agent_id"), + "runtime_type": entry.get("runtime_type"), + "transport": entry.get("transport", "gateway"), + "credential_source": entry.get("credential_source", "gateway"), + "sent_via": sent_via, + }, + } + if metadata_extra: + gateway_meta = metadata["gateway"] + if isinstance(gateway_meta, dict): + gateway_meta.update(metadata_extra) + result = client.send_message( + space_id, + message_content, + agent_id=str(entry.get("agent_id") or "") or None, + parent_id=parent_id or None, + metadata=metadata, + ) + payload = result.get("message", result) if isinstance(result, dict) else result + if isinstance(payload, dict): + record_gateway_activity( + "manual_message_sent", + entry=entry, + message_id=payload.get("id"), + reply_preview=message_content[:120] or None, + ) + return {"agent": entry.get("name"), "message": payload, "content": message_content} + + +def _gateway_test_sender_name(space_id: str) -> str: + normalized = "".join(ch for ch in str(space_id or "") if ch.isalnum()).lower() + suffix = normalized[:8] or "default" + return f"switchboard-{suffix}" + + +def _ensure_gateway_test_sender(target_entry: dict) -> dict: + target_space = str(target_entry.get("space_id") or "").strip() + if not target_space: + raise ValueError("Managed agent is missing a space id for Gateway test delivery.") + sender_name = _gateway_test_sender_name(target_space) + registry = load_gateway_registry() + existing = find_agent_entry(registry, sender_name) + if existing: + return annotate_runtime_health(existing, registry=registry) + return _register_managed_agent( + name=sender_name, + template_id="inbox", + space_id=target_space, + description="Gateway-managed passive sender for agent-authored tests.", + start=True, + ) + + def _status_payload(*, activity_limit: int = 10) -> dict: daemon = daemon_status() + ui = ui_status() session = load_gateway_session() registry = daemon["registry"] - agents = [annotate_runtime_health(agent) for agent in registry.get("agents", [])] - running_agents = [a for a in agents if str(a.get("effective_state")) == "running"] + agents = [annotate_runtime_health(agent, registry=registry) for agent in registry.get("agents", [])] + approvals = list_gateway_approvals() + pending_approvals = [item for item in approvals if str(item.get("status") or "") == "pending"] + live_agents = [a for a in agents if str(a.get("mode") or "") == "LIVE"] + on_demand_agents = [a for a in agents if str(a.get("mode") or "") == "ON-DEMAND"] + inbox_agents = [a for a in agents if str(a.get("mode") or "") == "INBOX"] connected_agents = [a for a in agents if bool(a.get("connected"))] - stale_agents = [a for a in agents if str(a.get("effective_state")) == "stale"] - errored_agents = [a for a in agents if str(a.get("effective_state")) == "error"] + stale_agents = [a for a in agents if str(a.get("presence") or "") == "STALE"] + offline_agents = [a for a in agents if str(a.get("presence") or "") == "OFFLINE"] + errored_agents = [a for a in agents if str(a.get("presence") or "") == "ERROR"] + low_confidence_agents = [a for a in agents if str(a.get("confidence") or "") in {"LOW", "BLOCKED"}] + blocked_agents = [a for a in agents if str(a.get("confidence") or "") == "BLOCKED"] gateway = dict(registry.get("gateway", {})) if not daemon["running"]: gateway["effective_state"] = "stopped" gateway["pid"] = None - return { + payload = { "gateway_dir": str(gateway_dir()), "connected": bool(session), "base_url": session.get("base_url") if session else None, "space_id": session.get("space_id") if session else None, + "space_name": session.get("space_name") if session else None, "user": session.get("username") if session else None, "daemon": { "running": daemon["running"], "pid": daemon["pid"], }, + "ui": { + "running": ui["running"], + "pid": ui["pid"], + "host": ui["host"], + "port": ui["port"], + "url": ui["url"], + "log_path": ui["log_path"], + }, "gateway": gateway, "agents": agents, + "approvals": approvals, "recent_activity": load_recent_gateway_activity(limit=activity_limit), "summary": { "managed_agents": len(agents), - "running_agents": len(running_agents), + "live_agents": len(live_agents), + "on_demand_agents": len(on_demand_agents), + "inbox_agents": len(inbox_agents), "connected_agents": len(connected_agents), "stale_agents": len(stale_agents), + "offline_agents": len(offline_agents), "errored_agents": len(errored_agents), + "low_confidence_agents": len(low_confidence_agents), + "blocked_agents": len(blocked_agents), + "pending_approvals": len(pending_approvals), + }, + } + alerts = _gateway_alerts(payload) + payload["alerts"] = alerts + payload["summary"]["alert_count"] = len(alerts) + return payload + + +def _gateway_alerts(payload: dict, *, limit: int = 6) -> list[dict]: + alerts: list[dict] = [] + seen: set[tuple[str, str, str]] = set() + + def push(severity: str, title: str, detail: str, *, agent_name: str | None = None) -> None: + key = (severity, title, agent_name or "") + if key in seen: + return + seen.add(key) + alerts.append( + { + "severity": severity, + "title": title, + "detail": detail, + "agent_name": agent_name, + } + ) + + if not payload.get("connected"): + push("error", "Gateway is not logged in", "Run `ax gateway login` to bootstrap the local control plane.") + elif not payload.get("daemon", {}).get("running"): + push("error", "Gateway daemon is stopped", "Start it with `uv run ax gateway start` or relaunch the local service.") + + if not payload.get("ui", {}).get("running"): + push("warning", "Gateway UI is stopped", "Start it with `uv run ax gateway start` to launch the local dashboard.") + + for agent in payload.get("agents", []): + name = str(agent.get("name") or "") + presence = str(agent.get("presence") or "").upper() + approval_state = str(agent.get("approval_state") or "").lower() + attestation_state = str(agent.get("attestation_state") or "").lower() + preview = str(agent.get("last_reply_preview") or "") + lowered_preview = preview.lower() + setup_error_preview = ( + preview.startswith("(stderr:") + or " repo not found" in lowered_preview + or lowered_preview.startswith("ollama bridge failed:") + ) + if approval_state == "pending": + detail = str(agent.get("confidence_detail") or "Gateway needs approval before this runtime can be trusted.") + push("warning", f"@{name} needs Gateway approval", detail, agent_name=name) + elif approval_state == "rejected" or attestation_state == "blocked": + detail = str(agent.get("confidence_detail") or "Gateway blocked this runtime.") + push("error", f"@{name} is blocked by Gateway", detail, agent_name=name) + elif attestation_state == "drifted": + detail = str(agent.get("confidence_detail") or "Runtime changed since approval and needs review.") + push("warning", f"@{name} changed since approval", detail, agent_name=name) + elif presence == "BLOCKED": + detail = str(agent.get("confidence_detail") or "Gateway blocked this runtime until identity, space, or approval state is fixed.") + push("error", f"@{name} is blocked", detail, agent_name=name) + elif presence == "ERROR": + if setup_error_preview: + push("error", f"@{name} has a runtime setup error", preview[:180], agent_name=name) + else: + detail = str(agent.get("confidence_detail") or agent.get("last_error") or "Runtime reported an error.") + push("error", f"@{name} hit an error", detail, agent_name=name) + elif presence == "STALE": + detail = f"No heartbeat for {_format_age(agent.get('last_seen_age_seconds'))}." + push("warning", f"@{name} looks stale", detail, agent_name=name) + elif presence == "OFFLINE" and str(agent.get("mode") or "") == "LIVE": + detail = str(agent.get("confidence_detail") or "Expected a live runtime, but Gateway does not currently have a working path.") + push("warning", f"@{name} is offline", detail, agent_name=name) + if setup_error_preview and presence != "ERROR": + push("error", f"@{name} has a runtime setup error", preview[:180], agent_name=name) + if int(agent.get("backlog_depth") or 0) > 0 and presence in {"OFFLINE", "ERROR", "STALE"}: + detail = f"{agent.get('backlog_depth')} queued item(s) may be stuck until the agent is healthy." + push("warning", f"@{name} has queued work", detail, agent_name=name) + + for item in reversed(payload.get("recent_activity", [])): + event = str(item.get("event") or "") + if event == "gateway_start_blocked": + existing = item.get("existing_pid") or item.get("existing_pids") + push("warning", "Another Gateway instance is already running", f"Existing process: {existing}.") + elif event in {"listener_error", "listener_timeout"}: + agent_name = str(item.get("agent_name") or "") + detail = str(item.get("error") or "Listener lost contact and is reconnecting.") + push("warning", f"@{agent_name} had a listener interruption", detail, agent_name=agent_name or None) + if len(alerts) >= limit: + break + + return alerts[:limit] + + +def _runtime_types_payload() -> dict: + return {"runtime_types": runtime_type_list(), "count": len(runtime_type_list())} + + +def _annotate_template_taxonomy(definition: dict) -> dict: + enriched = dict(definition) + descriptor = infer_asset_descriptor( + { + "template_id": definition.get("id"), + "template_label": definition.get("label"), + "runtime_type": definition.get("runtime_type"), + "telemetry_shape": definition.get("telemetry_shape"), + "asset_class": definition.get("asset_class"), + "intake_model": definition.get("intake_model"), + "worker_model": definition.get("worker_model"), + "trigger_sources": definition.get("trigger_sources"), + "return_paths": definition.get("return_paths"), + "tags": definition.get("tags"), + "capabilities": definition.get("capabilities"), + "constraints": definition.get("constraints"), + "addressable": definition.get("addressable"), + "messageable": definition.get("messageable"), + "schedulable": definition.get("schedulable"), + "externally_triggered": definition.get("externally_triggered"), + } + ) + enriched.update( + { + "asset_class": descriptor["asset_class"], + "intake_model": descriptor["intake_model"], + "worker_model": descriptor.get("worker_model"), + "trigger_sources": descriptor["trigger_sources"], + "return_paths": descriptor["return_paths"], + "telemetry_shape": descriptor["telemetry_shape"], + "asset_type_label": descriptor["type_label"], + "output_label": descriptor["output_label"], + "asset_descriptor": descriptor, + } + ) + return enriched + + +def _agent_templates_payload() -> dict: + templates = [_annotate_template_taxonomy(item) for item in agent_template_list()] + ollama_status = ollama_setup_status() + for item in templates: + if str(item.get("id") or "").strip().lower() != "ollama": + continue + defaults = dict(item.get("defaults") or {}) + recommended_model = str(ollama_status.get("recommended_model") or "").strip() or None + if recommended_model and not str(defaults.get("ollama_model") or "").strip(): + defaults["ollama_model"] = recommended_model + item["defaults"] = defaults + item["ollama_server_reachable"] = bool(ollama_status.get("server_reachable")) + item["ollama_available_models"] = list(ollama_status.get("available_models") or []) + item["ollama_local_models"] = list(ollama_status.get("local_models") or []) + item["ollama_recommended_model"] = recommended_model + item["ollama_summary"] = str(ollama_status.get("summary") or "") + return {"templates": templates, "count": len(templates)} + + +def _agent_detail_payload(name: str, *, activity_limit: int = 12) -> dict | None: + payload = _status_payload(activity_limit=activity_limit) + entry = next((agent for agent in payload["agents"] if str(agent.get("name") or "").lower() == name.lower()), None) + if not entry: + return None + activity = load_recent_gateway_activity(limit=activity_limit, agent_name=name) + return { + "gateway": { + "connected": payload["connected"], + "base_url": payload["base_url"], + "space_id": payload["space_id"], + "daemon": payload["daemon"], }, + "agent": entry, + "recent_activity": activity, + } + + +def _approval_rows_payload(*, status: str | None = None) -> dict: + approvals = list_gateway_approvals(status=status) + return { + "approvals": approvals, + "count": len(approvals), + "pending": len([item for item in approvals if str(item.get("status") or "") == "pending"]), + } + + +def _approval_detail_payload(approval_id: str) -> dict: + approval = get_gateway_approval(approval_id) + return {"approval": approval} + + +def _recommended_test_message(entry: dict) -> str: + template_id = str(entry.get("template_id") or "").strip() + if template_id: + try: + template = agent_template_definition(template_id) + message = str(template.get("recommended_test_message") or "").strip() + if message: + return message + except KeyError: + pass + runtime_type = str(entry.get("runtime_type") or "").lower() + if runtime_type == "echo": + return "gateway test ping" + if runtime_type == "inbox": + return "Queue this test job, mark it received, and do not reply inline." + return "Reply with exactly: Gateway test OK." + + +def _send_gateway_test_to_managed_agent( + name: str, + *, + content: str | None = None, + author: str = "agent", + sender_agent: str | None = None, +) -> dict: + entry = _load_managed_agent_or_exit(name) + space_id = str(entry.get("space_id") or "") + if not space_id: + raise ValueError(f"Managed agent is missing a space id: @{name}") + + prompt = (content or "").strip() or _recommended_test_message(entry) + target = str(entry.get("name") or "").lstrip("@") + normalized_author = str(author or "agent").strip().lower() + if normalized_author not in {"agent", "user"}: + raise ValueError("Gateway test author must be one of: agent, user.") + + sender_name = None + if normalized_author == "agent": + sender_name = str(sender_agent or "").strip() or str(_ensure_gateway_test_sender(entry).get("name") or "") + if not sender_name: + raise ValueError("Gateway could not resolve a managed sender for the test message.") + result = _send_from_managed_agent( + name=sender_name, + content=prompt, + to=target, + sent_via="gateway_test", + metadata_extra={ + "managed_target": True, + "target_agent_name": entry.get("name"), + "target_agent_id": entry.get("agent_id"), + "target_template": entry.get("template_id"), + "target_runtime_type": entry.get("runtime_type"), + "test_author": "agent", + }, + ) + payload = result.get("message", result) if isinstance(result, dict) else result + message_content = str(result.get("content") or f"@{target} {prompt}".strip()) + else: + client = _load_gateway_user_client() + message_content = f"@{target} {prompt}".strip() + metadata = { + "control_plane": "gateway", + "gateway": { + "managed_target": True, + "target_agent_name": entry.get("name"), + "target_agent_id": entry.get("agent_id"), + "target_template": entry.get("template_id"), + "target_runtime_type": entry.get("runtime_type"), + "sent_via": "gateway_test", + "test_author": "user", + }, + } + result = client.send_message(space_id, message_content, metadata=metadata) + payload = result.get("message", result) if isinstance(result, dict) else result + + if isinstance(payload, dict): + record_gateway_activity( + "gateway_test_sent", + entry=entry, + message_id=payload.get("id"), + reply_preview=message_content[:120] or None, + sender_agent_name=sender_name, + test_author=normalized_author, + ) + return { + "target_agent": entry.get("name"), + "sender_agent": sender_name, + "author": normalized_author, + "message": payload, + "content": message_content, + "recommended_prompt": prompt, + } + + +def _doctor_result_status(checks: list[dict]) -> str: + statuses = {str(item.get("status") or "").strip().lower() for item in checks} + if "failed" in statuses: + return "failed" + if "warning" in statuses: + return "warning" + return "passed" + + +def _doctor_summary(checks: list[dict], status: str) -> str: + failures = [str(item.get("detail") or item.get("name") or "").strip() for item in checks if str(item.get("status") or "").strip().lower() == "failed"] + warnings = [str(item.get("detail") or item.get("name") or "").strip() for item in checks if str(item.get("status") or "").strip().lower() == "warning"] + if status == "failed" and failures: + return failures[0] + if status == "warning" and warnings: + return warnings[0] + return "Gateway path looks healthy." + + +def _store_doctor_result(name: str, result: dict[str, object]) -> dict: + registry = load_gateway_registry() + entry = find_agent_entry(registry, name) + if not entry: + raise LookupError(f"Managed agent not found: {name}") + completed_at = str(result.get("completed_at") or datetime.now(timezone.utc).isoformat()) + entry["last_doctor_result"] = result + entry["last_doctor_at"] = completed_at + if str(result.get("status") or "").lower() != "failed": + entry["last_successful_doctor_at"] = completed_at + save_gateway_registry(registry) + record_gateway_activity( + "doctor_completed", + entry=entry, + activity_message=str(result.get("summary") or ""), + error=None if str(result.get("status") or "").lower() != "failed" else str(result.get("summary") or ""), + ) + return annotate_runtime_health(entry, registry=registry) + + +def _run_gateway_doctor(name: str, *, send_test: bool = False) -> dict: + registry = load_gateway_registry() + entry = find_agent_entry(registry, name) + if not entry: + raise LookupError(f"Managed agent not found: {name}") + ensure_gateway_identity_binding(registry, entry, session=load_gateway_session(), verify_spaces=False) + snapshot = annotate_runtime_health(entry, registry=registry) + checks: list[dict[str, str]] = [] + asset_class = str(snapshot.get("asset_class") or "") + intake_model = str(snapshot.get("intake_model") or "") + return_paths = [str(item) for item in (snapshot.get("return_paths") or []) if str(item)] + + def add_check(check_name: str, status: str, detail: str) -> None: + checks.append({"name": check_name, "status": status, "detail": detail}) + + def has_check(check_name: str) -> bool: + return any(str(item.get("name") or "") == check_name for item in checks) + + session = load_gateway_session() + add_check( + "gateway_auth", + "passed" if session else "failed", + "Gateway bootstrap session is present." if session else "Gateway is not logged in.", + ) + + identity_status = str(snapshot.get("identity_status") or "").lower() + if identity_status == "verified": + add_check("identity_binding", "passed", f"Gateway is acting as {snapshot.get('acting_agent_name') or entry.get('name')}.") + elif identity_status == "bootstrap_only": + add_check("identity_binding", "failed", "Gateway would need to use a bootstrap credential for an agent-authored action.") + else: + add_check("identity_binding", "failed", str(snapshot.get("confidence_detail") or "Gateway does not have a valid acting identity binding.")) + + environment_status = str(snapshot.get("environment_status") or "").lower() + if environment_status == "environment_allowed": + add_check("environment_binding", "passed", f"Requested environment matches {snapshot.get('environment_label') or snapshot.get('base_url') or entry.get('base_url')}.") + elif environment_status == "environment_mismatch": + add_check("environment_binding", "failed", str(snapshot.get("confidence_detail") or "Requested environment does not match the bound environment.")) + else: + add_check("environment_binding", "warning", "Gateway could not fully verify the bound environment.") + + allowed_spaces = snapshot.get("allowed_spaces") if isinstance(snapshot.get("allowed_spaces"), list) else [] + if allowed_spaces: + add_check("allowed_spaces", "passed", f"Gateway resolved {len(allowed_spaces)} allowed space(s).") + else: + add_check("allowed_spaces", "warning", "Gateway does not have a cached allowed-space list yet.") + + space_status = str(snapshot.get("space_status") or "").lower() + if space_status == "active_allowed": + add_check("space_binding", "passed", f"Active space is {snapshot.get('active_space_name') or snapshot.get('active_space_id')}.") + elif space_status == "no_active_space": + add_check("space_binding", "failed", "Gateway does not have an active space selected for this asset.") + elif space_status == "active_not_allowed": + add_check("space_binding", "failed", str(snapshot.get("confidence_detail") or "Active space is not allowed for this identity.")) + else: + add_check("space_binding", "warning", "Gateway could not fully verify the active space.") + + attestation_state = str(snapshot.get("attestation_state") or "").lower() + approval_state = str(snapshot.get("approval_state") or "").lower() + if approval_state == "pending": + add_check("binding_approval", "warning", str(snapshot.get("confidence_detail") or "Gateway needs approval before trusting this runtime binding.")) + elif approval_state == "rejected" or attestation_state == "blocked": + add_check("binding_approval", "failed", str(snapshot.get("confidence_detail") or "Gateway blocked this runtime binding.")) + elif attestation_state == "drifted": + add_check("binding_attestation", "failed", str(snapshot.get("confidence_detail") or "Runtime binding drifted from its approved launch spec.")) + elif attestation_state == "verified": + add_check("binding_attestation", "passed", "Runtime matches the approved local binding.") + + token_file = Path(str(entry.get("token_file") or "")).expanduser() + if token_file.exists() and token_file.read_text().strip(): + add_check("agent_token", "passed", "Managed agent token file is present.") + else: + add_check("agent_token", "failed", f"Managed agent token is missing or empty at {token_file}.") + + if asset_class == "background_worker" or intake_model == "queue_accept": + probe = agent_dir(name) / ".doctor-queue-check" + try: + probe.write_text("ok\n") + probe.unlink(missing_ok=True) + add_check("queue_writable", "passed", "Gateway queue is writable.") + except OSError as exc: + add_check("queue_writable", "failed", f"Gateway queue is not writable: {exc}") + if bool(snapshot.get("connected")): + add_check("worker_attached", "passed", "A queue worker is attached.") + else: + add_check("worker_attached", "warning", "Queue writable; no worker currently attached.") + if "summary_post" in return_paths: + add_check("summary_path", "passed", "Gateway is configured to post a summary after queued work completes.") + else: + exec_command = str(entry.get("exec_command") or "").strip() + runtime_type = str(entry.get("runtime_type") or "").strip().lower() + if intake_model == "live_listener": + if snapshot.get("activation") == "attach_only": + if str(snapshot.get("reachability") or "") == "attach_required": + add_check("session_attach", "warning", "Reconnect the attached session before sending.") + elif bool(snapshot.get("connected")): + add_check("session_attach", "passed", "Attached session is connected to Gateway.") + else: + add_check("session_attach", "failed", "Gateway does not currently have an attached session to supervise.") + elif runtime_type != "echo": + if exec_command: + add_check("runtime_launch", "passed", "Gateway has a launch command for this runtime.") + else: + add_check("runtime_launch", "failed", "Gateway does not have a launch command for this runtime.") + elif intake_model == "launch_on_send": + if runtime_type == "echo" or exec_command: + add_check("launch_ready", "passed", "Gateway can launch this runtime when work arrives.") + else: + add_check("launch_ready", "failed", "Gateway does not have a launch command for this on-demand runtime.") + elif intake_model == "scheduled_run": + add_check("schedule_ready", "warning", "Scheduled asset support is taxonomy-defined but not fully implemented in Gateway yet.") + elif intake_model == "event_triggered": + add_check("event_source", "warning", "Alert-driven asset support is taxonomy-defined but not fully implemented in Gateway yet.") + elif asset_class == "service_proxy": + if exec_command: + add_check("runtime_launch", "passed", "Gateway has a launch command for this runtime.") + else: + add_check("runtime_launch", "failed", "Gateway does not have a launch command for this runtime.") + + template_id = str(entry.get("template_id") or "").strip().lower() + if template_id == "hermes": + hermes_status = hermes_setup_status(entry) + if hermes_status.get("ready", True): + add_check("hermes_repo", "passed", str(hermes_status.get("summary") or "Hermes checkout found.")) + else: + add_check("hermes_repo", "failed", str(hermes_status.get("summary") or "Hermes checkout not found.")) + elif template_id == "ollama": + ollama_model = str(entry.get("ollama_model") or "").strip() + ollama_status = ollama_setup_status(preferred_model=ollama_model or None) + if bool(ollama_status.get("server_reachable")): + add_check("ollama_server", "passed", str(ollama_status.get("summary") or "Ollama server is reachable.")) + else: + add_check("ollama_server", "failed", str(ollama_status.get("summary") or "Ollama server is not reachable.")) + if ollama_model: + if bool(ollama_status.get("preferred_model_available")): + add_check("ollama_model", "passed", f"Gateway will launch Ollama with model {ollama_model}.") + else: + add_check("ollama_model", "failed", f"Configured Ollama model is not installed: {ollama_model}.") + else: + recommended_model = str(ollama_status.get("recommended_model") or "").strip() + if recommended_model: + add_check("ollama_model", "passed", f"Gateway will use the recommended local model {recommended_model}.") + else: + add_check("ollama_model", "warning", "No Ollama model is selected yet.") + add_check("launch_path", "passed", "Gateway can launch the Ollama bridge on send.") + + if str(snapshot.get("mode") or "") == "LIVE": + if str(snapshot.get("presence") or "") == "IDLE": + add_check("live_path", "passed", "Live listener is connected.") + elif str(snapshot.get("reachability") or "") == "attach_required": + add_check("live_path", "warning", "Reconnect the attached session before sending.") + elif str(snapshot.get("presence") or "") in {"STALE", "OFFLINE"}: + add_check("live_path", "failed", str(snapshot.get("confidence_detail") or _reachability_copy(snapshot))) + elif str(snapshot.get("mode") or "") == "ON-DEMAND" and not has_check("launch_ready"): + add_check("launch_ready", "passed", "Gateway can launch this runtime on send.") + + if send_test: + try: + sent = _send_gateway_test_to_managed_agent(name) + message_id = None + if isinstance(sent.get("message"), dict): + message_id = sent["message"].get("id") + add_check("test_send", "passed", f"Gateway test message sent{f' ({message_id})' if message_id else ''}.") + except Exception as exc: + add_check("test_send", "failed", f"Gateway test send failed: {exc}") + + status = _doctor_result_status(checks) + completed_at = datetime.now(timezone.utc).isoformat() + result = { + "status": status, + "completed_at": completed_at, + "checks": checks, + "summary": _doctor_summary(checks, status), + } + annotated = _store_doctor_result(name, result) + return { + "name": name, + "status": status, + "completed_at": completed_at, + "summary": result["summary"], + "checks": checks, + "agent": annotated, } @@ -213,20 +1170,80 @@ def _state_text(state: object) -> Text: return Text(f"● {label}", style=style) -def _metric_panel(label: str, value: object, *, tone: str = "cyan", subtitle: str | None = None) -> Panel: - body = Text() - body.append(str(value), style=f"bold {tone}") - body.append(f"\n{label}", style="dim") - if subtitle: - body.append(f"\n{subtitle}", style="dim") - return Panel(body, border_style=tone, padding=(1, 2)) +def _presence_text(presence: object) -> Text: + label = str(presence or "OFFLINE").upper() + style = _PRESENCE_STYLES.get(label, "white") + return Text(label, style=style) -def _sorted_agents(agents: list[dict]) -> list[dict]: - return sorted( +def _confidence_text(confidence: object) -> Text: + label = str(confidence or "MEDIUM").upper() + style = _CONFIDENCE_STYLES.get(label, "white") + return Text(label, style=style) + + +def _mode_text(mode: object) -> Text: + label = str(mode or "ON-DEMAND").upper() + style = { + "LIVE": "green", + "ON-DEMAND": "cyan", + "INBOX": "blue", + }.get(label, "white") + return Text(label, style=style) + + +def _reply_text(reply: object) -> Text: + label = str(reply or "REPLY").upper() + style = { + "REPLY": "green", + "SUMMARY": "yellow", + "SILENT": "dim", + }.get(label, "white") + return Text(label, style=style) + + +def _reachability_copy(agent: dict) -> str: + reachability = str(agent.get("reachability") or "unavailable") + mode = str(agent.get("mode") or "") + if reachability == "live_now": + return "Live listener ready to claim work." + if reachability == "queue_available": + return "Gateway can safely queue work now." + if reachability == "launch_available": + return "Gateway can launch this runtime on send." + if reachability == "attach_required": + return "Reconnect the attached session before sending." + if mode == "INBOX": + return "Queue path is unavailable." + return "Gateway does not currently have a working path." + + +def _agent_template_label(agent: dict) -> str: + return str(agent.get("template_label") or agent.get("runtime_type") or "-") + + +def _agent_type_label(agent: dict) -> str: + return str(agent.get("asset_type_label") or "Connected Asset") + + +def _agent_output_label(agent: dict) -> str: + return str(agent.get("output_label") or agent.get("reply") or "Reply") + + +def _metric_panel(label: str, value: object, *, tone: str = "cyan", subtitle: str | None = None) -> Panel: + body = Text() + body.append(str(value), style=f"bold {tone}") + body.append(f"\n{label}", style="dim") + if subtitle: + body.append(f"\n{subtitle}", style="dim") + return Panel(body, border_style=tone, padding=(1, 2)) + + +def _sorted_agents(agents: list[dict]) -> list[dict]: + return sorted( agents, key=lambda agent: ( - _STATE_ORDER.get(str(agent.get("effective_state") or "").lower(), 99), + _PRESENCE_ORDER.get(str(agent.get("presence") or "").upper(), 99), str(agent.get("name") or "").lower(), ), ) @@ -234,6 +1251,7 @@ def _sorted_agents(agents: list[dict]) -> list[dict]: def _render_gateway_overview(payload: dict) -> Panel: gateway = payload.get("gateway") or {} + ui = payload.get("ui") or {} grid = Table.grid(expand=True, padding=(0, 2)) grid.add_column(style="bold") grid.add_column(ratio=2) @@ -241,7 +1259,9 @@ def _render_gateway_overview(payload: dict) -> Panel: grid.add_column(ratio=2) grid.add_row("Gateway", str(gateway.get("gateway_id") or "-")[:8], "Daemon", "running" if payload["daemon"]["running"] else "stopped") grid.add_row("User", str(payload.get("user") or "-"), "Base URL", str(payload.get("base_url") or "-")) - grid.add_row("Space", str(payload.get("space_id") or "-"), "PID", str(payload["daemon"].get("pid") or "-")) + space_label = str(payload.get("space_name") or payload.get("space_id") or "-") + grid.add_row("Space", space_label, "PID", str(payload["daemon"].get("pid") or "-")) + grid.add_row("UI", str(ui.get("url") or "-"), "UI PID", str(ui.get("pid") or "-")) grid.add_row( "Session", "connected" if payload.get("connected") else "disconnected", @@ -255,25 +1275,37 @@ def _render_agent_table(agents: list[dict]) -> Table: table = Table(expand=True, box=box.SIMPLE_HEAVY) table.add_column("Agent", style="bold") table.add_column("Type") - table.add_column("State") - table.add_column("Phase") + table.add_column("Mode") + table.add_column("Presence") + table.add_column("Output") + table.add_column("Confidence") + table.add_column("Acting As") + table.add_column("Current Space") table.add_column("Queue", justify="right") table.add_column("Seen", justify="right") - table.add_column("Processed", justify="right") table.add_column("Activity", overflow="fold") if not agents: - table.add_row("No managed agents", "-", Text("● stopped", style="dim"), "-", "0", "-", "0", "-") + table.add_row("No managed agents", "-", Text("ON-DEMAND", style="dim"), Text("OFFLINE", style="dim"), Text("Reply", style="dim"), Text("MEDIUM", style="dim"), "-", "-", "0", "-", "-") return table for agent in _sorted_agents(agents): - activity = str(agent.get("current_activity") or agent.get("current_tool") or agent.get("last_reply_preview") or "-") + activity = str( + agent.get("current_activity") + or agent.get("confidence_detail") + or agent.get("current_tool") + or agent.get("last_reply_preview") + or "-" + ) table.add_row( f"@{agent.get('name')}", - str(agent.get("runtime_type") or "-"), - _state_text(agent.get("effective_state")), - str(agent.get("current_status") or "-"), + _agent_type_label(agent), + _mode_text(agent.get("mode")), + _presence_text(agent.get("presence")), + Text(_agent_output_label(agent), style="green" if str(agent.get("output_label") or "").lower() == "reply" else "yellow"), + _confidence_text(agent.get("confidence")), + str(agent.get("acting_agent_name") or agent.get("name") or "-"), + str(agent.get("active_space_name") or agent.get("active_space_id") or agent.get("space_id") or "-"), str(agent.get("backlog_depth") or 0), _format_age(agent.get("last_seen_age_seconds")), - str(agent.get("processed_count") or 0), activity, ) return table @@ -307,6 +1339,28 @@ def _render_activity_table(activity: list[dict]) -> Table: return table +def _render_alert_table(alerts: list[dict]) -> Table: + table = Table(expand=True, box=box.SIMPLE_HEAVY) + table.add_column("Level", no_wrap=True) + table.add_column("Alert", no_wrap=True) + table.add_column("Agent", no_wrap=True) + table.add_column("Detail", overflow="fold") + if not alerts: + table.add_row("info", "No active alerts", "-", "Gateway looks healthy.") + return table + for item in alerts: + severity = str(item.get("severity") or "info").lower() + style = {"error": "red", "warning": "yellow", "info": "cyan"}.get(severity, "white") + agent_name = str(item.get("agent_name") or "") + table.add_row( + Text(severity, style=style), + str(item.get("title") or "-"), + f"@{agent_name}" if agent_name else "-", + str(item.get("detail") or "-"), + ) + return table + + def _render_gateway_dashboard(payload: dict) -> Group: agents = payload.get("agents", []) summary = payload.get("summary", {}) @@ -314,9 +1368,12 @@ def _render_gateway_dashboard(payload: dict) -> Group: metrics = Columns( [ _metric_panel("managed agents", summary.get("managed_agents", 0), tone="cyan"), - _metric_panel("connected", summary.get("connected_agents", 0), tone="green"), - _metric_panel("stale", summary.get("stale_agents", 0), tone="yellow"), - _metric_panel("errors", summary.get("errored_agents", 0), tone="red"), + _metric_panel("live", summary.get("live_agents", 0), tone="green"), + _metric_panel("on-demand", summary.get("on_demand_agents", 0), tone="blue"), + _metric_panel("inbox", summary.get("inbox_agents", 0), tone="cyan"), + _metric_panel("pending approvals", summary.get("pending_approvals", 0), tone="yellow"), + _metric_panel("low confidence", summary.get("low_confidence_agents", 0), tone="yellow"), + _metric_panel("blocked", summary.get("blocked_agents", 0), tone="red"), _metric_panel("queue depth", queue_depth, tone="blue"), ], expand=True, @@ -325,28 +1382,1637 @@ def _render_gateway_dashboard(payload: dict) -> Group: return Group( _render_gateway_overview(payload), metrics, + Panel(_render_alert_table(payload.get("alerts", [])), title="Alerts", border_style="red"), Panel(_render_agent_table(agents), title="Managed Agents", border_style="green"), Panel(_render_activity_table(payload.get("recent_activity", [])), title="Recent Activity", border_style="magenta"), ) +def _render_gateway_ui_page(*, refresh_ms: int) -> str: + template = """ + + + + + ax gateway ui + + + +
+
+
+
+
Gateway Control Plane · Agent Operated
+

One local Gateway. Every agent in one place.

+

+ This dashboard is served locally by ax gateway ui and reads the + same Gateway state model as the terminal watch view. The browser is a human + view over the same local control plane that setup agents use through the CLI + and local API instead of maintaining separate logic. +

+
+
+
+
+ +
+ +
+
+ Alerts + loading… +
+
+
Waiting for Gateway alerts…
+
+
+ +
+
+
+ Gateway Agent Setup + agent skill · create +
+
+
+

+ This form mirrors the gateway-agent-setup skill. Agents and humans + should use the same Gateway-native setup, doctor, and update flow. +

+
+
+ + +
+
+ + +
+
+
+

Loading agent type…

+
+ +
+ + +
+
+
+
+
+ +
+
+ Custom Message + splunk · datadog · cron · manual +
+
+
+

+ Use Send Agent Test for the standard validation path. + Use this form for custom payloads, alerts, and scheduled-job style messages. +

+
+
+ + +
+
+ + +
+
+
+ + +
+
+ +
+
+
+
+
+
+ +
+
+
+ Managed Agents + loading… +
+
+ + + + + + + + + + + + + + + + + +
AgentTypeModePresenceOutputConfidenceQueueSeenActivity
Waiting for Gateway state…
+
+
+ +
+
+ Agent Drill-In +
+ + select an agent +
+
+
+
Choose a managed agent to inspect live detail.
+
+
+
+ +
+
+ Recent Activity + auto-refresh every __REFRESH_MS__ ms +
+
+
Waiting for activity…
+
+
+ +
+
+ Local status API: /api/status and /api/agents/<name> + Setup skill: skills/gateway-agent-setup/SKILL.md · Terminal parity: uv run ax gateway watch +
+
+
+ + + + +""" + return template.replace("__REFRESH_MS__", str(refresh_ms)) + + +class _GatewayUiServer(ThreadingHTTPServer): + allow_reuse_address = True + daemon_threads = True + + +def _write_json_response(handler: BaseHTTPRequestHandler, payload: dict, *, status: HTTPStatus = HTTPStatus.OK) -> None: + body = json.dumps(payload, indent=2, sort_keys=True).encode("utf-8") + handler.send_response(status.value) + handler.send_header("Content-Type", "application/json; charset=utf-8") + handler.send_header("Content-Length", str(len(body))) + handler.send_header("Cache-Control", "no-store") + handler.end_headers() + handler.wfile.write(body) + + +def _write_html_response(handler: BaseHTTPRequestHandler, payload: str) -> None: + body = payload.encode("utf-8") + handler.send_response(HTTPStatus.OK.value) + handler.send_header("Content-Type", "text/html; charset=utf-8") + handler.send_header("Content-Length", str(len(body))) + handler.send_header("Cache-Control", "no-store") + handler.end_headers() + handler.wfile.write(body) + + +def _read_json_request(handler: BaseHTTPRequestHandler) -> dict: + content_length = int(handler.headers.get("Content-Length", "0") or 0) + if content_length <= 0: + return {} + raw = handler.rfile.read(content_length) + if not raw: + return {} + try: + payload = json.loads(raw.decode("utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON body: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError("JSON body must be an object.") + return payload + + +def _build_gateway_ui_handler(*, activity_limit: int, refresh_ms: int): + class GatewayUiHandler(BaseHTTPRequestHandler): + def log_message(self, format: str, *args) -> None: # noqa: A003 + return + + def do_GET(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + if parsed.path == "/": + _write_html_response(self, _render_gateway_ui_page(refresh_ms=refresh_ms)) + return + if parsed.path == "/healthz": + _write_json_response(self, {"ok": True}) + return + if parsed.path == "/api/status": + _write_json_response(self, _status_payload(activity_limit=activity_limit)) + return + if parsed.path == "/api/runtime-types": + _write_json_response(self, _runtime_types_payload()) + return + if parsed.path == "/api/templates": + _write_json_response(self, _agent_templates_payload()) + return + if parsed.path.startswith("/api/agents/"): + name = unquote(parsed.path.removeprefix("/api/agents/")).strip() + payload = _agent_detail_payload(name, activity_limit=activity_limit) + if payload is None: + _write_json_response( + self, + {"error": f"Managed agent not found: {name}"}, + status=HTTPStatus.NOT_FOUND, + ) + return + _write_json_response(self, payload) + return + _write_json_response(self, {"error": "not found"}, status=HTTPStatus.NOT_FOUND) + + def do_POST(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + try: + body = _read_json_request(self) + if parsed.path == "/api/agents": + payload = _register_managed_agent( + name=str(body.get("name") or "").strip(), + template_id=str(body.get("template_id") or "").strip() or None, + runtime_type=str(body.get("runtime_type") or "").strip() or None, + exec_cmd=str(body.get("exec_command") or "").strip() or None, + workdir=str(body.get("workdir") or "").strip() or None, + ollama_model=str(body.get("ollama_model") or "").strip() or None, + space_id=str(body.get("space_id") or "").strip() or None, + audience=str(body.get("audience") or "both"), + description=str(body.get("description") or "").strip() or None, + model=str(body.get("model") or "").strip() or None, + start=bool(body.get("start", True)), + ) + _write_json_response(self, payload, status=HTTPStatus.CREATED) + return + if parsed.path.endswith("/start") and parsed.path.startswith("/api/agents/"): + name = unquote(parsed.path.removeprefix("/api/agents/").removesuffix("/start")).strip() + payload = _set_managed_agent_desired_state(name, "running") + _write_json_response(self, payload) + return + if parsed.path.endswith("/stop") and parsed.path.startswith("/api/agents/"): + name = unquote(parsed.path.removeprefix("/api/agents/").removesuffix("/stop")).strip() + payload = _set_managed_agent_desired_state(name, "stopped") + _write_json_response(self, payload) + return + if parsed.path.endswith("/send") and parsed.path.startswith("/api/agents/"): + name = unquote(parsed.path.removeprefix("/api/agents/").removesuffix("/send")).strip() + payload = _send_from_managed_agent( + name=name, + content=str(body.get("content") or ""), + to=str(body.get("to") or "").strip() or None, + parent_id=str(body.get("parent_id") or "").strip() or None, + ) + _write_json_response(self, payload, status=HTTPStatus.CREATED) + return + if parsed.path.endswith("/test") and parsed.path.startswith("/api/agents/"): + name = unquote(parsed.path.removeprefix("/api/agents/").removesuffix("/test")).strip() + payload = _send_gateway_test_to_managed_agent( + name, + content=str(body.get("content") or "").strip() or None, + author=str(body.get("author") or "agent").strip() or "agent", + sender_agent=str(body.get("sender_agent") or "").strip() or None, + ) + _write_json_response(self, payload, status=HTTPStatus.CREATED) + return + if parsed.path.endswith("/doctor") and parsed.path.startswith("/api/agents/"): + name = unquote(parsed.path.removeprefix("/api/agents/").removesuffix("/doctor")).strip() + payload = _run_gateway_doctor( + name, + send_test=bool(body.get("send_test", False)), + ) + _write_json_response(self, payload, status=HTTPStatus.CREATED) + return + _write_json_response(self, {"error": "not found"}, status=HTTPStatus.NOT_FOUND) + except LookupError as exc: + _write_json_response(self, {"error": str(exc)}, status=HTTPStatus.NOT_FOUND) + except ValueError as exc: + _write_json_response(self, {"error": str(exc)}, status=HTTPStatus.BAD_REQUEST) + except typer.Exit as exc: + status = HTTPStatus.BAD_REQUEST if int(exc.exit_code or 1) == 1 else HTTPStatus.OK + _write_json_response(self, {"error": "request failed"}, status=status) + except Exception as exc: + _write_json_response(self, {"error": str(exc)}, status=HTTPStatus.INTERNAL_SERVER_ERROR) + + def do_PUT(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + try: + body = _read_json_request(self) + if parsed.path.startswith("/api/agents/"): + name = unquote(parsed.path.removeprefix("/api/agents/")).strip() + payload = _update_managed_agent( + name=name, + template_id=str(body.get("template_id") or "").strip() or None, + runtime_type=str(body.get("runtime_type") or "").strip() or None, + exec_cmd=str(body.get("exec_command") or "") if "exec_command" in body else _UNSET, + workdir=str(body.get("workdir") or "") if "workdir" in body else _UNSET, + ollama_model=str(body.get("ollama_model") or "") if "ollama_model" in body else _UNSET, + description=str(body.get("description") or "").strip() or None, + model=str(body.get("model") or "").strip() or None, + desired_state=str(body.get("desired_state") or "").strip() or None, + ) + _write_json_response(self, payload) + return + _write_json_response(self, {"error": "not found"}, status=HTTPStatus.NOT_FOUND) + except LookupError as exc: + _write_json_response(self, {"error": str(exc)}, status=HTTPStatus.NOT_FOUND) + except ValueError as exc: + _write_json_response(self, {"error": str(exc)}, status=HTTPStatus.BAD_REQUEST) + except typer.Exit as exc: + status = HTTPStatus.BAD_REQUEST if int(exc.exit_code or 1) == 1 else HTTPStatus.OK + _write_json_response(self, {"error": "request failed"}, status=status) + except Exception as exc: + _write_json_response(self, {"error": str(exc)}, status=HTTPStatus.INTERNAL_SERVER_ERROR) + + def do_DELETE(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + if parsed.path.startswith("/api/agents/"): + name = unquote(parsed.path.removeprefix("/api/agents/")).strip() + try: + payload = _remove_managed_agent(name) + _write_json_response(self, payload) + except LookupError as exc: + _write_json_response(self, {"error": str(exc)}, status=HTTPStatus.NOT_FOUND) + return + _write_json_response(self, {"error": "not found"}, status=HTTPStatus.NOT_FOUND) + + return GatewayUiHandler + + def _render_agent_detail(entry: dict, *, activity: list[dict]) -> Group: overview = Table.grid(expand=True, padding=(0, 2)) overview.add_column(style="bold") overview.add_column(ratio=2) overview.add_column(style="bold") overview.add_column(ratio=2) - overview.add_row("Agent", f"@{entry.get('name')}", "Runtime", str(entry.get("runtime_type") or "-")) + overview.add_row("Agent", f"@{entry.get('name')}", "Type", _agent_type_label(entry)) + overview.add_row("Template", _agent_template_label(entry), "Output", _agent_output_label(entry)) + overview.add_row("Mode", str(entry.get("mode") or "-"), "Presence", str(entry.get("presence") or "-")) + overview.add_row("Reply", str(entry.get("reply") or "-"), "Confidence", str(entry.get("confidence") or "-")) + overview.add_row("Asset Class", str(entry.get("asset_class") or "-"), "Intake", str(entry.get("intake_model") or "-")) + overview.add_row("Trigger", str((entry.get("trigger_sources") or [None])[0] or "-"), "Return", str((entry.get("return_paths") or [None])[0] or "-")) + overview.add_row("Telemetry", str(entry.get("telemetry_shape") or "-"), "Worker", str(entry.get("worker_model") or "-")) + overview.add_row("Attestation", str(entry.get("attestation_state") or "-"), "Approval", str(entry.get("approval_state") or "-")) + overview.add_row("Acting As", str(entry.get("acting_agent_name") or "-"), "Identity", str(entry.get("identity_status") or "-")) + overview.add_row("Environment", str(entry.get("environment_label") or entry.get("base_url") or "-"), "Env Status", str(entry.get("environment_status") or "-")) + overview.add_row("Current Space", str(entry.get("active_space_name") or entry.get("active_space_id") or "-"), "Space Status", str(entry.get("space_status") or "-")) + overview.add_row("Default Space", str(entry.get("default_space_name") or entry.get("default_space_id") or "-"), "Allowed Spaces", str(entry.get("allowed_space_count") or 0)) + overview.add_row("Install", str(entry.get("install_id") or "-"), "Runtime Instance", str(entry.get("runtime_instance_id") or "-")) + overview.add_row("Reachability", _reachability_copy(entry), "Reason", str(entry.get("confidence_reason") or "-")) overview.add_row("Desired", str(entry.get("desired_state") or "-"), "Effective", str(entry.get("effective_state") or "-")) overview.add_row("Connected", "yes" if entry.get("connected") else "no", "Queue", str(entry.get("backlog_depth") or 0)) overview.add_row("Seen", _format_age(entry.get("last_seen_age_seconds")), "Reconnect", _format_age(entry.get("reconnect_backoff_seconds"))) overview.add_row("Processed", str(entry.get("processed_count") or 0), "Dropped", str(entry.get("dropped_count") or 0)) overview.add_row("Last Work", _format_timestamp(entry.get("last_work_received_at")), "Completed", _format_timestamp(entry.get("last_work_completed_at"))) overview.add_row("Phase", str(entry.get("current_status") or "-"), "Activity", str(entry.get("current_activity") or "-")) - overview.add_row("Tool", str(entry.get("current_tool") or "-"), "Transport", str(entry.get("transport") or "-")) + overview.add_row("Tool", str(entry.get("current_tool") or "-"), "Adapter", str(entry.get("runtime_type") or "-")) overview.add_row("Cred Source", str(entry.get("credential_source") or "-"), "Space", str(entry.get("space_id") or "-")) overview.add_row("Agent ID", str(entry.get("agent_id") or "-"), "Last Reply", str(entry.get("last_reply_preview") or "-")) - overview.add_row("Last Error", str(entry.get("last_error") or "-"), "", "") + overview.add_row("Last Error", str(entry.get("last_error") or "-"), "Confidence Detail", str(entry.get("confidence_detail") or "-")) + overview.add_row("Doctor", str(entry.get("last_successful_doctor_at") or "-"), "Doctor Status", str((entry.get("last_doctor_result") or {}).get("status") if isinstance(entry.get("last_doctor_result"), dict) else "-")) paths = Table.grid(expand=True, padding=(0, 2)) paths.add_column(style="bold") @@ -398,6 +3064,7 @@ def login( raise typer.Exit(1) selected_space = space_id + selected_space_name = None if not selected_space: try: spaces = client.list_spaces() @@ -405,14 +3072,30 @@ def login( selected = auth_cmd._select_login_space([s for s in space_list if isinstance(s, dict)]) if selected: selected_space = auth_cmd._candidate_space_id(selected) + selected_space_name = str(selected.get("name") or selected_space) except Exception: selected_space = None + elif selected_space: + try: + spaces = client.list_spaces() + space_list = spaces.get("spaces", spaces) if isinstance(spaces, dict) else spaces + selected_space_name = next( + ( + str(item.get("name") or selected_space) + for item in space_list + if isinstance(item, dict) and auth_cmd._candidate_space_id(item) == selected_space + ), + None, + ) + except Exception: + selected_space_name = None payload = { "token": resolved_token, "base_url": resolved_base_url, "principal_type": "user", "space_id": selected_space, + "space_name": selected_space_name, "username": me.get("username"), "email": me.get("email"), "saved_at": None, @@ -428,6 +3111,7 @@ def login( "session_path": str(path), "base_url": resolved_base_url, "space_id": selected_space, + "space_name": selected_space_name, "username": me.get("username"), "email": me.get("email"), } @@ -453,25 +3137,43 @@ def status(as_json: bool = JSON_OPTION): err_console.print(f" daemon = {'running' if payload['daemon']['running'] else 'stopped'}") if payload["daemon"]["pid"]: err_console.print(f" pid = {payload['daemon']['pid']}") + err_console.print(f" ui = {'running' if payload['ui']['running'] else 'stopped'}") + if payload["ui"]["pid"]: + err_console.print(f" ui_pid = {payload['ui']['pid']}") + err_console.print(f" ui_url = {payload['ui']['url']}") err_console.print(f" base_url = {payload['base_url']}") err_console.print(f" space_id = {payload['space_id']}") + if payload.get("space_name"): + err_console.print(f" space_name = {payload['space_name']}") err_console.print(f" user = {payload['user']}") err_console.print(f" agents = {payload['summary']['managed_agents']}") - err_console.print(f" connected = {payload['summary']['connected_agents']}") + err_console.print(f" live = {payload['summary']['live_agents']}") + err_console.print(f" on_demand = {payload['summary']['on_demand_agents']}") + err_console.print(f" inbox = {payload['summary']['inbox_agents']}") + err_console.print(f" alerts = {payload['summary'].get('alert_count', 0)}") + err_console.print(f" approvals = {payload['summary'].get('pending_approvals', 0)} pending") + if payload.get("alerts"): + print_table( + ["Level", "Alert", "Agent", "Detail"], + payload["alerts"], + keys=["severity", "title", "agent_name", "detail"], + ) if payload["agents"]: print_table( - ["Agent", "Type", "Desired", "Effective", "Phase", "Seen", "Backlog", "Activity", "Last Error"], - payload["agents"], + ["Agent", "Type", "Mode", "Presence", "Output", "Confidence", "Acting As", "Current Space", "Seen", "Backlog", "Reason"], + [{**agent, "type": _agent_type_label(agent), "output": _agent_output_label(agent)} for agent in payload["agents"]], keys=[ "name", - "runtime_type", - "desired_state", - "effective_state", - "current_status", + "type", + "mode", + "presence", + "output", + "confidence", + "acting_agent_name", + "active_space_name", "last_seen_age_seconds", "backlog_depth", - "current_activity", - "last_error", + "confidence_reason", ], ) if payload["recent_activity"]: @@ -482,16 +3184,322 @@ def status(as_json: bool = JSON_OPTION): ) -@app.command("watch") -def watch( - interval: float = typer.Option(2.0, "--interval", "-n", help="Dashboard refresh interval in seconds"), - activity_limit: int = typer.Option(8, "--activity-limit", help="Number of recent events to display"), - once: bool = typer.Option(False, "--once", help="Render one dashboard frame and exit"), -): - """Watch the Gateway in a live terminal dashboard.""" +@app.command("runtime-types") +def runtime_types(as_json: bool = JSON_OPTION): + """List advanced/internal Gateway runtime backends.""" + payload = _runtime_types_payload() + if as_json: + print_json(payload) + return + rows = [] + for item in payload["runtime_types"]: + rows.append( + { + "id": item["id"], + "label": item["label"], + "kind": item.get("kind"), + "activity": item.get("signals", {}).get("activity"), + "tools": item.get("signals", {}).get("tools"), + } + ) + print_table( + ["Type", "Label", "Kind", "Activity Signal", "Tool Signal"], + rows, + keys=["id", "label", "kind", "activity", "tools"], + ) - def render_dashboard() -> Group: - return _render_gateway_dashboard(_status_payload(activity_limit=activity_limit)) + +@app.command("templates") +def templates(as_json: bool = JSON_OPTION): + """List Gateway agent templates and what signals they provide.""" + payload = _agent_templates_payload() + if as_json: + print_json(payload) + return + rows = [] + for item in payload["templates"]: + rows.append( + { + "id": item["id"], + "label": item["label"], + "type": item.get("asset_type_label"), + "output": item.get("output_label"), + "availability": item.get("availability"), + "summary": item.get("operator_summary"), + "activity": item.get("signals", {}).get("activity"), + } + ) + print_table( + ["Template", "Label", "Type", "Output", "Status", "Why Pick It", "Activity Signal"], + rows, + keys=["id", "label", "type", "output", "availability", "summary", "activity"], + ) + + +def _gateway_cli_argv(*args: str) -> list[str]: + current_argv0 = str(sys.argv[0] or "").strip() + if current_argv0: + current_path = Path(current_argv0).expanduser() + if current_path.exists() and current_path.name in {"ax", "axctl"}: + return [str(current_path.resolve()), *args] + python_bin = Path(sys.executable).resolve().parent + for candidate in (python_bin / "ax", python_bin / "axctl"): + if candidate.exists(): + return [str(candidate), *args] + resolved = shutil.which("ax") or shutil.which("axctl") + if resolved: + return [resolved, *args] + command = ( + "import sys; " + "from ax_cli.main import main; " + "sys.argv = ['ax'] + sys.argv[1:]; " + "main()" + ) + return [sys.executable, "-c", command, *args] + + +def _spawn_gateway_background_process(command: list[str], *, log_path: Path) -> subprocess.Popen[bytes]: + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("ab") as handle: + process = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=handle, + stderr=subprocess.STDOUT, + cwd=str(Path.cwd()), + start_new_session=True, + close_fds=True, + ) + return process + + +def _tail_log_lines(path: Path, *, lines: int = 12) -> str: + if not path.exists(): + return "" + try: + text = path.read_text(errors="replace") + except OSError: + return "" + chunks = [line.rstrip() for line in text.splitlines() if line.strip()] + return "\n".join(chunks[-lines:]) + + +def _wait_for_daemon_ready(process: subprocess.Popen[bytes], *, timeout: float = 3.0) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + if process.poll() is not None: + return False + if daemon_status().get("running") or active_gateway_pid(): + return True + time.sleep(0.1) + return process.poll() is None and bool(daemon_status().get("running") or active_gateway_pid()) + + +def _wait_for_ui_ready(process: subprocess.Popen[bytes], *, host: str, port: int, timeout: float = 3.0) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + if process.poll() is not None: + return False + try: + with socket.create_connection((host, port), timeout=0.2): + return True + except OSError: + time.sleep(0.1) + try: + with socket.create_connection((host, port), timeout=0.2): + return True + except OSError: + return False + + +def _terminate_pids(pids: list[int], *, timeout: float = 3.0) -> tuple[list[int], list[int]]: + requested: list[int] = [] + forced: list[int] = [] + for pid in sorted(set(pids)): + try: + os.kill(pid, signal.SIGTERM) + requested.append(pid) + except ProcessLookupError: + continue + deadline = time.time() + timeout + while time.time() < deadline: + alive = [pid for pid in requested if gateway_core._pid_alive(pid)] + if not alive: + return requested, forced + time.sleep(0.1) + for pid in requested: + if not gateway_core._pid_alive(pid): + continue + try: + os.kill(pid, signal.SIGKILL) + forced.append(pid) + except ProcessLookupError: + continue + return requested, forced + + +@app.command("ui") +def ui( + host: str = typer.Option("127.0.0.1", "--host", help="Host interface to bind the local Gateway UI"), + port: int = typer.Option(8765, "--port", help="Port for the local Gateway UI"), + activity_limit: int = typer.Option(24, "--activity-limit", help="Number of recent events to expose in the UI"), + refresh: float = typer.Option(2.0, "--refresh", help="Browser auto-refresh interval in seconds"), + open_browser: bool = typer.Option(True, "--open/--no-open", help="Open the local UI in a browser"), +): + """Serve a local Gateway web UI.""" + refresh_ms = max(250, int(refresh * 1000)) + handler = _build_gateway_ui_handler(activity_limit=activity_limit, refresh_ms=refresh_ms) + try: + server = _GatewayUiServer((host, port), handler) + except OSError as exc: + err_console.print(f"[red]Failed to start Gateway UI:[/red] {exc}") + raise typer.Exit(1) + + url = f"http://{host}:{server.server_port}" + err_console.print("[bold]ax gateway ui[/bold] — local Gateway dashboard") + err_console.print(f" url = {url}") + err_console.print(f" refresh = {refresh_ms}ms") + err_console.print(f" source = {gateway_dir()}") + err_console.print(" stop = Ctrl-C") + write_gateway_ui_state(pid=os.getpid(), host=host, port=server.server_port) + record_gateway_activity("gateway_ui_started", pid=os.getpid(), host=host, port=server.server_port, url=url) + if open_browser: + try: + webbrowser.open_new_tab(url) + except Exception: + err_console.print("[yellow]Could not open a browser automatically.[/yellow]") + try: + server.serve_forever() + except KeyboardInterrupt: + err_console.print("[yellow]Gateway UI stopped.[/yellow]") + finally: + record_gateway_activity("gateway_ui_stopped", pid=os.getpid(), host=host, port=server.server_port, url=url) + clear_gateway_ui_state(os.getpid()) + server.server_close() + + +@app.command("start") +def start( + poll_interval: float = typer.Option(1.0, "--poll-interval", help="Registry reconcile interval in seconds"), + host: str = typer.Option("127.0.0.1", "--host", help="Host interface to bind the local Gateway UI"), + port: int = typer.Option(8765, "--port", help="Port for the local Gateway UI"), + activity_limit: int = typer.Option(24, "--activity-limit", help="Number of recent events to expose in the UI"), + refresh: float = typer.Option(2.0, "--refresh", help="Browser auto-refresh interval in seconds"), + open_browser: bool = typer.Option(True, "--open/--no-open", help="Open the local UI in a browser"), +): + """Start the Gateway daemon and local UI in the background.""" + session = load_gateway_session() + daemon_pid = active_gateway_pid() + ui_pid = active_gateway_ui_pid() + daemon_started = False + ui_started = False + daemon_note: str | None = None + + if daemon_pid is None: + if session: + daemon_process = _spawn_gateway_background_process( + _gateway_cli_argv("gateway", "run", "--poll-interval", str(poll_interval)), + log_path=daemon_log_path(), + ) + if _wait_for_daemon_ready(daemon_process): + daemon_pid = active_gateway_pid() or daemon_process.pid + daemon_started = True + else: + detail = _tail_log_lines(daemon_log_path()) + err_console.print(f"[red]Failed to start Gateway daemon.[/red] {detail or 'Check gateway.log for details.'}") + raise typer.Exit(1) + else: + daemon_note = "Gateway is not logged in yet; the UI can still start in disconnected mode." + + if ui_pid is None: + ui_process = _spawn_gateway_background_process( + _gateway_cli_argv( + "gateway", + "ui", + "--host", + host, + "--port", + str(port), + "--activity-limit", + str(activity_limit), + "--refresh", + str(refresh), + "--no-open", + ), + log_path=ui_log_path(), + ) + if _wait_for_ui_ready(ui_process, host=host, port=port): + ui_pid = active_gateway_ui_pid() or ui_process.pid + ui_started = True + else: + detail = _tail_log_lines(ui_log_path()) + if daemon_started and daemon_pid: + _terminate_pids([daemon_pid]) + gateway_core.clear_gateway_pid() + err_console.print(f"[red]Failed to start Gateway UI.[/red] {detail or 'Check gateway-ui.log for details.'}") + raise typer.Exit(1) + + ui_meta = ui_status() + if open_browser and ui_meta.get("running"): + try: + webbrowser.open_new_tab(str(ui_meta.get("url") or f"http://{host}:{port}")) + except Exception: + err_console.print("[yellow]Could not open a browser automatically.[/yellow]") + + err_console.print("[bold]ax gateway start[/bold]") + err_console.print(f" daemon = {'started' if daemon_started else 'running' if daemon_pid else 'not started'}") + if daemon_pid: + err_console.print(f" daemon_pid= {daemon_pid}") + err_console.print(f" ui = {'started' if ui_started else 'running' if ui_pid else 'not started'}") + if ui_pid: + err_console.print(f" ui_pid = {ui_pid}") + err_console.print(f" url = {ui_meta.get('url') or f'http://{host}:{port}'}") + err_console.print(f" logs = {daemon_log_path()}") + err_console.print(f" ui_logs = {ui_log_path()}") + if daemon_note: + err_console.print(f"[yellow]{daemon_note}[/yellow]") + + +@app.command("stop") +def stop(): + """Stop the background Gateway daemon and local UI.""" + daemon_pids = active_gateway_pids() + ui_pids = active_gateway_ui_pids() + if not daemon_pids and not ui_pids: + clear_gateway_ui_state() + gateway_core.clear_gateway_pid() + err_console.print("[yellow]Gateway daemon and UI are already stopped.[/yellow]") + return + + ui_requested, ui_forced = _terminate_pids(ui_pids) + daemon_requested, daemon_forced = _terminate_pids(daemon_pids) + clear_gateway_ui_state() + gateway_core.clear_gateway_pid() + record_gateway_activity( + "gateway_services_stopped", + daemon_pids=daemon_requested, + ui_pids=ui_requested, + daemon_forced=daemon_forced, + ui_forced=ui_forced, + ) + + err_console.print("[bold]ax gateway stop[/bold]") + err_console.print(f" daemon = {daemon_requested or []}") + err_console.print(f" ui = {ui_requested or []}") + if daemon_forced or ui_forced: + err_console.print(f"[yellow]Forced kill:[/yellow] daemon={daemon_forced or []} ui={ui_forced or []}") + + +@app.command("watch") +def watch( + interval: float = typer.Option(2.0, "--interval", "-n", help="Dashboard refresh interval in seconds"), + activity_limit: int = typer.Option(8, "--activity-limit", help="Number of recent events to display"), + once: bool = typer.Option(False, "--once", help="Render one dashboard frame and exit"), +): + """Watch the Gateway in a live terminal dashboard.""" + + def render_dashboard() -> Group: + return _render_gateway_dashboard(_status_payload(activity_limit=activity_limit)) if once: console.print(render_dashboard()) @@ -528,12 +3536,124 @@ def run( err_console.print("[yellow]Gateway stopped.[/yellow]") +@approvals_app.command("list") +def list_approvals( + status: str | None = typer.Option(None, "--status", help="Optional filter: pending | approved | rejected"), + as_json: bool = JSON_OPTION, +): + """List local Gateway approval requests.""" + payload = _approval_rows_payload(status=status) + if as_json: + print_json(payload) + return + err_console.print("[bold]ax gateway approvals list[/bold]") + err_console.print(f" approvals = {payload['count']}") + err_console.print(f" pending = {payload['pending']}") + if not payload["approvals"]: + err_console.print("[dim]No Gateway approvals found.[/dim]") + return + print_table( + ["Approval", "Asset", "Kind", "Status", "Risk", "Reason", "Requested"], + payload["approvals"], + keys=["approval_id", "asset_id", "approval_kind", "status", "risk", "reason", "requested_at"], + ) + + +@approvals_app.command("show") +def show_approval( + approval_id: str = typer.Argument(..., help="Approval request id"), + as_json: bool = JSON_OPTION, +): + """Show one local Gateway approval request.""" + try: + payload = _approval_detail_payload(approval_id) + except LookupError as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) + if as_json: + print_json(payload) + return + approval = payload["approval"] + print_table( + ["Field", "Value"], + [ + {"field": "approval_id", "value": approval.get("approval_id")}, + {"field": "asset_id", "value": approval.get("asset_id")}, + {"field": "gateway_id", "value": approval.get("gateway_id")}, + {"field": "install_id", "value": approval.get("install_id")}, + {"field": "kind", "value": approval.get("approval_kind")}, + {"field": "status", "value": approval.get("status")}, + {"field": "risk", "value": approval.get("risk")}, + {"field": "action", "value": approval.get("action")}, + {"field": "resource", "value": approval.get("resource")}, + {"field": "reason", "value": approval.get("reason")}, + {"field": "requested_at", "value": approval.get("requested_at")}, + {"field": "decided_at", "value": approval.get("decided_at")}, + {"field": "decision_scope", "value": approval.get("decision_scope")}, + ], + keys=["field", "value"], + ) + candidate = approval.get("candidate_binding") if isinstance(approval.get("candidate_binding"), dict) else None + if candidate: + print_table( + ["Candidate Field", "Value"], + [ + {"field": "path", "value": candidate.get("path")}, + {"field": "binding_type", "value": candidate.get("binding_type")}, + {"field": "launch_spec_hash", "value": candidate.get("launch_spec_hash")}, + {"field": "candidate_signature", "value": candidate.get("candidate_signature")}, + ], + keys=["field", "value"], + ) + + +@approvals_app.command("approve") +def approve_approval( + approval_id: str = typer.Argument(..., help="Approval request id"), + scope: str = typer.Option("asset", "--scope", help="Recorded approval scope: once | asset | gateway"), + as_json: bool = JSON_OPTION, +): + """Approve a local Gateway binding request.""" + try: + payload = approve_gateway_approval(approval_id, scope=scope) + except (LookupError, ValueError) as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) + if as_json: + print_json(payload) + return + approval = payload["approval"] + err_console.print(f"[green]Approved:[/green] {approval['approval_id']}") + err_console.print(f" asset = {approval.get('asset_id')}") + err_console.print(f" scope = {approval.get('decision_scope')}") + + +@approvals_app.command("deny") +def deny_approval( + approval_id: str = typer.Argument(..., help="Approval request id"), + as_json: bool = JSON_OPTION, +): + """Deny a local Gateway binding request.""" + try: + payload = deny_gateway_approval(approval_id) + except LookupError as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) + if as_json: + print_json({"approval": payload}) + return + err_console.print(f"[yellow]Denied:[/yellow] {payload['approval_id']}") + err_console.print(f" asset = {payload.get('asset_id')}") + + @agents_app.command("add") def add_agent( name: str = typer.Argument(..., help="Managed agent name"), - runtime_type: str = typer.Option("echo", "--type", help="Runtime type: echo | exec | inbox"), - exec_cmd: str = typer.Option(None, "--exec", help="Per-mention command for exec runtimes"), - workdir: str = typer.Option(None, "--workdir", help="Working directory for exec runtimes"), + template_id: str = typer.Option(None, "--template", help="Agent template: echo_test | ollama | hermes | claude_code_channel"), + runtime_type: str = typer.Option(None, "--type", help="Advanced/internal runtime backend: echo | exec | inbox"), + exec_cmd: str = typer.Option(None, "--exec", help="Advanced override for exec-based templates"), + workdir: str = typer.Option(None, "--workdir", help="Advanced working directory override"), + ollama_model: str = typer.Option(None, "--ollama-model", help="Ollama model override for the Ollama template"), space_id: str = typer.Option(None, "--space-id", help="Target space (defaults to gateway session)"), audience: str = typer.Option("both", "--audience", help="Minted PAT audience"), description: str = typer.Option(None, "--description", help="Create/update description"), @@ -542,90 +3662,73 @@ def add_agent( as_json: bool = JSON_OPTION, ): """Register a managed agent and mint a Gateway-owned PAT for it.""" - runtime_type = runtime_type.lower().strip() - if runtime_type not in {"echo", "exec", "command", "inbox"}: - err_console.print("[red]Unsupported runtime type.[/red] Use echo, exec, or inbox.") - raise typer.Exit(1) - if runtime_type in {"exec", "command"} and not exec_cmd: - err_console.print("[red]Exec runtimes require --exec.[/red]") - raise typer.Exit(1) - if runtime_type in {"echo", "inbox"} and exec_cmd: - err_console.print("[red]Echo and inbox runtimes do not accept --exec.[/red]") - raise typer.Exit(1) - - session = _load_gateway_session_or_exit() - selected_space = space_id or session.get("space_id") - if not selected_space: - err_console.print("[red]No space selected.[/red] Use --space-id or re-run `ax gateway login` with one.") - raise typer.Exit(1) - - client = _load_gateway_user_client() - existing = _find_agent_in_space(client, name, selected_space) - if existing: - agent = existing - if description or model: - client.update_agent(name, **{k: v for k, v in {"description": description, "model": model}.items() if v}) - else: - agent = _create_agent_in_space( - client, + selected_template = template_id or ("echo_test" if not runtime_type else None) + try: + entry = _register_managed_agent( name=name, - space_id=selected_space, + template_id=selected_template, + runtime_type=runtime_type, + exec_cmd=exec_cmd, + workdir=workdir, + ollama_model=ollama_model, + space_id=space_id, + audience=audience, description=description, model=model, + start=start, ) - _polish_metadata(client, name=name, bio=None, specialization=None, system_prompt=None) - - agent_id = str(agent.get("id") or agent.get("agent_id") or "") - token, pat_source = _mint_agent_pat( - client, - agent_id=agent_id, - agent_name=name, - audience=audience, - expires_in_days=90, - pat_name=f"gateway-{name}", - space_id=selected_space, - ) - token_file = _save_agent_token(name, token) - - registry = load_gateway_registry() - entry = upsert_agent_entry( - registry, - { - "name": name, - "agent_id": agent_id, - "space_id": selected_space, - "base_url": session["base_url"], - "runtime_type": "exec" if runtime_type == "command" else runtime_type, - "exec_command": exec_cmd, - "workdir": workdir, - "token_file": str(token_file), - "desired_state": "running" if start else "stopped", - "effective_state": "stopped", - "transport": "gateway", - "credential_source": "gateway", - "last_error": None, - "backlog_depth": 0, - "processed_count": 0, - "dropped_count": 0, - "pat_source": pat_source, - "added_at": __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat(), - }, - ) - save_gateway_registry(registry) - record_gateway_activity( - "managed_agent_added", - entry=entry, - space_id=selected_space, - token_file=str(token_file), - ) + except (ValueError, LookupError) as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) if as_json: print_json(entry) else: err_console.print(f"[green]Managed agent ready:[/green] @{name}") - err_console.print(f" runtime_type = {entry['runtime_type']}") + if entry.get("template_label"): + err_console.print(f" type = {entry['template_label']}") + if entry.get("asset_type_label"): + err_console.print(f" asset = {entry['asset_type_label']}") err_console.print(f" desired_state = {entry['desired_state']}") - err_console.print(f" token_file = {token_file}") + err_console.print(f" token_file = {entry['token_file']}") + + +@agents_app.command("update") +def update_agent( + name: str = typer.Argument(..., help="Managed agent name"), + template_id: str = typer.Option(None, "--template", help="Replace the agent template"), + runtime_type: str = typer.Option(None, "--type", help="Advanced/internal runtime backend override: echo | exec | inbox"), + exec_cmd: str = typer.Option(None, "--exec", help="Advanced override for exec-based templates"), + workdir: str = typer.Option(None, "--workdir", help="Advanced working directory override"), + ollama_model: str = typer.Option(None, "--ollama-model", help="Ollama model override for the Ollama template"), + description: str = typer.Option(None, "--description", help="Update platform agent description"), + model: str = typer.Option(None, "--model", help="Update platform agent model"), + desired_state: str = typer.Option(None, "--desired-state", help="running | stopped"), + as_json: bool = JSON_OPTION, +): + """Update a managed agent without redoing Gateway bootstrap.""" + try: + entry = _update_managed_agent( + name=name, + template_id=template_id, + runtime_type=runtime_type, + exec_cmd=exec_cmd if exec_cmd is not None else _UNSET, + workdir=workdir if workdir is not None else _UNSET, + ollama_model=ollama_model if ollama_model is not None else _UNSET, + description=description, + model=model, + desired_state=desired_state, + ) + except (LookupError, ValueError) as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) + + if as_json: + print_json(entry) + return + err_console.print(f"[green]Managed agent updated:[/green] @{name}") + err_console.print(f" type = {entry.get('template_label') or entry.get('runtime_type')}") + err_console.print(f" desired_state = {entry.get('desired_state')}") @agents_app.command("list") @@ -636,9 +3739,9 @@ def list_agents(as_json: bool = JSON_OPTION): print_json({"agents": agents, "count": len(agents)}) return print_table( - ["Agent", "Type", "Desired", "Effective", "Space"], - agents, - keys=["name", "runtime_type", "desired_state", "effective_state", "space_id"], + ["Agent", "Type", "Mode", "Presence", "Output", "Confidence", "Space"], + [{**agent, "type": _agent_type_label(agent), "output": _agent_output_label(agent)} for agent in agents], + keys=["name", "type", "mode", "presence", "output", "confidence", "space_id"], ) @@ -649,26 +3752,63 @@ def show_agent( as_json: bool = JSON_OPTION, ): """Show one managed agent in detail.""" - payload = _status_payload(activity_limit=activity_limit) - entry = next((agent for agent in payload["agents"] if str(agent.get("name") or "").lower() == name.lower()), None) - if not entry: + result = _agent_detail_payload(name, activity_limit=activity_limit) + if result is None: err_console.print(f"[red]Managed agent not found:[/red] {name}") raise typer.Exit(1) - activity = load_recent_gateway_activity(limit=activity_limit, agent_name=name) - result = { - "gateway": { - "connected": payload["connected"], - "base_url": payload["base_url"], - "space_id": payload["space_id"], - "daemon": payload["daemon"], - }, - "agent": entry, - "recent_activity": activity, - } if as_json: print_json(result) return - console.print(_render_agent_detail(entry, activity=activity)) + console.print(_render_agent_detail(result["agent"], activity=result["recent_activity"])) + + +@agents_app.command("test") +def test_agent( + name: str = typer.Argument(..., help="Managed agent name"), + message: str = typer.Option(None, "--message", help="Override the recommended Gateway test prompt"), + author: str = typer.Option("agent", "--author", help="Who should author the test message: agent | user"), + sender_agent: str = typer.Option(None, "--sender-agent", help="Managed sender identity to use when --author agent"), + as_json: bool = JSON_OPTION, +): + """Send a Gateway-authored test message to one managed agent.""" + try: + result = _send_gateway_test_to_managed_agent(name, content=message, author=author, sender_agent=sender_agent) + except ValueError as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) + + if as_json: + print_json(result) + return + + err_console.print(f"[green]Gateway test sent:[/green] @{result['target_agent']}") + err_console.print(f" prompt = {result['recommended_prompt']}") + message_payload = result.get("message") or {} + if isinstance(message_payload, dict) and message_payload.get("id"): + err_console.print(f" message_id = {message_payload['id']}") + + +@agents_app.command("doctor") +def doctor_agent( + name: str = typer.Argument(..., help="Managed agent name"), + send_test: bool = typer.Option(False, "--send-test", help="Also send a Gateway-authored smoke test"), + as_json: bool = JSON_OPTION, +): + """Run Gateway Doctor checks for one managed agent.""" + try: + result = _run_gateway_doctor(name, send_test=send_test) + except (LookupError, ValueError) as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) + + if as_json: + print_json(result) + return + + tone = {"passed": "green", "warning": "yellow", "failed": "red"}.get(result["status"], "cyan") + err_console.print(f"[{tone}]Gateway Doctor {result['status']}:[/{tone}] @{name}") + err_console.print(f" summary = {result['summary']}") + print_table(["Check", "Status", "Detail"], result["checks"], keys=["name", "status", "detail"]) @agents_app.command("send") @@ -680,95 +3820,49 @@ def send_as_agent( as_json: bool = JSON_OPTION, ): """Send a message as a Gateway-managed agent.""" - entry = _load_managed_agent_or_exit(name) - client = _load_managed_agent_client(entry) - space_id = str(entry.get("space_id") or "") - if not space_id: - err_console.print(f"[red]Managed agent is missing a space id:[/red] @{name}") + try: + result = _send_from_managed_agent(name=name, content=content, to=to, parent_id=parent_id) + except ValueError as exc: + err_console.print(f"[red]{exc}[/red]") raise typer.Exit(1) - message_content = content.strip() - mention = str(to or "").strip().lstrip("@") - if mention: - prefix = f"@{mention}" - if not message_content.startswith(prefix): - message_content = f"{prefix} {message_content}".strip() - - metadata = { - "control_plane": "gateway", - "gateway": { - "managed": True, - "agent_name": entry.get("name"), - "agent_id": entry.get("agent_id"), - "runtime_type": entry.get("runtime_type"), - "transport": entry.get("transport", "gateway"), - "credential_source": entry.get("credential_source", "gateway"), - "sent_via": "gateway_cli", - }, - } - result = client.send_message( - space_id, - message_content, - agent_id=str(entry.get("agent_id") or "") or None, - parent_id=parent_id or None, - metadata=metadata, - ) - payload = result.get("message", result) if isinstance(result, dict) else result - if isinstance(payload, dict): - record_gateway_activity( - "manual_message_sent", - entry=entry, - message_id=payload.get("id"), - reply_preview=message_content[:120] or None, - ) if as_json: - print_json({"agent": entry.get("name"), "message": payload, "content": message_content}) + print_json(result) return - err_console.print(f"[green]Sent as managed agent:[/green] @{entry.get('name')}") - if isinstance(payload, dict) and payload.get("id"): - err_console.print(f" id = {payload['id']}") - err_console.print(f" content = {message_content}") + err_console.print(f"[green]Sent as managed agent:[/green] @{result['agent']}") + if isinstance(result["message"], dict) and result["message"].get("id"): + err_console.print(f" id = {result['message']['id']}") + err_console.print(f" content = {result['content']}") @agents_app.command("start") def start_agent(name: str = typer.Argument(..., help="Managed agent name")): """Set a managed agent's desired state to running.""" - registry = load_gateway_registry() - entry = find_agent_entry(registry, name) - if not entry: + try: + _set_managed_agent_desired_state(name, "running") + except LookupError: err_console.print(f"[red]Managed agent not found:[/red] {name}") raise typer.Exit(1) - entry["desired_state"] = "running" - save_gateway_registry(registry) - record_gateway_activity("managed_agent_desired_running", entry=entry) err_console.print(f"[green]Desired state set to running:[/green] @{name}") @agents_app.command("stop") def stop_agent(name: str = typer.Argument(..., help="Managed agent name")): """Set a managed agent's desired state to stopped.""" - registry = load_gateway_registry() - entry = find_agent_entry(registry, name) - if not entry: + try: + _set_managed_agent_desired_state(name, "stopped") + except LookupError: err_console.print(f"[red]Managed agent not found:[/red] {name}") raise typer.Exit(1) - entry["desired_state"] = "stopped" - save_gateway_registry(registry) - record_gateway_activity("managed_agent_desired_stopped", entry=entry) err_console.print(f"[green]Desired state set to stopped:[/green] @{name}") @agents_app.command("remove") def remove_agent(name: str = typer.Argument(..., help="Managed agent name")): """Remove a managed agent from local Gateway control.""" - registry = load_gateway_registry() - entry = remove_agent_entry(registry, name) - if not entry: + try: + _remove_managed_agent(name) + except LookupError: err_console.print(f"[red]Managed agent not found:[/red] {name}") raise typer.Exit(1) - save_gateway_registry(registry) - token_file = Path(str(entry.get("token_file") or "")) - if token_file.exists(): - token_file.unlink() - record_gateway_activity("managed_agent_removed", entry=entry) err_console.print(f"[green]Removed managed agent:[/green] @{name}") diff --git a/ax_cli/gateway.py b/ax_cli/gateway.py index abff838..15c6782 100644 --- a/ax_cli/gateway.py +++ b/ax_cli/gateway.py @@ -13,6 +13,7 @@ import hashlib import json import os +import platform import queue import re import shlex @@ -23,6 +24,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable +from urllib.parse import urlparse import httpx @@ -46,6 +48,7 @@ SSE_IDLE_TIMEOUT_SECONDS = 45.0 RUNTIME_STALE_AFTER_SECONDS = 75.0 GATEWAY_EVENT_PREFIX = "AX_GATEWAY_EVENT " +DEFAULT_OLLAMA_BASE_URL = os.environ.get("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/") ENV_DENYLIST = { "AX_AGENT_ID", "AX_AGENT_NAME", @@ -60,13 +63,1702 @@ "AX_USER_TOKEN", } _ACTIVITY_LOCK = threading.Lock() -_GATEWAY_PROCESS_RE = re.compile(r"(?:^|\s)(?:uv\s+run\s+ax\s+gateway\s+run|.+?/ax\s+gateway\s+run)(?:\s|$)") +_GATEWAY_PROCESS_RE = re.compile( + r"(?:uv\s+run\s+ax\s+gateway\s+run|(?:^|\s).+?/ax(?:ctl)?\s+gateway\s+run(?:\s|$)|-m\s+ax_cli\.main\s+gateway\s+run(?:\s|$))" +) +_GATEWAY_UI_PROCESS_RE = re.compile( + r"(?:uv\s+run\s+ax\s+gateway\s+ui|(?:^|\s).+?/ax(?:ctl)?\s+gateway\s+ui(?:\s|$)|-m\s+ax_cli\.main\s+gateway\s+ui(?:\s|$))" +) + +_CONTROLLED_PLACEMENTS = {"hosted", "attached", "brokered", "mailbox"} +_CONTROLLED_ACTIVATIONS = {"persistent", "on_demand", "attach_only", "queue_worker"} +_CONTROLLED_LIVENESS = {"connected", "stale", "offline", "setup_error"} +_CONTROLLED_WORK_STATES = {"idle", "queued", "working", "blocked"} +_CONTROLLED_REPLY_MODES = {"interactive", "background", "summary_only", "silent"} +_CONTROLLED_TELEMETRY_LEVELS = {"rich", "basic", "silent"} +_CONTROLLED_ASSET_CLASSES = {"interactive_agent", "background_worker", "scheduled_job", "alert_listener", "service_proxy"} +_CONTROLLED_INTAKE_MODELS = {"live_listener", "launch_on_send", "queue_accept", "queue_drain", "scheduled_run", "event_triggered", "manual_only"} +_CONTROLLED_TRIGGER_SOURCES = {"direct_message", "queued_job", "scheduled_invocation", "external_alert", "manual_trigger", "tool_call"} +_CONTROLLED_RETURN_PATHS = {"inline_reply", "sender_inbox", "summary_post", "task_update", "event_log", "silent"} +_CONTROLLED_TELEMETRY_SHAPES = {"rich", "basic", "heartbeat_only", "opaque"} +_CONTROLLED_WORKER_MODELS = {"queue_drain"} +_CONTROLLED_ATTESTATION_STATES = {"verified", "drifted", "unknown", "blocked"} +_CONTROLLED_APPROVAL_STATES = {"not_required", "pending", "approved", "rejected"} +_CONTROLLED_IDENTITY_STATUSES = {"verified", "unknown_identity", "credential_mismatch", "fallback_blocked", "bootstrap_only", "blocked"} +_CONTROLLED_SPACE_STATUSES = {"active_allowed", "active_not_allowed", "no_active_space", "unknown"} +_CONTROLLED_ENVIRONMENT_STATUSES = {"environment_allowed", "environment_mismatch", "environment_unknown", "environment_blocked"} +_CONTROLLED_ACTIVE_SPACE_SOURCES = {"explicit_request", "gateway_binding", "visible_default", "none"} +_CONTROLLED_MODES = {"LIVE", "ON-DEMAND", "INBOX"} +_CONTROLLED_PRESENCE = {"IDLE", "QUEUED", "WORKING", "BLOCKED", "STALE", "OFFLINE", "ERROR"} +_CONTROLLED_REPLY = {"REPLY", "SUMMARY", "SILENT"} +_CONTROLLED_CONFIDENCE = {"HIGH", "MEDIUM", "LOW", "BLOCKED"} +_CONTROLLED_REACHABILITY = {"live_now", "queue_available", "launch_available", "attach_required", "unavailable"} +_CONTROLLED_CONFIDENCE_REASONS = { + "live_now", + "queue_available", + "launch_available", + "attach_required", + "unavailable", + "setup_blocked", + "recent_test_failed", + "completion_degraded", + "approval_required", + "binding_drift", + "new_gateway", + "unknown_asset", + "asset_mismatch", + "approval_denied", + "identity_unbound", + "identity_mismatch", + "fallback_blocked", + "bootstrap_only", + "active_space_not_allowed", + "no_active_space", + "space_unknown", + "environment_mismatch", + "unknown", + "other", +} +_WORKING_STATUSES = {"accepted", "started", "processing", "thinking", "tool_call", "tool_started", "streaming", "working"} +_BLOCKED_STATUSES = {"rate_limited"} + + +def _normalized_controlled(value: object, allowed: set[str], *, fallback: str) -> str: + normalized = str(value or "").strip() + if normalized in allowed: + return normalized + lowered_map = {item.lower(): item for item in allowed} + lowered = normalized.lower() + if lowered in lowered_map: + return lowered_map[lowered] + return fallback + + +def _normalized_controlled_list(value: object, allowed: set[str], *, fallback: list[str]) -> list[str]: + raw_items: list[str] = [] + if isinstance(value, str): + parts = value.split(",") if "," in value else [value] + raw_items = [part.strip() for part in parts if part.strip()] + elif isinstance(value, (list, tuple, set)): + raw_items = [str(item).strip() for item in value if str(item).strip()] + + lowered_map = {item.lower(): item for item in allowed} + normalized: list[str] = [] + seen: set[str] = set() + for item in raw_items: + candidate = item if item in allowed else lowered_map.get(item.lower()) + if not candidate or candidate in seen: + continue + seen.add(candidate) + normalized.append(candidate) + return normalized or list(fallback) + + +def _normalized_optional_controlled(value: object, allowed: set[str]) -> str | None: + normalized = str(value or "").strip() + if not normalized: + return None + if normalized in allowed: + return normalized + lowered_map = {item.lower(): item for item in allowed} + return lowered_map.get(normalized.lower()) + + +def _normalized_string_list(value: object, *, fallback: list[str]) -> list[str]: + if isinstance(value, str): + items = [part.strip() for part in value.split(",") if part.strip()] + return items or list(fallback) + if isinstance(value, (list, tuple, set)): + items = [str(item).strip() for item in value if str(item).strip()] + return items or list(fallback) + return list(fallback) + + +def _bool_with_fallback(value: object, *, fallback: bool) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"true", "1", "yes", "y", "on"}: + return True + if lowered in {"false", "0", "no", "n", "off"}: + return False + return fallback + + +def _override_fields(snapshot: dict[str, Any], *, domain: str) -> set[str]: + names: set[str] = set() + nested = snapshot.get("user_overrides") + if isinstance(nested, dict): + scoped = nested.get(domain) + if isinstance(scoped, dict): + names.update(str(key).strip() for key in scoped.keys() if str(key).strip()) + elif isinstance(scoped, (list, tuple, set)): + names.update(str(item).strip() for item in scoped if str(item).strip()) + + direct_key = f"{domain}_overrides" + direct = snapshot.get(direct_key) + if isinstance(direct, dict): + names.update(str(key).strip() for key in direct.keys() if str(key).strip()) + elif isinstance(direct, (list, tuple, set)): + names.update(str(item).strip() for item in direct if str(item).strip()) + return names + + +def _template_operator_defaults(template_id: str | None, runtime_type: object) -> dict[str, str]: + template_key = str(template_id or "").strip().lower() + runtime_key = str(runtime_type or "").strip().lower() + defaults_by_template = { + "echo_test": { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "telemetry_level": "basic", + }, + "ollama": { + "placement": "hosted", + "activation": "on_demand", + "reply_mode": "interactive", + "telemetry_level": "basic", + }, + "hermes": { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "telemetry_level": "rich", + }, + "claude_code_channel": { + "placement": "attached", + "activation": "attach_only", + "reply_mode": "interactive", + "telemetry_level": "basic", + }, + "inbox": { + "placement": "mailbox", + "activation": "queue_worker", + "reply_mode": "summary_only", + "telemetry_level": "basic", + }, + } + defaults_by_runtime = { + "echo": { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "telemetry_level": "basic", + }, + "exec": { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "telemetry_level": "basic", + }, + "inbox": { + "placement": "mailbox", + "activation": "queue_worker", + "reply_mode": "summary_only", + "telemetry_level": "basic", + }, + } + return dict(defaults_by_template.get(template_key) or defaults_by_runtime.get(runtime_key) or defaults_by_runtime["exec"]) + + +def _template_asset_defaults(template_id: str | None, runtime_type: object) -> dict[str, Any]: + template_key = str(template_id or "").strip().lower() + runtime_key = str(runtime_type or "").strip().lower() + defaults_by_template: dict[str, dict[str, Any]] = { + "echo_test": { + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "basic", + "worker_model": None, + "addressable": True, + "messageable": True, + "schedulable": False, + "externally_triggered": False, + "tags": ["local", "live-listener", "test-agent"], + "capabilities": ["reply"], + "constraints": [], + }, + "ollama": { + "asset_class": "interactive_agent", + "intake_model": "launch_on_send", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "basic", + "worker_model": None, + "addressable": True, + "messageable": True, + "schedulable": False, + "externally_triggered": False, + "tags": ["local", "on-demand", "cold-start"], + "capabilities": ["reply"], + "constraints": ["requires-model"], + }, + "hermes": { + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "rich", + "worker_model": None, + "addressable": True, + "messageable": True, + "schedulable": False, + "externally_triggered": False, + "tags": ["local", "live-listener", "hosted-by-gateway", "rich-telemetry", "repo-bound"], + "capabilities": ["reply", "progress", "tool_events"], + "constraints": ["requires-repo", "requires-provider-auth"], + }, + "claude_code_channel": { + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "basic", + "worker_model": None, + "addressable": True, + "messageable": True, + "schedulable": False, + "externally_triggered": False, + "tags": ["attached-session", "live-listener", "basic-telemetry"], + "capabilities": ["reply"], + "constraints": ["requires-attached-session"], + }, + "inbox": { + "asset_class": "background_worker", + "intake_model": "queue_accept", + "trigger_sources": ["queued_job", "manual_trigger"], + "return_paths": ["summary_post"], + "telemetry_shape": "basic", + "worker_model": "queue_drain", + "addressable": True, + "messageable": True, + "schedulable": False, + "externally_triggered": False, + "tags": ["queue-backed", "summary-later"], + "capabilities": ["queue_work", "post_summary"], + "constraints": [], + }, + } + defaults_by_runtime: dict[str, dict[str, Any]] = { + "echo": defaults_by_template["echo_test"], + "exec": { + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "basic", + "worker_model": None, + "addressable": True, + "messageable": True, + "schedulable": False, + "externally_triggered": False, + "tags": ["local", "custom-bridge"], + "capabilities": ["reply"], + "constraints": [], + }, + "inbox": defaults_by_template["inbox"], + } + resolved = defaults_by_template.get(template_key) or defaults_by_runtime.get(runtime_key) or defaults_by_runtime["exec"] + return { + "asset_class": resolved["asset_class"], + "intake_model": resolved["intake_model"], + "trigger_sources": list(resolved["trigger_sources"]), + "return_paths": list(resolved["return_paths"]), + "telemetry_shape": resolved["telemetry_shape"], + "worker_model": resolved.get("worker_model"), + "addressable": bool(resolved.get("addressable", True)), + "messageable": bool(resolved.get("messageable", True)), + "schedulable": bool(resolved.get("schedulable", False)), + "externally_triggered": bool(resolved.get("externally_triggered", False)), + "tags": list(resolved.get("tags", [])), + "capabilities": list(resolved.get("capabilities", [])), + "constraints": list(resolved.get("constraints", [])), + } + + +def _asset_type_label(*, asset_class: str, intake_model: str, worker_model: str | None = None) -> str: + if asset_class == "interactive_agent": + if intake_model == "live_listener": + return "Live Listener" + if intake_model == "launch_on_send": + return "On-Demand Agent" + if asset_class == "background_worker": + if intake_model == "queue_accept" or worker_model == "queue_drain": + return "Inbox Worker" + return "Background Worker" + if asset_class == "scheduled_job": + return "Scheduled Job" + if asset_class == "alert_listener": + return "Alert Listener" + if asset_class == "service_proxy": + return "Service / Tool Proxy" + return "Connected Asset" + + +def _output_label(return_paths: list[str]) -> str: + primary = return_paths[0] if return_paths else "inline_reply" + return { + "inline_reply": "Reply", + "sender_inbox": "Inbox", + "summary_post": "Summary", + "task_update": "Task", + "event_log": "Event Log", + "silent": "Silent", + }.get(primary, "Reply") + + +def infer_asset_descriptor(snapshot: dict[str, Any], *, operator_profile: dict[str, str] | None = None) -> dict[str, Any]: + defaults = _template_asset_defaults(str(snapshot.get("template_id") or "").strip() or None, snapshot.get("runtime_type")) + overrides = _override_fields(snapshot, domain="asset") + telemetry_fallback = defaults["telemetry_shape"] + if operator_profile: + telemetry_fallback = { + "rich": "rich", + "basic": "basic", + "silent": "opaque", + }.get(operator_profile.get("telemetry_level", ""), telemetry_fallback) + + asset_class = defaults["asset_class"] + if "asset_class" in overrides: + asset_class = _normalized_controlled(snapshot.get("asset_class"), _CONTROLLED_ASSET_CLASSES, fallback=defaults["asset_class"]) + + intake_model = defaults["intake_model"] + if "intake_model" in overrides: + intake_model = _normalized_controlled(snapshot.get("intake_model"), _CONTROLLED_INTAKE_MODELS, fallback=defaults["intake_model"]) + + worker_model = defaults.get("worker_model") + if "worker_model" in overrides: + worker_model = _normalized_optional_controlled(snapshot.get("worker_model"), _CONTROLLED_WORKER_MODELS) or defaults.get("worker_model") + + trigger_sources = list(defaults["trigger_sources"]) + if "trigger_sources" in overrides or "trigger_source" in overrides: + trigger_sources = _normalized_controlled_list( + snapshot.get("trigger_sources") if snapshot.get("trigger_sources") is not None else snapshot.get("trigger_source"), + _CONTROLLED_TRIGGER_SOURCES, + fallback=defaults["trigger_sources"], + ) + + return_paths = list(defaults["return_paths"]) + if "return_paths" in overrides or "return_path" in overrides: + return_paths = _normalized_controlled_list( + snapshot.get("return_paths") if snapshot.get("return_paths") is not None else snapshot.get("return_path"), + _CONTROLLED_RETURN_PATHS, + fallback=defaults["return_paths"], + ) + + telemetry_shape = telemetry_fallback + if "telemetry_shape" in overrides: + telemetry_shape = _normalized_controlled( + snapshot.get("telemetry_shape"), + _CONTROLLED_TELEMETRY_SHAPES, + fallback=telemetry_fallback, + ) + + tags = list(defaults["tags"]) + if "tags" in overrides: + tags = _normalized_string_list(snapshot.get("tags"), fallback=defaults["tags"]) + + capabilities = list(defaults["capabilities"]) + if "capabilities" in overrides: + capabilities = _normalized_string_list(snapshot.get("capabilities"), fallback=defaults["capabilities"]) + + constraints = list(defaults["constraints"]) + if "constraints" in overrides: + constraints = _normalized_string_list(snapshot.get("constraints"), fallback=defaults["constraints"]) + + descriptor = { + "asset_id": str(snapshot.get("asset_id") or snapshot.get("agent_id") or snapshot.get("name") or "").strip() or None, + "gateway_id": str(snapshot.get("gateway_id") or "").strip() or None, + "display_name": str(snapshot.get("display_name") or snapshot.get("name") or snapshot.get("template_label") or snapshot.get("runtime_type") or "Managed Asset"), + "asset_class": asset_class, + "intake_model": intake_model, + "worker_model": worker_model, + "trigger_sources": trigger_sources, + "return_paths": return_paths, + "telemetry_shape": telemetry_shape, + "addressable": _bool_with_fallback(snapshot.get("addressable"), fallback=defaults["addressable"]) + if "addressable" in overrides + else defaults["addressable"], + "messageable": _bool_with_fallback(snapshot.get("messageable"), fallback=defaults["messageable"]) + if "messageable" in overrides + else defaults["messageable"], + "schedulable": _bool_with_fallback(snapshot.get("schedulable"), fallback=defaults["schedulable"]) + if "schedulable" in overrides + else defaults["schedulable"], + "externally_triggered": _bool_with_fallback(snapshot.get("externally_triggered"), fallback=defaults["externally_triggered"]) + if "externally_triggered" in overrides + else defaults["externally_triggered"], + "tags": tags, + "capabilities": capabilities, + "constraints": constraints, + } + descriptor["type_label"] = _asset_type_label( + asset_class=descriptor["asset_class"], + intake_model=descriptor["intake_model"], + worker_model=descriptor.get("worker_model"), + ) + descriptor["output_label"] = _output_label(descriptor["return_paths"]) + descriptor["primary_trigger_source"] = descriptor["trigger_sources"][0] if descriptor["trigger_sources"] else None + descriptor["primary_return_path"] = descriptor["return_paths"][0] if descriptor["return_paths"] else None + return descriptor + + +def _hermes_repo_candidates(entry: dict[str, Any] | None = None) -> list[Path]: + entry = entry or {} + candidates: list[Path] = [] + seen: set[str] = set() + + def add(path_value: object) -> None: + raw = str(path_value or "").strip() + if not raw: + return + expanded = Path(raw).expanduser() + key = str(expanded) + if key in seen: + return + seen.add(key) + candidates.append(expanded) + + add(entry.get("hermes_repo_path")) + add(os.environ.get("HERMES_REPO_PATH")) + + workdir_raw = str(entry.get("workdir") or "").strip() + if workdir_raw: + workdir = Path(workdir_raw).expanduser() + add(workdir.parent / "hermes-agent") + + add(Path.home() / "hermes-agent") + return candidates + + +def hermes_setup_status(entry: dict[str, Any]) -> dict[str, Any]: + template_id = str(entry.get("template_id") or "").strip().lower() + if template_id != "hermes": + return {"ready": True, "template_id": template_id} + + candidates = _hermes_repo_candidates(entry) + resolved = next((candidate for candidate in candidates if candidate.exists()), None) + if resolved is not None: + return { + "ready": True, + "template_id": template_id, + "resolved_path": str(resolved), + "summary": f"Hermes checkout found at {resolved}.", + } + + expected = candidates[0] if candidates else (Path.home() / "hermes-agent") + return { + "ready": False, + "template_id": template_id, + "resolved_path": None, + "expected_path": str(expected), + "summary": f"Hermes checkout not found at {expected}.", + "detail": ( + f"Hermes checkout not found at {expected}. " + "Set HERMES_REPO_PATH or clone hermes-agent to ~/hermes-agent." + ), + } + + +def _ollama_model_rows(payload: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + models = payload.get("models") + if not isinstance(models, list): + return rows + for item in models: + if not isinstance(item, dict): + continue + name = str(item.get("name") or item.get("model") or "").strip() + if not name: + continue + details = item.get("details") if isinstance(item.get("details"), dict) else {} + families = details.get("families") if isinstance(details.get("families"), list) else [] + family_values = [str(value).strip() for value in families if str(value).strip()] + remote_host = str(item.get("remote_host") or "").strip() or None + lowered_name = name.lower() + is_embedding = "embed" in lowered_name or any("bert" in family.lower() for family in family_values) + rows.append( + { + "name": name, + "family": str(details.get("family") or "").strip() or None, + "families": family_values, + "parameter_size": str(details.get("parameter_size") or "").strip() or None, + "modified_at": str(item.get("modified_at") or "").strip() or None, + "remote_host": remote_host, + "is_cloud": bool(remote_host or lowered_name.endswith(":cloud") or lowered_name.endswith("-cloud")), + "is_embedding": is_embedding, + } + ) + return rows + + +def _recommended_ollama_model(rows: list[dict[str, Any]]) -> str | None: + if not rows: + return None + + def pick(candidates: list[dict[str, Any]]) -> str | None: + if not candidates: + return None + ordered = sorted( + candidates, + key=lambda item: ( + str(item.get("modified_at") or ""), + str(item.get("parameter_size") or ""), + str(item.get("name") or ""), + ), + reverse=True, + ) + return str(ordered[0].get("name") or "").strip() or None + + local_rows = [item for item in rows if not bool(item.get("is_cloud"))] + local_chat_rows = [item for item in local_rows if not bool(item.get("is_embedding"))] + chat_rows = [item for item in rows if not bool(item.get("is_embedding"))] + return pick(local_chat_rows) or pick(local_rows) or pick(chat_rows) or pick(rows) + + +def ollama_setup_status(*, preferred_model: str | None = None) -> dict[str, Any]: + base_url = DEFAULT_OLLAMA_BASE_URL + endpoint = f"{base_url}/api/tags" + preferred = str(preferred_model or "").strip() or None + try: + response = httpx.get(endpoint, timeout=3.0) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict): + raise ValueError("Ollama returned a non-object response.") + except Exception as exc: + return { + "ready": False, + "server_reachable": False, + "base_url": base_url, + "endpoint": endpoint, + "preferred_model": preferred, + "preferred_model_available": False, + "recommended_model": None, + "available_models": [], + "local_models": [], + "models": [], + "summary": f"Ollama server not reachable at {base_url}.", + "detail": str(exc), + } + + rows = _ollama_model_rows(payload) + available_models = [str(item.get("name") or "") for item in rows if str(item.get("name") or "").strip()] + local_models = [str(item.get("name") or "") for item in rows if not bool(item.get("is_cloud"))] + preferred_available = bool(preferred and preferred in available_models) + recommended_model = preferred if preferred_available else _recommended_ollama_model(rows) + ready = bool(available_models) + if preferred and not preferred_available: + summary = f"Ollama is reachable, but {preferred} is not installed locally." + elif recommended_model: + summary = f"Ollama is reachable. Recommended model: {recommended_model}." + elif available_models: + summary = f"Ollama is reachable with {len(available_models)} model(s) available." + else: + summary = "Ollama is reachable, but no models are installed yet." + return { + "ready": ready, + "server_reachable": True, + "base_url": base_url, + "endpoint": endpoint, + "preferred_model": preferred, + "preferred_model_available": preferred_available, + "recommended_model": recommended_model, + "available_models": available_models, + "local_models": local_models, + "models": rows, + "summary": summary, + "detail": None, + } + + +def infer_operator_profile(snapshot: dict[str, Any]) -> dict[str, str]: + defaults = _template_operator_defaults(str(snapshot.get("template_id") or "").strip() or None, snapshot.get("runtime_type")) + overrides = _override_fields(snapshot, domain="operator") + return { + "placement": _normalized_controlled(snapshot.get("placement"), _CONTROLLED_PLACEMENTS, fallback=defaults["placement"]) + if "placement" in overrides + else defaults["placement"], + "activation": _normalized_controlled(snapshot.get("activation"), _CONTROLLED_ACTIVATIONS, fallback=defaults["activation"]) + if "activation" in overrides + else defaults["activation"], + "reply_mode": _normalized_controlled(snapshot.get("reply_mode"), _CONTROLLED_REPLY_MODES, fallback=defaults["reply_mode"]) + if "reply_mode" in overrides + else defaults["reply_mode"], + "telemetry_level": _normalized_controlled( + snapshot.get("telemetry_level"), + _CONTROLLED_TELEMETRY_LEVELS, + fallback=defaults["telemetry_level"], + ) + if "telemetry_level" in overrides + else defaults["telemetry_level"], + } + + +def _looks_like_setup_error(snapshot: dict[str, Any], raw_state: str) -> bool: + if raw_state == "error": + return True + last_error = str(snapshot.get("last_error") or "").lower() + preview = str(snapshot.get("last_reply_preview") or "").lower() + if "repo not found" in last_error or "repo not found" in preview: + return True + if preview.startswith("(stderr:") or last_error.startswith("stderr:"): + return True + return False + + +def _derive_liveness(snapshot: dict[str, Any], *, raw_state: str, last_seen_age: int | None) -> tuple[str, bool]: + if _looks_like_setup_error(snapshot, raw_state): + return "setup_error", False + if raw_state == "running": + if last_seen_age is None or last_seen_age > RUNTIME_STALE_AFTER_SECONDS: + return "stale", False + return "connected", True + if raw_state in {"starting", "reconnecting", "stale"}: + return "stale", False + return "offline", False + + +def _derive_work_state(snapshot: dict[str, Any], *, liveness: str) -> str: + attestation_state = _normalized_optional_controlled(snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + approval_state = _normalized_optional_controlled(snapshot.get("approval_state"), _CONTROLLED_APPROVAL_STATES) + identity_status = _normalized_optional_controlled(snapshot.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) + environment_status = _normalized_optional_controlled(snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + space_status = _normalized_optional_controlled(snapshot.get("space_status"), _CONTROLLED_SPACE_STATUSES) + if liveness == "setup_error": + return "blocked" + if attestation_state in {"drifted", "unknown", "blocked"} or approval_state in {"pending", "rejected"}: + return "blocked" + if identity_status in {"unknown_identity", "credential_mismatch", "fallback_blocked", "bootstrap_only", "blocked"}: + return "blocked" + if environment_status in {"environment_mismatch", "environment_blocked"}: + return "blocked" + if space_status in {"active_not_allowed", "no_active_space"}: + return "blocked" + status = str(snapshot.get("current_status") or "").strip().lower() + backlog_depth = int(snapshot.get("backlog_depth") or 0) + if status in _WORKING_STATUSES: + return "working" + if status == "queued" or backlog_depth > 0: + return "queued" + if status in _BLOCKED_STATUSES: + return "blocked" + return "idle" + + +def _doctor_has_failed(snapshot: dict[str, Any]) -> bool: + result = snapshot.get("last_doctor_result") + if not isinstance(result, dict): + return False + status = str(result.get("status") or "").strip().lower() + if status in {"failed", "error"}: + return True + checks = result.get("checks") + if isinstance(checks, list): + return any(isinstance(item, dict) and str(item.get("status") or "").strip().lower() == "failed" for item in checks) + return False + + +def _derive_mode(profile: dict[str, str]) -> str: + if profile["placement"] == "mailbox": + return "INBOX" + if profile["activation"] in {"persistent", "attach_only"}: + return "LIVE" + return "ON-DEMAND" + + +def _derive_presence(*, mode: str, liveness: str, work_state: str) -> str: + if liveness == "setup_error": + return "ERROR" + if work_state == "blocked": + return "BLOCKED" + if liveness == "stale": + return "STALE" + if liveness == "offline" and mode == "LIVE": + return "OFFLINE" + if work_state == "working": + return "WORKING" + if work_state == "queued": + return "QUEUED" + return "IDLE" + + +def _derive_reply(reply_mode: str) -> str: + if reply_mode == "interactive": + return "REPLY" + if reply_mode == "silent": + return "SILENT" + return "SUMMARY" + + +def _derive_reachability(*, snapshot: dict[str, Any], mode: str, liveness: str, activation: str) -> str: + attestation_state = _normalized_optional_controlled(snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + approval_state = _normalized_optional_controlled(snapshot.get("approval_state"), _CONTROLLED_APPROVAL_STATES) + identity_status = _normalized_optional_controlled(snapshot.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) + environment_status = _normalized_optional_controlled(snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + space_status = _normalized_optional_controlled(snapshot.get("space_status"), _CONTROLLED_SPACE_STATUSES) + if liveness == "setup_error": + return "unavailable" + if attestation_state in {"drifted", "unknown", "blocked"} or approval_state in {"pending", "rejected"}: + return "unavailable" + if identity_status in {"unknown_identity", "credential_mismatch", "fallback_blocked", "bootstrap_only", "blocked"}: + return "unavailable" + if environment_status in {"environment_mismatch", "environment_blocked"}: + return "unavailable" + if space_status in {"active_not_allowed", "no_active_space"}: + return "unavailable" + if mode == "INBOX": + return "queue_available" + if activation == "attach_only" and liveness in {"stale", "offline"}: + return "attach_required" + if mode == "LIVE" and liveness == "connected": + return "live_now" + if mode == "ON-DEMAND" and liveness != "setup_error": + return "launch_available" + return "unavailable" + + +def _setup_error_detail(snapshot: dict[str, Any]) -> str: + if _doctor_has_failed(snapshot): + summary = _doctor_summary(snapshot) + if summary: + return summary + return str(snapshot.get("last_error") or snapshot.get("last_reply_preview") or "Setup must be fixed before Gateway can send work.") + + +def _doctor_summary(snapshot: dict[str, Any]) -> str: + result = snapshot.get("last_doctor_result") + if not isinstance(result, dict): + return "" + summary = str(result.get("summary") or "").strip() + if summary: + return summary + checks = result.get("checks") + if isinstance(checks, list): + failed = [str(item.get("name") or "").strip() for item in checks if isinstance(item, dict) and str(item.get("status") or "").strip().lower() == "failed"] + if failed: + return f"Doctor failed: {', '.join(filter(None, failed))}." + return "" + + +def _derive_confidence( + snapshot: dict[str, Any], + *, + mode: str, + liveness: str, + reachability: str, +) -> tuple[str, str, str]: + attestation_state = _normalized_optional_controlled(snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + approval_state = _normalized_optional_controlled(snapshot.get("approval_state"), _CONTROLLED_APPROVAL_STATES) + governance_reason = _normalized_optional_controlled(snapshot.get("confidence_reason"), _CONTROLLED_CONFIDENCE_REASONS) + governance_detail = str(snapshot.get("confidence_detail") or "").strip() or "Gateway blocked this runtime until its binding is approved." + identity_status = _normalized_optional_controlled(snapshot.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) + environment_status = _normalized_optional_controlled(snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + space_status = _normalized_optional_controlled(snapshot.get("space_status"), _CONTROLLED_SPACE_STATUSES) + if liveness == "setup_error": + return ("BLOCKED", "setup_blocked", _setup_error_detail(snapshot)) + if identity_status == "unknown_identity": + return ("BLOCKED", "identity_unbound", "Gateway does not have a bound acting identity for this asset in the requested environment.") + if identity_status in {"credential_mismatch", "fallback_blocked"}: + return ("BLOCKED", "identity_mismatch", "Gateway blocked a mismatched acting identity instead of borrowing another identity.") + if identity_status == "bootstrap_only": + return ("BLOCKED", "bootstrap_only", "Gateway bootstrap credentials can only be used for setup, verification, or repair flows.") + if environment_status == "environment_mismatch": + return ("BLOCKED", "environment_mismatch", "Requested environment does not match the bound Gateway environment for this asset.") + if environment_status == "environment_blocked": + return ("BLOCKED", "environment_mismatch", "Gateway blocked this asset in the requested environment.") + if space_status == "active_not_allowed": + return ("BLOCKED", "active_space_not_allowed", "The resolved target space is not allowed for this acting identity.") + if space_status == "no_active_space": + return ("BLOCKED", "no_active_space", "Gateway does not have an active space selected for this asset.") + if space_status == "unknown": + return ("LOW", "space_unknown", "Gateway could not verify the allowed-space list for this acting identity.") + if approval_state == "rejected": + return ("BLOCKED", governance_reason or "approval_denied", governance_detail) + if attestation_state in {"blocked", "unknown", "drifted"} or approval_state == "pending": + return ("BLOCKED", governance_reason or "approval_required", governance_detail) + if _doctor_has_failed(snapshot): + detail = _doctor_summary(snapshot) or "Gateway Doctor reported a failed send path." + return ("LOW", "recent_test_failed", detail) + completion_rate = snapshot.get("completion_rate") + try: + if completion_rate is not None and float(completion_rate) < 0.5: + return ("LOW", "completion_degraded", "Recent completion rate is below the healthy threshold.") + except (TypeError, ValueError): + pass + if mode == "INBOX": + return ("HIGH", "queue_available", "Gateway can safely accept and queue work now.") + if mode == "ON-DEMAND" and reachability == "launch_available": + return ("MEDIUM", "launch_available", "Gateway can launch this runtime on send. Cold start possible.") + if liveness in {"offline", "stale"}: + if reachability == "attach_required": + return ("LOW", "attach_required", "Reconnect the attached session before sending.") + return ("LOW", "unavailable", "Gateway does not currently have a healthy live path.") + if liveness == "connected": + return ("HIGH", "live_now", "A live runtime is attached and ready to claim work.") + return ("MEDIUM", "unknown", "Gateway has partial health signals but no stronger confidence signal yet.") def _now_iso() -> str: return datetime.now(timezone.utc).isoformat() +def _gateway_id_from_registry(registry: dict[str, Any]) -> str: + gateway = registry.setdefault("gateway", {}) + gateway_id = str(gateway.get("gateway_id") or "").strip() + if gateway_id: + return gateway_id + gateway_id = str(uuid.uuid4()) + gateway["gateway_id"] = gateway_id + return gateway_id + + +def _asset_id_for_entry(entry: dict[str, Any]) -> str: + return str(entry.get("agent_id") or entry.get("asset_id") or entry.get("name") or "").strip() + + +def _binding_type_for_entry(entry: dict[str, Any]) -> str: + activation = str(entry.get("activation") or "").strip() + if activation == "attach_only": + return "attached_session" + if activation == "queue_worker" or str(entry.get("runtime_type") or "").strip().lower() == "inbox": + return "queue_worker" + return "local_runtime" + + +def _launch_spec_for_entry(entry: dict[str, Any]) -> dict[str, Any]: + return { + "runtime_type": str(entry.get("runtime_type") or "").strip() or None, + "template_id": str(entry.get("template_id") or "").strip() or None, + "command": str(entry.get("exec_command") or "").strip() or None, + "workdir": str(entry.get("workdir") or "").strip() or None, + "ollama_model": str(entry.get("ollama_model") or "").strip() or None, + "transport": str(entry.get("transport") or "").strip() or None, + } + + +def _payload_hash(payload: dict[str, Any]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _host_fingerprint() -> str: + host = platform.node() or "unknown-host" + return f"host:{hashlib.sha256(host.encode('utf-8')).hexdigest()[:16]}" + + +def _normalized_base_url(value: object) -> str: + return str(value or "").strip().rstrip("/") + + +def _environment_label_for_base_url(base_url: object) -> str: + normalized = _normalized_base_url(base_url) + if not normalized: + return "unknown" + parsed = urlparse(normalized if "://" in normalized else f"https://{normalized}") + host = str(parsed.netloc or parsed.path or "").lower() + if host == "paxai.app": + return "prod" + if host == "dev.paxai.app": + return "dev" + if host in {"localhost", "127.0.0.1"} or host.startswith("localhost:") or host.startswith("127.0.0.1:"): + return "local" + return host or "custom" + + +def _redacted_path(value: object) -> str | None: + raw = str(value or "").strip() + if not raw: + return None + try: + path = Path(raw).expanduser().resolve() + home = Path.home().resolve() + try: + rel = path.relative_to(home) + return str(Path("~") / rel) + except ValueError: + return str(path) + except Exception: + return raw + + +def _space_cache_rows(value: object) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + items = value if isinstance(value, list) else [] + seen: set[str] = set() + for item in items: + if not isinstance(item, dict): + continue + space_id = str(item.get("space_id") or item.get("id") or "").strip() + if not space_id or space_id in seen: + continue + seen.add(space_id) + rows.append( + { + "space_id": space_id, + "name": str(item.get("name") or item.get("space_name") or space_id), + "is_default": bool(item.get("is_default", False)), + } + ) + return rows + + +def _space_name_from_cache(allowed_spaces: list[dict[str, Any]], space_id: str | None) -> str | None: + if not space_id: + return None + for item in allowed_spaces: + if str(item.get("space_id") or "") == str(space_id): + return str(item.get("name") or space_id) + return None + + +def _fallback_allowed_spaces(entry: dict[str, Any], session: dict[str, Any] | None = None) -> list[dict[str, Any]]: + session = session or {} + default_id = str(entry.get("default_space_id") or entry.get("space_id") or session.get("space_id") or "").strip() + active_id = str(entry.get("active_space_id") or entry.get("space_id") or default_id).strip() + rows: list[dict[str, Any]] = [] + if default_id: + rows.append( + { + "space_id": default_id, + "name": str(entry.get("default_space_name") or entry.get("space_name") or session.get("space_name") or default_id), + "is_default": True, + } + ) + if active_id and active_id != default_id: + rows.append( + { + "space_id": active_id, + "name": str(entry.get("active_space_name") or entry.get("space_name") or active_id), + "is_default": False, + } + ) + return _space_cache_rows(rows) + + +def _space_id_allowed(allowed_spaces: list[dict[str, Any]], space_id: str | None) -> bool: + if not space_id: + return False + return any(str(item.get("space_id") or "") == str(space_id) for item in allowed_spaces) + + +def _binding_candidate_for_entry(entry: dict[str, Any], registry: dict[str, Any]) -> dict[str, Any]: + asset_id = _asset_id_for_entry(entry) + install_id = str(entry.get("install_id") or "").strip() or str(uuid.uuid4()) + launch_spec = _launch_spec_for_entry(entry) + workdir = str(entry.get("workdir") or "").strip() or None + path = str(Path(workdir).expanduser()) if workdir else None + candidate = { + "asset_id": asset_id, + "gateway_id": _gateway_id_from_registry(registry), + "install_id": install_id, + "binding_type": _binding_type_for_entry(entry), + "path": path, + "launch_spec": launch_spec, + "launch_spec_hash": _payload_hash(launch_spec), + "created_from": str(entry.get("created_from") or ("ax_template" if entry.get("template_id") else "custom_bridge")), + "created_via": str(entry.get("created_via") or "cli"), + "approved_state": str(entry.get("approved_state") or "approved"), + "first_seen_at": str(entry.get("first_seen_at") or _now_iso()), + "last_verified_at": str(entry.get("last_verified_at") or _now_iso()), + } + candidate["candidate_signature"] = _payload_hash( + { + "asset_id": candidate["asset_id"], + "gateway_id": candidate["gateway_id"], + "install_id": candidate["install_id"], + "path": candidate["path"], + "launch_spec_hash": candidate["launch_spec_hash"], + } + ) + return candidate + + +def _ensure_registry_lists(registry: dict[str, Any]) -> None: + registry.setdefault("bindings", []) + registry.setdefault("identity_bindings", []) + registry.setdefault("approvals", []) + + +def find_binding( + registry: dict[str, Any], + *, + asset_id: str | None = None, + install_id: str | None = None, + gateway_id: str | None = None, +) -> dict[str, Any] | None: + _ensure_registry_lists(registry) + for binding in registry.get("bindings", []): + if asset_id and str(binding.get("asset_id") or "") != asset_id: + continue + if install_id and str(binding.get("install_id") or "") != install_id: + continue + if gateway_id and str(binding.get("gateway_id") or "") != gateway_id: + continue + return binding + return None + + +def _bindings_for_asset(registry: dict[str, Any], asset_id: str) -> list[dict[str, Any]]: + _ensure_registry_lists(registry) + return [binding for binding in registry.get("bindings", []) if str(binding.get("asset_id") or "") == asset_id] + + +def upsert_binding(registry: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]: + _ensure_registry_lists(registry) + bindings = registry["bindings"] + target_install_id = str(binding.get("install_id") or "") + for idx, existing in enumerate(bindings): + if str(existing.get("install_id") or "") == target_install_id and target_install_id: + merged = dict(existing) + merged.update(binding) + bindings[idx] = merged + return merged + bindings.append(binding) + return binding + + +def find_identity_binding( + registry: dict[str, Any], + *, + identity_binding_id: str | None = None, + install_id: str | None = None, + asset_id: str | None = None, + gateway_id: str | None = None, + base_url: str | None = None, +) -> dict[str, Any] | None: + _ensure_registry_lists(registry) + normalized_base_url = _normalized_base_url(base_url) + for binding in registry.get("identity_bindings", []): + if identity_binding_id and str(binding.get("identity_binding_id") or "") != identity_binding_id: + continue + if install_id and str(binding.get("install_id") or "") != install_id: + continue + if asset_id and str(binding.get("asset_id") or "") != asset_id: + continue + if gateway_id and str(binding.get("gateway_id") or "") != gateway_id: + continue + if normalized_base_url and _normalized_base_url(((binding.get("environment") or {}) if isinstance(binding.get("environment"), dict) else {}).get("base_url")) != normalized_base_url: + continue + return binding + return None + + +def _identity_bindings_for_asset(registry: dict[str, Any], asset_id: str, *, gateway_id: str | None = None) -> list[dict[str, Any]]: + _ensure_registry_lists(registry) + rows = [binding for binding in registry.get("identity_bindings", []) if str(binding.get("asset_id") or "") == asset_id] + if gateway_id: + rows = [binding for binding in rows if str(binding.get("gateway_id") or "") == gateway_id] + return rows + + +def upsert_identity_binding(registry: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]: + _ensure_registry_lists(registry) + bindings = registry["identity_bindings"] + target_id = str(binding.get("identity_binding_id") or "") + target_install_id = str(binding.get("install_id") or "") + target_base_url = _normalized_base_url(((binding.get("environment") or {}) if isinstance(binding.get("environment"), dict) else {}).get("base_url")) + for idx, existing in enumerate(bindings): + existing_base_url = _normalized_base_url(((existing.get("environment") or {}) if isinstance(existing.get("environment"), dict) else {}).get("base_url")) + if target_id and str(existing.get("identity_binding_id") or "") == target_id: + merged = dict(existing) + merged.update(binding) + bindings[idx] = merged + return merged + if target_install_id and str(existing.get("install_id") or "") == target_install_id and existing_base_url == target_base_url: + merged = dict(existing) + merged.update(binding) + bindings[idx] = merged + return merged + bindings.append(binding) + return binding + + +def _normalize_allowed_spaces_payload(payload: object) -> list[dict[str, Any]]: + if isinstance(payload, dict): + if isinstance(payload.get("spaces"), list): + return _space_cache_rows(payload.get("spaces")) + if isinstance(payload.get("items"), list): + return _space_cache_rows(payload.get("items")) + if isinstance(payload.get("results"), list): + return _space_cache_rows(payload.get("results")) + return _space_cache_rows(payload) + + +def _fetch_allowed_spaces_for_entry(entry: dict[str, Any]) -> list[dict[str, Any]] | None: + token_file = Path(str(entry.get("token_file") or "")).expanduser() + base_url = _normalized_base_url(entry.get("base_url")) + if not token_file.exists() or not base_url: + return None + token = token_file.read_text().strip() + if not token: + return None + client = AxClient( + base_url=base_url, + token=token, + agent_name=str(entry.get("name") or "") or None, + agent_id=str(entry.get("agent_id") or "") or None, + ) + try: + return _normalize_allowed_spaces_payload(client.list_spaces()) + except Exception: + return None + finally: + try: + client.close() + except Exception: + pass + + +def ensure_gateway_identity_binding( + registry: dict[str, Any], + entry: dict[str, Any], + *, + session: dict[str, Any] | None = None, + created_via: str | None = None, + verify_spaces: bool = False, +) -> dict[str, Any]: + _ensure_registry_lists(registry) + gateway_id = _gateway_id_from_registry(registry) + asset_id = _asset_id_for_entry(entry) + install_id = str(entry.get("install_id") or "").strip() + if not install_id: + install_id = str(uuid.uuid4()) + entry["install_id"] = install_id + base_url = _normalized_base_url(entry.get("base_url") or (session or {}).get("base_url")) + existing = find_identity_binding( + registry, + identity_binding_id=str(entry.get("identity_binding_id") or "").strip() or None, + install_id=install_id, + base_url=base_url or None, + ) + allowed_spaces = _space_cache_rows(entry.get("allowed_spaces")) + if not allowed_spaces and existing: + allowed_spaces = _space_cache_rows(existing.get("allowed_spaces_cache")) + if verify_spaces: + fetched = _fetch_allowed_spaces_for_entry(entry) + if fetched: + allowed_spaces = fetched + if not allowed_spaces: + allowed_spaces = _fallback_allowed_spaces(entry, session=session) + default_space_id = str( + entry.get("default_space_id") + or ((existing or {}).get("default_space_id") if isinstance(existing, dict) else "") + or next((item.get("space_id") for item in allowed_spaces if bool(item.get("is_default"))), None) + or entry.get("space_id") + or (session or {}).get("space_id") + or "" + ).strip() or None + active_space_id = str( + entry.get("active_space_id") + or ((existing or {}).get("active_space_id") if isinstance(existing, dict) else "") + or entry.get("space_id") + or default_space_id + or "" + ).strip() or None + default_space_name = _space_name_from_cache(allowed_spaces, default_space_id) or str(entry.get("default_space_name") or entry.get("space_name") or default_space_id or "") + active_space_name = _space_name_from_cache(allowed_spaces, active_space_id) or str(entry.get("active_space_name") or entry.get("space_name") or active_space_id or "") + binding = { + "identity_binding_id": str((existing or {}).get("identity_binding_id") or f"idbind_{str(uuid.uuid4())}"), + "asset_id": asset_id, + "gateway_id": gateway_id, + "install_id": install_id, + "environment": { + "base_url": base_url or None, + "label": _environment_label_for_base_url(base_url), + "host": urlparse(base_url).netloc if base_url else None, + }, + "acting_identity": ( + dict(existing.get("acting_identity") or {}) + if isinstance(existing, dict) and isinstance(existing.get("acting_identity"), dict) + else { + "agent_id": str(entry.get("agent_id") or asset_id or "") or None, + "agent_name": str(entry.get("name") or "") or None, + "principal_type": "agent", + } + ), + "credential_ref": { + "kind": "token_file" if str(entry.get("token_file") or "").strip() else "unknown", + "id": str((existing or {}).get("credential_ref", {}).get("id") if isinstance((existing or {}).get("credential_ref"), dict) else "") or f"cred_{str(entry.get('name') or asset_id or 'asset')}_{_environment_label_for_base_url(base_url)}", + "display": "Gateway-managed agent token" if str(entry.get("credential_source") or "gateway") == "gateway" else "Non-gateway credential", + "path_redacted": _redacted_path(entry.get("token_file")), + }, + "active_space_id": active_space_id, + "active_space_name": active_space_name or None, + "default_space_id": default_space_id, + "default_space_name": default_space_name or None, + "allowed_spaces_cache": allowed_spaces, + "binding_state": "verified" if base_url and str(entry.get("agent_id") or "") else "unbound", + "created_via": str(created_via or entry.get("created_via") or "gateway_setup"), + "last_verified_at": _now_iso(), + } + stored = upsert_identity_binding(registry, binding) + entry["identity_binding_id"] = stored["identity_binding_id"] + entry["default_space_id"] = stored.get("default_space_id") + entry["default_space_name"] = stored.get("default_space_name") + if stored.get("active_space_id"): + entry["active_space_id"] = stored.get("active_space_id") + if stored.get("active_space_name"): + entry["active_space_name"] = stored.get("active_space_name") + return stored + + +def evaluate_identity_space_binding( + registry: dict[str, Any], + entry: dict[str, Any], + *, + explicit_space_id: str | None = None, + requested_base_url: str | None = None, +) -> dict[str, Any]: + _ensure_registry_lists(registry) + gateway_id = _gateway_id_from_registry(registry) + asset_id = _asset_id_for_entry(entry) + install_id = str(entry.get("install_id") or "").strip() or None + requested_url = _normalized_base_url(requested_base_url or entry.get("base_url")) + binding = find_identity_binding( + registry, + identity_binding_id=str(entry.get("identity_binding_id") or "").strip() or None, + install_id=install_id, + base_url=requested_url or None, + ) + asset_bindings = _identity_bindings_for_asset(registry, asset_id, gateway_id=gateway_id) if asset_id else [] + fallback_binding = asset_bindings[0] if asset_bindings else None + acting_identity = ( + (binding.get("acting_identity") if isinstance(binding, dict) and isinstance(binding.get("acting_identity"), dict) else None) + or (fallback_binding.get("acting_identity") if isinstance(fallback_binding, dict) and isinstance(fallback_binding.get("acting_identity"), dict) else None) + or {} + ) + bound_base_url = _normalized_base_url(((binding.get("environment") or {}) if isinstance(binding, dict) and isinstance(binding.get("environment"), dict) else {}).get("base_url")) + environment_status = "environment_unknown" + if binding: + environment_status = "environment_allowed" + if requested_url and bound_base_url and requested_url != bound_base_url: + environment_status = "environment_mismatch" + elif requested_url and asset_bindings: + environment_status = "environment_mismatch" + + identity_status = "verified" + if not binding: + identity_status = "verified" if asset_bindings else "unknown_identity" + elif str(entry.get("credential_source") or "gateway").strip().lower() not in {"gateway", ""}: + identity_status = "bootstrap_only" + elif not str(entry.get("token_file") or "").strip(): + identity_status = "bootstrap_only" + else: + bound_agent_id = str(acting_identity.get("agent_id") or "").strip() + bound_agent_name = str(acting_identity.get("agent_name") or "").strip().lower() + entry_agent_id = str(entry.get("agent_id") or "").strip() + entry_agent_name = str(entry.get("name") or "").strip().lower() + if bound_agent_id and entry_agent_id and bound_agent_id != entry_agent_id: + identity_status = "credential_mismatch" + elif bound_agent_name and entry_agent_name and bound_agent_name != entry_agent_name: + identity_status = "fallback_blocked" + + allowed_spaces = _space_cache_rows((binding or {}).get("allowed_spaces_cache")) + if not allowed_spaces and binding: + allowed_spaces = _fallback_allowed_spaces(entry) + active_space_source = "none" + active_space_id = str(explicit_space_id or "").strip() or None + if active_space_id: + active_space_source = "explicit_request" + elif binding and str(binding.get("active_space_id") or "").strip(): + active_space_id = str(binding.get("active_space_id") or "").strip() + active_space_source = "gateway_binding" + elif binding and str(binding.get("default_space_id") or "").strip(): + active_space_id = str(binding.get("default_space_id") or "").strip() + active_space_source = "visible_default" + + default_space_id = str((binding or {}).get("default_space_id") or "").strip() or None + default_space_name = str((binding or {}).get("default_space_name") or _space_name_from_cache(allowed_spaces, default_space_id) or default_space_id or "").strip() or None + active_space_name = _space_name_from_cache(allowed_spaces, active_space_id) or str((binding or {}).get("active_space_name") or active_space_id or "").strip() or None + + if not active_space_id: + space_status = "no_active_space" + elif not allowed_spaces: + space_status = "unknown" + elif _space_id_allowed(allowed_spaces, active_space_id): + space_status = "active_allowed" + else: + space_status = "active_not_allowed" + + return { + "identity_binding_id": str((binding or {}).get("identity_binding_id") or entry.get("identity_binding_id") or "") or None, + "asset_id": asset_id or None, + "gateway_id": gateway_id, + "install_id": install_id, + "acting_agent_id": str(acting_identity.get("agent_id") or entry.get("agent_id") or "").strip() or None, + "acting_agent_name": str(acting_identity.get("agent_name") or entry.get("name") or "").strip() or None, + "principal_type": str(acting_identity.get("principal_type") or "agent"), + "base_url": bound_base_url or requested_url or None, + "environment_label": _environment_label_for_base_url(bound_base_url or requested_url), + "environment_status": environment_status, + "active_space_id": active_space_id, + "active_space_name": active_space_name, + "active_space_source": active_space_source, + "default_space_id": default_space_id, + "default_space_name": default_space_name, + "allowed_spaces": allowed_spaces, + "allowed_space_count": len(allowed_spaces), + "identity_status": identity_status, + "space_status": space_status, + "last_space_verification_at": str((binding or {}).get("last_verified_at") or ""), + "identity_binding_state": str((binding or {}).get("binding_state") or "unbound"), + "credential_ref": dict((binding or {}).get("credential_ref") or {}) if isinstance((binding or {}).get("credential_ref"), dict) else None, + } + + +def _approval_status(approval: dict[str, Any]) -> str: + status = str(approval.get("status") or "").strip().lower() + if status == "denied": + return "rejected" + return status + + +def _find_approval_by_id(registry: dict[str, Any], approval_id: str) -> dict[str, Any] | None: + _ensure_registry_lists(registry) + for approval in registry.get("approvals", []): + if str(approval.get("approval_id") or "") == approval_id: + return approval + return None + + +def _find_approval_for_signature(registry: dict[str, Any], candidate_signature: str) -> dict[str, Any] | None: + _ensure_registry_lists(registry) + matches = [ + approval + for approval in registry.get("approvals", []) + if str(approval.get("candidate_signature") or "") == candidate_signature + ] + if not matches: + return None + return sorted(matches, key=lambda item: str(item.get("requested_at") or ""))[-1] + + +def list_gateway_approvals(*, status: str | None = None) -> list[dict[str, Any]]: + registry = load_gateway_registry() + _ensure_registry_lists(registry) + normalized_status = _normalized_optional_controlled(status, {"pending", "approved", "rejected"}) + approvals: list[dict[str, Any]] = [] + for approval in registry.get("approvals", []): + row = dict(approval) + row["status"] = _approval_status(row) + if normalized_status and row["status"] != normalized_status: + continue + approvals.append(row) + approvals.sort(key=lambda item: str(item.get("requested_at") or ""), reverse=True) + return approvals + + +def get_gateway_approval(approval_id: str) -> dict[str, Any]: + registry = load_gateway_registry() + approval = _find_approval_by_id(registry, approval_id) + if approval is None: + raise LookupError(f"Approval not found: {approval_id}") + result = dict(approval) + result["status"] = _approval_status(result) + return result + + +def _refresh_attestation_for_matching_entries( + registry: dict[str, Any], + *, + install_id: str | None = None, + asset_id: str | None = None, +) -> None: + for entry in registry.get("agents", []): + if install_id and str(entry.get("install_id") or "") != install_id: + continue + if asset_id and _asset_id_for_entry(entry) != asset_id: + continue + ensure_gateway_identity_binding(registry, entry) + entry.update(evaluate_identity_space_binding(registry, entry)) + entry.update(evaluate_runtime_attestation(registry, entry)) + + +def approve_gateway_approval(approval_id: str, *, scope: str = "asset", decided_by: str | None = None) -> dict[str, Any]: + normalized_scope = str(scope or "asset").strip().lower() + if normalized_scope not in {"once", "asset", "gateway"}: + raise ValueError("Approval scope must be one of: once, asset, gateway.") + registry = load_gateway_registry() + approval = _find_approval_by_id(registry, approval_id) + if approval is None: + raise LookupError(f"Approval not found: {approval_id}") + candidate_binding = approval.get("candidate_binding") if isinstance(approval.get("candidate_binding"), dict) else None + if not candidate_binding: + raise ValueError("Approval is missing its candidate binding.") + now = _now_iso() + approval["status"] = "approved" + approval["decision"] = "approve" + approval["decision_scope"] = normalized_scope + approval["decided_at"] = now + approval["decided_by"] = decided_by or "local_gateway_operator" + binding = dict(candidate_binding) + binding["approved_state"] = "approved" + binding["approved_at"] = now + binding["approval_scope"] = normalized_scope + binding["last_verified_at"] = now + stored_binding = upsert_binding(registry, binding) + _refresh_attestation_for_matching_entries( + registry, + install_id=str(approval.get("install_id") or "") or None, + asset_id=str(approval.get("asset_id") or "") or None, + ) + save_gateway_registry(registry) + _record_governance_activity( + "approval_granted", + asset_id=approval.get("asset_id"), + install_id=approval.get("install_id"), + approval_id=approval.get("approval_id"), + decision_scope=normalized_scope, + decided_by=approval["decided_by"], + gateway_id=approval.get("gateway_id"), + path=stored_binding.get("path"), + ) + result = dict(approval) + result["status"] = _approval_status(result) + return {"approval": result, "binding": stored_binding} + + +def deny_gateway_approval(approval_id: str, *, decided_by: str | None = None) -> dict[str, Any]: + registry = load_gateway_registry() + approval = _find_approval_by_id(registry, approval_id) + if approval is None: + raise LookupError(f"Approval not found: {approval_id}") + now = _now_iso() + approval["status"] = "rejected" + approval["decision"] = "deny" + approval["decided_at"] = now + approval["decided_by"] = decided_by or "local_gateway_operator" + _refresh_attestation_for_matching_entries( + registry, + install_id=str(approval.get("install_id") or "") or None, + asset_id=str(approval.get("asset_id") or "") or None, + ) + save_gateway_registry(registry) + _record_governance_activity( + "approval_denied", + asset_id=approval.get("asset_id"), + install_id=approval.get("install_id"), + approval_id=approval.get("approval_id"), + decided_by=approval["decided_by"], + gateway_id=approval.get("gateway_id"), + ) + result = dict(approval) + result["status"] = _approval_status(result) + return result + + +def _record_governance_activity(event: str, *, entry: dict[str, Any] | None = None, **fields: Any) -> dict[str, Any]: + return record_gateway_activity(event, entry=entry, **fields) + + +def ensure_local_asset_binding( + registry: dict[str, Any], + entry: dict[str, Any], + *, + created_via: str | None = None, + auto_approve: bool = True, +) -> dict[str, Any]: + _ensure_registry_lists(registry) + gateway_id = _gateway_id_from_registry(registry) + asset_id = _asset_id_for_entry(entry) + install_id = str(entry.get("install_id") or "").strip() + if not install_id: + install_id = str(uuid.uuid4()) + entry["install_id"] = install_id + existing = find_binding(registry, install_id=install_id) or find_binding(registry, asset_id=asset_id, gateway_id=gateway_id) + if existing: + entry.setdefault("install_id", str(existing.get("install_id") or install_id)) + return existing + candidate = _binding_candidate_for_entry({**entry, "created_via": created_via or entry.get("created_via")}, registry) + if auto_approve: + candidate["approved_state"] = "approved" + candidate["approved_at"] = _now_iso() + binding = upsert_binding(registry, candidate) + entry["install_id"] = str(binding.get("install_id") or install_id) + _record_governance_activity( + "asset_bound", + entry=entry, + asset_id=asset_id, + install_id=entry["install_id"], + binding_type=binding.get("binding_type"), + gateway_id=gateway_id, + path=binding.get("path"), + ) + return binding + + +def _create_binding_approval( + registry: dict[str, Any], + entry: dict[str, Any], + *, + candidate_binding: dict[str, Any], + action: str, + reason: str, + risk: str, + approval_kind: str, +) -> dict[str, Any]: + existing = _find_approval_for_signature(registry, str(candidate_binding.get("candidate_signature") or "")) + if existing: + return existing + approval = { + "approval_id": str(uuid.uuid4()), + "asset_id": candidate_binding.get("asset_id"), + "gateway_id": candidate_binding.get("gateway_id"), + "install_id": candidate_binding.get("install_id"), + "action": action, + "resource": candidate_binding.get("path") or candidate_binding.get("launch_spec_hash"), + "reason": reason, + "risk": risk, + "status": "pending", + "decision": None, + "requested_at": _now_iso(), + "expires_at": None, + "candidate_signature": candidate_binding.get("candidate_signature"), + "candidate_binding": candidate_binding, + "approval_kind": approval_kind, + } + registry.setdefault("approvals", []).append(approval) + _record_governance_activity( + "approval_requested", + entry=entry, + approval_id=approval["approval_id"], + asset_id=approval["asset_id"], + install_id=approval["install_id"], + approval_kind=approval_kind, + reason=reason, + risk=risk, + ) + return approval + + +def evaluate_runtime_attestation(registry: dict[str, Any], entry: dict[str, Any]) -> dict[str, Any]: + _ensure_registry_lists(registry) + gateway_id = _gateway_id_from_registry(registry) + asset_id = _asset_id_for_entry(entry) + install_id = str(entry.get("install_id") or "").strip() + candidate = _binding_candidate_for_entry(entry, registry) + latest_approval = _find_approval_for_signature(registry, candidate["candidate_signature"]) + + def blocked(reason: str, detail: str, *, approval: dict[str, Any] | None = None, state: str = "blocked") -> dict[str, Any]: + return { + "asset_id": asset_id or None, + "gateway_id": gateway_id, + "install_id": install_id or candidate["install_id"], + "binding": None, + "candidate_binding": candidate, + "runtime_instance_id": str(entry.get("runtime_instance_id") or "") or None, + "attestation_state": state, + "drift_reason": reason, + "approval_state": "rejected" if approval and _approval_status(approval) == "rejected" else ("pending" if approval and _approval_status(approval) == "pending" else "not_required"), + "approval_id": approval.get("approval_id") if approval else None, + "confidence_reason": reason, + "confidence_detail": detail, + } + + if not asset_id: + return blocked("unknown_asset", "Runtime is missing a registered asset identity.") + + install_binding = find_binding(registry, install_id=install_id) if install_id else None + asset_bindings = _bindings_for_asset(registry, asset_id) + + if install_binding and str(install_binding.get("asset_id") or "") != asset_id: + return blocked("asset_mismatch", "Runtime install is bound to a different asset id than the one it claimed.") + + if latest_approval and _approval_status(latest_approval) == "rejected": + return blocked("approval_denied", "A prior approval request for this runtime binding was denied.", approval=latest_approval) + + if not install_binding: + if asset_bindings: + same_gateway = next((binding for binding in asset_bindings if str(binding.get("gateway_id") or "") == gateway_id), None) + if same_gateway is None: + approval = latest_approval or _create_binding_approval( + registry, + entry, + candidate_binding=candidate, + action="runtime.bind", + reason="Asset is requesting access from a different Gateway than the approved binding.", + risk="high", + approval_kind="new_gateway", + ) + return blocked("new_gateway", "This asset is requesting access from a new Gateway and needs approval.", approval=approval, state="unknown") + approval = latest_approval or _create_binding_approval( + registry, + entry, + candidate_binding=candidate, + action="runtime.bind", + reason="Gateway discovered a runtime binding that has not been approved yet.", + risk="medium", + approval_kind="new_binding", + ) + return blocked("approval_required", "Gateway needs approval before trusting this new asset binding.", approval=approval, state="unknown") + + binding = install_binding + if str(binding.get("gateway_id") or "") != gateway_id: + approval = latest_approval or _create_binding_approval( + registry, + entry, + candidate_binding=candidate, + action="runtime.bind", + reason="Asset binding is attempting to run from a different Gateway than the approved one.", + risk="high", + approval_kind="new_gateway", + ) + return blocked("new_gateway", "This asset binding is tied to a different Gateway and needs approval.", approval=approval, state="unknown") + + if str(binding.get("approved_state") or "approved").lower() == "rejected": + return blocked("approval_denied", "This asset binding was previously rejected.") + + current_path = str(candidate.get("path") or "") + bound_path = str(binding.get("path") or "") + current_hash = str(candidate.get("launch_spec_hash") or "") + bound_hash = str(binding.get("launch_spec_hash") or "") + if current_path != bound_path or current_hash != bound_hash: + approval = latest_approval or _create_binding_approval( + registry, + entry, + candidate_binding=candidate, + action="runtime.attest", + reason="Runtime launch path or launch spec changed since approval.", + risk="high", + approval_kind="binding_drift", + ) + detail = "Runtime launch path or spec changed since approval. Review and approve the new binding before Gateway will trust it." + return { + "asset_id": asset_id, + "gateway_id": gateway_id, + "install_id": str(binding.get("install_id") or candidate["install_id"]), + "binding": binding, + "candidate_binding": candidate, + "runtime_instance_id": str(entry.get("runtime_instance_id") or "") or None, + "attestation_state": "drifted", + "drift_reason": "binding_drift", + "approval_state": "pending" if approval and _approval_status(approval) == "pending" else "not_required", + "approval_id": approval.get("approval_id") if approval else None, + "confidence_reason": "binding_drift", + "confidence_detail": detail, + } + + return { + "asset_id": asset_id, + "gateway_id": gateway_id, + "install_id": str(binding.get("install_id") or candidate["install_id"]), + "binding": binding, + "candidate_binding": candidate, + "runtime_instance_id": str(entry.get("runtime_instance_id") or "") or None, + "attestation_state": "verified", + "drift_reason": None, + "approval_state": "not_required", + "approval_id": None, + "confidence_reason": None, + "confidence_detail": "Runtime matches the approved local binding.", + } + + def _parse_iso8601(value: object) -> datetime | None: if not value or not isinstance(value, str): return None @@ -87,8 +1779,26 @@ def _age_seconds(value: object, *, now: datetime | None = None) -> int | None: return max(0, int(delta.total_seconds())) -def annotate_runtime_health(snapshot: dict[str, Any], *, now: datetime | None = None) -> dict[str, Any]: +def annotate_runtime_health( + snapshot: dict[str, Any], + *, + now: datetime | None = None, + registry: dict[str, Any] | None = None, + explicit_space_id: str | None = None, +) -> dict[str, Any]: enriched = dict(snapshot) + resolved_registry = registry + if resolved_registry is None: + try: + resolved_registry = load_gateway_registry() + except Exception: + resolved_registry = None + if resolved_registry and ( + resolved_registry.get("identity_bindings") + or enriched.get("identity_binding_id") + ): + identity_space = evaluate_identity_space_binding(resolved_registry, enriched, explicit_space_id=explicit_space_id) + enriched.update(identity_space) last_seen_age = _age_seconds(enriched.get("last_seen_at"), now=now) last_error_age = _age_seconds(enriched.get("last_listener_error_at"), now=now) if last_seen_age is not None: @@ -96,15 +1806,73 @@ def annotate_runtime_health(snapshot: dict[str, Any], *, now: datetime | None = if last_error_age is not None: enriched["last_listener_error_age_seconds"] = last_error_age + profile = infer_operator_profile(enriched) + asset_descriptor = infer_asset_descriptor(enriched, operator_profile=profile) state = str(enriched.get("effective_state") or "stopped").lower() - connected = False - if state == "running": - if last_seen_age is None or last_seen_age > RUNTIME_STALE_AFTER_SECONDS: - state = "stale" - else: - connected = True + raw_state = state + liveness, connected = _derive_liveness(enriched, raw_state=state, last_seen_age=last_seen_age) + if liveness == "stale" and raw_state == "running": + state = "stale" + elif liveness == "setup_error": + state = "error" + elif liveness == "offline" and state not in {"stopped", "error"}: + state = "stopped" + + work_state = _derive_work_state(enriched, liveness=liveness) + mode = _derive_mode(profile) + presence = _derive_presence(mode=mode, liveness=liveness, work_state=work_state) + reply = _derive_reply(profile["reply_mode"]) + reachability = _derive_reachability(snapshot=enriched, mode=mode, liveness=liveness, activation=profile["activation"]) + confidence, confidence_reason, confidence_detail = _derive_confidence( + enriched, + mode=mode, + liveness=liveness, + reachability=reachability, + ) + + enriched.update(profile) + enriched["asset_class"] = _normalized_controlled(asset_descriptor["asset_class"], _CONTROLLED_ASSET_CLASSES, fallback="interactive_agent") + enriched["intake_model"] = _normalized_controlled(asset_descriptor["intake_model"], _CONTROLLED_INTAKE_MODELS, fallback="launch_on_send") + if asset_descriptor.get("worker_model"): + enriched["worker_model"] = asset_descriptor["worker_model"] + enriched["trigger_sources"] = list(asset_descriptor.get("trigger_sources") or []) + enriched["return_paths"] = list(asset_descriptor.get("return_paths") or []) + enriched["telemetry_shape"] = _normalized_controlled( + asset_descriptor.get("telemetry_shape"), + _CONTROLLED_TELEMETRY_SHAPES, + fallback="basic", + ) + enriched["asset_type_label"] = str(asset_descriptor.get("type_label") or "Connected Asset") + enriched["output_label"] = str(asset_descriptor.get("output_label") or "Reply") + enriched["tags"] = list(asset_descriptor.get("tags") or []) + enriched["capabilities"] = list(asset_descriptor.get("capabilities") or []) + enriched["constraints"] = list(asset_descriptor.get("constraints") or []) + enriched["asset_descriptor"] = asset_descriptor enriched["effective_state"] = state enriched["connected"] = connected + enriched["liveness"] = _normalized_controlled(liveness, _CONTROLLED_LIVENESS, fallback="offline") + enriched["work_state"] = _normalized_controlled(work_state, _CONTROLLED_WORK_STATES, fallback="idle") + enriched["mode"] = _normalized_controlled(mode, _CONTROLLED_MODES, fallback="ON-DEMAND") + enriched["presence"] = _normalized_controlled(presence, _CONTROLLED_PRESENCE, fallback="OFFLINE") + enriched["reply"] = _normalized_controlled(reply, _CONTROLLED_REPLY, fallback="REPLY") + enriched["reachability"] = _normalized_controlled(reachability, _CONTROLLED_REACHABILITY, fallback="unavailable") + enriched["confidence"] = _normalized_controlled(confidence, _CONTROLLED_CONFIDENCE, fallback="MEDIUM") + enriched["confidence_reason"] = _normalized_controlled( + confidence_reason, + _CONTROLLED_CONFIDENCE_REASONS, + fallback="unknown", + ) + enriched["confidence_detail"] = str(confidence_detail or "").strip() or None + enriched["attestation_state"] = _normalized_optional_controlled(enriched.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + enriched["approval_state"] = _normalized_optional_controlled(enriched.get("approval_state"), _CONTROLLED_APPROVAL_STATES) + enriched["identity_status"] = _normalized_optional_controlled(enriched.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) + enriched["space_status"] = _normalized_optional_controlled(enriched.get("space_status"), _CONTROLLED_SPACE_STATUSES) + enriched["environment_status"] = _normalized_optional_controlled(enriched.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + enriched["active_space_source"] = _normalized_optional_controlled(enriched.get("active_space_source"), _CONTROLLED_ACTIVE_SPACE_SOURCES) + enriched["queue_capable"] = profile["placement"] == "mailbox" + enriched["queue_depth"] = int(enriched.get("backlog_depth") or 0) + enriched.setdefault("last_successful_doctor_at", None) + enriched.setdefault("last_doctor_result", None) return enriched @@ -134,6 +1902,18 @@ def pid_path() -> Path: return gateway_dir() / "gateway.pid" +def ui_state_path() -> Path: + return gateway_dir() / "gateway-ui.json" + + +def daemon_log_path() -> Path: + return gateway_dir() / "gateway.log" + + +def ui_log_path() -> Path: + return gateway_dir() / "gateway-ui.log" + + def activity_log_path() -> Path: return gateway_dir() / "activity.jsonl" @@ -162,6 +1942,9 @@ def _default_registry() -> dict[str, Any]: "last_reconcile_at": None, }, "agents": [], + "bindings": [], + "identity_bindings": [], + "approvals": [], } @@ -196,6 +1979,9 @@ def load_gateway_registry() -> dict[str, Any]: registry.setdefault("version", 1) registry.setdefault("gateway", {}) registry.setdefault("agents", []) + registry.setdefault("bindings", []) + registry.setdefault("identity_bindings", []) + registry.setdefault("approvals", []) gateway = registry["gateway"] gateway.setdefault("gateway_id", str(uuid.uuid4())) gateway.setdefault("desired_state", "stopped") @@ -229,18 +2015,23 @@ def daemon_status() -> dict[str, Any]: pid = int(pid_path().read_text().strip()) except ValueError: pid = None + running = _pid_alive(pid) + if not running: + scanned = _scan_gateway_process_pids() + if scanned: + pid = scanned[0] + running = True registry = load_gateway_registry() return { "pid": pid, - "running": _pid_alive(pid), + "running": running, "registry_path": str(registry_path()), "session_path": str(session_path()), "registry": registry, } -def _scan_gateway_process_pids() -> list[int]: - """Best-effort fallback for live daemons that predate the pid file.""" +def _scan_process_pids(pattern: re.Pattern[str]) -> list[int]: current_pid = os.getpid() parent_pid = os.getppid() try: @@ -265,11 +2056,118 @@ def _scan_gateway_process_pids() -> list[int]: if pid in {current_pid, parent_pid} or not _pid_alive(pid): continue command = command.strip() - if command and _GATEWAY_PROCESS_RE.search(command): + if command and pattern.search(command): pids.append(pid) return sorted(set(pids)) +def _scan_gateway_process_pids() -> list[int]: + """Best-effort fallback for live daemons that predate the pid file.""" + return _scan_process_pids(_GATEWAY_PROCESS_RE) + + +def _default_ui_state() -> dict[str, Any]: + return { + "pid": None, + "host": "127.0.0.1", + "port": 8765, + "last_started_at": None, + } + + +def load_gateway_ui_state() -> dict[str, Any]: + state = _read_json(ui_state_path(), default=_default_ui_state()) + state.setdefault("pid", None) + state.setdefault("host", "127.0.0.1") + state.setdefault("port", 8765) + state.setdefault("last_started_at", None) + return state + + +def save_gateway_ui_state(data: dict[str, Any]) -> Path: + payload = _default_ui_state() + payload.update(data) + _write_json(ui_state_path(), payload) + return ui_state_path() + + +def ui_status() -> dict[str, Any]: + state = load_gateway_ui_state() + pid = state.get("pid") + try: + pid_value = int(pid) if pid is not None else None + except (TypeError, ValueError): + pid_value = None + host = str(state.get("host") or "127.0.0.1") + try: + port = int(state.get("port") or 8765) + except (TypeError, ValueError): + port = 8765 + running = _pid_alive(pid_value) + if not running: + scanned = _scan_gateway_ui_process_pids() + if scanned: + pid_value = scanned[0] + running = True + return { + "pid": pid_value, + "running": running, + "host": host, + "port": port, + "url": f"http://{host}:{port}", + "state_path": str(ui_state_path()), + "log_path": str(ui_log_path()), + "last_started_at": state.get("last_started_at"), + } + + +def _scan_gateway_ui_process_pids() -> list[int]: + """Best-effort fallback for live UIs that predate the ui state file.""" + return _scan_process_pids(_GATEWAY_UI_PROCESS_RE) + + +def active_gateway_ui_pids() -> list[int]: + """Return all known live Gateway UI PIDs except the current process.""" + status = ui_status() + pids: list[int] = [] + pid = status.get("pid") + if isinstance(pid, int) and status.get("running") and pid != os.getpid(): + pids.append(pid) + pids.extend(_scan_gateway_ui_process_pids()) + return sorted(set(pids)) + + +def active_gateway_ui_pid() -> int | None: + """Return the PID of a live Gateway UI, if one is already running.""" + pids = active_gateway_ui_pids() + return pids[0] if pids else None + + +def write_gateway_ui_state(*, pid: int, host: str, port: int) -> None: + save_gateway_ui_state( + { + "pid": pid, + "host": host, + "port": port, + "last_started_at": _now_iso(), + } + ) + + +def clear_gateway_ui_state(pid: int | None = None) -> None: + if not ui_state_path().exists(): + return + if pid is not None: + try: + state = load_gateway_ui_state() + existing_pid = int(state.get("pid")) if state.get("pid") is not None else None + except (TypeError, ValueError): + existing_pid = None + if existing_pid not in {None, pid}: + return + ui_state_path().unlink() + + def active_gateway_pids() -> list[int]: """Return all known live Gateway daemon PIDs except the current process.""" status = daemon_status() @@ -324,6 +2222,9 @@ def record_gateway_activity( { "agent_name": entry.get("name"), "agent_id": entry.get("agent_id"), + "asset_id": _asset_id_for_entry(entry) or None, + "install_id": entry.get("install_id"), + "runtime_instance_id": entry.get("runtime_instance_id"), "runtime_type": entry.get("runtime_type"), "transport": entry.get("transport", "gateway"), "credential_source": entry.get("credential_source", "gateway"), @@ -409,6 +2310,12 @@ def sanitize_exec_env(prompt: str, entry: dict[str, Any]) -> dict[str, str]: env["AX_GATEWAY_AGENT_NAME"] = str(entry.get("name") or "") env["AX_GATEWAY_RUNTIME_TYPE"] = str(entry.get("runtime_type") or "") env["AX_MENTION_CONTENT"] = prompt + ollama_model = str(entry.get("ollama_model") or "").strip() + if ollama_model: + env["OLLAMA_MODEL"] = ollama_model + hermes_repo_path = str(entry.get("hermes_repo_path") or "").strip() + if hermes_repo_path: + env["HERMES_REPO_PATH"] = hermes_repo_path return env @@ -547,6 +2454,7 @@ def __init__( self._stream_response = None self._state: dict[str, Any] = { "effective_state": "stopped", + "runtime_instance_id": None, "backlog_depth": 0, "dropped_count": 0, "processed_count": 0, @@ -627,7 +2535,7 @@ def _consume_completed_seen(self, message_id: str) -> bool: def snapshot(self) -> dict[str, Any]: with self._state_lock: - return annotate_runtime_health(dict(self._state)) + return dict(self._state) def start(self) -> None: if self._listener_thread and self._listener_thread.is_alive(): @@ -637,8 +2545,11 @@ def start(self) -> None: self._reply_anchor_ids = set() self._seen_ids = set() self._completed_seen_ids = set() + runtime_instance_id = str(uuid.uuid4()) + self.entry["runtime_instance_id"] = runtime_instance_id self._update_state( effective_state="starting", + runtime_instance_id=runtime_instance_id, backlog_depth=0, current_status=None, current_activity=None, @@ -664,7 +2575,7 @@ def start(self) -> None: if self._worker_thread is not None: self._worker_thread.start() self._listener_thread.start() - record_gateway_activity("runtime_started", entry=self.entry) + record_gateway_activity("runtime_started", entry=self.entry, runtime_instance_id=runtime_instance_id) self._log("started") def stop(self, timeout: float = 5.0) -> None: @@ -690,8 +2601,10 @@ def stop(self, timeout: float = 5.0) -> None: self._stream_client = None self._send_client = None self._stream_response = None + self.entry["runtime_instance_id"] = None self._update_state( effective_state="stopped", + runtime_instance_id=None, backlog_depth=0, current_status=None, current_activity=None, @@ -1243,8 +3156,41 @@ def stop(self) -> None: def _reconcile_runtime(self, entry: dict[str, Any]) -> None: name = str(entry.get("name") or "") desired_state = str(entry.get("desired_state") or "stopped").lower() + attestation_state = _normalized_optional_controlled(entry.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + approval_state = _normalized_optional_controlled(entry.get("approval_state"), _CONTROLLED_APPROVAL_STATES) + identity_status = _normalized_optional_controlled(entry.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) + environment_status = _normalized_optional_controlled(entry.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + space_status = _normalized_optional_controlled(entry.get("space_status"), _CONTROLLED_SPACE_STATUSES) runtime = self._runtimes.get(name) - if desired_state == "running": + hermes_status = hermes_setup_status(entry) + if not hermes_status.get("ready", True): + if runtime is not None: + runtime.stop() + self._runtimes.pop(name, None) + entry.update( + { + "effective_state": "error", + "runtime_instance_id": None, + "last_error": str(hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete."), + "current_status": None, + "current_activity": str(hermes_status.get("summary") or "Hermes setup is incomplete."), + "current_tool": None, + "current_tool_call_id": None, + "backlog_depth": 0, + } + ) + return + if hermes_status.get("resolved_path"): + entry["hermes_repo_path"] = str(hermes_status["resolved_path"]) + allowed_to_run = ( + desired_state == "running" + and attestation_state in {None, "verified"} + and approval_state not in {"pending", "rejected"} + and identity_status in {None, "verified"} + and environment_status not in {"environment_mismatch", "environment_blocked"} + and space_status not in {"active_not_allowed", "no_active_space"} + ) + if allowed_to_run: if runtime is None: runtime = ManagedAgentRuntime(entry, client_factory=self.client_factory, logger=self.logger) self._runtimes[name] = runtime @@ -1258,6 +3204,7 @@ def _reconcile_runtime(self, entry: dict[str, Any]) -> None: self._runtimes.pop(name, None) def _reconcile_registry(self, registry: dict[str, Any], session: dict[str, Any]) -> dict[str, Any]: + _ensure_registry_lists(registry) agents = registry.setdefault("agents", []) agent_names = {str(entry.get("name") or "") for entry in agents} for name, runtime in list(self._runtimes.items()): @@ -1270,10 +3217,85 @@ def _reconcile_registry(self, registry: dict[str, Any], session: dict[str, Any]) entry.setdefault("credential_source", "gateway") entry.setdefault("runtime_type", "echo") entry.setdefault("desired_state", "stopped") + if not str(entry.get("install_id") or "").strip(): + entry["install_id"] = str(uuid.uuid4()) + + asset_id = _asset_id_for_entry(entry) + existing_binding = find_binding(registry, install_id=str(entry.get("install_id") or "").strip()) if asset_id else None + if not existing_binding and asset_id and not _bindings_for_asset(registry, asset_id): + ensure_local_asset_binding( + registry, + entry, + created_via=str(entry.get("created_via") or "legacy_registry"), + auto_approve=True, + ) + ensure_gateway_identity_binding( + registry, + entry, + session=session, + created_via=str(entry.get("created_via") or "legacy_registry"), + ) + entry.update(evaluate_identity_space_binding(registry, entry)) + + previous_attestation = ( + str(entry.get("attestation_state") or ""), + str(entry.get("approval_state") or ""), + str(entry.get("approval_id") or ""), + str(entry.get("drift_reason") or ""), + ) + attestation = evaluate_runtime_attestation(registry, entry) + entry.update(attestation) + current_attestation = ( + str(entry.get("attestation_state") or ""), + str(entry.get("approval_state") or ""), + str(entry.get("approval_id") or ""), + str(entry.get("drift_reason") or ""), + ) + if current_attestation != previous_attestation: + state = str(entry.get("attestation_state") or "") + if state == "verified": + record_gateway_activity( + "runtime_attested", + entry=entry, + install_id=entry.get("install_id"), + attestation_state=state, + ) + elif state == "drifted": + record_gateway_activity( + "attestation_drift_detected", + entry=entry, + install_id=entry.get("install_id"), + attestation_state=state, + approval_id=entry.get("approval_id"), + drift_reason=entry.get("drift_reason"), + ) + elif state in {"unknown", "blocked"}: + record_gateway_activity( + "invocation_blocked", + entry=entry, + install_id=entry.get("install_id"), + attestation_state=state, + approval_id=entry.get("approval_id"), + reason=entry.get("confidence_reason"), + ) self._reconcile_runtime(entry) runtime = self._runtimes.get(str(entry.get("name") or "")) - snapshot = runtime.snapshot() if runtime is not None else annotate_runtime_health({"effective_state": "stopped"}) + snapshot = ( + runtime.snapshot() + if runtime is not None + else { + "effective_state": entry.get("effective_state") or "stopped", + "runtime_instance_id": None, + "last_error": entry.get("last_error"), + "current_status": entry.get("current_status"), + "current_activity": entry.get("current_activity"), + "current_tool": entry.get("current_tool"), + "current_tool_call_id": entry.get("current_tool_call_id"), + "backlog_depth": int(entry.get("backlog_depth") or 0), + } + ) entry.update(snapshot) + entry.update(annotate_runtime_health(entry, registry=registry)) gateway = registry.setdefault("gateway", {}) gateway.update( @@ -1337,6 +3359,7 @@ def run(self, *, once: bool = False) -> None: runtime = self._runtimes.get(name) if runtime is not None: entry.update(runtime.snapshot()) + entry.update(annotate_runtime_health(entry, registry=final_registry)) save_gateway_registry(final_registry) record_gateway_activity("gateway_stopped") clear_gateway_pid(os.getpid()) diff --git a/ax_cli/gateway_runtime_types.py b/ax_cli/gateway_runtime_types.py new file mode 100644 index 0000000..7fe5fb2 --- /dev/null +++ b/ax_cli/gateway_runtime_types.py @@ -0,0 +1,313 @@ +"""Gateway runtime backends and operator-facing templates. + +Runtime types are the low-level execution adapters used by the Gateway. +Templates are the higher-level, user-facing choices presented in CLI and UI. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +def _repo_root() -> Path: + return Path(__file__).resolve().parent.parent + + +def _gateway_setup_skill_path() -> Path: + return _repo_root() / "skills" / "gateway-agent-setup" / "SKILL.md" + + +def _shared_signals() -> dict[str, str]: + return { + "delivery": "Gateway confirms when a message was queued or claimed.", + "liveness": "Gateway heartbeat and reconnect logic determine connected or stale state.", + } + + +def runtime_type_catalog() -> dict[str, dict[str, Any]]: + repo_root = _repo_root() + return { + "echo": { + "id": "echo", + "label": "Echo", + "description": "Built-in test runtime for proving delivery, queueing, and reply flow.", + "kind": "builtin", + "passive": False, + "requires": [], + "form_fields": [], + "examples": [], + "signals": { + **_shared_signals(), + "activity": "Gateway emits built-in working and completed phases for echo replies.", + "tools": "No tool-call telemetry. Echo is intentionally simple.", + }, + }, + "exec": { + "id": "exec", + "label": "Command Bridge", + "description": ( + "Gateway-owned command execution for bridges and adapters that print " + "AX_GATEWAY_EVENT lines." + ), + "kind": "exec", + "passive": False, + "requires": ["exec_command"], + "form_fields": [ + { + "name": "exec_command", + "label": "Exec Command", + "required": True, + "placeholder": "python3 examples/hermes_sentinel/hermes_bridge.py", + }, + { + "name": "workdir", + "label": "Workdir", + "required": False, + "placeholder": str(repo_root), + }, + ], + "examples": [ + { + "label": "Gateway Probe", + "exec_command": "python3 examples/gateway_probe/probe_bridge.py", + "workdir": str(repo_root), + }, + { + "label": "Codex Bridge", + "exec_command": "python3 examples/codex_gateway/codex_bridge.py", + "workdir": str(repo_root), + }, + { + "label": "Hermes Sentinel", + "exec_command": "python3 examples/hermes_sentinel/hermes_bridge.py", + "workdir": str(repo_root), + "note": "Requires a local hermes-agent checkout plus auth setup.", + }, + { + "label": "Ollama", + "exec_command": "python3 examples/gateway_ollama/ollama_bridge.py", + "workdir": str(repo_root), + "note": "Requires a local Ollama server and model.", + }, + ], + "signals": { + **_shared_signals(), + "activity": ( + "Gateway can surface live activity when the bridge prints AX_GATEWAY_EVENT lines. " + "Without that, the operator still gets pickup and final completion." + ), + "tools": "Gateway can record tool usage when the bridge emits tool events.", + }, + }, + "inbox": { + "id": "inbox", + "label": "Passive Inbox", + "description": "Passive Gateway-managed identity that receives and queues work without auto-replying.", + "kind": "builtin", + "passive": True, + "requires": [], + "form_fields": [], + "examples": [], + "signals": { + **_shared_signals(), + "activity": "Gateway reports queued state only. This runtime is passive by design.", + "tools": "No tool-call telemetry. Inbox runtimes do not execute work.", + }, + }, + } + + +def runtime_type_definition(runtime_type: str) -> dict[str, Any]: + normalized = runtime_type.lower().strip() + if normalized == "command": + normalized = "exec" + catalog = runtime_type_catalog() + if normalized not in catalog: + raise KeyError(runtime_type) + return catalog[normalized] + + +def runtime_type_list() -> list[dict[str, Any]]: + catalog = runtime_type_catalog() + ordered_ids = ["echo", "exec", "inbox"] + return [catalog[runtime_id] for runtime_id in ordered_ids if runtime_id in catalog] + + +def agent_template_catalog() -> dict[str, dict[str, Any]]: + repo_root = _repo_root() + skill_path = _gateway_setup_skill_path() + runtime_signals = {key: runtime_type_definition(key)["signals"] for key in ("echo", "exec", "inbox")} + return { + "echo_test": { + "id": "echo_test", + "label": "Echo (Test)", + "description": "Fastest way to prove the Gateway is connected and replying correctly.", + "availability": "ready", + "launchable": True, + "runtime_type": "echo", + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "basic", + "suggested_name": "echo-bot", + "operator_summary": "Best first test. No local setup required.", + "recommended_test_message": "gateway test ping", + "what_you_need": [], + "setup_skill": "gateway-agent-setup", + "setup_skill_path": str(skill_path), + "defaults": { + "runtime_type": "echo", + }, + "signals": runtime_signals["echo"], + "advanced": { + "adapter_label": "Built-in echo runtime", + "supports_command_override": False, + }, + }, + "ollama": { + "id": "ollama", + "label": "Ollama", + "description": "Local model runtime managed by Gateway.", + "availability": "ready", + "launchable": True, + "runtime_type": "exec", + "asset_class": "interactive_agent", + "intake_model": "launch_on_send", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "basic", + "suggested_name": "ollama-bot", + "operator_summary": "Good for a local model with pickup, liveness, and streaming activity.", + "recommended_test_message": "Reply with exactly: Gateway test OK. Then mention which local model answered.", + "what_you_need": [ + "Run a local Ollama server on this machine.", + "Have at least one Ollama model pulled locally. Gateway can suggest an installed model when the server is reachable.", + ], + "setup_skill": "gateway-agent-setup", + "setup_skill_path": str(skill_path), + "defaults": { + "runtime_type": "exec", + "exec_command": "python3 examples/gateway_ollama/ollama_bridge.py", + "workdir": str(repo_root), + }, + "signals": runtime_signals["exec"], + "advanced": { + "adapter_label": "Gateway command bridge", + "supports_command_override": True, + }, + }, + "hermes": { + "id": "hermes", + "label": "Hermes", + "description": "Local Hermes agent bridge with strong activity and tool telemetry.", + "availability": "setup_required", + "launchable": True, + "runtime_type": "exec", + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "rich", + "suggested_name": "hermes-bot", + "operator_summary": "Best path for a capable local agent with tool use and rich progress.", + "recommended_test_message": "Pause for 5 seconds, narrate activity as you go, and end with: Gateway test OK.", + "what_you_need": [ + "A local hermes-agent checkout, usually at ~/hermes-agent or via HERMES_REPO_PATH.", + "Hermes auth or model credentials such as ~/.hermes/auth.json or provider env vars.", + ], + "setup_skill": "gateway-agent-setup", + "setup_skill_path": str(skill_path), + "defaults": { + "runtime_type": "exec", + "exec_command": "python3 examples/hermes_sentinel/hermes_bridge.py", + "workdir": str(repo_root), + }, + "signals": runtime_signals["exec"], + "advanced": { + "adapter_label": "Gateway command bridge", + "supports_command_override": True, + }, + }, + "claude_code_channel": { + "id": "claude_code_channel", + "label": "Claude Code Channel", + "description": "Live Claude Code session bridged through aX channel delivery.", + "availability": "coming_soon", + "launchable": False, + "runtime_type": "exec", + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "basic", + "suggested_name": "cc-channel", + "operator_summary": "Planned managed channel adapter. Pickup and liveness first, richer activity where possible.", + "recommended_test_message": "Reply with exactly: Gateway test OK.", + "what_you_need": [ + "A dedicated managed-daemon adapter so Gateway can supervise a live ax channel session cleanly.", + ], + "setup_skill": "gateway-agent-setup", + "setup_skill_path": str(skill_path), + "defaults": { + "runtime_type": "exec", + }, + "signals": { + **runtime_signals["exec"], + "activity": ( + "Today the channel is usually sparse while working. Gateway should still provide reliable " + "pickup and liveness even when the adapter emits little activity." + ), + }, + "advanced": { + "adapter_label": "Managed daemon adapter", + "supports_command_override": False, + }, + }, + "inbox": { + "id": "inbox", + "label": "Passive Inbox", + "description": "Passive receiver identity for queue demos, operator flows, and non-replying endpoints.", + "availability": "advanced", + "launchable": True, + "runtime_type": "inbox", + "asset_class": "background_worker", + "intake_model": "queue_accept", + "worker_model": "queue_drain", + "trigger_sources": ["queued_job", "manual_trigger"], + "return_paths": ["summary_post"], + "telemetry_shape": "basic", + "suggested_name": "inbox-bot", + "operator_summary": "Advanced testing and operator-only flow.", + "recommended_test_message": "Queue this test job, mark it received, and do not reply inline.", + "what_you_need": [], + "setup_skill": "gateway-agent-setup", + "setup_skill_path": str(skill_path), + "defaults": { + "runtime_type": "inbox", + }, + "signals": runtime_signals["inbox"], + "advanced": { + "adapter_label": "Built-in passive inbox runtime", + "supports_command_override": False, + }, + }, + } + + +def agent_template_definition(template_id: str) -> dict[str, Any]: + normalized = template_id.lower().strip() + catalog = agent_template_catalog() + if normalized not in catalog: + raise KeyError(template_id) + return catalog[normalized] + + +def agent_template_list(*, include_advanced: bool = False) -> list[dict[str, Any]]: + catalog = agent_template_catalog() + ordered_ids = ["echo_test", "ollama", "hermes", "claude_code_channel", "inbox"] + templates = [catalog[template_id] for template_id in ordered_ids if template_id in catalog] + if include_advanced: + return templates + return [item for item in templates if str(item.get("availability") or "") != "advanced"] diff --git a/examples/gateway_ollama/ollama_bridge.py b/examples/gateway_ollama/ollama_bridge.py new file mode 100644 index 0000000..a1b049f --- /dev/null +++ b/examples/gateway_ollama/ollama_bridge.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Gateway-managed bridge for a local Ollama model. + +This bridge is designed for `ax gateway agents add ... --template ollama`. +It emits Gateway progress events while making a streaming call to a local +Ollama server, then prints the final text reply to stdout. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from typing import Any +from urllib import error, request + +EVENT_PREFIX = "AX_GATEWAY_EVENT " +DEFAULT_OLLAMA_BASE_URL = os.environ.get("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/") +DEFAULT_OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "llama3.2") + + +def emit_event(payload: dict[str, Any]) -> None: + print(f"{EVENT_PREFIX}{json.dumps(payload, sort_keys=True)}", flush=True) + + +def _read_prompt() -> str: + if len(sys.argv) > 1 and sys.argv[-1] != "-": + return sys.argv[-1] + env_prompt = os.environ.get("AX_MENTION_CONTENT", "").strip() + if env_prompt: + return env_prompt + return sys.stdin.read().strip() + + +def _generate(prompt: str) -> str: + model = DEFAULT_OLLAMA_MODEL + endpoint = f"{DEFAULT_OLLAMA_BASE_URL}/api/generate" + body = { + "model": model, + "prompt": prompt, + "stream": True, + } + emit_event({"kind": "status", "status": "thinking", "message": f"Preparing Ollama request ({model})"}) + emit_event({"kind": "status", "status": "processing", "message": f"Calling Ollama ({model})"}) + + req = request.Request( + endpoint, + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + + started = time.monotonic() + chunks: list[str] = [] + first_token_seen = False + last_activity_at = 0.0 + try: + with request.urlopen(req, timeout=300) as response: + for raw in response: + line = raw.decode("utf-8", errors="replace").strip() + if not line: + continue + payload = json.loads(line) + if not isinstance(payload, dict): + continue + if payload.get("error"): + raise RuntimeError(str(payload["error"])) + text = str(payload.get("response") or "") + if text: + chunks.append(text) + now = time.monotonic() + if not first_token_seen: + first_token_seen = True + emit_event({"kind": "status", "status": "processing", "message": f"Ollama is responding ({model})"}) + if now - last_activity_at >= 1.0: + emit_event({"kind": "activity", "activity": f"Streaming response from {model}..."}) + last_activity_at = now + if payload.get("done"): + break + except error.URLError as exc: + raise RuntimeError(f"Failed to reach Ollama at {endpoint}: {exc.reason}") from exc + + duration_ms = int((time.monotonic() - started) * 1000) + emit_event( + { + "kind": "status", + "status": "completed", + "message": f"Ollama completed in {duration_ms}ms", + "detail": {"model": model, "duration_ms": duration_ms}, + } + ) + return "".join(chunks).strip() + + +def main() -> int: + prompt = _read_prompt() + if not prompt: + print("(no mention content received)", file=sys.stderr) + return 1 + + try: + reply = _generate(prompt) + except Exception as exc: + emit_event({"kind": "status", "status": "error", "error_message": str(exc)}) + print(f"Ollama bridge failed: {exc}") + return 1 + + print(reply or f"Ollama ({DEFAULT_OLLAMA_MODEL}) finished without text.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/SKILL.md b/skills/SKILL.md index 59230ef..2674872 100644 --- a/skills/SKILL.md +++ b/skills/SKILL.md @@ -461,6 +461,22 @@ axctl token mint name --create --audience both # create/mint agent PAT (user PA axctl handoff agent "bounded task" --loop --max-rounds 5 --completion-promise DONE ``` +## Gateway-Managed Agent Setup + +When the job is to create or modify a managed local runtime, treat Gateway as +the control plane and use the companion skill +[`gateway-agent-setup`](gateway-agent-setup/SKILL.md). + +That flow is for: +- `ax gateway start` +- `ax gateway agents add ...` +- `ax gateway agents update ...` +- `ax gateway agents doctor ...` +- `ax gateway approvals ...` + +The browser UI is a human-readable view over the same Gateway state, but the +setup flow itself must stay agent-operable through the CLI and local API. + ## Troubleshooting | Error | Meaning | Fix | diff --git a/skills/gateway-agent-setup/SKILL.md b/skills/gateway-agent-setup/SKILL.md new file mode 100644 index 0000000..2a7d9c6 --- /dev/null +++ b/skills/gateway-agent-setup/SKILL.md @@ -0,0 +1,159 @@ +--- +name: gateway-agent-setup +description: | + Create, update, doctor, and supervise Gateway-managed aX assets through the + local Gateway control plane. Use when an agent needs to set up or modify a + managed Hermes, Ollama, Echo, or inbox-backed asset without falling back to + ad hoc local state. +--- + +# Gateway Agent Setup + +This skill is the setup and maintenance wrapper for Gateway-managed agents. +Gateway is the control plane. The browser UI is a human-readable view of the +same control plane; it is not the only place setup happens. + +Use this skill when the task is: +- creating a managed agent +- updating a managed agent's template or launch settings +- running Gateway Doctor after setup +- checking approval, identity, environment, or space state +- verifying a persistent runtime such as Hermes stays healthy + +## Principles + +1. Bootstrap is human-scoped; runtime is agent-scoped. + - User PAT/bootstrap login stays in Gateway. + - Managed agents get their own Gateway-owned runtime token and identity. + +2. Prefer Gateway-native templates first. + - `echo_test` + - `ollama` + - `hermes` + - `inbox` + +3. Treat setup as an agent-operable workflow. + - CLI and local API are the primary control surface. + - UI is a review and intervention surface over the same state. + +4. Do not hide setup gaps. + - If Hermes checkout or identity/space binding is wrong, keep the asset + blocked and explain why. + +## Golden Path + +### 1. Make sure the Gateway is up + +```bash +uv run ax gateway start +uv run ax gateway status +``` + +If Gateway is not logged in yet: + +```bash +uv run ax gateway login +``` + +### 2. Inspect templates + +```bash +uv run ax gateway templates +``` + +Pick the template that matches the asset class and intake model you want. + +### 3. Add the managed asset + +Examples: + +```bash +uv run ax gateway agents add echo-bot --template echo_test +uv run ax gateway agents add northstar --template hermes +uv run ax gateway agents add ollama-bot --template ollama +``` + +### 4. Update instead of recreating when possible + +Use update when the identity should stay the same but the setup needs to +change. + +```bash +uv run ax gateway agents update northstar --template hermes +uv run ax gateway agents update northstar --workdir /absolute/path/to/ax-cli +uv run ax gateway agents update ollama-bot --desired-state stopped +``` + +### 5. Run Gateway Doctor + +```bash +uv run ax gateway agents doctor northstar +``` + +Doctor is the canonical preflight and repair surface. It should be used after +create/update and before asking humans to trust the asset. + +### 5a. Prefer agent-authored tests + +Gateway test sends should default to an agent-authored path, not the bootstrap +user identity. + +```bash +uv run ax gateway agents test northstar +``` + +For diagnostics, a user-authored test is still allowed explicitly: + +```bash +uv run ax gateway agents test northstar --author user +``` + +Custom payloads should use the normal send path, not the test path. This is +how to simulate alerting and scheduled inputs such as Splunk, Datadog, or cron +jobs: + +```bash +uv run ax gateway agents send switchboard- "Datadog alert: api latency is above threshold" --to northstar +``` + +### 6. Check approvals when Gateway detects drift or a new binding + +```bash +uv run ax gateway approvals list +uv run ax gateway approvals show +uv run ax gateway approvals approve --scope asset +uv run ax gateway approvals deny +``` + +## Hermes Notes + +Hermes is a persistent live-listener asset when healthy. It should stay +running, not cold-start on every send. + +Hermes requires: +- a local `hermes-agent` checkout, typically at `~/hermes-agent` or resolved + through `HERMES_REPO_PATH` +- provider auth or Hermes auth material + +If Hermes setup is incomplete: +- Gateway should keep the asset blocked +- Doctor should show a clean setup error +- the runtime should not start and then answer with raw stderr in chat + +## Output Standard + +When using this skill, always leave the operator with: +- the managed agent name +- current `Mode + Presence + Reply + Confidence` +- whether Doctor passed, warned, or failed +- the exact blocking setup gap if one exists +- the next command or UI surface to use + +## Review Checklist + +Before handing off: +- the asset exists in Gateway registry +- identity and space binding are visible +- Doctor is current +- Hermes/Ollama setup gaps are explicit +- no user bootstrap token is being used as the acting runtime identity diff --git a/specs/CONNECTED-ASSET-GOVERNANCE-001/spec.md b/specs/CONNECTED-ASSET-GOVERNANCE-001/spec.md new file mode 100644 index 0000000..6c20491 --- /dev/null +++ b/specs/CONNECTED-ASSET-GOVERNANCE-001/spec.md @@ -0,0 +1,759 @@ +# CONNECTED-ASSET-GOVERNANCE-001: Registry, Provenance, Capabilities, Grants, Secrets, and Approval + +**Status:** Draft +**Owner:** @madtank +**Date:** 2026-04-22 +**Related:** GATEWAY-CONNECTIVITY-001, GATEWAY-ASSET-TAXONOMY-001, AGENT-PAT-001, DEVICE-TRUST-001, RUNTIME-CONFIG-001, AX-SCHEDULE-001 + +## Purpose + +Define the governance and registry layer above Gateway connectivity and asset +taxonomy. + +The existing Gateway specs answer: + +- what kind of connected asset this is, +- how work flows through it, and +- whether Gateway can safely route work through it right now. + +This spec answers the next control-plane questions: + +- Who or what is this asset? +- Where did it come from? +- What is it allowed to do? +- What secrets, tools, and context can it access? +- Which Gateway or device is allowed to run it? +- What changed since approval? +- When is human approval required? +- Which policy, grant, or approval decision allowed an action? + +This spec establishes the governance frame for aX as the canonical registry and +Gateway as the trusted local enforcement edge. + +## Core Framing + +The architectural split is: + +- **aX** is the canonical registry, collaboration, context, policy, and audit + plane. +- **Gateway** is the local execution, enforcement, credential boundary, + runtime supervision, and real-time signal plane. +- **Connected assets** are agents, workers, jobs, listeners, tools, and + proxies registered in aX and enforced through Gateway. + +Gateway must be treated as an **agent-operable control plane**. The local UI is +a human-readable view over that control plane, not the only place where lifecycle, +approval, doctor, or binding actions exist. + +The default setup and maintenance path for managed assets should be an +agent-facing Gateway skill built on these same primitives, not a privileged UI +side path with different semantics. + +### Relationship to existing Gateway specs + +- [GATEWAY-ASSET-TAXONOMY-001](../GATEWAY-ASSET-TAXONOMY-001/spec.md) + defines what kind of connected asset this is and how work flows through it. +- [GATEWAY-CONNECTIVITY-001](../GATEWAY-CONNECTIVITY-001/spec.md) + defines whether the path is safe, healthy, live, stale, queued, blocked, or + expected to reply right now. +- This governance spec defines who controls the asset, how it is approved, + what it can access, and how drift, grants, secrets, and approvals are + enforced and audited. + +These three layers must remain separate: + +- `AssetDescriptor` says **what the asset is**. +- `AgentStatusSnapshot` says **whether its Gateway path is safe/healthy now**. +- `InvocationStatusSnapshot` says **what is happening to one work item**. +- Governance objects say **why this asset is allowed, what it can do, and who + approved it**. + +## Goals + +- Make aX the source of truth for connected asset identity, ownership, + provenance, grants, policy, and audit. +- Make Gateway the local trusted edge that enforces policy, protects + credentials, validates runtime provenance, and emits real-time evidence. +- Prevent assets from being implicitly trusted just because they exist. +- Allow arbitrary local agents, workers, jobs, listeners, and proxies to be + registered and governed without forcing them all into one interaction model. +- Support seamless user flows where assets are created in aX, locally bound + through Gateway, and then enforced and observed without requiring users to + reason about two disconnected systems. + +## Non-goals + +- Replacing the current Gateway status model. +- Replacing the asset taxonomy model. +- Making every decision online-only. Gateway may enforce cached policy within a + defined offline window. +- Building a full production secret manager in v1. +- Designing third-party marketplace governance in v1. +- Supporting arbitrary cross-machine HA Gateways in v1. + +## Control-plane Responsibilities + +### aX responsibilities + +aX is the source of truth for: + +- asset identity +- ownership +- workspace/project membership +- asset taxonomy +- capability declarations +- tool manifests +- policy templates +- grants +- secret references +- context access rules +- approval rules +- human-readable registry UX +- audit history + +### Gateway responsibilities + +Gateway is the local enforcement plane for: + +- local runtime launch and attach +- local runtime fingerprinting +- path/process validation +- local secret materialization +- capability-token issuance +- policy enforcement before execution +- tool-call enforcement +- doctor/preflight checks +- revocation enforcement +- real-time telemetry emission +- runtime attestation and drift detection + +Gateway should enforce policy received from aX and report evidence back. It +should not become the long-term policy authority. + +### Runtime responsibilities + +The runtime should remain least-privileged: + +- receives assigned work +- receives only scoped local capability or secret access +- emits Gateway events +- returns results +- cannot mint identities +- cannot impersonate another asset +- cannot directly call aX as the user +- cannot self-expand permissions + +## Identity Hierarchy + +The governance model needs a stable identity chain: + +```text +user_id / org_id / workspace_id + -> gateway_id + -> asset_id + -> install_id + -> runtime_instance_id + -> invocation_id + -> tool_call_id +``` + +### Definitions + +- `asset_id` + - logical registered asset in aX, such as `@hermes-bot` +- `gateway_id` + - registered Gateway/device/controller +- `install_id` + - specific local install/binding of an asset to a Gateway +- `runtime_instance_id` + - concrete running process/session/container +- `invocation_id` + - one unit of work +- `tool_call_id` + - one capability/tool action within an invocation + +### Identity invariants + +- Same `asset_id`, same `gateway_id`, same `install_id`, new + `runtime_instance_id` is a normal restart. +- Same `asset_id`, same `gateway_id`, different path/hash/launch spec is + drift. +- Same `asset_id`, different `gateway_id` is a new device/server binding and + requires approval or policy allowance. +- Unknown `gateway_id` cannot claim a known `asset_id`. +- A runtime cannot claim a different `asset_id` than the one bound to its + install. + +## Registry Objects + +### `AssetDescriptor` + +Defines what the asset is. This spec extends the taxonomy layer by making the +descriptor a canonical registry object in aX. + +### `AssetBinding` + +Defines where and how an asset is installed or attached. + +Example: + +```json +{ + "asset_id": "asset_hermes", + "gateway_id": "gw_jacob_macbook", + "install_id": "inst_456", + "binding_type": "local_runtime", + "path": "/Users/jacob/hermes-agent", + "launch_spec": { + "runtime_type": "exec", + "command": "python3 examples/hermes_sentinel/hermes_bridge.py", + "workdir": "/Users/jacob/claude_home/ax-cli" + }, + "created_by": "user_123", + "created_via": "web_ui", + "created_from": "ax_template", + "approved_state": "approved", + "first_seen_at": "2026-04-22T18:00:00Z", + "last_verified_at": "2026-04-22T18:15:00Z" +} +``` + +### `RuntimeAttestation` + +Records what Gateway observed about a concrete runtime instance. + +```json +{ + "runtime_instance_id": "rt_789", + "asset_id": "asset_hermes", + "gateway_id": "gw_jacob_macbook", + "install_id": "inst_456", + "host_fingerprint": "host_sha256:...", + "path": "/Users/jacob/hermes-agent", + "launch_spec_hash": "sha256:...", + "executable_hash": "sha256:...", + "repo_remote": "git@github.com:...", + "repo_commit": "abc123", + "working_tree_dirty": false, + "environment_profile_hash": "sha256:...", + "capability_manifest_hash": "sha256:...", + "attestation_state": "verified", + "observed_at": "2026-04-22T18:20:00Z" +} +``` + +### `CapabilityManifest` + +Declares broad things the asset can do. + +### `ToolManifest` + +Declares callable actions exposed by or granted to the asset. + +### `Grant` + +Represents a permission allowing an asset/runtime/invocation to use a +capability, tool, context, or secret under conditions. + +### `Policy` + +Represents a decision rule: + +- `allow` +- `deny` +- `require_approval` +- `allow_with_limits` +- `allow_once` + +### `SecretRef` + +Represents a secret without exposing the secret value. + +### `ApprovalRequest` + +Represents a first-class human approval decision. + +### `AuditEvent` + +Captures what happened, who or what requested it, and which policy/grant or +approval allowed or blocked it. + +## Provenance and Creation + +Every asset should have provenance metadata. + +Example: + +```json +{ + "asset_id": "asset_hermes", + "created_by": "user_123", + "created_from": "ax_template", + "created_via": "web_ui", + "template_id": "hermes", + "gateway_id": "gw_jacob_macbook", + "install_id": "inst_456", + "source": { + "kind": "local_repo", + "path": "/Users/jacob/hermes-agent", + "repo_remote": "git@github.com:example/hermes-agent.git", + "commit": "abc123", + "branch": "staging", + "launch_spec_hash": "sha256:..." + }, + "first_seen_at": "2026-04-22T18:00:00Z", + "last_verified_at": "2026-04-22T18:15:00Z" +} +``` + +### `created_from` + +Must support at least: + +- `ax_template` +- `custom_bridge` +- `imported_local` +- `agent_created` +- `api_created` +- `schedule_created` +- `external_integration` +- `mcp_server` + +### `created_via` + +Must support at least: + +- `web_ui` +- `desktop_client` +- `cli` +- `gateway_discovery` +- `agent_assistant` +- `api` + +## Runtime Attestation and Drift Detection + +Gateway must compare what is running against what was approved. + +### Minimum attestation inputs + +- `gateway_id` +- host/device fingerprint +- `asset_id` +- `install_id` +- canonical path +- launch command hash +- executable hash +- repo remote +- repo commit +- working tree dirty flag +- container image digest, when applicable +- environment profile hash +- declared tool manifest hash +- declared capability manifest hash + +### `attestation_state` + +- `verified` +- `drifted` +- `unknown` +- `blocked` + +### Attestation semantics + +- `verified` + - runtime matches the approved binding +- `drifted` + - same asset and gateway, but path/hash/manifest changed +- `unknown` + - runtime claims an asset but has no approved install binding +- `blocked` + - runtime violates policy or appears from an unapproved gateway/path + +### User-facing copy + +- `Verified local install` +- `Changed since approval` +- `New machine requesting access` +- `Unknown runtime blocked` + +## Capabilities and Tools + +Capabilities and tools must be separated. + +- **capability** + - broad thing the asset can do +- **tool** + - callable action exposed by or granted to the asset +- **grant** + - permission allowing a capability/tool under conditions +- **policy** + - rule deciding whether the request is allowed, denied, or needs approval + +### Example capability declaration + +```json +{ + "capabilities": [ + { + "id": "read_repo", + "risk": "low", + "resources": ["repo:/Users/jacob/hermes-agent"], + "requires_approval": false + }, + { + "id": "run_shell", + "risk": "high", + "resources": ["host:local"], + "requires_approval": true + }, + { + "id": "send_email", + "risk": "high", + "resources": ["gmail:jacob"], + "requires_approval": true + } + ] +} +``` + +### Example tool inventory + +- `read_file` +- `search_files` +- `write_file` +- `run_command` +- `open_browser` +- `send_message` +- `create_task` +- `update_task` +- `read_secret` +- `call_mcp_tool` +- `send_email` +- `post_summary` + +Important invariant: + +**A tool is not a permission by itself. A tool must be backed by a grant.** + +## Grants and Policy + +### Policy evaluation model + +Every request should evaluate: + +- `subject` + - user, asset, runtime instance, gateway +- `action` + - invoke, claim, read_context, use_tool, read_secret, write_file, post_message +- `resource` + - workspace, thread, repo, file path, secret, task, external integration +- `conditions` + - gateway, install, attestation, asset class, risk, time window, branch/path, + approval state, spend/runtime limits, human presence, and more + +### Example policy + +```json +{ + "policy_id": "pol_shell_high_risk", + "effect": "require_approval", + "subject": {"asset_id": "asset_hermes"}, + "action": "tool.run_shell", + "resource": "host:local", + "conditions": { + "attestation_state": "verified", + "gateway_id": "gw_jacob_macbook", + "risk": "high" + } +} +``` + +### Toggle rule + +Do not model governance only as booleans. UI toggles are acceptable, but they +must compile down to explicit policy or grant decisions. + +Examples of user-facing toggles: + +- `Allow this once` +- `Always allow for this asset` +- `Allow only on this Gateway` +- `Require approval for shell commands` +- `Block external sends` + +These must be backed by persistent policy/grant objects rather than opaque UI +state. + +## Vault and Secret Materialization + +The user-facing model should remain simple: + +- **aX Vault** + - stores secret references, grants, metadata, policy, and audit +- **Gateway Vault** + - local encrypted cache/materializer for secrets approved for a specific + Gateway +- **Runtime** + - receives only narrow ephemeral material or Gateway-mediated access, not + broad vault access + +### Example secret grant + +```json +{ + "secret_ref": "vault://openai/project-key", + "granted_to": "asset_hermes", + "gateway_id": "gw_jacob_macbook", + "scope": ["read"], + "conditions": { + "install_id": "inst_456", + "attestation_state": "verified", + "allowed_tools": ["llm_call"], + "expires_at": "2026-04-22T20:00:00Z" + } +} +``` + +### Secret invariants + +- Runtime should not receive long-lived broad secrets by default. +- Secret access should prefer: + - short-lived local capability token, or + - Gateway-mediated access. +- Revoked grants must stop future materialization immediately. + +## Human Approval Model + +Approval must be policy-driven, not hardcoded per runtime or tool. + +### Typical approval triggers + +- new asset +- new gateway/device +- runtime drift +- first use of a capability +- first use of a secret +- shell command execution +- writing outside approved path +- sending external messages +- spending money or API quota +- accessing sensitive context +- destructive action +- production environment access + +### Example approval object + +```json +{ + "approval_id": "appr_123", + "requested_by": "asset_hermes", + "gateway_id": "gw_jacob_macbook", + "action": "tool.run_shell", + "resource": "repo:/Users/jacob/hermes-agent", + "risk": "high", + "reason": "Command writes files", + "requested_at": "2026-04-22T18:30:00Z", + "status": "pending", + "decision": null, + "expires_at": "2026-04-22T19:00:00Z" +} +``` + +### Approval decisions + +- `approve_once` +- `approve_for_session` +- `approve_for_asset` +- `approve_for_gateway` +- `approve_for_workspace` +- `deny_once` +- `deny_always` + +## Context Access Model + +The registry must support context permissions alongside tool and secret grants. + +Context may include: + +- threads and conversations +- documents and repo trees +- task boards +- project notes and memories +- external system mirrors + +Grants should define whether an asset may: + +- read context +- write context +- summarize or transform context +- cross-post context into another workspace or thread + +## Client Creation Flow + +Users should not feel like they are operating two disconnected applications. + +### Target flow + +1. User starts in aX and chooses `Create Agent / Asset`. +2. aX asks: + - what type is it? + - what should it do? + - where should it run? + - what tools, context, and secrets does it need? +3. aX creates: + - `AssetDescriptor` + - initial taxonomy + - draft policies and grants + - Gateway install or attach plan +4. Gateway receives: + - local binding instructions + - doctor checks + - approval requirements +5. User confirms local requirements only when needed: + - local path + - repo checkout + - model/runtime + - secret materialization +6. Gateway reports: + - `verified`, `drifted`, or `blocked` + - current `Mode + Presence + Reply + Confidence` + +### Agent-assisted creation + +Agent-assisted creation is allowed. + +Agent self-registration without registry approval is not. + +An assistant or agent may generate a proposed asset definition, install plan, +policy draft, and grant request. The user or workspace policy must still +approve it in aX/Gateway before it becomes active. + +## Audit Events + +Every governance-relevant decision should emit an audit event with evidence. + +Examples: + +- asset created +- asset bound to gateway +- runtime attested +- attestation drift detected +- grant issued +- grant revoked +- secret materialized +- approval requested +- approval granted/denied +- policy evaluated +- invocation blocked +- tool call allowed/blocked + +Each audit event should include: + +- actor/subject +- target resource +- decision +- policy/grant/approval identifiers +- gateway/runtime evidence +- timestamp + +## UI Surfaces + +### aX asset page + +Should show: + +- asset identity +- taxonomy +- provenance +- current Gateway bindings +- capabilities and tools +- grants and policies +- secret references +- approval history +- audit trail +- current connectivity snapshot + +### Gateway local UI + +Should show: + +- local binding health +- attestation state +- drift reason +- doctor results +- local approvals waiting +- current secret/materialization state +- local capability token scope + +Every action exposed in the Gateway UI must also be available through a stable +CLI and/or local API so managed agents can operate the same control plane under +policy, not through UI-only affordances. + +### Invocation approval modal + +Should clearly answer: + +- what is asking? +- from which Gateway/device/path? +- what action/resource is requested? +- why is approval required? +- what scope does approval cover? +- when does it expire? + +## Acceptance Tests + +Minimum tests for this governance model: + +- same asset starts from approved path on approved Gateway -> allowed +- same asset starts from different path -> drifted, approval required for + sensitive capabilities +- same asset starts from different Gateway -> blocked or approval required +- runtime claims wrong `asset_id` -> blocked and audited +- runtime requests tool not in manifest -> blocked +- runtime requests capability without grant -> blocked or approval required +- runtime requests secret with valid grant -> Gateway materializes scoped access +- runtime requests secret after grant revoked -> blocked +- high-risk shell command -> approval required +- low-risk read-only context access with valid grant -> allowed +- agent-assisted creation -> draft asset and policy request, not active runtime + power +- Gateway offline from aX -> cached policy enforced only within allowed offline + window +- approval denied -> invocation blocked with structured reason +- policy changed in aX -> Gateway receives update and revokes local access + +## Roadmap + +### v1 + +- Make aX the canonical asset registry. +- Make Gateway enforce asset binding, grants, and local approval decisions. +- Keep PAT bootstrap acceptable, but never expose PATs or broad secrets to + runtimes. +- Introduce provenance, attestation, grants, secret refs, and approval objects + in the registry model. + +### Later + +- richer org/workspace policy templates +- stronger local attestation proof formats +- broader vault providers +- deeper MCP/service proxy governance +- multi-Gateway asset migration and failover policy + +## Key Product Rule + +Connected assets are not trusted because they exist. + +They are trusted because they are: + +- registered, +- bound, +- attested, +- granted, +- observed, +- auditable, and +- revocable. + +That is the governance layer that lets Gateway and aX stay flexible without +becoming unpredictable. diff --git a/specs/GATEWAY-ASSET-TAXONOMY-001/spec.md b/specs/GATEWAY-ASSET-TAXONOMY-001/spec.md new file mode 100644 index 0000000..82ce2d9 --- /dev/null +++ b/specs/GATEWAY-ASSET-TAXONOMY-001/spec.md @@ -0,0 +1,586 @@ +# GATEWAY-ASSET-TAXONOMY-001: Gateway Asset Taxonomy and Flow Semantics + +**Status:** Draft +**Owner:** @madtank +**Date:** 2026-04-22 +**Related:** GATEWAY-CONNECTIVITY-001, AGENT-CONTACT-001, LISTENER-001, AX-SCHEDULE-001, ATTACHMENT-FLOW-001 + +## Purpose + +Define the taxonomy for **connected runtime assets** managed by Gateway. + +Gateway manages more than interactive chat agents. A connected asset may be: + +- an interactive agent, +- a background worker, +- a scheduled job, +- an alert listener, or +- a service/tool proxy. + +This spec answers a different question than +[GATEWAY-CONNECTIVITY-001](../GATEWAY-CONNECTIVITY-001/spec.md): + +- **Asset taxonomy** explains what kind of thing this is and how work flows + through it. +- **Gateway connectivity** explains whether the current path is safe, live, + stale, queued, blocked, or expected to reply. + +The two specs are complementary and must not be collapsed into one overloaded +status model. + +## Goals + +- Give the product a stable language for connected assets that are not all + "agents" in the same sense. +- Make intake, trigger, return, and observability semantics explicit. +- Keep the user mental model simple enough that aX can explain whether an + asset listens live, launches on send, drains a queue, runs on a schedule, or + waits for external alerts. +- Provide a canonical mapping from taxonomy fields into the connectivity model + defined by GATEWAY-CONNECTIVITY-001. +- Support current starter templates and future runtime classes without forcing + them all into the word `agent`. + +## Non-goals + +- Replacing the canonical status model from GATEWAY-CONNECTIVITY-001. +- Introducing new v1 primary status chips beyond `Mode`, `Presence`, `Reply`, + and `Confidence`. +- Forcing every asset into a live-listener model. +- Finalizing the future v2 user-facing `SCHEDULED` or `EVENT` mode values. +- Defining exact transport payloads for alerts or schedules. Those belong in + source-specific specs. + +## Relationship to GATEWAY-CONNECTIVITY-001 + +This taxonomy spec sits one layer above the connectivity contract. + +### Taxonomy tells the user and operator: + +- what kind of connected thing this is, +- how work enters it, +- what wakes it up, +- where results go, and +- how much Gateway can observe while it works. + +### Connectivity tells the user and operator: + +- whether Gateway can safely route work to it right now, +- whether the asset is live, stale, offline, blocked, or queued, +- what kind of outcome to expect, and +- how much operational confidence the sender should have right now. + +### Important invariant + +**Do not infer asset class from live status.** + +Examples: + +- A stale live listener is still a live-listener asset. +- An inbox-backed worker with no queued jobs is still an inbox worker. +- A scheduled job between runs is not offline just because it is idle. +- An alert listener with no recent alerts is not disconnected by default. +- A service proxy may be healthy even if it never emits replies. + +## Definitions + +- **Asset**: anything Gateway can register, supervise, invoke, queue work for, + or expose through a predictable local contract. +- **Asset taxonomy**: the stable identity and flow semantics of an asset. +- **Connectivity state**: the current health and trust state of the path + between Gateway, the asset, and aX. +- **Intake model**: how work enters the asset from Gateway. +- **Trigger source**: what kind of external or internal event caused an + invocation. +- **Return path**: where a user-visible or operator-visible outcome lands. +- **Telemetry shape**: how much runtime activity Gateway can observe. + +## Canonical Asset Axes + +### `asset_class` + +What kind of thing this is. + +| Value | Meaning | +| --- | --- | +| `interactive_agent` | Message-oriented runtime that is expected to do work and usually reply | +| `background_worker` | Queue-backed worker that processes jobs and may summarize later | +| `scheduled_job` | Asset that runs because of a timer, cron, or schedule | +| `alert_listener` | Asset woken by external alerts or event sources | +| `service_proxy` | Capability or tool surface that may not reply like a normal agent | + +### `intake_model` + +How Gateway gets work into the asset. + +| Value | Meaning | +| --- | --- | +| `live_listener` | Asset is already listening and can claim work now | +| `launch_on_send` | Gateway launches or invokes the asset when work arrives | +| `queue_accept` | Gateway can durably accept work for later handling | +| `queue_drain` | Worker process drains already-queued work | +| `scheduled_run` | Gateway invokes the asset because of a schedule | +| `event_triggered` | Gateway invokes the asset because of an external event | +| `manual_only` | Asset only runs when an operator explicitly triggers it | + +### `trigger_source` + +What started a particular invocation. + +| Value | Meaning | +| --- | --- | +| `direct_message` | User or agent sent a normal message | +| `queued_job` | Work item was pulled from a queue | +| `scheduled_invocation` | Scheduler fired | +| `external_alert` | External system event or alert fired | +| `manual_trigger` | Human or operator manually started it | +| `tool_call` | Another asset invoked it as a capability | + +### `return_path` + +Where the outcome is expected to go. + +| Value | Meaning | +| --- | --- | +| `inline_reply` | Normal reply in the current conversation or thread | +| `sender_inbox` | Result lands in the sender's inbox or notification stream | +| `summary_post` | Background result is summarized later | +| `task_update` | Outcome updates a task/job record rather than posting a chat reply | +| `event_log` | Outcome is recorded operationally but not posted as chat output | +| `silent` | No user-visible output is expected unless there is an error | + +### `telemetry_shape` + +How observable the asset is while it works. + +| Value | Meaning | +| --- | --- | +| `rich` | Progress, tool events, and intermediate activity are available | +| `basic` | Pickup and coarse progress are available | +| `heartbeat_only` | Gateway can prove liveness/freshness but not much activity | +| `opaque` | Gateway sees only invocation boundaries or errors | + +### Optional `worker_model` + +Queue-backed assets may also declare how queued work is later processed. + +| Value | Meaning | +| --- | --- | +| `queue_drain` | One or more workers later claim work from the durable queue | + +This field is only relevant when `intake_model=queue_accept`. + +## Queue-backed Semantics + +Queue-backed assets need two separate concepts: + +- `queue_accept` + - Gateway can durably accept work for the asset. +- `queue_drain` + - a worker process later drains queued work. + +These must not be collapsed into a single `queued` concept. + +The connectivity contract already distinguishes: + +- agent-level queue capability, and +- invocation-level queued state. + +This taxonomy spec must preserve that distinction. + +## User-facing Categories + +The product should present these simple starter categories in setup and fleet +UX: + +- `Live Listener` +- `On-Demand Agent` +- `Inbox Worker` +- `Scheduled Job` +- `Alert Listener` + +`Service / Tool Proxy` should exist as an advanced/internal category in v1. + +### Category descriptions + +#### `Live Listener` + +Already listening now. Messages can be picked up immediately. + +#### `On-Demand Agent` + +Starts or attaches when work arrives. Cold start may apply. + +#### `Inbox Worker` + +Queue-backed. Work can be accepted safely even when no live worker is attached. + +#### `Scheduled Job` + +Runs because of a timer or schedule. It is not expected to behave like a live +listener between runs. + +#### `Alert Listener` + +Runs because an external event source wakes it up. It is not expected to +receive normal direct messages in the same way as a chat agent. + +## Mapping to Gateway Connectivity Fields + +The taxonomy does not replace the connectivity model. It maps into it. + +| Taxonomy field | Connectivity implication | +| --- | --- | +| `asset_class` | template identity and UX category | +| `intake_model` | hints for `placement`, `activation`, and `mode` | +| `trigger_source` | invocation source and event story | +| `return_path` | maps into `reply_mode` and output expectations | +| `telemetry_shape` | maps into `telemetry_level` and observability guarantees | + +### Canonical mapping examples + +#### Interactive live listener + +```text +asset_class=interactive_agent +intake_model=live_listener +-> placement=hosted or attached +-> activation=persistent or attach_only +-> mode=LIVE +``` + +#### Interactive on-demand agent + +```text +asset_class=interactive_agent +intake_model=launch_on_send +-> placement=hosted or brokered +-> activation=on_demand +-> mode=ON-DEMAND +``` + +#### Inbox-backed background worker + +```text +asset_class=background_worker +intake_model=queue_accept +worker_model=queue_drain +-> placement=mailbox +-> activation=queue_worker +-> mode=INBOX +``` + +#### Scheduled job + +```text +asset_class=scheduled_job +intake_model=scheduled_run +-> trigger_source=scheduled_invocation +-> mode remains ON-DEMAND in v1 UI unless a future SCHEDULED mode is added +``` + +#### Alert listener + +```text +asset_class=alert_listener +intake_model=event_triggered +-> trigger_source=external_alert +-> mode remains ON-DEMAND in v1 UI unless a future EVENT mode is added +``` + +## Asset Descriptor Schema + +Each registered asset should have a stable descriptor beside, not inside, its +current status snapshot. + +### `AssetDescriptor` + +```json +{ + "asset_id": "asset_123", + "gateway_id": "gw_123", + "display_name": "Docs Worker", + "asset_class": "background_worker", + "intake_model": "queue_accept", + "worker_model": "queue_drain", + "trigger_sources": ["queued_job", "manual_trigger"], + "return_paths": ["summary_post", "task_update"], + "telemetry_shape": "basic", + "addressable": true, + "messageable": true, + "schedulable": false, + "externally_triggered": false, + "tags": ["queue-backed", "summary-later", "repo-bound"], + "capabilities": ["summarize", "update_task"], + "constraints": ["requires-repo"] +} +``` + +### Descriptor rules + +- `AssetDescriptor` tells the product what the asset is. +- `AgentStatusSnapshot` tells the product whether Gateway can safely route to + it right now. +- `InvocationStatusSnapshot` tells the product what is happening to one piece + of work. + +These objects must stay separate even when the UI renders them together. + +### Descriptor additions + +The following extensibility fields are part of the taxonomy layer: + +- `tags` +- `capabilities` +- `constraints` + +They explain and filter assets, but they must never replace the canonical +connectivity fields from GATEWAY-CONNECTIVITY-001. + +## Template Examples + +### Hermes + +```yaml +asset_class: interactive_agent +intake_model: live_listener +trigger_source: direct_message +return_path: inline_reply +telemetry_shape: rich +mode: LIVE +reply: REPLY +``` + +### Codex through Gateway + +```yaml +asset_class: interactive_agent +intake_model: launch_on_send +trigger_source: direct_message +return_path: inline_reply +telemetry_shape: basic +mode: ON-DEMAND +reply: REPLY +``` + +If Codex later keeps a live attached session instead of launching on send, the +taxonomy changes to `intake_model=live_listener` but it remains an +`interactive_agent`. + +### Inbox docs worker + +```yaml +asset_class: background_worker +intake_model: queue_accept +worker_model: queue_drain +trigger_source: queued_job +return_path: summary_post +telemetry_shape: basic +mode: INBOX +reply: SUMMARY +``` + +### Reminder bot + +```yaml +asset_class: scheduled_job +intake_model: scheduled_run +trigger_source: scheduled_invocation +return_path: task_update +telemetry_shape: basic +reply: SUMMARY +``` + +### PagerDuty or Datadog bridge + +```yaml +asset_class: alert_listener +intake_model: event_triggered +trigger_source: external_alert +return_path: sender_inbox +telemetry_shape: heartbeat_only +reply: SUMMARY +``` + +### Tool proxy + +```yaml +asset_class: service_proxy +intake_model: manual_only +trigger_source: tool_call +return_path: event_log +telemetry_shape: opaque +reply: SILENT +``` + +## UI and Onboarding Implications + +### Fleet view + +Fleet UX should evolve from: + +```text +Agent | Mode | Presence | Reply | Confidence +``` + +to: + +```text +Asset | Type | Mode | Presence | Output | Confidence +``` + +Example: + +```text +@hermes-bot Live Listener LIVE WORKING Reply HIGH +@ollama-bot On-Demand Agent ON-DEMAND IDLE Reply MEDIUM +@docs-worker Inbox Worker INBOX IDLE Summary HIGH +@reminder-bot Scheduled Job ON-DEMAND IDLE Task HIGH +@datadog-bridge Alert Listener ON-DEMAND IDLE Inbox HIGH +``` + +For v1, `Type` carries the richer mental model while `Mode` remains one of: + +- `LIVE` +- `ON-DEMAND` +- `INBOX` + +### Connect wizard + +Setup should lead with asset categories rather than low-level backends. + +The first visible cards should be: + +- `Live Listener` +- `On-Demand Agent` +- `Inbox Worker` +- `Scheduled Job` +- `Alert Listener` + +The template-specific form can then specialize: + +- Hermes under `Live Listener` +- Ollama under `On-Demand Agent` +- Inbox docs worker under `Inbox Worker` +- Reminder bot under `Scheduled Job` +- PagerDuty bridge under `Alert Listener` + +### Composer expectations + +Pre-send UX should combine the taxonomy layer with the connectivity layer. + +Example: + +```text +@docs-worker +Inbox Worker · INBOX · IDLE · SUMMARY · HIGH +Queue-backed. Work can be accepted now and summarized later. +``` + +### Custom Bridge + +Custom Bridge should become the advanced escape hatch for arbitrary local +assets. In addition to the connectivity contract, setup should ask: + +- what class of asset this is, +- how work enters it, +- what triggers it, +- where results go, and +- how much telemetry Gateway should expect. + +## Gateway Doctor Requirements by Asset Class + +Gateway Doctor must become asset-class aware. + +### Live Listener + +- identity +- Gateway auth +- runtime launch or attach +- heartbeat +- test claim +- inline reply + +### On-Demand Agent + +- identity +- launch preflight +- cold-start test +- test claim +- return path validation + +### Inbox Worker + +- queue writable +- worker config valid +- optional worker attached +- test job queued +- summary viability + +### Scheduled Job + +- schedule registered +- next run computed +- dry run succeeds +- return path valid + +### Alert Listener + +- webhook or event source configured +- signing secret valid +- test event accepted +- task or inbox return path valid + +### Service Proxy + +- capability exposed +- auth boundary valid +- test tool call succeeds +- output path valid + +## Acceptance Tests + +Minimum acceptance tests for this taxonomy: + +- Hermes renders as `Live Listener` and maps to `LIVE + REPLY + rich`. +- Ollama renders as `On-Demand Agent` and maps to `ON-DEMAND + REPLY`. +- Inbox docs worker renders as `Inbox Worker` even with `queue_depth=0`. +- Inbox docs worker with `queue_depth>0` changes presence to `QUEUED` but not + asset class. +- Scheduled job with `next_run_at` in the future is healthy and not considered + offline just because it is waiting. +- Scheduled job dry-run failure surfaces as blocked or error according to the + connectivity contract. +- Alert listener with valid webhook config but no recent alerts is healthy. +- Alert listener failed signature validation surfaces as a security/setup + problem, not as a missing reply. +- Service proxy can be `SILENT` and still healthy. +- Changing `telemetry_shape` from `rich` to `heartbeat_only` changes + observability copy, not asset class. + +## Roadmap + +### v1 + +- Keep taxonomy as a descriptor layer beside the connectivity model. +- Use `Type` in setup and fleet UX before introducing new primary mode values. +- Keep `Mode` user-facing values limited to `LIVE`, `ON-DEMAND`, and `INBOX`. + +### Later + +- Add explicit user-facing `SCHEDULED` and `EVENT` modes if those asset classes + become common enough to deserve their own top-level chips. +- Add source-specific specs for alert/webhook contracts. +- Add richer service/tool proxy and MCP-facing asset classes once Gateway MCP + mode is further along. + +## Key Product Rule + +Gateway should make this promise: + +> I can connect many kinds of runtime assets, and I will tell you honestly what +> kind of connection this is, how work gets in, what wakes it up, where results +> go, and how observable it is. + +That honesty is the point of the taxonomy layer. It lets aX stay trustworthy +even when not every asset is a live chat agent. diff --git a/specs/GATEWAY-CONNECTIVITY-001/mockups.md b/specs/GATEWAY-CONNECTIVITY-001/mockups.md new file mode 100644 index 0000000..9b731e8 --- /dev/null +++ b/specs/GATEWAY-CONNECTIVITY-001/mockups.md @@ -0,0 +1,299 @@ +# GATEWAY-CONNECTIVITY-001 Mockups + +These wireframes evolve the existing Gateway presentation into a v1 operator +surface that uses **Mode + Presence + Reply + Confidence** instead of +`running`. + +## 1. Connect Agent Wizard + +### Step 1 — Pick a template + +```text ++--------------------------------------------------------------------------+ +| Add Managed Agent | +| Connect this agent through your local Gateway | ++--------------------------------------------------------------------------+ +| [ Echo (Test) ] LIVE-ready test bot | +| Replies inline · Basic activity · Best first smoke test | +| | +| [ Hermes ] LIVE local agent | +| Replies inline · Rich activity · Tool telemetry | +| | +| [ Claude Code Channel ] | +| LIVE attached session · Replies inline · Basic activity | +| | +| [ Ollama ] ON-DEMAND local model | +| Replies inline · Basic activity · Launches when needed | +| | +| [ Advanced ] Inbox / Background Worker · Custom bridge | ++--------------------------------------------------------------------------+ +``` + +### Step 2 — Expectations before connect + +```text ++--------------------------------------------------------------------------+ +| Hermes | +| LIVE · IDLE · REPLY · HIGH | +| Reachability: live_now | +| You can expect an inline reply. | +| | +| What you need | +| - local hermes-agent checkout | +| - provider auth/model credentials | +| | +| Guaranteed signals | +| - Received by Gateway | +| - Claimed by runtime | +| - Working | +| - Completed / Error | +| | +| Optional signals | +| - tool call / tool result | +| - richer progress | +| | +| Safe by default | +| Gateway keeps your aX credential. This runtime receives only a local | +| scoped capability. It cannot impersonate you or mint other agents. | +| | +| [ Send test ] [ Run doctor ] [ Connect agent ] | ++--------------------------------------------------------------------------+ +``` + +## 2. Fleet View + +```text ++----------------------------------------------------------------------------------------------------------------------+ +| Gateway Overview | +| Connected to dev.paxai.app · Gateway healthy · Alerts: 1 | ++----------------------------------------------------------------------------------------------------------------------+ +| Agent | Mode | Presence | Reply | Confidence | Queue | Typical Claim | First Activity | Last Out | +|--------------------+-----------+----------+---------+------------+-------+---------------+----------------+----------| +| @hermes-bot | LIVE | WORKING | REPLY | HIGH | 0 | 1.2s p50 | 1.8s p50 | tool | +| @cc-channel | LIVE | STALE | REPLY | LOW | 0 | 2.5s p50 | 8.0s p50 | reply | +| @ollama-bot | ON-DEMAND | IDLE | REPLY | MEDIUM | 0 | 4.8s p50 | 6.2s p50 | reply | +| @docs-worker | INBOX | IDLE | SUMMARY | HIGH | 0 | 12s p50 | 45s p50 | summary | +| @broken-hermes | LIVE | ERROR | REPLY | BLOCKED | 0 | - | - | setup | ++----------------------------------------------------------------------------------------------------------------------+ +| Alerts | +| - @broken-hermes setup error: hermes-agent repo missing | +| - @cc-channel needs reconnect before pickup confidence returns | ++----------------------------------------------------------------------------------------------------------------------+ +``` + +### Fleet row semantics + +- `Mode` tells the operator how the agent connects or is invoked. +- `Presence` tells the operator whether it is healthy, queued, actively + working, stale, or blocked. +- `Reply` tells the sender what kind of outcome to expect. +- `Confidence` summarizes how likely the current path is to work right now. +- `Typical Completion` is rendered as `Typical Summary Time` for summary-only + or background agents. + +## 3. Agent Drill-In + +```text ++-----------------------------------------------------------------------------------------------------+ +| @hermes-bot | +| LIVE · WORKING · REPLY · HIGH | ++-----------------------------------------------------------------------------------------------------+ +| Placement: hosted Activation: persistent Telemetry: rich | +| Liveness: connected Reachability: live_now Last seen: 4s ago | +| Heartbeat source: runtime Confidence reason: recent heartbeat + passing test | +| Work state: working Current invocation: inv_123 Current tool: search_files | ++-----------------------------------------------------------------------------------------------------+ +| Typical Claim: 1.2s p50 / 2.6s p95 Typical First Activity: 1.8s / 3.9s | +| Typical Completion: 24s p50 / 63s p95 Completion rate: 96% | ++-----------------------------------------------------------------------------------------------------+ +| Recent Lifecycle | +| 19:15:30 message_received msg_123 | +| 19:15:31 message_claimed inv_123 | +| 19:15:32 working "Preparing runtime" | +| 19:15:34 tool_call search_files | +| 19:15:37 tool_result success | +| 19:15:55 completed reply_sent | ++-----------------------------------------------------------------------------------------------------+ +| Setup | +| - hermes-agent checkout: /Users/jacob/hermes-agent | +| - auth present: yes | +| - launch path healthy: yes | +| Tags: local · hosted-by-gateway · rich-telemetry · repo-bound | +| Capabilities: reply · progress · tool_events · read_files · bash_tools | +| Constraints: requires-repo · requires-provider-auth | +| [ Send test ] [ Run doctor ] [ Restart ] [ Pause ] | ++-----------------------------------------------------------------------------------------------------+ +``` + +## 4. Gateway Doctor + +```text ++--------------------------------------------------------------------------------------+ +| ax gateway doctor @hermes-bot | ++--------------------------------------------------------------------------------------+ +| ✓ Gateway connected to dev.paxai.app | +| ✓ Agent identity minted | +| ✓ Hermes checkout found | +| ✓ Runtime starts | +| ✓ Heartbeat received | +| ✓ Test message claimed | +| ✓ Inline reply received | +| | +| Status: LIVE · IDLE · REPLY · HIGH | +| Reachability: live_now | ++--------------------------------------------------------------------------------------+ +``` + +Inbox-backed example: + +```text ++--------------------------------------------------------------------------------------+ +| ax gateway doctor @docs-worker | ++--------------------------------------------------------------------------------------+ +| ✓ Gateway connected | +| ✓ Inbox queue writable | +| ✓ Worker config valid | +| ! No worker currently attached | +| ✓ Test job queued | +| | +| Status: INBOX · IDLE · SUMMARY · HIGH | +| Reachability: queue_available | +| Expectation: Work can be queued now. Summary will post when a worker drains inbox. | ++--------------------------------------------------------------------------------------+ +``` + +## 5. Sender Composer Expectations + +```text ++--------------------------------------------------------------------------------------+ +| To: @ollama-bot | +| ON-DEMAND · IDLE · REPLY · MEDIUM | +| Reachability: launch_available | +| Typical Claim 4.8s · Typical Completion 18s | +| Note: cold start possible | ++--------------------------------------------------------------------------------------+ +| Message | +| "Summarize the docs in this folder." | ++--------------------------------------------------------------------------------------+ +``` + +## 6. Post-Send Activity Bubble + +### Interactive agent + +```text +To @hermes-bot · Claimed by runtime + +Summarize the docs in this folder. + +status: +- Received by Gateway +- Claimed by runtime +- Using tool: search_files +``` + +Later: + +```text +To @hermes-bot · Completed + +Summarize the docs in this folder. + +status: +- Received by Gateway +- Claimed by runtime +- Working +- Completed +``` + +### Background / inbox agent + +```text +To @docs-worker · Summary pending + +Check these docs and file the findings. + +status: +- Received by Gateway +- Queued in inbox +- Claimed by worker +- Summary pending +``` + +Later, a new summary lands at the bottom of the transcript: + +```text +@docs-worker + +Summary of document check: +- 2 broken links +- 1 outdated section +- 0 missing files +``` + +## 7. Custom Bridge Setup + +```text ++--------------------------------------------------------------------------------------+ +| Custom Bridge | +| For agents you control locally | ++--------------------------------------------------------------------------------------+ +| Reply behavior | +| [x] Replies inline [ ] Summary later [ ] Silent completion | +| | +| Activation | +| [x] Gateway launches command | +| [ ] Gateway invokes on demand | +| [ ] Agent drains inbox | +| [ ] Gateway attaches to existing session | +| | +| Telemetry | +| [x] Heartbeat only [x] Progress events [x] Tool events [x] Completion | +| | +| Generated assets | +| - command template | +| - expected env vars | +| - AX_GATEWAY_EVENT examples | +| - local capability token | +| - smoke test / doctor checks | ++--------------------------------------------------------------------------------------+ +``` + +## 8. Topology Reference + +### v1 hybrid topology + +```mermaid +flowchart LR + U["User / CLI / Web UI"] --> G["Local Gateway"] + G --> M["Gateway registry + local API"] + G <-->|per-agent upstream listeners + API| AX["aX platform"] + G --> H["Hermes runtime"] + G --> C["Claude Code Channel session"] + G --> O["Ollama bridge"] + G --> I["Inbox queue / worker"] +``` + +### later single-pipe direction + +```mermaid +flowchart LR + U["User / CLI / Web UI"] --> G["Local Gateway"] + G <-->|single control stream| AX["aX platform"] + G --> H["Hosted runtimes"] + G --> A["Attached sessions"] + G --> B["Brokered invocations"] + G --> I["Inbox / queue workers"] + G --> MCP["MCP Gateway mode"] +``` + +## 9. Design Notes + +- Never show `RUNNING` as the primary status chip. +- Use `Mode + Presence + Reply + Confidence` consistently in both CLI and local + UI. +- Keep `placement` and `activation` available in drill-in because operators will + need that truth once live and on-demand agents coexist. +- Treat inbox-backed agents as queue-capable, not inherently queued. +- Treat `Typical Claim`, `Typical First Activity`, and `Typical Completion` as + first-class UX elements, not hidden metrics. diff --git a/specs/GATEWAY-CONNECTIVITY-001/spec.md b/specs/GATEWAY-CONNECTIVITY-001/spec.md new file mode 100644 index 0000000..ff017bd --- /dev/null +++ b/specs/GATEWAY-CONNECTIVITY-001/spec.md @@ -0,0 +1,1050 @@ +# GATEWAY-CONNECTIVITY-001: Gateway Connectivity, Signal Model, and Sender Confidence + +**Status:** Draft +**Owner:** @madtank +**Date:** 2026-04-22 +**Related:** LISTENER-001, AGENT-CONTACT-001, MESH-SPAWN-001, [docs/mcp-remote-oauth.md](../../docs/mcp-remote-oauth.md) +**Companion Mockups:** [mockups.md](mockups.md) + +## Purpose + +Define the v1 contract for the local Gateway as the execution and control plane +between aX and managed runtimes. The goal is to make agent connectivity, +pickup, progress, and reply expectations explicit enough that: + +1. senders trust that a message really reached the Gateway, +2. operators can tell whether an agent is live, on-demand, inbox-backed, or in + error, +3. aX can render durable sender confidence without depending on transient SSE + alone, and +4. later OAuth, MCP Gateway mode, and single-pipe Gateway ownership can replace + transport details without renaming the core model. + +The product must never use `running` as the primary status. User-facing state +is always expressed as **Mode + Presence + Reply behavior + Confidence**, +derived from a more precise internal model. + +## Goals + +- Make the Gateway the local source of truth for delivery, queueing, claim, + progress, and completion semantics. +- Distinguish persistent live agents from on-demand, inbox, and attached + agents. +- Preserve rich telemetry where adapters can provide it, while still making + sparse runtimes trustworthy. +- Persist both **latest status snapshots** and **recent event timelines** in aX + for fleet views, drill-ins, and sender activity bubbles. +- Keep user PAT bootstrap acceptable for v1 while making OAuth a login-provider + swap rather than a runtime-contract rewrite. +- Make it almost impossible for a user to misunderstand whether an agent is + listening live, queue-backed, on-demand, stale, or blocked. + +## Non-goals + +- Exact-once delivery. +- Direct child-runtime authentication to aX. +- Skill Gateway in v1. +- Cross-machine HA Gateway. +- Arbitrary third-party template marketplace. +- Hard service-level guarantees. +- Production OAuth requirement in v1. +- Direct shipping from `main` as an implementation constraint. + +## Product Questions Answered + +### 1. How does a user connect agents to the platform? + +Users connect agents through Gateway-owned templates, not raw runtime backends. +The first visible starter kit is: + +- `Echo (Test)` +- `Hermes` +- `Claude Code Channel` +- `Ollama` + +`Inbox / Background Worker` is fully specified but advanced in v1. The product +should also name this clearly as an **inbox-backed agent** pattern so it is not +mistaken for a broken or disconnected live agent. + +The user picks a template, sees its reply behavior and expected signals, runs a +Gateway-authored smoke test, and then sees the resulting mode, presence, +typical timing, and last outcome in the fleet view. + +### 2. How does Gateway know whether an agent is really reachable? + +Gateway tracks six immutable internal dimensions: + +- `placement` +- `activation` +- `liveness` +- `work_state` +- `reply_mode` +- `telemetry_level` + +The operator never sees `running` directly. The operator sees: + +- `mode` +- `presence` +- `reply` +- `confidence` + +And the UI may also surface: + +- `reachability` + +Gateway is an **agent-operable control plane**. The UI is a human-readable +surface over that control plane, but lifecycle, doctor, send, and approval +actions must also be available through stable CLI and local API paths so +agents can operate Gateway without UI-only dependencies. + +### 3. How does aX learn what is happening? + +Gateway emits append-only lifecycle events to aX and also updates two derived +snapshots: + +- `AgentStatusSnapshot` keyed by `agent_id` for fleet and drill-in views +- `InvocationStatusSnapshot` keyed by `invocation_id` and indexed by + `message_id + agent_id` for message bubbles and recent activity + +The timeline is the history. The snapshot is the latest truth. + +### 4. How does the sender know what to expect? + +Expectation is shown both **before** sending and **after** sending: + +- Pre-send: Mode, Presence, Reply behavior, telemetry richness, and current + confidence. +- Post-send: compact activity bubble phases such as `Received by Gateway`, + `Claimed by runtime`, `Working`, `Summary pending`, or `No reply expected`. + +## Topology and Upstream Model + +### v1 Hybrid Upstream Model + +v1 keeps the current hybrid pattern: + +- Gateway stores one bootstrap user credential for management/login. +- Gateway mints and stores per-agent Gateway-managed credentials. +- Each managed agent may still hold its own upstream listen/send relationship to + aX through Gateway's supervision. +- Child runtimes do not receive user PATs or raw platform JWTs. + +This is acceptable for v1 because the product semantics are normalized through +Gateway even if the upstream transport is still per-agent. + +### Later Single-Pipe Direction + +The protocol defined here must also support a later model where: + +- aX sees the Gateway as the authoritative connected object, +- Gateway owns the single upstream control relationship, +- all managed agents exist behind Gateway, and +- child runtimes never authenticate directly to aX. + +The lifecycle event names and status model in this spec must remain valid under +both topologies. + +## Internal Model + +### Internal Fields + +| Field | Values | Meaning | +| --- | --- | --- | +| `placement` | `hosted`, `attached`, `brokered`, `mailbox` | Where the runtime actually lives relative to Gateway | +| `activation` | `persistent`, `on_demand`, `attach_only`, `queue_worker` | How the runtime becomes active | +| `liveness` | `connected`, `stale`, `offline`, `setup_error` | Health of the active execution path | +| `work_state` | `idle`, `queued`, `working`, `blocked` | Current work ownership | +| `reply_mode` | `interactive`, `background`, `summary_only`, `silent` | Expected outcome behavior | +| `telemetry_level` | `rich`, `basic`, `silent` | Signal richness the adapter can provide | + +### Internal Semantics + +- `placement=hosted` means Gateway owns the runtime process or launch path. +- `placement=attached` means the runtime lives elsewhere, but Gateway has a + live session/claim path to it. +- `placement=brokered` means Gateway invokes an external client or service on + demand and no persistent listener is expected. +- `placement=mailbox` means Gateway accepts durable work but there may be no + live runtime attached. + +- `activation=persistent` means the runtime is expected to stay live. +- `activation=on_demand` means Gateway launches or invokes it per task. +- `activation=attach_only` means Gateway supervises an already-existing session + but does not own the lifecycle. +- `activation=queue_worker` means work is first queued durably and later + claimed by a worker. + +## Derived Operator Model + +### Operator Fields + +| Field | Values | Meaning | +| --- | --- | --- | +| `mode` | `LIVE`, `ON-DEMAND`, `INBOX` | What kind of connectivity the user should assume | +| `presence` | `IDLE`, `QUEUED`, `WORKING`, `BLOCKED`, `STALE`, `OFFLINE`, `ERROR` | Current operational truth | +| `reply` | `REPLY`, `SUMMARY`, `SILENT` | What sort of result the sender should expect | +| `confidence` | `HIGH`, `MEDIUM`, `LOW`, `BLOCKED` | How safe it is to send work through this path right now | + +`BROKERED` remains an internal placement detail in v1 and maps to +`mode=ON-DEMAND` for user-facing UX. + +### Reachability Helper + +In addition to the primary operator fields, the product should derive a +human-readable `reachability` helper for wizard copy, composer expectations, +and drill-ins: + +- `live_now` +- `queue_available` +- `launch_available` +- `attach_required` +- `unavailable` + +This is explanatory text, not a primary fleet chip. + +### Deterministic Derivation Rules + +Implementations must use the same precedence rules everywhere. + +#### Mode derivation + +```text +if placement == mailbox: + mode = INBOX +else if activation in {persistent, attach_only}: + mode = LIVE +else if activation == on_demand: + mode = ON-DEMAND +else if placement == brokered: + mode = ON-DEMAND +else: + mode = ON-DEMAND +``` + +#### Presence derivation + +```text +if liveness == setup_error: + presence = ERROR +else if work_state == blocked: + presence = BLOCKED +else if work_state == working: + presence = WORKING +else if work_state == queued: + presence = QUEUED +else if liveness == stale: + presence = STALE +else if liveness == offline: + presence = OFFLINE +else if liveness == connected: + presence = IDLE +else: + presence = OFFLINE +``` + +#### Reply derivation + +```text +interactive -> REPLY +background or summary_only -> SUMMARY +silent -> SILENT +``` + +#### Confidence derivation + +```text +if liveness == setup_error: + confidence = BLOCKED +else if recent_test_failed or completion_rate_below_threshold: + confidence = LOW +else if liveness in {offline, stale}: + confidence = LOW +else if placement == mailbox and queue_health == healthy: + confidence = HIGH +else if activation == on_demand and launch_health == healthy: + confidence = MEDIUM +else if recent_smoke_test_passed and heartbeat_recent and queue_or_listener_healthy: + confidence = HIGH +else: + confidence = MEDIUM +``` + +`confidence` is specifically about **safe to send now through Gateway**. For +inbox-backed agents, `HIGH` confidence means Gateway can safely accept and +queue work. It does not imply that a worker is attached or that completion is +immediate. + +#### Confidence reason + +Every derived confidence value must include a machine-readable reason code and +human-readable explanation: + +- `confidence_reason`: short code such as `recent_smoke_test_passed`, + `queue_writable`, `cold_start_possible`, `session_stale`, `setup_missing_repo` +- `confidence_detail`: short operator-facing string such as `Queue writable`, + `Cold start possible`, `Reconnect local session`, `Missing Hermes checkout` + +#### Reachability derivation + +```text +if liveness == setup_error: + reachability = unavailable +else if placement == mailbox and queue_health == healthy: + reachability = queue_available +else if activation in {persistent, attach_only} and liveness == connected: + reachability = live_now +else if activation == on_demand and launch_health == healthy: + reachability = launch_available +else if activation == attach_only: + reachability = attach_required +else: + reachability = unavailable +``` + +#### Invariants + +- `ERROR` always overrides `IDLE`, `QUEUED`, or `WORKING`. +- `OFFLINE` and `STALE` are distinct user-facing states in v1. +- `QUEUED` never implies ownership by a runtime. +- `WORKING` always implies that a runtime or worker has already claimed the + invocation. +- `INBOX` describes an agent's queue-backed connectivity class, not whether a + specific invocation is already queued. + +### Queue Capability vs Queued Work + +The spec must distinguish: + +- **agent-level queue capability** + - `placement=mailbox` + - `activation=queue_worker` + - `queue_capable=true` + - `queue_depth=n` +- **invocation-level queued state** + - `message_queued` + - `work_state=queued` + +An inbox-backed agent with zero pending work should usually render as: + +- `mode=INBOX` +- `presence=IDLE` +- `reply=SUMMARY` + +An inbox-backed agent with pending work should render as: + +- `mode=INBOX` +- `presence=QUEUED` +- `reply=SUMMARY` + +The product must not imply that every inbox-backed agent is always already +queued. + +## Heartbeats, Health, and Staleness + +### Heartbeat Sources + +Gateway may derive health from any of these sources depending on template: + +- runtime heartbeat +- upstream listener heartbeat +- attached-session heartbeat +- queue worker heartbeat +- successful preflight check for on-demand runtimes +- last successful claim/completion for brokered or sparse adapters + +### v1 Timing Defaults + +- Target heartbeat interval for persistent live runtimes: **15 seconds** +- `connected`: heartbeat seen within **30 seconds** +- `stale`: no heartbeat for **>45 seconds** +- `offline`: no heartbeat for **>120 seconds**, process exit, session detach, or + repeated launch/preflight failure +- `setup_error`: dependency, config, auth, or launch validation failed before a + runtime ever became healthy + +### Special Cases + +- `Claude Code Channel` may emit sparse work telemetry and still be healthy if + pickup and completion remain reliable. +- `on_demand` runtimes such as default Ollama are considered `connected` only + if their preflight passes and the launch path is currently healthy. +- `mailbox` agents do not require a live runtime heartbeat to accept work; they + are healthy if the queue can durably accept work and no setup error blocks + drain workers. +- `mailbox` agents may have `reachability=queue_available` even when no live + worker is attached. They are not `OFFLINE` unless the queue path itself is + unavailable. + +## Lifecycle and Protocol Invariants + +### Live Interactive Lifecycle + +1. `message_received` +2. `message_claimed` +3. `working` +4. optional `progress` +5. optional `tool_call` +6. optional `tool_result` +7. `completed` or `error` + +### Inbox / Background Lifecycle + +1. `message_received` +2. `message_queued` +3. `message_claimed` +4. `working` +5. optional `progress` +6. optional `summary_pending` +7. `summary_posted` or `completed` or `error` + +### Required Meanings + +- `message_received`: Gateway accepted responsibility to evaluate the message. +- `message_queued`: the message is durably accepted into a Gateway queue or + mailbox and is safe but not yet owned by a worker. +- `message_claimed`: a specific runtime or worker accepted ownership. +- `working`: the claimant has started execution. +- `summary_pending`: background work is done or nearly done, and the sender + should expect a summary instead of an inline assistant reply. + +### Terminal States + +- `completed` +- `error` +- `cancelled` +- `expired` + +Late or duplicate events after a terminal state are ignored for snapshot +derivation but preserved in the local Gateway log as protocol anomalies. + +### Events vs Status + +- Events are append-only facts. +- Status is a derived snapshot. +- aX and Gateway UIs render the current snapshot plus recent timeline. +- No UI may infer durable state from a single transient event without snapshot + derivation. + +## Event Envelope and Delivery Semantics + +### Canonical Envelope + +Every Gateway↔aX lifecycle event must use this envelope: + +```json +{ + "schema_version": "gateway.event.v1", + "event_id": "evt_01H...", + "event_type": "message_claimed", + "gateway_id": "gw_123", + "agent_id": "agt_123", + "message_id": "msg_123", + "invocation_id": "inv_123", + "runtime_id": "rt_123", + "attempt": 1, + "sequence": 3, + "observed_at": "2026-04-22T19:15:30Z", + "emitted_at": "2026-04-22T19:15:31Z", + "payload": { + "backlog_depth": 0 + } +} +``` + +### Delivery Rules + +- Delivery semantics are **at least once**. +- `event_id` is the dedupe key. +- `invocation_id + sequence` is the ordering key. +- `attempt` increments when Gateway retries the same target message after a + failed prior attempt. +- Each retry gets a **new** `invocation_id`. +- Consumers must accept duplicate delivery and late arrival. + +### Snapshot Persistence in aX + +aX must persist: + +1. `AgentStatusSnapshot` + - keyed by `agent_id` + - used for fleet view and agent drill-in + - includes `mode`, `presence`, `reply`, `confidence`, queue capability, + queue depth, tags, capabilities, constraints, latest health details, + `confidence_reason`, `confidence_detail`, + `last_successful_doctor_at`, and `last_doctor_result` +2. `InvocationStatusSnapshot` + - keyed by `invocation_id` + - indexed by `message_id + agent_id` + - used for sender confidence bubbles and recent activity + - includes current invocation `presence`, `reply`, queue/claim timestamps, + and final outcome +3. `GatewayEventTimeline` + - append-only recent event stream keyed by `invocation_id` + +## Gateway ↔ Runtime Adapter Contract + +### Adapter Event Types + +All adapters must map to these logical events: + +- `hello` +- `heartbeat` +- `claim` +- `progress` +- `tool_call` +- `tool_result` +- `complete` +- `error` + +### Command Bridge v1 + +The v1 command-bridge protocol is line-oriented JSON. + +- One JSON object per line +- Canonical prefix: `AX_GATEWAY_EVENT=` +- Compatibility prefix accepted during migration: `AX_GATEWAY_EVENT ` +- `schema_version` required +- `stderr` is treated as logs, not protocol +- Process exit before `complete` or `error` maps to `invocation_failed` + +Canonical example: + +```text +AX_GATEWAY_EVENT={"schema_version":"gateway.runtime.v1","type":"progress","message":"Indexing repo","percent":40} +``` + +### Runtime Envelope + +```json +{ + "schema_version": "gateway.runtime.v1", + "type": "tool_call", + "invocation_id": "inv_123", + "agent_id": "agt_123", + "emitted_at": "2026-04-22T19:16:00Z", + "payload": { + "tool_name": "read_file", + "detail": { + "path": "README.md" + } + } +} +``` + +### Runtime Protocol Rules + +- Unknown `invocation_id` events are rejected and logged. +- Wrong `agent_id` is rejected and logged as a protocol violation. +- `tool_result` without a prior `tool_call` is rejected and surfaced as an + adapter warning. +- Duplicate `complete` is ignored after the first terminal event. +- Events after terminal state do not mutate snapshots. +- Invalid JSON and oversized events are dropped and surfaced as adapter + warnings. + +## Template Capability Matrix + +Every template definition should expose both: + +- the canonical core model (`placement`, `activation`, `reply_mode`, + `telemetry_level`) +- extensible metadata: + - `tags` + - `capabilities` + - `constraints` + +Tags explain and filter. They must not replace the core state model. + +### Echo (Test) + +| Field | Value | +| --- | --- | +| Placement | `hosted` | +| Activation | `persistent` | +| Reply mode | `interactive` | +| Telemetry | `basic` | +| Gateway launches runtime | Yes | +| Gateway only attaches | No | +| Guaranteed signals | `message_received`, `message_claimed`, `working`, `completed`, `error` | +| Optional signals | None | +| Healthy means | Gateway listener is healthy and built-in runtime is available | +| Disconnected means | Gateway listener is stale/offline | +| Inline reply expected | Yes | +| Tags | `local`, `hosted-by-gateway`, `inline-reply`, `basic-telemetry` | +| Capabilities | `reply`, `smoke_test` | +| Constraints | `test_only` | + +### Hermes + +| Field | Value | +| --- | --- | +| Placement | `hosted` | +| Activation | `persistent` | +| Reply mode | `interactive` | +| Telemetry | `rich` | +| Gateway launches runtime | Yes | +| Gateway only attaches | No | +| Guaranteed signals | `message_received`, `message_claimed`, `working`, `completed`, `error` | +| Optional signals | `progress`, `tool_call`, `tool_result`, richer activity messages | +| Healthy means | Hermes checkout, auth, and launch path validate; heartbeats continue | +| Disconnected means | Launch failure, heartbeat expiry, or runtime exit | +| Inline reply expected | Yes | +| Tags | `local`, `hosted-by-gateway`, `live-listener`, `rich-telemetry`, `filesystem-capable`, `repo-bound` | +| Capabilities | `reply`, `progress`, `tool_events`, `read_files`, `bash_tools` | +| Constraints | `requires-repo`, `requires-provider-auth` | + +### Claude Code Channel + +| Field | Value | +| --- | --- | +| Placement | `attached` | +| Activation | `attach_only` | +| Reply mode | `interactive` | +| Telemetry | `basic` | +| Gateway launches runtime | No | +| Gateway only attaches | Yes | +| Guaranteed signals | `message_received`, `message_claimed`, `completed`, `error`, connection health | +| Optional signals | `working`, sparse `progress`, sparse tool telemetry | +| Healthy means | Active channel session, identity match, pickup test succeeds | +| Disconnected means | Channel closed, session heartbeat expired, pickup test fails | +| Inline reply expected | Yes | +| Tags | `attached-session`, `inline-reply`, `basic-telemetry`, `user-launched` | +| Capabilities | `reply`, `claim_work` | +| Constraints | `requires-live-session`, `attach-required` | + +### Ollama + +| Field | Value | +| --- | --- | +| Placement | `hosted` | +| Activation | `on_demand` | +| Reply mode | `interactive` | +| Telemetry | `basic` | +| Gateway launches runtime | Yes | +| Gateway only attaches | No | +| Guaranteed signals | `message_received`, `message_claimed`, `working`, `completed`, `error` | +| Optional signals | basic `progress`, model selection detail | +| Healthy means | launch preflight passes, Ollama server reachable, model present | +| Disconnected means | preflight fails, launch fails, or repeated invocation errors | +| Inline reply expected | Yes | +| Tags | `local`, `on-demand`, `cold-start`, `inline-reply`, `basic-telemetry` | +| Capabilities | `reply`, `launch_on_send`, `model_inference` | +| Constraints | `requires-model`, `requires-local-server` | + +### Inbox / Background Worker (Inbox-backed agent) + +| Field | Value | +| --- | --- | +| Placement | `mailbox` | +| Activation | `queue_worker` | +| Reply mode | `summary_only` by default | +| Telemetry | `basic` | +| Gateway launches runtime | Optional drain worker | +| Gateway only attaches | N/A | +| Guaranteed signals | `message_received`, `message_queued`, `message_claimed`, `completed`, `error` | +| Optional signals | `working`, `summary_pending`, `summary_posted` | +| Healthy means | queue accepts work durably and at least one worker can claim | +| Disconnected means | queue unavailable, drain workers permanently offline, or setup error | +| Inline reply expected | No; summary later by default | +| Tags | `queue-backed`, `summary-later`, `background`, `mailbox`, `basic-telemetry` | +| Capabilities | `queue_work`, `claim_work`, `post_summary` | +| Constraints | `not-live-listener` | + +## Sender Experience + +### Pre-send Expectations + +The composer and agent picker must show Mode, Presence, Reply behavior, and +telemetry richness before sending, plus current confidence and natural-language +reachability help. + +Examples: + +- `Hermes — LIVE · IDLE · REPLY · HIGH` +- `Claude Code Channel — LIVE · IDLE · REPLY · MEDIUM` +- `Ollama — ON-DEMAND · IDLE · REPLY · MEDIUM` +- `Inbox-backed Worker — INBOX · IDLE · SUMMARY · HIGH` + +Supporting copy examples: + +- `You can expect an inline reply.` +- `Gateway will start this runtime when you send.` +- `This agent is inbox-backed. Work can be queued safely even without a live worker.` +- `Reconnect the local session before sending.` + +### Post-send Inline Activity Bubble + +Interactive agents: + +- `Received by Gateway` +- `Claimed by runtime` +- `Working` +- `Using tool` +- `Responding` +- `Completed` + +Background/inbox agents: + +- `Queued in inbox` +- `Claimed by worker` +- `Working` +- `Summary pending` +- `Summary posted` +- `Completed with no reply expected` + +Failure states: + +- `No active runtime attached` +- `Setup error` +- `Stale listener` +- `Invocation failed` + +The originating message keeps a compact completed status. The final reply or +summary still lands at the bottom of the transcript. + +### Confidence Surface + +The sender surface should expose a deterministic operational confidence label: + +- `HIGH` +- `MEDIUM` +- `LOW` +- `BLOCKED` + +Examples: + +- `Hermes — LIVE · IDLE · REPLY · HIGH` +- `Ollama — ON-DEMAND · IDLE · REPLY · MEDIUM` +- `Inbox-backed Worker — INBOX · IDLE · SUMMARY · HIGH` +- `Claude Code Channel — LIVE · STALE · REPLY · LOW` +- `Broken Hermes — LIVE · ERROR · REPLY · BLOCKED` + +## Operator Experience + +### Fleet View + +The fleet view must show: + +- Agent +- Mode +- Presence +- Reply +- Confidence +- Telemetry +- Queue depth +- Typical Claim +- Typical First Activity +- Typical Completion +- Typical Summary Time when applicable +- Last Seen +- Last Outcome + +### Drill-In + +The drill-in must show: + +- placement and activation +- reachability explanation +- connection health and heartbeat source +- recent lifecycle timeline +- latest invocation state +- setup requirements and missing dependencies +- test-send controls +- recent errors and alerts + +### Gateway Doctor + +`Gateway Doctor` is a required setup primitive in both CLI and UI. Every +template must expose a deterministic setup report covering: + +- Identity +- Gateway auth +- Local path or dependency checks +- Runtime launch or attach validation +- Heartbeat or queue health +- Test claim +- Test reply or summary viability when applicable + +Doctor results must update `AgentStatusSnapshot` with: + +- `last_successful_doctor_at` +- `last_doctor_result` +- any changed `confidence_reason` / `confidence_detail` + +The default agent-facing wrapper for this flow should be a Gateway-native setup +skill, such as `gateway-agent-setup`, built on top of the same CLI and local +API primitives rather than a separate browser-only wizard. + +Canonical CLI shape: + +```text +ax gateway doctor @hermes-bot +✓ Gateway connected to dev.paxai.app +✓ Agent identity minted +✓ Hermes checkout found +✓ Runtime starts +✓ Heartbeat received +✓ Test message claimed +✓ Inline reply received +Status: +LIVE · IDLE · REPLY · HIGH +``` + +Inbox-backed example: + +```text +ax gateway doctor @docs-worker +✓ Gateway connected +✓ Inbox queue writable +✓ Worker config valid +! No worker currently attached +✓ Test job queued +Status: +INBOX · IDLE · SUMMARY · HIGH +Expectation: +Work can be queued now. Summary will post when a worker drains the inbox. +``` + +### First-run Contract + +The first-run Gateway experience should strongly recommend or require: + +1. Install Gateway +2. Log in / bootstrap +3. Run Echo smoke test +4. See message reach aX and return +5. Add a real agent from a template +6. Run Gateway Doctor +7. Send a template-specific test message +8. Verify the agent in Fleet View + +## Auth and Credential Boundary + +### v1 Rules + +- User PAT is stored only by Gateway. +- User PAT is bootstrap/enrollment only. +- Child runtimes never receive user PAT, raw platform JWT, or another agent's + credentials through env, args, config, stdin, logs, or protocol events. +- Gateway local API binds to loopback by default. +- Gateway local API requires a local session token or capability boundary. +- Gateway redacts PATs, JWTs, and local capability tokens from logs and event + payloads. +- Child runtimes may emit Gateway events, receive assigned work, and return + results. +- Child runtimes may not mint identities, impersonate another agent, or call aX + as the user. + +### Safe by Default Setup Copy + +The setup UX should visibly explain the trust boundary: + +- `Gateway keeps your aX credential.` +- `This runtime receives only a local scoped capability.` +- `It cannot impersonate you or mint other agents.` + +### OAuth Later + +OAuth to `paxai.app` is a later login-provider swap. It must not change: + +- lifecycle event names +- internal model fields +- sender bubble semantics +- runtime adapter contract + +## Metrics and Confidence Signals + +### Canonical Metrics + +- `time_to_gateway_ack` +- `time_to_claim` +- `time_to_first_activity` +- `time_to_completion` +- `reply_rate` +- `summary_rate` +- `completion_rate` + +### UX Labels + +Do not use `SLA` or `response time` as the generic label. + +Use: + +- `Typical Claim` +- `Typical First Activity` +- `Typical Completion` +- `Typical Summary Time` + +### Confidence Inputs + +`confidence` is deterministic, not a vague heuristic. It should be derived +from: + +- last successful smoke test +- last heartbeat +- queue health +- launch/preflight health +- recent completion rate +- recent claim latency p95 +- setup errors + +`confidence` answers: **how safe is it to send work through this path right +now?** + +It does not promise immediate completion. For inbox-backed agents especially, +high confidence can mean the queue path is healthy even if no worker is +currently attached. + +### Windows and Denominators + +- Windows: last `24h`, `7d`, `30d` +- `p50` and `p95` computed over successful attempts unless explicitly labeled + otherwise +- in-flight attempts excluded from completion latency +- timeout/failure contributes to failure and timeout counts, not latency + +Denominators: + +- `reply_rate`: completed invocations where `reply_mode=interactive` +- `summary_rate`: completed invocations where summary behavior is possible +- `completion_rate`: claimed invocations +- `claim_latency`: received messages that later claimed successfully + +If a message never claims, it counts toward claim timeout/failure, not claim +latency. + +## Acceptance and Adversarial Tests + +### Happy Paths + +- Echo test proves end-to-end Gateway receipt, claim, progress, and reply. +- Hermes emits rich activity and tool telemetry. +- Claude Code Channel shows reliable pickup and completion even with sparse + activity. +- Ollama launches on demand, claims work, and completes. +- Inbox accepts work without inline reply and later posts summary. +- Inbox-backed agent with empty queue renders `INBOX · IDLE · SUMMARY`. +- Inbox-backed agent with pending work renders `INBOX · QUEUED · SUMMARY`. +- Gateway Doctor produces deterministic pass/fail output for each template. + +### Adversarial Cases + +- Gateway restarts after `message_received` before `message_claimed`. +- Gateway restarts after `message_claimed` before terminal event. +- Runtime crashes after claim. +- Adapter emits malformed JSON. +- Adapter emits duplicate `complete`. +- Adapter emits `tool_result` without `tool_call`. +- aX delivers the same message twice. +- Two workers race for the same inbox job. +- PAT revoked while Gateway is running. +- SSE/channel disconnect during active work. +- On-demand runtime fails to launch. +- Local dependency disappears after prior healthy state. +- Gateway reconnects after stale local state. +- Background job finishes with summary later and no inline reply. +- Queue is healthy but no live worker is attached. +- Queue is unhealthy and inbox-backed agent becomes `ERROR`. + +## Custom Bridge Contract + +Gateway must support custom/local agents without requiring them to fit a single +blessed framework. + +The custom bridge flow should let an operator declare: + +- reply behavior: + - inline reply + - summary later + - silent completion +- activation: + - Gateway launches command + - Gateway invokes on demand + - agent drains inbox + - Gateway attaches to existing session +- telemetry: + - heartbeat only + - progress events + - tool events + - completion only + +Gateway should then provide: + +- command template +- expected env vars +- `AX_GATEWAY_EVENT` examples +- local capability boundary +- smoke test / doctor checks + +## Extensible Metadata + +In addition to the core state model, the product should support extensible: + +- `tags` +- `capabilities` +- `constraints` + +These are used for filtering, discovery, and explanation. They must not replace +the canonical state model. + +Suggested tag families: + +- Connectivity: `live-listener`, `attached-session`, `on-demand`, + `queue-backed`, `external-broker` +- Execution: `local`, `remote`, `hosted-by-gateway`, `user-launched`, + `cold-start` +- Reply behavior: `inline-reply`, `summary-later`, `silent-completion` +- Telemetry: `rich-telemetry`, `basic-telemetry`, `heartbeat-only`, + `no-tool-events` +- Risk/setup: `requires-repo`, `requires-model`, `requires-provider-auth`, + `experimental` + +## Roadmap + +### v1 Minimum + +- normalized internal and operator state model +- template-first connection flow +- Gateway Doctor +- pre-send expectation chips +- sender confidence bubble +- latest snapshot + recent timeline persisted in aX +- p50/p95 metrics for claim, first activity, and completion +- PAT bootstrap with hardened local auth boundary + +### Phase 2 + +- stronger lease and claim semantics +- queue worker hardening +- better dependency/setup diagnostics +- auth/session hardening + +### Phase 3 + +- richer telemetry +- fleet analytics and trends +- alerting and escalations +- production rollout hardening + +### Phase 4 + +- single upstream Gateway control stream +- Gateway as authoritative connected object to aX + +### Phase 5 + +- MCP Gateway mode +- MCP Jam SDK coverage + +### Phase 6 + +- skill/capability layer on top of stable CLI + MCP foundations + +## Deliverables + +- this primary spec +- companion [mockups.md](mockups.md) +- lifecycle/state derivation tables in this spec +- event envelope definition in this spec +- template capability matrix in this spec +- acceptance/adversarial test matrix in this spec diff --git a/specs/GATEWAY-IDENTITY-SPACE-001/spec.md b/specs/GATEWAY-IDENTITY-SPACE-001/spec.md new file mode 100644 index 0000000..4f87b28 --- /dev/null +++ b/specs/GATEWAY-IDENTITY-SPACE-001/spec.md @@ -0,0 +1,659 @@ +# GATEWAY-IDENTITY-SPACE-001: Gateway Identity, Space Binding, and Visibility + +**Status:** Draft +**Owner:** @madtank +**Date:** 2026-04-22 +**Related:** GATEWAY-CONNECTIVITY-001, CONNECTED-ASSET-GOVERNANCE-001, RUNTIME-CONFIG-001, AGENT-PAT-001, IDENTIFIER-DISPLAY-001, CONTRACT-QA-001 + +## Purpose + +Define the v1 contract for how Gateway resolves, verifies, shows, and enforces: + +- which identity it is acting as, +- which environment it is targeting, +- which space that identity is currently operating in, +- which spaces that identity is allowed to access, and +- when Gateway must block a send/listen action because those facts do not line + up. + +This spec exists because a multi-agent machine can hold many local identities, +many spaces, and many environments at once. The Gateway must make that visible +and safe. + +The central product rule is: + +> Gateway must never silently borrow another identity or silently target an +> unexpected space. + +If an asset does not have a valid identity binding for the requested +environment and space, Gateway should block the action and surface the setup gap +explicitly. + +## Relationship to other specs + +- [GATEWAY-CONNECTIVITY-001](../GATEWAY-CONNECTIVITY-001/spec.md) + defines whether Gateway can safely route work right now using `Mode + + Presence + Reply + Confidence`. +- [CONNECTED-ASSET-GOVERNANCE-001](../CONNECTED-ASSET-GOVERNANCE-001/spec.md) + defines asset registry, provenance, approvals, attestation, grants, and + audit. +- This spec defines the missing identity-and-space layer between them: + - who Gateway is acting as, + - where it is acting, + - whether that space is allowed for that identity, + - and how that should be shown to the operator and sender. + +These layers must remain separate: + +- **AssetDescriptor** says what the asset is. +- **AgentStatusSnapshot** says whether Gateway can route to it now. +- **Identity/Space binding** says who it is acting as and where that identity is + valid. + +## Goals + +- Make acting identity explicit in Gateway UI, CLI, and local API. +- Make active space, default space, and allowed spaces visible everywhere an + operator can send or review work. +- Ensure every managed asset has its own identity binding per environment. +- Prevent hidden fallback from one local identity to another. +- Prevent hidden fallback from a user bootstrap credential to an agent runtime + identity. +- Block sends/listens when the active space is not allowed for the acting + identity. +- Give Doctor and onboarding a deterministic contract for verifying identity and + space access. + +## Non-goals + +- Designing the full agent registry or policy engine. +- Replacing `Mode + Presence + Reply + Confidence`. +- Solving full org-level RBAC in this spec. +- Defining the full OAuth model. +- Defining all multi-org or cross-tenant behavior. + +## Core framing + +Gateway must track three separate layers: + +1. **Bootstrap credential** + - the human login or trusted device credential used to provision agent + identities +2. **Acting identity** + - the asset identity Gateway is currently using for sends, listens, or + status updates +3. **Space binding** + - the space Gateway is currently targeting and the list of spaces that + acting identity is allowed to access + +These must not be conflated. + +### Wrong behavior + +- `codex` cannot speak in prod, so Gateway silently reuses `night_owl` +- a repo-local home config changes the active identity without the operator + noticing +- Gateway shows an agent as healthy but hides that it is pointing at the wrong + space +- sender actions succeed through a user credential path when the operator + expected agent-authored behavior + +### Correct behavior + +- Gateway shows `acting as codex` +- Gateway shows `environment: prod` +- Gateway shows `current space: ax-cli-dev` +- Gateway shows `allowed spaces: ax-cli-dev, madtank's Workspace` +- if `codex` has no prod binding or no access to `ax-cli-dev`, Gateway blocks + the action and says so + +## Definitions + +### Acting identity + +The concrete asset identity Gateway is using for an action such as: + +- listening for messages +- sending a direct message +- replying in a thread +- posting processing status +- claiming work + +Examples: + +- `codex` +- `night_owl` +- `cli-managed-bot` + +### Environment + +The backend host and auth context where the identity is operating. + +Examples: + +- `dev.paxai.app` +- `paxai.app` +- local + +### Active space + +The concrete space a send/listen action is currently targeting. + +### Default space + +The space the identity would use if no explicit target space is chosen. + +### Allowed spaces + +The list of spaces the acting identity is permitted to operate in according to +the backend. + +### Active space source + +How Gateway resolved the active target space for the current action: + +- `explicit_request` +- `gateway_binding` +- `visible_default` +- `none` + +### Space status + +The evaluation of whether the active space is valid for the acting identity: + +- `active_allowed` +- `active_not_allowed` +- `no_active_space` +- `unknown` + +### Environment status + +The evaluation of whether the requested environment and the bound environment +line up: + +- `environment_allowed` +- `environment_mismatch` +- `environment_unknown` +- `environment_blocked` + +## Canonical model + +### `GatewayIdentityBinding` + +Describes the identity Gateway should use for one asset in one environment. + +```json +{ + "identity_binding_id": "idbind_codex_prod_ax_cli_dev", + "asset_id": "db1f2a10-cdb2-4fce-a028-7fe0edb2d08f", + "gateway_id": "gw_jacob_macbook", + "install_id": "inst_codex_prod_local", + "environment": { + "base_url": "https://paxai.app", + "label": "prod", + "host": "paxai.app" + }, + "acting_identity": { + "agent_id": "e9877470-5e3a-4f08-869d-22fc86b2e063", + "agent_name": "codex", + "principal_type": "agent" + }, + "active_space_id": "ed81ae98-50cb-4268-b986-1b9fe76df742", + "default_space_id": "ed81ae98-50cb-4268-b986-1b9fe76df742", + "credential_ref": { + "kind": "token_file", + "id": "cred_codex_prod", + "display": "Gateway-managed agent token", + "path_redacted": "~/.ax/runtimes/codex-prod/.ax/token" + }, + "binding_state": "verified", + "created_via": "gateway_setup", + "created_from": "ax_template", + "last_verified_at": "2026-04-22T23:00:00Z" +} +``` + +`credential_ref` is a non-sensitive display contract for UI, audit, and local +API use. Full local token paths or other secret material references should +remain in Gateway-private state only and must not be casually exposed through +operator-facing surfaces. + +### `IdentitySpaceBindingSnapshot` + +Derived status object for UI/CLI/runtime decisions. + +```json +{ + "identity_binding_id": "idbind_codex_prod_ax_cli_dev", + "asset_id": "db1f2a10-cdb2-4fce-a028-7fe0edb2d08f", + "gateway_id": "gw_jacob_macbook", + "install_id": "inst_codex_prod_local", + "acting_agent_id": "e9877470-5e3a-4f08-869d-22fc86b2d08f", + "acting_agent_name": "codex", + "principal_type": "agent", + "base_url": "https://paxai.app", + "environment_label": "prod", + "environment_status": "environment_allowed", + "active_space_id": "ed81ae98-50cb-4268-b986-1b9fe76df742", + "active_space_name": "ax-cli-dev", + "active_space_source": "gateway_binding", + "default_space_id": "ed81ae98-50cb-4268-b986-1b9fe76df742", + "default_space_name": "ax-cli-dev", + "allowed_spaces": [ + { + "space_id": "ed81ae98-50cb-4268-b986-1b9fe76df742", + "name": "ax-cli-dev", + "is_default": true + } + ], + "allowed_space_count": 1, + "space_status": "active_allowed", + "identity_status": "verified", + "last_space_verification_at": "2026-04-22T23:00:00Z" +} +``` + +### Optional status values + +#### `binding_state` + +- `verified` +- `unbound` +- `mismatch` +- `blocked` + +#### `identity_status` + +- `verified` +- `unknown_identity` +- `credential_mismatch` +- `fallback_blocked` +- `bootstrap_only` +- `blocked` + +#### `environment_status` + +- `environment_allowed` +- `environment_mismatch` +- `environment_unknown` +- `environment_blocked` + +#### `space_status` + +- `active_allowed` +- `active_not_allowed` +- `no_active_space` +- `unknown` + +#### `active_space_source` + +- `explicit_request` +- `gateway_binding` +- `visible_default` +- `none` + +## Required rules and invariants + +### 1. Explicit acting identity + +Gateway must always know and show which identity an action will use. + +Required surfaces: + +- fleet view drill-in +- message send composer +- doctor output +- local status API + +### 2. Per-asset identity bindings + +Each managed asset must have its own binding per environment. + +Examples: + +- `codex` on prod +- `codex` on dev +- `night_owl` on prod + +These are distinct bindings and must not be merged implicitly. + +### 3. No silent cross-identity fallback + +Gateway must not silently fall back from: + +- one agent identity to another +- a managed agent identity to a human bootstrap identity +- one environment binding to another environment binding + +If the requested identity is missing, invalid, or blocked, the action must fail +closed and show the problem. + +### 4. No hidden space fallback + +Gateway must not silently rely on: + +- browser/session "current space" +- a stale home-level default space +- an unrelated repo-local space + +The active space used for send/listen must be explicit or derivable from a +visible binding rule. + +Gateway should resolve active space using this deterministic precedence: + +1. explicit send/listen target space, if provided +2. explicit Gateway identity binding `active_space_id` +3. visible binding `default_space_id` +4. otherwise `no_active_space` + +Every resolved `active_space_id` must still be checked against +`allowed_spaces`. A visible default is convenience only; it does not authorize +access by itself. + +### 5. Active space must be allowed + +Before Gateway sends, listens, or claims work, it must verify that the active +space is in the acting identity's allowed-space list. + +If not: + +- block the action +- mark `space_status = active_not_allowed` +- surface `confidence = BLOCKED` +- set a structured reason + +### 6. Default space is informational, not a hidden permission bypass + +`default_space_id` is useful for setup and convenience, but it does not +override explicit target spaces and it does not authorize anything by itself. + +### 7. User bootstrap credential is not a runtime identity + +The user PAT or user login session may bootstrap, verify, or mint agent +credentials, but after binding it must not be used for normal agent-authored +operations. + +If Gateway is about to perform an agent action through a user identity, it must +surface that as a blocked or warning state rather than silently proceeding. + +For normal agent-authored operations, `bootstrap_only` should be treated as +blocked. Bootstrap credentials may be used only for setup, verification, +minting, or repair flows. + +### 8. Identity-space binding must be verified before agent-authored actions + +Gateway must verify identity-space binding before: + +- sending +- listening +- claiming work +- posting status +- replying +- emitting agent-authored lifecycle events + +Identity-space verification is not optional just because runtime presence or +delivery plumbing is healthy. + +## Integration with connectivity + +This spec extends the operator truth from: + +- `Mode + Presence + Reply + Confidence` + +to include visible identity and space context: + +- `Acting As` +- `Environment` +- `Current Space` +- `Allowed Spaces` + +### Confidence integration + +The following structured `confidence_reason` values should exist or be added: + +- `identity_unbound` +- `identity_mismatch` +- `bootstrap_only` +- `environment_mismatch` +- `active_space_not_allowed` +- `no_active_space` +- `space_unknown` + +Recommended mapping: + +- `identity_unbound` -> `BLOCKED` +- `identity_mismatch` -> `BLOCKED` +- `bootstrap_only` -> `BLOCKED` +- `environment_mismatch` -> `BLOCKED` +- `active_space_not_allowed` -> `BLOCKED` +- `no_active_space` -> `LOW` or `BLOCKED` +- `space_unknown` -> `LOW` + +### Reachability integration + +Even if `mode = LIVE`, Gateway should not claim the asset is safely reachable if +the identity or space binding is invalid. + +Examples: + +- `LIVE · IDLE · REPLY · HIGH` + - valid live listener, valid acting identity, active space allowed +- `LIVE · OFFLINE · REPLY · LOW` + - valid identity binding, active space allowed, runtime not live +- `LIVE · BLOCKED · REPLY · BLOCKED` + - acting identity mismatch or active space not allowed + +## UI and CLI requirements + +### Fleet view + +Fleet rows should show: + +- `Asset` +- `Type` +- `Mode` +- `Presence` +- `Output` +- `Confidence` +- `Acting As` +- `Current Space` +- `Allowed Spaces` + +Compact example: + +```text +codex Live Listener LIVE IDLE Reply HIGH +acting as codex · prod · ax-cli-dev · 1 allowed space +``` + +### Drill-in + +Drill-in should show: + +- acting identity name and id +- environment/base URL +- active space name and id +- default space name and id +- allowed spaces list +- last verification time +- identity status +- space status + +### Composer / send controls + +Before send, the UI should show: + +- `Sending as codex` +- `Environment: prod` +- `Target space: ax-cli-dev` + +If the target is not allowed, the send button must block. + +### Doctor + +Doctor must verify: + +- acting identity exists +- acting identity matches expected asset +- requested environment matches the bound environment/base URL +- active space resolves +- active space is allowed +- allowed spaces list is fetchable +- default space, if present, is consistent with backend response +- the credential in use is agent-authored rather than bootstrap-only + +## Setup and onboarding + +### First bind + +When connecting an asset to Gateway in a new environment, setup should produce: + +- identity binding +- active/default space selection +- allowed-space verification +- a visible summary before enabling send/listen + +### Existing asset, new environment + +If `codex` exists in prod but has no usable local runtime credential yet: + +- Gateway should say `prod binding missing` or `prod runtime credential missing` +- Gateway should not fall back to another identity such as `night_owl` + +### Existing asset, wrong current space + +If the identity is valid but the target space is not in `allowed_spaces`: + +- show the target space +- show the allowed spaces +- block the action + +## Audit expectations + +Gateway should emit audit-worthy events for: + +- `identity_binding_verified` +- `identity_binding_missing` +- `identity_mismatch_detected` +- `space_binding_verified` +- `space_binding_blocked` +- `fallback_blocked` +- `bootstrap_identity_blocked` + +Every event should include: + +- `gateway_id` +- `asset_id` +- `install_id` +- `identity_binding_id` +- `acting_agent_id` +- `acting_agent_name` +- `runtime_instance_id`, when available +- `base_url` +- `environment_status` +- `active_space_id` +- `active_space_source` +- `default_space_id` +- `allowed_space_ids` +- `decision` +- `reason` +- `observed_at` + +Additional audit-worthy events: + +- `environment_mismatch_detected` +- `active_space_resolved` +- `identity_space_snapshot_updated` + +## Acceptance tests + +### Identity correctness + +- A managed `codex` binding for prod resolves `acting as codex`, not + `night_owl`. +- A managed `codex` binding for prod never silently uses a valid dev binding. +- Gateway blocks a `codex` send if only `night_owl` has a valid binding. +- Two installs for the same asset on the same Gateway are distinguished by + `install_id`, not only `asset_id`. +- A user bootstrap token can verify or mint but cannot be used as the acting + runtime identity for normal sends. + +### Space correctness + +- A valid prod binding with `active_space = ax-cli-dev` and + `allowed_spaces = [ax-cli-dev]` is `active_allowed`. +- A binding whose target space is not in the allowed-space list is blocked. +- A missing active space is surfaced explicitly and does not silently use a + hidden backend current-space fallback. +- An explicit `--space` target overrides the visible default but must still be + in `allowed_spaces`. +- A backend or browser "current space" that is not present in the Gateway + binding is ignored. + +### Visibility + +- Fleet view shows acting identity and current space. +- Drill-in shows active, default, and allowed spaces. +- Composer shows `Sending as in `. + +### Fallback blocking + +- If a home-level config for `night_owl` exists and `codex` is requested, + Gateway blocks instead of silently using `night_owl`. +- If a repo-local config points to the wrong environment, Gateway blocks or + warns rather than silently crossing environments. +- An environment mismatch emits `environment_mismatch_detected`. + +### Multi-space agents + +- An identity with two allowed spaces shows both spaces and a count. +- Changing the active space updates the visible target before send/listen. + +### Enforcement + +- Gateway blocks status posting if the acting identity binding is invalid. +- Gateway blocks claim if the resolved active space is not allowed. +- `bootstrap_only` is allowed for doctor/setup flows but blocked for + send/listen/claim/reply/status. + +## First implementation slice + +The first implementation slice for this spec should be: + +- `IdentitySpaceBindingSnapshot` +- Doctor identity/space verification +- send/listen/claim blocking on invalid identity or space binding + +Minimum local objects: + +- `GatewayIdentityBinding` +- `IdentitySpaceBindingSnapshot` + +First enforcement cases: + +- requested `codex` + valid prod `codex` binding + allowed space -> allow +- requested `codex` + only `night_owl` binding exists -> block +- requested `codex` + valid credential + disallowed active space -> block +- requested `codex` + requested prod but repo/home config points to dev -> block +- requested agent action would use bootstrap credential -> block + +This slice should not require the full grants, vault, or policy engine before +it is useful. + +## Roadmap + +### v1 minimum + +- Local identity-space snapshot in Gateway state +- Visible acting identity and current space in fleet/drill-in +- Doctor checks for identity and allowed spaces +- Block silent fallback to another identity +- Block sends to disallowed spaces + +### Later + +- Full aX-backed canonical identity binding registry +- Approval policy for first-time space bindings +- UI-based identity/space switching flows +- Cross-environment identity dashboards +- Richer org/workspace policy overlays diff --git a/specs/README.md b/specs/README.md index 78fc56c..3b78ded 100644 --- a/specs/README.md +++ b/specs/README.md @@ -11,11 +11,15 @@ agent skills. - [AGENT-PAT-001: Agent PAT Minting and JWT Exchange](AGENT-PAT-001/spec.md) - [IDENTIFIER-DISPLAY-001: Human-Readable Identifier Display](IDENTIFIER-DISPLAY-001/spec.md) - [RUNTIME-CONFIG-001: Shared Agent Runtime Configuration](RUNTIME-CONFIG-001/spec.md) +- [GATEWAY-IDENTITY-SPACE-001: Gateway Identity, Space Binding, and Visibility](GATEWAY-IDENTITY-SPACE-001/spec.md) ## Workflow and Delivery - [CONTRACT-QA-001: API-First Regression Harness](CONTRACT-QA-001/spec.md) - [CLI-WORKFLOW-001: Smart Workflow Flags on Existing Commands](CLI-WORKFLOW-001/spec.md) +- [GATEWAY-CONNECTIVITY-001: Gateway Connectivity, Signal Model, and Sender Confidence](GATEWAY-CONNECTIVITY-001/spec.md) +- [GATEWAY-ASSET-TAXONOMY-001: Gateway Asset Taxonomy and Flow Semantics](GATEWAY-ASSET-TAXONOMY-001/spec.md) +- [CONNECTED-ASSET-GOVERNANCE-001: Registry, Provenance, Capabilities, Grants, Secrets, and Approval](CONNECTED-ASSET-GOVERNANCE-001/spec.md) - [AGENT-CONTACT-001: Agent Contact Modes](AGENT-CONTACT-001/spec.md) - [AGENT-MESH-PATTERNS-001: Shared-State Agent Mesh](AGENT-MESH-PATTERNS-001/spec.md) - [MESH-SPAWN-001: User-Bootstrapped Agent Credential Spawning](MESH-SPAWN-001/spec.md) diff --git a/tests/test_gateway_commands.py b/tests/test_gateway_commands.py index 868bd87..6603568 100644 --- a/tests/test_gateway_commands.py +++ b/tests/test_gateway_commands.py @@ -1,10 +1,14 @@ import json +import socket import sys +import threading import time +from contextlib import closing from datetime import datetime, timedelta, timezone from pathlib import Path import httpx +import pytest from typer.testing import CliRunner from ax_cli import gateway as gateway_core @@ -39,6 +43,23 @@ class _FakeUserClient: def update_agent(self, *args, **kwargs): return {"ok": True} + def send_message(self, space_id, content, *, agent_id=None, parent_id=None, metadata=None): + return { + "message": { + "id": "gateway-test-1", + "space_id": space_id, + "content": content, + "agent_id": agent_id, + "parent_id": parent_id, + "metadata": metadata, + } + } + + +def _fake_create_agent_in_space(*args, **kwargs): + name = kwargs.get("name", "agent") + return {"id": f"agent-{name}", "name": name} + class _FakeSseResponse: status_code = 200 @@ -229,6 +250,194 @@ def test_scan_gateway_process_pids_ignores_current_parent_wrapper(monkeypatch): assert gateway_core._scan_gateway_process_pids() == [5514] +def test_gateway_start_launches_background_daemon_and_ui(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "madtank", + } + ) + + state = {"daemon_pid": None, "ui_pid": None} + spawned: list[tuple[list[str], str]] = [] + + class _FakeProcess: + def __init__(self, pid: int): + self.pid = pid + + def poll(self): + return None + + def fake_spawn(command, *, log_path): + spawned.append((command, str(log_path))) + if "run" in command: + state["daemon_pid"] = 5514 + return _FakeProcess(5514) + state["ui_pid"] = 5515 + return _FakeProcess(5515) + + monkeypatch.setattr(gateway_cmd, "_spawn_gateway_background_process", fake_spawn) + monkeypatch.setattr(gateway_cmd, "_wait_for_daemon_ready", lambda process, timeout=3.0: True) + monkeypatch.setattr(gateway_cmd, "_wait_for_ui_ready", lambda process, host, port, timeout=3.0: True) + monkeypatch.setattr(gateway_cmd, "active_gateway_pid", lambda: state["daemon_pid"]) + monkeypatch.setattr(gateway_cmd, "active_gateway_ui_pid", lambda: state["ui_pid"]) + monkeypatch.setattr( + gateway_cmd, + "ui_status", + lambda: { + "running": True, + "pid": state["ui_pid"], + "host": "127.0.0.1", + "port": 8765, + "url": "http://127.0.0.1:8765", + "log_path": str(gateway_core.ui_log_path()), + }, + ) + opened: list[str] = [] + monkeypatch.setattr(gateway_cmd.webbrowser, "open_new_tab", lambda url: opened.append(url)) + + result = runner.invoke(app, ["gateway", "start", "--no-open"]) + + assert result.exit_code == 0, result.output + assert "daemon = started" in result.output + assert "ui = started" in result.output + assert len(spawned) == 2 + assert "gateway" in spawned[0][0] and "run" in spawned[0][0] + assert spawned[0][0][-2:] == ["--poll-interval", "1.0"] + assert "gateway" in spawned[1][0] and "ui" in spawned[1][0] + assert opened == [] + + +def test_gateway_cli_argv_prefers_current_ax_script(monkeypatch, tmp_path): + current_ax = tmp_path / "bin" / "ax" + current_ax.parent.mkdir(parents=True) + current_ax.write_text("#!/bin/sh\n") + current_ax.chmod(0o755) + + monkeypatch.setattr(gateway_cmd.sys, "argv", [str(current_ax), "gateway", "start"]) + monkeypatch.setattr(gateway_cmd.sys, "executable", "/opt/homebrew/bin/python3") + monkeypatch.setattr(gateway_cmd.shutil, "which", lambda name: f"/opt/homebrew/bin/{name}") + + argv = gateway_cmd._gateway_cli_argv("gateway", "run") + + assert argv == [str(current_ax.resolve()), "gateway", "run"] + + +def test_gateway_start_without_login_starts_ui_only(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + + state = {"ui_pid": None} + spawned: list[list[str]] = [] + + class _FakeProcess: + def __init__(self, pid: int): + self.pid = pid + + def poll(self): + return None + + def fake_spawn(command, *, log_path): + spawned.append(command) + state["ui_pid"] = 6615 + return _FakeProcess(6615) + + monkeypatch.setattr(gateway_cmd, "_spawn_gateway_background_process", fake_spawn) + monkeypatch.setattr(gateway_cmd, "_wait_for_ui_ready", lambda process, host, port, timeout=3.0: True) + monkeypatch.setattr(gateway_cmd, "active_gateway_pid", lambda: None) + monkeypatch.setattr(gateway_cmd, "active_gateway_ui_pid", lambda: state["ui_pid"]) + monkeypatch.setattr( + gateway_cmd, + "ui_status", + lambda: { + "running": True, + "pid": state["ui_pid"], + "host": "127.0.0.1", + "port": 8765, + "url": "http://127.0.0.1:8765", + "log_path": str(gateway_core.ui_log_path()), + }, + ) + + result = runner.invoke(app, ["gateway", "start", "--no-open"]) + + assert result.exit_code == 0, result.output + assert "Gateway is not logged in yet" in result.output + assert len(spawned) == 1 + assert "gateway" in spawned[0] and "ui" in spawned[0] + + +def test_gateway_stop_terminates_daemon_and_ui(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + monkeypatch.setattr(gateway_cmd, "active_gateway_pids", lambda: [7714]) + monkeypatch.setattr(gateway_cmd, "active_gateway_ui_pids", lambda: [7715]) + monkeypatch.setattr( + gateway_cmd, + "_terminate_pids", + lambda pids, timeout=3.0: (list(pids), [pids[0]] if pids and pids[0] == 7714 else []), + ) + + result = runner.invoke(app, ["gateway", "stop"]) + + assert result.exit_code == 0, result.output + assert "daemon = [7714]" in result.output + assert "ui = [7715]" in result.output + assert "Forced kill:" in result.output + + +def test_gateway_start_rolls_back_daemon_when_ui_start_fails(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "madtank", + } + ) + + state = {"daemon_pid": None, "ui_pid": None} + + class _FakeProcess: + def __init__(self, pid: int): + self.pid = pid + + def poll(self): + return None + + def fake_spawn(command, *, log_path): + if "run" in command: + state["daemon_pid"] = 8814 + return _FakeProcess(8814) + state["ui_pid"] = 8815 + return _FakeProcess(8815) + + terminated: list[list[int]] = [] + cleared: list[int | None] = [] + + monkeypatch.setattr(gateway_cmd, "_spawn_gateway_background_process", fake_spawn) + monkeypatch.setattr(gateway_cmd, "_wait_for_daemon_ready", lambda process, timeout=3.0: True) + monkeypatch.setattr(gateway_cmd, "_wait_for_ui_ready", lambda process, host, port, timeout=3.0: False) + monkeypatch.setattr(gateway_cmd, "active_gateway_pid", lambda: state["daemon_pid"]) + monkeypatch.setattr(gateway_cmd, "active_gateway_ui_pid", lambda: state["ui_pid"]) + monkeypatch.setattr(gateway_cmd, "_tail_log_lines", lambda path, lines=12: "address already in use") + monkeypatch.setattr(gateway_cmd, "_terminate_pids", lambda pids, timeout=3.0: (terminated.append(list(pids)) or (list(pids), []))) + monkeypatch.setattr(gateway_core, "clear_gateway_pid", lambda pid=None: cleared.append(pid)) + + result = runner.invoke(app, ["gateway", "start", "--no-open"]) + + assert result.exit_code == 1, result.output + assert "Failed to start Gateway UI." in result.output + assert terminated == [[8814]] + assert cleared == [None] + + def test_gateway_agents_add_mints_token_and_writes_registry(monkeypatch, tmp_path): config_dir = tmp_path / "config" monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) @@ -261,6 +470,9 @@ def test_gateway_agents_add_mints_token_and_writes_registry(monkeypatch, tmp_pat assert payload["transport"] == "gateway" registry = gateway_core.load_gateway_registry() assert registry["agents"][0]["name"] == "echo-bot" + assert registry["bindings"][0]["asset_id"] == "agent-1" + assert registry["bindings"][0]["approved_state"] == "approved" + assert registry["agents"][0]["install_id"] == registry["bindings"][0]["install_id"] token_file = Path(registry["agents"][0]["token_file"]) assert token_file.exists() assert token_file.read_text().strip() == "axp_a_agent.secret" @@ -269,6 +481,275 @@ def test_gateway_agents_add_mints_token_and_writes_registry(monkeypatch, tmp_pat assert recent[-1]["agent_name"] == "echo-bot" +def test_gateway_daemon_reconcile_normalizes_legacy_inbox_metadata(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + registry = gateway_core.load_gateway_registry() + entry = { + "name": "dev_channel_alpha", + "agent_id": "agent-dev-channel-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "inbox", + "desired_state": "stopped", + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "telemetry_level": "basic", + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "tags": ["local", "custom-bridge"], + "capabilities": ["reply"], + "created_via": "legacy_registry", + } + gateway_core.ensure_local_asset_binding(registry, entry, created_via="legacy_registry", auto_approve=True) + registry["agents"] = [entry] + + daemon = gateway_core.GatewayDaemon(client_factory=lambda **kwargs: _SharedRuntimeClient({})) + reconciled = daemon._reconcile_registry(registry, {"token": "axp_u_test.token"}) + agent = reconciled["agents"][0] + + assert agent["placement"] == "mailbox" + assert agent["activation"] == "queue_worker" + assert agent["reply_mode"] == "summary_only" + assert agent["mode"] == "INBOX" + assert agent["reply"] == "SUMMARY" + assert agent["asset_class"] == "background_worker" + assert agent["intake_model"] == "queue_accept" + assert agent["worker_model"] == "queue_drain" + assert agent["return_paths"] == ["summary_post"] + assert agent["asset_type_label"] == "Inbox Worker" + assert agent["output_label"] == "Summary" + + +def test_annotate_runtime_health_respects_explicit_user_overrides(): + snapshot = { + "name": "custom-inbox-ish", + "agent_id": "agent-custom-1", + "runtime_type": "inbox", + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "user_overrides": { + "operator": { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + }, + "asset": { + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + }, + }, + "effective_state": "stopped", + } + + annotated = gateway_core.annotate_runtime_health(snapshot) + + assert annotated["placement"] == "hosted" + assert annotated["activation"] == "persistent" + assert annotated["reply_mode"] == "interactive" + assert annotated["mode"] == "LIVE" + assert annotated["reply"] == "REPLY" + assert annotated["asset_class"] == "interactive_agent" + assert annotated["intake_model"] == "live_listener" + assert annotated["return_paths"] == ["inline_reply"] + assert annotated["asset_type_label"] == "Live Listener" + + +def test_evaluate_runtime_attestation_detects_binding_drift_and_creates_approval(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + registry = gateway_core.load_gateway_registry() + entry = { + "name": "docs-worker", + "agent_id": "agent-docs-1", + "runtime_type": "exec", + "exec_command": "python3 worker.py", + "workdir": str(tmp_path / "repo-a"), + "created_via": "cli", + } + gateway_core.ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) + + drifted = dict(entry) + drifted["workdir"] = str(tmp_path / "repo-b") + + attestation = gateway_core.evaluate_runtime_attestation(registry, drifted) + snapshot = gateway_core.annotate_runtime_health({**drifted, **attestation, "effective_state": "stopped"}) + + assert attestation["attestation_state"] == "drifted" + assert attestation["approval_state"] == "pending" + assert attestation["approval_id"] + assert registry["approvals"][0]["approval_kind"] == "binding_drift" + assert snapshot["presence"] == "BLOCKED" + assert snapshot["confidence"] == "BLOCKED" + assert snapshot["confidence_reason"] == "binding_drift" + + +def test_evaluate_runtime_attestation_blocks_asset_mismatch(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + registry = gateway_core.load_gateway_registry() + entry = { + "name": "codex", + "agent_id": "agent-codex-1", + "runtime_type": "exec", + "exec_command": "python3 codex_bridge.py", + "workdir": str(tmp_path / "repo"), + "install_id": "install-1", + "created_via": "cli", + } + gateway_core.ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) + + mismatched = dict(entry) + mismatched["agent_id"] = "agent-other-2" + + attestation = gateway_core.evaluate_runtime_attestation(registry, mismatched) + snapshot = gateway_core.annotate_runtime_health({**mismatched, **attestation, "effective_state": "stopped"}) + + assert attestation["attestation_state"] == "blocked" + assert attestation["confidence_reason"] == "asset_mismatch" + assert snapshot["presence"] == "BLOCKED" + assert snapshot["confidence"] == "BLOCKED" + + +def test_gateway_daemon_reconcile_blocks_drifted_runtime(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + registry = gateway_core.load_gateway_registry() + entry = { + "name": "drift-bot", + "agent_id": "agent-drift-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "exec", + "exec_command": "python3 drift.py", + "workdir": str(tmp_path / "repo-a"), + "token_file": str(tmp_path / "token"), + "desired_state": "running", + "created_via": "cli", + } + Path(entry["token_file"]).write_text("axp_a_agent.secret") + gateway_core.ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) + entry["workdir"] = str(tmp_path / "repo-b") + registry["agents"] = [entry] + + daemon = gateway_core.GatewayDaemon(client_factory=lambda **kwargs: _SharedRuntimeClient({})) + reconciled = daemon._reconcile_registry(registry, {"token": "axp_u_test.token"}) + agent = reconciled["agents"][0] + + assert agent["attestation_state"] == "drifted" + assert agent["approval_state"] == "pending" + assert agent["presence"] == "BLOCKED" + assert "drift-bot" not in daemon._runtimes + + +def test_gateway_daemon_reconcile_blocks_hermes_without_repo(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + workdir = tmp_path / "workspace" / "ax-cli" + workdir.mkdir(parents=True) + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + monkeypatch.delenv("HERMES_REPO_PATH", raising=False) + monkeypatch.setattr(gateway_core.Path, "home", classmethod(lambda cls: tmp_path / "home")) + + registry = gateway_core.load_gateway_registry() + entry = { + "name": "hermes-2", + "agent_id": "agent-hermes-2", + "space_id": "space-1", + "base_url": "https://paxai.app", + "template_id": "hermes", + "runtime_type": "exec", + "exec_command": "python3 examples/hermes_sentinel/hermes_bridge.py", + "workdir": str(workdir), + "token_file": str(tmp_path / "token"), + "desired_state": "running", + "created_via": "cli", + } + Path(entry["token_file"]).write_text("axp_a_agent.secret") + gateway_core.ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) + registry["agents"] = [entry] + + daemon = gateway_core.GatewayDaemon(client_factory=lambda **kwargs: _SharedRuntimeClient({})) + reconciled = daemon._reconcile_registry(registry, {"token": "axp_u_test.token", "base_url": "https://paxai.app"}) + agent = reconciled["agents"][0] + + assert daemon._runtimes == {} + assert agent["effective_state"] == "error" + assert agent["presence"] == "ERROR" + assert agent["confidence"] == "BLOCKED" + assert agent["confidence_reason"] == "setup_blocked" + assert "Hermes checkout not found" in str(agent["last_error"]) + assert "Hermes checkout not found" in str(agent["confidence_detail"]) + + +def test_gateway_approvals_approve_updates_binding(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + registry = gateway_core.load_gateway_registry() + entry = { + "name": "approve-bot", + "agent_id": "agent-approve-1", + "runtime_type": "exec", + "exec_command": "python3 worker.py", + "workdir": str(tmp_path / "repo-a"), + "created_via": "cli", + } + gateway_core.ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) + drifted = dict(entry) + drifted["workdir"] = str(tmp_path / "repo-b") + attestation = gateway_core.evaluate_runtime_attestation(registry, drifted) + gateway_core.save_gateway_registry(registry) + + result = runner.invoke(app, ["gateway", "approvals", "approve", attestation["approval_id"], "--scope", "gateway", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["approval"]["status"] == "approved" + assert payload["approval"]["decision_scope"] == "gateway" + stored = gateway_core.load_gateway_registry() + binding = gateway_core.find_binding(stored, install_id=entry["install_id"]) + assert binding is not None + assert binding["path"] == str(Path(drifted["workdir"]).expanduser()) + assert binding["approval_scope"] == "gateway" + + +def test_gateway_approvals_deny_marks_request_rejected(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + registry = gateway_core.load_gateway_registry() + entry = { + "name": "deny-bot", + "agent_id": "agent-deny-1", + "runtime_type": "exec", + "exec_command": "python3 worker.py", + "workdir": str(tmp_path / "repo-a"), + "created_via": "cli", + } + gateway_core.ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) + drifted = dict(entry) + drifted["workdir"] = str(tmp_path / "repo-b") + attestation = gateway_core.evaluate_runtime_attestation(registry, drifted) + gateway_core.save_gateway_registry(registry) + + result = runner.invoke(app, ["gateway", "approvals", "deny", attestation["approval_id"], "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["approval"]["status"] == "rejected" + stored = gateway_core.load_gateway_registry() + approval = next(item for item in stored["approvals"] if item["approval_id"] == attestation["approval_id"]) + assert approval["status"] == "rejected" + + def test_sanitize_exec_env_strips_ax_credentials(monkeypatch): monkeypatch.setenv("AX_TOKEN", "secret-token") monkeypatch.setenv("AX_USER_TOKEN", "secret-user") @@ -477,55 +958,7 @@ def test_annotate_runtime_health_marks_stale_after_missed_heartbeat(): assert snapshot["last_seen_age_seconds"] >= gateway_core.RUNTIME_STALE_AFTER_SECONDS -def test_listener_timeout_enters_reconnecting_state(tmp_path, monkeypatch): - config_dir = tmp_path / "config" - config_dir.mkdir() - monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) - token_file = tmp_path / "token" - token_file.write_text("axp_a_agent.secret") - - class _TimeoutRuntimeClient: - def __init__(self): - self.timeout = None - - def connect_sse(self, *, space_id, timeout=None): - self.timeout = timeout - raise httpx.ReadTimeout("boom", request=httpx.Request("GET", "https://paxai.app/api/v1/sse/messages")) - - def close(self): - return None - - shared = _TimeoutRuntimeClient() - runtime = gateway_core.ManagedAgentRuntime( - { - "name": "echo-bot", - "agent_id": "agent-1", - "space_id": "space-1", - "base_url": "https://paxai.app", - "runtime_type": "echo", - "token_file": str(token_file), - }, - client_factory=lambda **kwargs: shared, - ) - - runtime.start() - deadline = time.time() + 1.0 - snapshot = runtime.snapshot() - while time.time() < deadline and snapshot["effective_state"] != "reconnecting": - time.sleep(0.05) - snapshot = runtime.snapshot() - runtime.stop() - - assert shared.timeout is not None - assert shared.timeout.read == gateway_core.SSE_IDLE_TIMEOUT_SECONDS - assert snapshot["effective_state"] == "reconnecting" - assert snapshot["last_error"] == "idle timeout after 45s without SSE heartbeat" - recent = gateway_core.load_recent_gateway_activity() - assert recent[-1]["event"] in {"runtime_stopped", "listener_timeout"} - assert any(row["event"] == "listener_timeout" for row in recent) - - -def test_gateway_watch_once_renders_dashboard(monkeypatch, tmp_path): +def test_annotate_runtime_health_derives_identity_space_snapshot(monkeypatch, tmp_path): config_dir = tmp_path / "config" monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) gateway_core.save_gateway_session( @@ -533,72 +966,887 @@ def test_gateway_watch_once_renders_dashboard(monkeypatch, tmp_path): "token": "axp_u_test.token", "base_url": "https://paxai.app", "space_id": "space-1", + "space_name": "ax-cli-dev", "username": "codex", } ) + token_file = tmp_path / "identity.token" + token_file.write_text("axp_a_agent.secret") registry = gateway_core.load_gateway_registry() - registry["gateway"].update( - { - "gateway_id": "gw-12345678", - "desired_state": "running", - "effective_state": "running", - "last_reconcile_at": datetime.now(timezone.utc).isoformat(), - } - ) registry["agents"] = [ { - "name": "echo-bot", + "name": "identity-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", "runtime_type": "echo", + "credential_source": "gateway", + "token_file": str(token_file), "desired_state": "running", "effective_state": "running", - "backlog_depth": 2, - "processed_count": 7, "last_seen_at": datetime.now(timezone.utc).isoformat(), - "last_reply_preview": "Echo: ping", + "install_id": "inst-identity-1", } ] - gateway_core.save_gateway_registry(registry) - gateway_core.record_gateway_activity("message_received", entry=registry["agents"][0], message_id="msg-1") + gateway_core.ensure_gateway_identity_binding(registry, registry["agents"][0], session=gateway_core.load_gateway_session()) - result = runner.invoke(app, ["gateway", "watch", "--once"]) + snapshot = gateway_core.annotate_runtime_health(registry["agents"][0], registry=registry) - assert result.exit_code == 0, result.output - assert "Gateway Overview" in result.output - assert "Managed Agents" in result.output - assert "@echo-bot" in result.output - assert "Recent Activity" in result.output + assert snapshot["acting_agent_name"] == "identity-bot" + assert snapshot["environment_label"] == "prod" + assert snapshot["environment_status"] == "environment_allowed" + assert snapshot["active_space_id"] == "space-1" + assert snapshot["active_space_source"] == "gateway_binding" + assert snapshot["space_status"] == "active_allowed" + assert snapshot["identity_status"] == "verified" + assert snapshot["confidence"] == "HIGH" -def test_gateway_agents_show_json_filters_activity(monkeypatch, tmp_path): +def test_annotate_runtime_health_blocks_environment_mismatch(monkeypatch, tmp_path): config_dir = tmp_path / "config" monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) - gateway_core.save_gateway_session( - { - "token": "axp_u_test.token", - "base_url": "https://paxai.app", - "space_id": "space-1", - "username": "codex", - } - ) + token_file = tmp_path / "identity.token" + token_file.write_text("axp_a_agent.secret") registry = gateway_core.load_gateway_registry() registry["agents"] = [ { - "name": "echo-bot", + "name": "identity-bot", "agent_id": "agent-1", "space_id": "space-1", + "base_url": "https://paxai.app", "runtime_type": "echo", + "credential_source": "gateway", + "token_file": str(token_file), "desired_state": "running", "effective_state": "running", "last_seen_at": datetime.now(timezone.utc).isoformat(), - "last_reply_preview": "Echo: ping", - "token_file": "/tmp/echo-token", - }, - { - "name": "other-bot", - "agent_id": "agent-2", - "space_id": "space-1", - "runtime_type": "exec", - "desired_state": "running", + "install_id": "inst-identity-1", + } + ] + gateway_core.ensure_gateway_identity_binding(registry, registry["agents"][0]) + + snapshot = gateway_core.annotate_runtime_health( + {**registry["agents"][0], "base_url": "https://dev.paxai.app"}, + registry=registry, + ) + + assert snapshot["environment_status"] == "environment_mismatch" + assert snapshot["presence"] == "BLOCKED" + assert snapshot["confidence"] == "BLOCKED" + assert snapshot["confidence_reason"] == "environment_mismatch" + + +@pytest.mark.parametrize( + ("input_snapshot", "expected"), + [ + ( + { + "template_id": "claude_code_channel", + "placement": "attached", + "activation": "attach_only", + "reply_mode": "interactive", + "effective_state": "stale", + }, + { + "mode": "LIVE", + "presence": "STALE", + "reply": "REPLY", + "confidence": "LOW", + "reachability": "attach_required", + }, + ), + ( + { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "effective_state": "stopped", + }, + { + "mode": "LIVE", + "presence": "OFFLINE", + "reply": "REPLY", + "confidence": "LOW", + "reachability": "unavailable", + }, + ), + ( + { + "runtime_type": "inbox", + "placement": "mailbox", + "activation": "queue_worker", + "reply_mode": "summary_only", + "effective_state": "running", + "last_seen_at": datetime.now(timezone.utc).isoformat(), + "backlog_depth": 0, + }, + { + "mode": "INBOX", + "presence": "IDLE", + "reply": "SUMMARY", + "confidence": "HIGH", + "reachability": "queue_available", + }, + ), + ( + { + "runtime_type": "inbox", + "placement": "mailbox", + "activation": "queue_worker", + "reply_mode": "summary_only", + "effective_state": "running", + "last_seen_at": datetime.now(timezone.utc).isoformat(), + "backlog_depth": 3, + "last_doctor_result": { + "status": "failed", + "summary": "Queue writable but worker smoke test failed.", + "checks": [{"name": "test_claim", "status": "failed"}], + }, + }, + { + "mode": "INBOX", + "presence": "QUEUED", + "reply": "SUMMARY", + "confidence": "LOW", + "reachability": "queue_available", + }, + ), + ( + { + "template_id": "ollama", + "placement": "hosted", + "activation": "on_demand", + "reply_mode": "interactive", + "effective_state": "stopped", + }, + { + "mode": "ON-DEMAND", + "presence": "IDLE", + "reply": "REPLY", + "confidence": "MEDIUM", + "reachability": "launch_available", + }, + ), + ( + { + "template_id": "hermes", + "effective_state": "error", + "reply_mode": "interactive", + "last_error": "missing repo", + }, + { + "mode": "LIVE", + "presence": "ERROR", + "reply": "REPLY", + "confidence": "BLOCKED", + "reachability": "unavailable", + }, + ), + ], +) +def test_annotate_runtime_health_derives_gateway_operator_model(input_snapshot, expected): + snapshot = gateway_core.annotate_runtime_health(input_snapshot) + + assert snapshot["mode"] == expected["mode"] + assert snapshot["presence"] == expected["presence"] + assert snapshot["reply"] == expected["reply"] + assert snapshot["confidence"] == expected["confidence"] + assert snapshot["reachability"] == expected["reachability"] + + +def test_annotate_runtime_health_prefers_doctor_summary_for_setup_error_detail(): + snapshot = gateway_core.annotate_runtime_health( + { + "template_id": "hermes", + "effective_state": "error", + "last_reply_preview": "(stderr: ERROR: hermes-agent repo not found at /Users/jacob/hermes-agent. Set HERMES_REPO_PATH or clone hermes-agent.)", + "last_doctor_result": { + "status": "failed", + "summary": "Hermes checkout not found at /Users/jacob/hermes-agent.", + "checks": [{"name": "hermes_repo", "status": "failed"}], + }, + } + ) + + assert snapshot["confidence"] == "BLOCKED" + assert snapshot["confidence_reason"] == "setup_blocked" + assert snapshot["confidence_detail"] == "Hermes checkout not found at /Users/jacob/hermes-agent." + + +def test_hermes_setup_status_prefers_sibling_checkout(monkeypatch, tmp_path): + workdir = tmp_path / "workspace" / "ax-cli" + sibling = tmp_path / "workspace" / "hermes-agent" + workdir.mkdir(parents=True) + sibling.mkdir(parents=True) + monkeypatch.delenv("HERMES_REPO_PATH", raising=False) + monkeypatch.setattr(gateway_core.Path, "home", classmethod(lambda cls: tmp_path / "home")) + + status = gateway_core.hermes_setup_status({"template_id": "hermes", "workdir": str(workdir)}) + + assert status["ready"] is True + assert status["resolved_path"] == str(sibling) + + +def test_sanitize_exec_env_sets_resolved_hermes_repo_path(): + env = gateway_core.sanitize_exec_env( + "Gateway test OK.", + { + "agent_id": "agent-hermes-2", + "name": "hermes-2", + "runtime_type": "exec", + "hermes_repo_path": "/tmp/hermes-agent", + }, + ) + + assert env["HERMES_REPO_PATH"] == "/tmp/hermes-agent" + + +def test_sanitize_exec_env_sets_ollama_model_override(): + env = gateway_core.sanitize_exec_env( + "Gateway test OK.", + { + "agent_id": "agent-ember-1", + "name": "ember", + "runtime_type": "exec", + "ollama_model": "gemma4:latest", + }, + ) + + assert env["OLLAMA_MODEL"] == "gemma4:latest" + + +def test_ollama_setup_status_recommends_recent_local_chat_model(monkeypatch): + class _FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return { + "models": [ + { + "name": "nomic-embed-text:latest", + "modified_at": "2026-01-06T21:04:28.576252397-08:00", + "details": {"family": "nomic-bert", "families": ["nomic-bert"], "parameter_size": "137M"}, + }, + { + "name": "nemotron-3-nano:latest", + "modified_at": "2025-12-16T14:03:52.946489046-08:00", + "details": {"family": "nemotron_h_moe", "families": ["nemotron_h_moe"], "parameter_size": "31.6B"}, + }, + { + "name": "gemma4:latest", + "modified_at": "2026-04-02T19:28:17.519867961-07:00", + "details": {"family": "gemma4", "families": ["gemma4"], "parameter_size": "8.0B"}, + }, + { + "name": "gpt-oss:120b-cloud", + "modified_at": "2025-11-11T16:50:56.418111483-08:00", + "remote_host": "https://ollama.com:443", + "details": {"family": "gptoss", "families": ["gptoss"], "parameter_size": "116.8B"}, + }, + ] + } + + monkeypatch.setattr(gateway_core.httpx, "get", lambda *args, **kwargs: _FakeResponse()) + + status = gateway_core.ollama_setup_status() + + assert status["server_reachable"] is True + assert status["recommended_model"] == "gemma4:latest" + assert status["available_models"] == [ + "nomic-embed-text:latest", + "nemotron-3-nano:latest", + "gemma4:latest", + "gpt-oss:120b-cloud", + ] + assert status["local_models"] == [ + "nomic-embed-text:latest", + "nemotron-3-nano:latest", + "gemma4:latest", + ] + + +@pytest.mark.parametrize( + ("input_snapshot", "expected"), + [ + ( + { + "template_id": "hermes", + "effective_state": "running", + "last_seen_at": datetime.now(timezone.utc).isoformat(), + }, + { + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "asset_type_label": "Live Listener", + "output_label": "Reply", + "telemetry_shape": "rich", + }, + ), + ( + { + "template_id": "ollama", + "effective_state": "stopped", + }, + { + "asset_class": "interactive_agent", + "intake_model": "launch_on_send", + "asset_type_label": "On-Demand Agent", + "output_label": "Reply", + "telemetry_shape": "basic", + }, + ), + ( + { + "runtime_type": "inbox", + "template_id": "inbox", + "effective_state": "running", + "last_seen_at": datetime.now(timezone.utc).isoformat(), + }, + { + "asset_class": "background_worker", + "intake_model": "queue_accept", + "asset_type_label": "Inbox Worker", + "output_label": "Summary", + "telemetry_shape": "basic", + "worker_model": "queue_drain", + }, + ), + ], +) +def test_annotate_runtime_health_derives_asset_taxonomy_fields(input_snapshot, expected): + snapshot = gateway_core.annotate_runtime_health(input_snapshot) + + for key, value in expected.items(): + assert snapshot[key] == value + assert isinstance(snapshot["asset_descriptor"], dict) + assert snapshot["asset_descriptor"]["asset_class"] == expected["asset_class"] + + +def test_listener_timeout_enters_reconnecting_state(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + token_file = tmp_path / "token" + token_file.write_text("axp_a_agent.secret") + + class _TimeoutRuntimeClient: + def __init__(self): + self.timeout = None + + def connect_sse(self, *, space_id, timeout=None): + self.timeout = timeout + raise httpx.ReadTimeout("boom", request=httpx.Request("GET", "https://paxai.app/api/v1/sse/messages")) + + def close(self): + return None + + shared = _TimeoutRuntimeClient() + runtime = gateway_core.ManagedAgentRuntime( + { + "name": "echo-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "echo", + "token_file": str(token_file), + }, + client_factory=lambda **kwargs: shared, + ) + + runtime.start() + deadline = time.time() + 1.0 + snapshot = runtime.snapshot() + while time.time() < deadline and snapshot["effective_state"] != "reconnecting": + time.sleep(0.05) + snapshot = runtime.snapshot() + runtime.stop() + + assert shared.timeout is not None + assert shared.timeout.read == gateway_core.SSE_IDLE_TIMEOUT_SECONDS + assert snapshot["effective_state"] == "reconnecting" + assert snapshot["last_error"] == "idle timeout after 45s without SSE heartbeat" + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] in {"runtime_stopped", "listener_timeout"} + assert any(row["event"] == "listener_timeout" for row in recent) + + +def test_gateway_watch_once_renders_dashboard(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + registry = gateway_core.load_gateway_registry() + registry["gateway"].update( + { + "gateway_id": "gw-12345678", + "desired_state": "running", + "effective_state": "running", + "last_reconcile_at": datetime.now(timezone.utc).isoformat(), + } + ) + registry["agents"] = [ + { + "name": "echo-bot", + "runtime_type": "echo", + "desired_state": "running", + "effective_state": "running", + "backlog_depth": 2, + "processed_count": 7, + "last_seen_at": datetime.now(timezone.utc).isoformat(), + "last_reply_preview": "Echo: ping", + } + ] + gateway_core.save_gateway_registry(registry) + gateway_core.record_gateway_activity("message_received", entry=registry["agents"][0], message_id="msg-1") + + result = runner.invoke(app, ["gateway", "watch", "--once"]) + + assert result.exit_code == 0, result.output + assert "Gateway Overview" in result.output + assert "Managed Agents" in result.output + assert "@echo-bot" in result.output + assert "Recent Activity" in result.output + + +def test_render_gateway_ui_page_contains_local_dashboard_shell(): + page = gateway_cmd._render_gateway_ui_page(refresh_ms=2000) + + assert "Gateway Control Plane" in page + assert "Agent Operated" in page + assert "/api/status" in page + assert "/api/templates" in page + assert "/api/agents/<name>" in page + assert "refreshMs = 2000" in page + assert "Gateway Agent Setup" in page + assert "gateway-agent-setup" in page + assert "Agent Type" in page + assert "Output" in page + assert "Advanced launch settings" in page + assert "Alerts" in page + + +def test_gateway_templates_command_json(): + result = runner.invoke(app, ["gateway", "templates", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + ids = [item["id"] for item in payload["templates"]] + assert ids[:4] == ["echo_test", "ollama", "hermes", "claude_code_channel"] + assert "inbox" not in ids + assert payload["count"] == 4 + ollama = next(item for item in payload["templates"] if item["id"] == "ollama") + assert ollama["runtime_type"] == "exec" + assert ollama["launchable"] is True + assert ollama["asset_type_label"] == "On-Demand Agent" + assert ollama["output_label"] == "Reply" + assert ollama["setup_skill"] == "gateway-agent-setup" + assert ollama["setup_skill_path"].endswith("skills/gateway-agent-setup/SKILL.md") + + +def test_gateway_templates_command_json_includes_ollama_catalog(monkeypatch): + monkeypatch.setattr( + gateway_cmd, + "ollama_setup_status", + lambda preferred_model=None: { + "server_reachable": True, + "recommended_model": "gemma4:latest", + "available_models": ["gemma4:latest", "nemotron-3-nano:latest"], + "local_models": ["gemma4:latest", "nemotron-3-nano:latest"], + "summary": "Ollama is reachable. Recommended model: gemma4:latest.", + }, + ) + + result = runner.invoke(app, ["gateway", "templates", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + ollama = next(item for item in payload["templates"] if item["id"] == "ollama") + assert ollama["defaults"]["ollama_model"] == "gemma4:latest" + assert ollama["ollama_recommended_model"] == "gemma4:latest" + assert ollama["ollama_available_models"] == ["gemma4:latest", "nemotron-3-nano:latest"] + + +def test_gateway_runtime_types_command_json(): + result = runner.invoke(app, ["gateway", "runtime-types", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + ids = [item["id"] for item in payload["runtime_types"]] + assert ids == ["echo", "exec", "inbox"] + exec_type = next(item for item in payload["runtime_types"] if item["id"] == "exec") + assert exec_type["signals"]["activity"] + assert exec_type["examples"] + + +def test_gateway_ui_handler_serves_status_and_agent_detail(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + monkeypatch.setattr(gateway_core, "_scan_gateway_process_pids", lambda: []) + monkeypatch.setattr(gateway_core, "_scan_gateway_ui_process_pids", lambda: []) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://dev.paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + registry = gateway_core.load_gateway_registry() + registry["gateway"].update( + { + "gateway_id": "gw-ui-12345678", + "desired_state": "running", + "effective_state": "running", + "last_reconcile_at": datetime.now(timezone.utc).isoformat(), + } + ) + registry["agents"] = [ + { + "name": "echo-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "runtime_type": "echo", + "desired_state": "running", + "effective_state": "running", + "last_seen_at": datetime.now(timezone.utc).isoformat(), + "last_reply_preview": "Echo: ping", + "token_file": "/tmp/echo-token", + "transport": "gateway", + "credential_source": "gateway", + } + ] + gateway_core.save_gateway_registry(registry) + gateway_core.record_gateway_activity("reply_sent", entry=registry["agents"][0], reply_preview="Echo: ping") + + handler = gateway_cmd._build_gateway_ui_handler(activity_limit=5, refresh_ms=1500) + with closing(socket.socket()) as probe: + probe.bind(("127.0.0.1", 0)) + host, port = probe.getsockname() + server = gateway_cmd._GatewayUiServer((host, port), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with httpx.Client(base_url=f"http://{host}:{port}", timeout=2.0) as client: + status = client.get("/api/status") + assert status.status_code == 200 + status_payload = status.json() + assert status_payload["gateway"]["gateway_id"] == "gw-ui-12345678" + assert status_payload["agents"][0]["name"] == "echo-bot" + assert status_payload["agents"][0]["mode"] == "LIVE" + assert status_payload["agents"][0]["presence"] == "IDLE" + assert status_payload["agents"][0]["reply"] == "REPLY" + assert status_payload["agents"][0]["confidence"] == "HIGH" + assert status_payload["summary"]["alert_count"] >= 1 + assert status_payload["alerts"][0]["title"] == "Gateway daemon is stopped" + + runtime_types = client.get("/api/runtime-types") + assert runtime_types.status_code == 200 + runtime_payload = runtime_types.json() + assert runtime_payload["count"] == 3 + assert runtime_payload["runtime_types"][1]["id"] == "exec" + + templates = client.get("/api/templates") + assert templates.status_code == 200 + template_payload = templates.json() + assert template_payload["templates"][0]["id"] == "echo_test" + assert template_payload["templates"][3]["launchable"] is False + assert template_payload["count"] == 4 + + detail = client.get("/api/agents/echo-bot") + assert detail.status_code == 200 + detail_payload = detail.json() + assert detail_payload["agent"]["name"] == "echo-bot" + assert detail_payload["recent_activity"][0]["event"] == "reply_sent" + + page = client.get("/") + assert page.status_code == 200 + assert "Gateway Control Plane" in page.text + assert "refreshMs = 1500" in page.text + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2.0) + + +def test_gateway_ui_handler_supports_agent_mutations(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://dev.paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + monkeypatch.setattr(gateway_cmd, "_load_gateway_user_client", lambda: _FakeUserClient()) + monkeypatch.setattr(gateway_cmd, "_find_agent_in_space", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_create_agent_in_space", _fake_create_agent_in_space) + monkeypatch.setattr(gateway_cmd, "_polish_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_mint_agent_pat", lambda *args, **kwargs: ("axp_a_agent.secret", "mgmt")) + monkeypatch.setattr(gateway_cmd, "AxClient", _FakeManagedSendClient) + + handler = gateway_cmd._build_gateway_ui_handler(activity_limit=5, refresh_ms=1500) + with closing(socket.socket()) as probe: + probe.bind(("127.0.0.1", 0)) + host, port = probe.getsockname() + server = gateway_cmd._GatewayUiServer((host, port), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with httpx.Client(base_url=f"http://{host}:{port}", timeout=2.0) as client: + created = client.post( + "/api/agents", + json={ + "name": "ui-bot", + "template_id": "echo_test", + }, + ) + assert created.status_code == 201 + assert created.json()["name"] == "ui-bot" + assert created.json()["template_label"] == "Echo (Test)" + + updated = client.put( + "/api/agents/ui-bot", + json={ + "template_id": "ollama", + "workdir": str(tmp_path), + "exec_command": "python3 examples/gateway_ollama/ollama_bridge.py", + }, + ) + assert updated.status_code == 200 + updated_payload = updated.json() + assert updated_payload["template_id"] == "ollama" + assert updated_payload["workdir"] == str(tmp_path) + + stopped = client.post("/api/agents/ui-bot/stop", json={}) + assert stopped.status_code == 200 + assert stopped.json()["desired_state"] == "stopped" + + started = client.post("/api/agents/ui-bot/start", json={}) + assert started.status_code == 200 + assert started.json()["desired_state"] == "running" + + sent = client.post( + "/api/agents/ui-bot/send", + json={"content": "hello there", "to": "codex"}, + ) + assert sent.status_code == 201 + sent_payload = sent.json() + assert sent_payload["agent"] == "ui-bot" + assert sent_payload["content"] == "@codex hello there" + + tested = client.post("/api/agents/ui-bot/test", json={}) + assert tested.status_code == 201 + tested_payload = tested.json() + assert tested_payload["target_agent"] == "ui-bot" + assert tested_payload["author"] == "agent" + assert tested_payload["sender_agent"].startswith("switchboard-") + assert tested_payload["content"] == "@ui-bot Reply with exactly: Gateway test OK. Then mention which local model answered." + + doctored = client.post("/api/agents/ui-bot/doctor", json={}) + assert doctored.status_code == 201 + doctor_payload = doctored.json() + assert doctor_payload["name"] == "ui-bot" + assert doctor_payload["status"] in {"passed", "warning", "failed"} + + removed = client.delete("/api/agents/ui-bot") + assert removed.status_code == 200 + assert removed.json()["name"] == "ui-bot" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2.0) + + +def test_gateway_agents_update_changes_template_and_workdir(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + token_file = tmp_path / "echo.token" + token_file.write_text("axp_a_agent.secret") + registry = gateway_core.load_gateway_registry() + entry = { + "name": "northstar", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "echo", + "template_id": "echo_test", + "template_label": "Echo (Test)", + "desired_state": "running", + "effective_state": "running", + "token_file": str(token_file), + "transport": "gateway", + "credential_source": "gateway", + "created_via": "cli", + } + registry["agents"] = [entry] + gateway_core.ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) + gateway_core.ensure_gateway_identity_binding(registry, entry, session=gateway_core.load_gateway_session()) + gateway_core.save_gateway_registry(registry) + monkeypatch.setattr(gateway_cmd, "_load_gateway_user_client", lambda: _FakeUserClient()) + + result = runner.invoke( + app, + [ + "gateway", + "agents", + "update", + "northstar", + "--template", + "ollama", + "--workdir", + str(tmp_path), + "--exec", + "python3 examples/gateway_ollama/ollama_bridge.py", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["template_id"] == "ollama" + assert payload["runtime_type"] == "exec" + assert payload["workdir"] == str(tmp_path) + stored = gateway_core.load_gateway_registry()["agents"][0] + assert stored["template_id"] == "ollama" + assert stored["workdir"] == str(tmp_path) + + +def test_gateway_agents_add_ollama_persists_model_override(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + monkeypatch.setattr(gateway_cmd, "_load_gateway_user_client", lambda: _FakeUserClient()) + monkeypatch.setattr(gateway_cmd, "_find_agent_in_space", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_create_agent_in_space", _fake_create_agent_in_space) + monkeypatch.setattr(gateway_cmd, "_polish_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_mint_agent_pat", lambda *args, **kwargs: ("axp_a_agent.secret", "mgmt")) + + result = runner.invoke( + app, + [ + "gateway", + "agents", + "add", + "ember", + "--template", + "ollama", + "--ollama-model", + "gemma4:latest", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["template_id"] == "ollama" + assert payload["ollama_model"] == "gemma4:latest" + stored = gateway_core.load_gateway_registry()["agents"][0] + assert stored["ollama_model"] == "gemma4:latest" + + +def test_gateway_agents_add_ollama_uses_recommended_model_when_unspecified(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + monkeypatch.setattr(gateway_cmd, "_load_gateway_user_client", lambda: _FakeUserClient()) + monkeypatch.setattr(gateway_cmd, "_find_agent_in_space", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_create_agent_in_space", _fake_create_agent_in_space) + monkeypatch.setattr(gateway_cmd, "_polish_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_mint_agent_pat", lambda *args, **kwargs: ("axp_a_agent.secret", "mgmt")) + monkeypatch.setattr( + gateway_cmd, + "ollama_setup_status", + lambda preferred_model=None: { + "recommended_model": "gemma4:latest", + "server_reachable": True, + "available_models": ["gemma4:latest"], + "local_models": ["gemma4:latest"], + "summary": "Ollama is reachable. Recommended model: gemma4:latest.", + }, + ) + + result = runner.invoke( + app, + [ + "gateway", + "agents", + "add", + "ember-default", + "--template", + "ollama", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["template_id"] == "ollama" + assert payload["ollama_model"] == "gemma4:latest" + stored = gateway_core.load_gateway_registry()["agents"][0] + assert stored["ollama_model"] == "gemma4:latest" + + +def test_gateway_agents_show_json_filters_activity(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "echo-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "runtime_type": "echo", + "desired_state": "running", + "effective_state": "running", + "last_seen_at": datetime.now(timezone.utc).isoformat(), + "last_reply_preview": "Echo: ping", + "token_file": "/tmp/echo-token", + }, + { + "name": "other-bot", + "agent_id": "agent-2", + "space_id": "space-1", + "runtime_type": "exec", + "desired_state": "running", "effective_state": "running", "last_seen_at": datetime.now(timezone.utc).isoformat(), "token_file": "/tmp/other-token", @@ -657,3 +1905,237 @@ def test_gateway_agents_send_uses_managed_identity(monkeypatch, tmp_path): assert payload["message"]["metadata"]["gateway"]["sent_via"] == "gateway_cli" recent = gateway_core.load_recent_gateway_activity() assert recent[-1]["event"] == "manual_message_sent" + + +def test_gateway_agents_send_blocks_identity_mismatch(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + token_file = tmp_path / "sender.token" + token_file.write_text("axp_a_agent.secret") + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "sender-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "inbox", + "desired_state": "running", + "effective_state": "running", + "token_file": str(token_file), + "transport": "gateway", + "credential_source": "gateway", + "install_id": "inst-sender-1", + } + ] + gateway_core.ensure_gateway_identity_binding(registry, registry["agents"][0], session=gateway_core.load_gateway_session()) + registry["identity_bindings"][0]["acting_identity"]["agent_name"] = "night_owl" + gateway_core.save_gateway_registry(registry) + monkeypatch.setattr(gateway_cmd, "AxClient", _FakeManagedSendClient) + + result = runner.invoke(app, ["gateway", "agents", "send", "sender-bot", "hello there", "--to", "codex", "--json"]) + + assert result.exit_code == 1, result.output + assert "identity_mismatch" in result.output.lower() or "mismatched acting identity" in result.output.lower() + + +def test_gateway_agents_test_sends_gateway_authored_probe(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "echo-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "echo", + "template_id": "echo_test", + "desired_state": "running", + "effective_state": "running", + "transport": "gateway", + "credential_source": "gateway", + } + ] + gateway_core.save_gateway_registry(registry) + monkeypatch.setattr(gateway_cmd, "_load_gateway_user_client", lambda: _FakeUserClient()) + monkeypatch.setattr(gateway_cmd, "_find_agent_in_space", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_create_agent_in_space", _fake_create_agent_in_space) + monkeypatch.setattr(gateway_cmd, "_polish_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(gateway_cmd, "_mint_agent_pat", lambda *args, **kwargs: ("axp_a_agent.secret", "mgmt")) + monkeypatch.setattr(gateway_cmd, "AxClient", _FakeManagedSendClient) + + result = runner.invoke(app, ["gateway", "agents", "test", "echo-bot", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["target_agent"] == "echo-bot" + assert payload["author"] == "agent" + assert payload["sender_agent"] == "switchboard-space1" + assert payload["recommended_prompt"] == "gateway test ping" + assert payload["content"] == "@echo-bot gateway test ping" + assert payload["message"]["metadata"]["gateway"]["sent_via"] == "gateway_test" + assert payload["message"]["metadata"]["gateway"]["test_author"] == "agent" + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] == "gateway_test_sent" + + +def test_gateway_agents_test_can_send_as_user(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "echo-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "echo", + "template_id": "echo_test", + "desired_state": "running", + "effective_state": "running", + "transport": "gateway", + "credential_source": "gateway", + } + ] + gateway_core.save_gateway_registry(registry) + monkeypatch.setattr(gateway_cmd, "_load_gateway_user_client", lambda: _FakeUserClient()) + + result = runner.invoke(app, ["gateway", "agents", "test", "echo-bot", "--author", "user", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["author"] == "user" + assert payload["sender_agent"] is None + assert payload["message"]["metadata"]["gateway"]["test_author"] == "user" + + +def test_gateway_agents_doctor_persists_structured_result(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + token_file = tmp_path / "inbox.token" + token_file.write_text("axp_a_agent.secret") + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "docs-worker", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "inbox", + "template_id": "inbox", + "desired_state": "running", + "effective_state": "stopped", + "token_file": str(token_file), + "transport": "gateway", + "credential_source": "gateway", + } + ] + gateway_core.save_gateway_registry(registry) + + result = runner.invoke(app, ["gateway", "agents", "doctor", "docs-worker", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["status"] == "warning" + check_names = [item["name"] for item in payload["checks"]] + assert "gateway_auth" in check_names + assert "queue_writable" in check_names + assert "worker_attached" in check_names + assert isinstance(payload["agent"]["last_doctor_result"], dict) + assert payload["agent"]["last_doctor_result"]["status"] == "warning" + assert payload["agent"]["last_doctor_result"]["checks"] + assert payload["agent"]["last_successful_doctor_at"] + + stored = gateway_core.load_gateway_registry()["agents"][0] + assert stored["last_doctor_result"]["status"] == "warning" + assert stored["last_successful_doctor_at"] + + +def test_gateway_status_payload_surfaces_alerts(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "stale-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "runtime_type": "exec", + "desired_state": "running", + "effective_state": "running", + "last_seen_at": (datetime.now(timezone.utc) - timedelta(seconds=gateway_core.RUNTIME_STALE_AFTER_SECONDS + 5)).isoformat(), + "backlog_depth": 2, + "last_error": None, + "token_file": "/tmp/stale-token", + }, + { + "name": "broken-bot", + "agent_id": "agent-2", + "space_id": "space-1", + "runtime_type": "exec", + "desired_state": "running", + "effective_state": "error", + "last_error": "bridge crashed", + "token_file": "/tmp/broken-token", + }, + { + "name": "setup-bot", + "agent_id": "agent-3", + "space_id": "space-1", + "runtime_type": "exec", + "desired_state": "running", + "effective_state": "running", + "last_reply_preview": "(stderr: ERROR: hermes-agent repo not found at /Users/jacob/hermes-agent.)", + "token_file": "/tmp/setup-token", + }, + ] + gateway_core.save_gateway_registry(registry) + + payload = gateway_cmd._status_payload(activity_limit=5) + + assert payload["summary"]["alert_count"] >= 2 + titles = [item["title"] for item in payload["alerts"]] + assert any("@stale-bot looks stale" == title for title in titles) + assert any("@broken-bot hit an error" == title for title in titles) + assert any("@setup-bot has a runtime setup error" == title for title in titles) From db088f3c68f915b2fac38e12762ff94a676cac68 Mon Sep 17 00:00:00 2001 From: anvil Date: Thu, 23 Apr 2026 23:09:21 +0000 Subject: [PATCH 3/3] feat(gateway): preserve hermes sentinel runtime --- README.md | 10 +- ax_cli/commands/gateway.py | 386 ++++++-- ax_cli/gateway.py | 1375 +++++++++++++++++++++++++--- ax_cli/gateway_runtime_types.py | 143 ++- docs/gateway-agent-runtimes.md | 198 ++++ examples/hermes_sentinel/README.md | 10 + tests/test_gateway_commands.py | 339 ++++++- 7 files changed, 2240 insertions(+), 221 deletions(-) create mode 100644 docs/gateway-agent-runtimes.md diff --git a/README.md b/README.md index 2aa6f35..f89c6c2 100644 --- a/README.md +++ b/README.md @@ -131,8 +131,8 @@ The primary agent types start with: - `Echo (Test)` — built-in ping/echo bot for proving the control plane. - `Ollama` — a local-model runtime managed through Gateway. -- `Hermes` — a local Hermes bridge with rich activity and tool telemetry. -- `Claude Code Channel` — planned managed channel adapter. +- `Hermes Sentinel` — a Gateway-managed long-running Hermes coding agent. +- `Claude Code Channel` — an attached Claude Code session managed and observed by Gateway. The lower-level runtime backends still exist under `ax gateway runtime-types` for advanced/debug use, but they are not the main operator-facing choices. @@ -140,6 +140,12 @@ Advanced users can still override launch commands and working directories from the CLI or the UI's advanced launch section when they are building a custom bridge. +See [Gateway Agent Runtimes](docs/gateway-agent-runtimes.md) for the operating +model. The key migration from the original CLI setup is management, not a new +agent brain: Gateway owns credentials, launch specs, lifecycle, status, and +liveness, while Hermes sentinels and Claude Code channels keep the runtime +patterns that already worked. + Managed `exec` runtimes can now emit structured progress lines back to Gateway while they are still working. The bridge prints lines prefixed with `AX_GATEWAY_EVENT ` and Gateway turns them into control-plane activity, diff --git a/ax_cli/commands/gateway.py b/ax_cli/commands/gateway.py index 5131481..49ad317 100644 --- a/ax_cli/commands/gateway.py +++ b/ax_cli/commands/gateway.py @@ -52,6 +52,7 @@ deny_gateway_approval, ensure_gateway_identity_binding, ensure_local_asset_binding, + evaluate_runtime_attestation, find_agent_entry, gateway_dir, get_gateway_approval, @@ -191,7 +192,7 @@ def _normalize_runtime_type(runtime_type: str) -> str: try: return str(runtime_type_definition(runtime_type)["id"]) except KeyError as exc: - raise ValueError("Unsupported runtime type. Use echo, exec, or inbox.") from exc + raise ValueError("Unsupported runtime type. Use echo, exec, hermes_sentinel, sentinel_cli, or inbox.") from exc def _validate_runtime_registration(runtime_type: str, exec_cmd: str | None) -> None: @@ -200,7 +201,7 @@ def _validate_runtime_registration(runtime_type: str, exec_cmd: str | None) -> N if "exec_command" in required and not exec_cmd: raise ValueError("Exec runtimes require --exec.") if "exec_command" not in required and exec_cmd: - raise ValueError("Echo and inbox runtimes do not accept --exec.") + raise ValueError("This runtime does not accept --exec.") def _register_managed_agent( @@ -304,10 +305,13 @@ def _register_managed_agent( ) ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True) ensure_gateway_identity_binding(registry, entry, session=session, created_via="cli") + entry.update(evaluate_runtime_attestation(registry, entry)) hermes_status = hermes_setup_status(entry) if not hermes_status.get("ready", True): entry["effective_state"] = "error" - entry["last_error"] = str(hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete.") + entry["last_error"] = str( + hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete." + ) entry["current_activity"] = str(hermes_status.get("summary") or "Hermes setup is incomplete.") elif hermes_status.get("resolved_path"): entry["hermes_repo_path"] = str(hermes_status["resolved_path"]) @@ -351,7 +355,9 @@ def _update_managed_agent( if not bool(template.get("launchable", True)): raise ValueError(f"Template {template['label']} is not launchable yet.") - runtime_candidate = runtime_type or (template.get("defaults") or {}).get("runtime_type") if template else runtime_type + runtime_candidate = ( + runtime_type or (template.get("defaults") or {}).get("runtime_type") if template else runtime_type + ) runtime_effective = str(runtime_candidate or entry.get("runtime_type") or "echo") runtime_effective = _normalize_runtime_type(runtime_effective) template_effective_id = str(template.get("id") if template else entry.get("template_id") or "").strip().lower() @@ -375,9 +381,7 @@ def _update_managed_agent( else (str(exec_cmd).strip() or None) ) workdir_effective = ( - str(entry.get("workdir") or "").strip() or None - if workdir is _UNSET - else (str(workdir).strip() or None) + str(entry.get("workdir") or "").strip() or None if workdir is _UNSET else (str(workdir).strip() or None) ) if ollama_model is _UNSET: @@ -420,10 +424,14 @@ def _update_managed_agent( entry.pop("hermes_repo_path", None) ensure_gateway_identity_binding(registry, entry, session=session) + ensure_local_asset_binding(registry, entry, created_via="cli", auto_approve=True, replace_existing=True) + entry.update(evaluate_runtime_attestation(registry, entry)) hermes_status = hermes_setup_status(entry) if not hermes_status.get("ready", True): entry["effective_state"] = "error" - entry["last_error"] = str(hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete.") + entry["last_error"] = str( + hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete." + ) entry["current_activity"] = str(hermes_status.get("summary") or "Hermes setup is incomplete.") elif hermes_status.get("resolved_path"): entry["hermes_repo_path"] = str(hermes_status["resolved_path"]) @@ -482,6 +490,54 @@ def _identity_space_send_guard(entry: dict, *, explicit_space_id: str | None = N return snapshot +def _sync_passive_queue_after_manual_send( + *, + entry: dict, + handled_message_id: str | None, + reply_message_id: str | None, + reply_preview: str | None, +) -> None: + runtime_type = str(entry.get("runtime_type") or "").lower() + if runtime_type not in {"inbox", "passive", "monitor"}: + return + + pending_items = gateway_core.remove_agent_pending_message(str(entry.get("name") or ""), handled_message_id) + registry = load_gateway_registry() + stored = find_agent_entry(registry, str(entry.get("name") or "")) or entry + backlog_depth = len(pending_items) + last_pending = pending_items[-1] if pending_items else {} + + if handled_message_id: + stored["processed_count"] = int(stored.get("processed_count") or 0) + 1 + stored["last_work_completed_at"] = datetime.now(timezone.utc).isoformat() + + stored["backlog_depth"] = backlog_depth + stored["current_status"] = "queued" if backlog_depth > 0 else None + stored["current_activity"] = ( + gateway_core._gateway_pickup_activity(runtime_type, backlog_depth)[:240] if backlog_depth > 0 else None + ) + stored["last_reply_message_id"] = reply_message_id or stored.get("last_reply_message_id") + stored["last_reply_preview"] = reply_preview or stored.get("last_reply_preview") + if last_pending: + stored["last_received_message_id"] = last_pending.get("message_id") + stored["last_work_received_at"] = ( + last_pending.get("queued_at") or last_pending.get("created_at") or stored.get("last_work_received_at") + ) + elif handled_message_id: + stored["last_received_message_id"] = None + stored["last_work_received_at"] = None + + save_gateway_registry(registry) + if handled_message_id: + record_gateway_activity( + "manual_queue_acknowledged", + entry=stored, + message_id=handled_message_id, + reply_message_id=reply_message_id, + backlog_depth=backlog_depth, + ) + + def _send_from_managed_agent( *, name: str, @@ -538,6 +594,12 @@ def _send_from_managed_agent( message_id=payload.get("id"), reply_preview=message_content[:120] or None, ) + _sync_passive_queue_after_manual_send( + entry=entry, + handled_message_id=parent_id, + reply_message_id=str(payload.get("id") or "") or None, + reply_preview=message_content[:120] or None, + ) return {"agent": entry.get("name"), "message": payload, "content": message_content} @@ -650,10 +712,16 @@ def push(severity: str, title: str, detail: str, *, agent_name: str | None = Non if not payload.get("connected"): push("error", "Gateway is not logged in", "Run `ax gateway login` to bootstrap the local control plane.") elif not payload.get("daemon", {}).get("running"): - push("error", "Gateway daemon is stopped", "Start it with `uv run ax gateway start` or relaunch the local service.") + push( + "error", + "Gateway daemon is stopped", + "Start it with `uv run ax gateway start` or relaunch the local service.", + ) if not payload.get("ui", {}).get("running"): - push("warning", "Gateway UI is stopped", "Start it with `uv run ax gateway start` to launch the local dashboard.") + push( + "warning", "Gateway UI is stopped", "Start it with `uv run ax gateway start` to launch the local dashboard." + ) for agent in payload.get("agents", []): name = str(agent.get("name") or "") @@ -677,7 +745,10 @@ def push(severity: str, title: str, detail: str, *, agent_name: str | None = Non detail = str(agent.get("confidence_detail") or "Runtime changed since approval and needs review.") push("warning", f"@{name} changed since approval", detail, agent_name=name) elif presence == "BLOCKED": - detail = str(agent.get("confidence_detail") or "Gateway blocked this runtime until identity, space, or approval state is fixed.") + detail = str( + agent.get("confidence_detail") + or "Gateway blocked this runtime until identity, space, or approval state is fixed." + ) push("error", f"@{name} is blocked", detail, agent_name=name) elif presence == "ERROR": if setup_error_preview: @@ -689,7 +760,10 @@ def push(severity: str, title: str, detail: str, *, agent_name: str | None = Non detail = f"No heartbeat for {_format_age(agent.get('last_seen_age_seconds'))}." push("warning", f"@{name} looks stale", detail, agent_name=name) elif presence == "OFFLINE" and str(agent.get("mode") or "") == "LIVE": - detail = str(agent.get("confidence_detail") or "Expected a live runtime, but Gateway does not currently have a working path.") + detail = str( + agent.get("confidence_detail") + or "Expected a live runtime, but Gateway does not currently have a working path." + ) push("warning", f"@{name} is offline", detail, agent_name=name) if setup_error_preview and presence != "ERROR": push("error", f"@{name} has a runtime setup error", preview[:180], agent_name=name) @@ -909,8 +983,16 @@ def _doctor_result_status(checks: list[dict]) -> str: def _doctor_summary(checks: list[dict], status: str) -> str: - failures = [str(item.get("detail") or item.get("name") or "").strip() for item in checks if str(item.get("status") or "").strip().lower() == "failed"] - warnings = [str(item.get("detail") or item.get("name") or "").strip() for item in checks if str(item.get("status") or "").strip().lower() == "warning"] + failures = [ + str(item.get("detail") or item.get("name") or "").strip() + for item in checks + if str(item.get("status") or "").strip().lower() == "failed" + ] + warnings = [ + str(item.get("detail") or item.get("name") or "").strip() + for item in checks + if str(item.get("status") or "").strip().lower() == "warning" + ] if status == "failed" and failures: return failures[0] if status == "warning" and warnings: @@ -965,17 +1047,37 @@ def has_check(check_name: str) -> bool: identity_status = str(snapshot.get("identity_status") or "").lower() if identity_status == "verified": - add_check("identity_binding", "passed", f"Gateway is acting as {snapshot.get('acting_agent_name') or entry.get('name')}.") + add_check( + "identity_binding", + "passed", + f"Gateway is acting as {snapshot.get('acting_agent_name') or entry.get('name')}.", + ) elif identity_status == "bootstrap_only": - add_check("identity_binding", "failed", "Gateway would need to use a bootstrap credential for an agent-authored action.") + add_check( + "identity_binding", + "failed", + "Gateway would need to use a bootstrap credential for an agent-authored action.", + ) else: - add_check("identity_binding", "failed", str(snapshot.get("confidence_detail") or "Gateway does not have a valid acting identity binding.")) + add_check( + "identity_binding", + "failed", + str(snapshot.get("confidence_detail") or "Gateway does not have a valid acting identity binding."), + ) environment_status = str(snapshot.get("environment_status") or "").lower() if environment_status == "environment_allowed": - add_check("environment_binding", "passed", f"Requested environment matches {snapshot.get('environment_label') or snapshot.get('base_url') or entry.get('base_url')}.") + add_check( + "environment_binding", + "passed", + f"Requested environment matches {snapshot.get('environment_label') or snapshot.get('base_url') or entry.get('base_url')}.", + ) elif environment_status == "environment_mismatch": - add_check("environment_binding", "failed", str(snapshot.get("confidence_detail") or "Requested environment does not match the bound environment.")) + add_check( + "environment_binding", + "failed", + str(snapshot.get("confidence_detail") or "Requested environment does not match the bound environment."), + ) else: add_check("environment_binding", "warning", "Gateway could not fully verify the bound environment.") @@ -987,22 +1089,42 @@ def has_check(check_name: str) -> bool: space_status = str(snapshot.get("space_status") or "").lower() if space_status == "active_allowed": - add_check("space_binding", "passed", f"Active space is {snapshot.get('active_space_name') or snapshot.get('active_space_id')}.") + add_check( + "space_binding", + "passed", + f"Active space is {snapshot.get('active_space_name') or snapshot.get('active_space_id')}.", + ) elif space_status == "no_active_space": add_check("space_binding", "failed", "Gateway does not have an active space selected for this asset.") elif space_status == "active_not_allowed": - add_check("space_binding", "failed", str(snapshot.get("confidence_detail") or "Active space is not allowed for this identity.")) + add_check( + "space_binding", + "failed", + str(snapshot.get("confidence_detail") or "Active space is not allowed for this identity."), + ) else: add_check("space_binding", "warning", "Gateway could not fully verify the active space.") attestation_state = str(snapshot.get("attestation_state") or "").lower() approval_state = str(snapshot.get("approval_state") or "").lower() if approval_state == "pending": - add_check("binding_approval", "warning", str(snapshot.get("confidence_detail") or "Gateway needs approval before trusting this runtime binding.")) + add_check( + "binding_approval", + "warning", + str(snapshot.get("confidence_detail") or "Gateway needs approval before trusting this runtime binding."), + ) elif approval_state == "rejected" or attestation_state == "blocked": - add_check("binding_approval", "failed", str(snapshot.get("confidence_detail") or "Gateway blocked this runtime binding.")) + add_check( + "binding_approval", + "failed", + str(snapshot.get("confidence_detail") or "Gateway blocked this runtime binding."), + ) elif attestation_state == "drifted": - add_check("binding_attestation", "failed", str(snapshot.get("confidence_detail") or "Runtime binding drifted from its approved launch spec.")) + add_check( + "binding_attestation", + "failed", + str(snapshot.get("confidence_detail") or "Runtime binding drifted from its approved launch spec."), + ) elif attestation_state == "verified": add_check("binding_attestation", "passed", "Runtime matches the approved local binding.") @@ -1036,7 +1158,9 @@ def has_check(check_name: str) -> bool: elif bool(snapshot.get("connected")): add_check("session_attach", "passed", "Attached session is connected to Gateway.") else: - add_check("session_attach", "failed", "Gateway does not currently have an attached session to supervise.") + add_check( + "session_attach", "failed", "Gateway does not currently have an attached session to supervise." + ) elif runtime_type != "echo": if exec_command: add_check("runtime_launch", "passed", "Gateway has a launch command for this runtime.") @@ -1046,11 +1170,21 @@ def has_check(check_name: str) -> bool: if runtime_type == "echo" or exec_command: add_check("launch_ready", "passed", "Gateway can launch this runtime when work arrives.") else: - add_check("launch_ready", "failed", "Gateway does not have a launch command for this on-demand runtime.") + add_check( + "launch_ready", "failed", "Gateway does not have a launch command for this on-demand runtime." + ) elif intake_model == "scheduled_run": - add_check("schedule_ready", "warning", "Scheduled asset support is taxonomy-defined but not fully implemented in Gateway yet.") + add_check( + "schedule_ready", + "warning", + "Scheduled asset support is taxonomy-defined but not fully implemented in Gateway yet.", + ) elif intake_model == "event_triggered": - add_check("event_source", "warning", "Alert-driven asset support is taxonomy-defined but not fully implemented in Gateway yet.") + add_check( + "event_source", + "warning", + "Alert-driven asset support is taxonomy-defined but not fully implemented in Gateway yet.", + ) elif asset_class == "service_proxy": if exec_command: add_check("runtime_launch", "passed", "Gateway has a launch command for this runtime.") @@ -1079,7 +1213,9 @@ def has_check(check_name: str) -> bool: else: recommended_model = str(ollama_status.get("recommended_model") or "").strip() if recommended_model: - add_check("ollama_model", "passed", f"Gateway will use the recommended local model {recommended_model}.") + add_check( + "ollama_model", "passed", f"Gateway will use the recommended local model {recommended_model}." + ) else: add_check("ollama_model", "warning", "No Ollama model is selected yet.") add_check("launch_path", "passed", "Gateway can launch the Ollama bridge on send.") @@ -1257,7 +1393,12 @@ def _render_gateway_overview(payload: dict) -> Panel: grid.add_column(ratio=2) grid.add_column(style="bold") grid.add_column(ratio=2) - grid.add_row("Gateway", str(gateway.get("gateway_id") or "-")[:8], "Daemon", "running" if payload["daemon"]["running"] else "stopped") + grid.add_row( + "Gateway", + str(gateway.get("gateway_id") or "-")[:8], + "Daemon", + "running" if payload["daemon"]["running"] else "stopped", + ) grid.add_row("User", str(payload.get("user") or "-"), "Base URL", str(payload.get("base_url") or "-")) space_label = str(payload.get("space_name") or payload.get("space_id") or "-") grid.add_row("Space", space_label, "PID", str(payload["daemon"].get("pid") or "-")) @@ -1285,7 +1426,19 @@ def _render_agent_table(agents: list[dict]) -> Table: table.add_column("Seen", justify="right") table.add_column("Activity", overflow="fold") if not agents: - table.add_row("No managed agents", "-", Text("ON-DEMAND", style="dim"), Text("OFFLINE", style="dim"), Text("Reply", style="dim"), Text("MEDIUM", style="dim"), "-", "-", "0", "-", "-") + table.add_row( + "No managed agents", + "-", + Text("ON-DEMAND", style="dim"), + Text("OFFLINE", style="dim"), + Text("Reply", style="dim"), + Text("MEDIUM", style="dim"), + "-", + "-", + "0", + "-", + "-", + ) return table for agent in _sorted_agents(agents): activity = str( @@ -1300,7 +1453,10 @@ def _render_agent_table(agents: list[dict]) -> Table: _agent_type_label(agent), _mode_text(agent.get("mode")), _presence_text(agent.get("presence")), - Text(_agent_output_label(agent), style="green" if str(agent.get("output_label") or "").lower() == "reply" else "yellow"), + Text( + _agent_output_label(agent), + style="green" if str(agent.get("output_label") or "").lower() == "reply" else "yellow", + ), _confidence_text(agent.get("confidence")), str(agent.get("acting_agent_name") or agent.get("name") or "-"), str(agent.get("active_space_name") or agent.get("active_space_id") or agent.get("space_id") or "-"), @@ -1384,7 +1540,9 @@ def _render_gateway_dashboard(payload: dict) -> Group: metrics, Panel(_render_alert_table(payload.get("alerts", [])), title="Alerts", border_style="red"), Panel(_render_agent_table(agents), title="Managed Agents", border_style="green"), - Panel(_render_activity_table(payload.get("recent_activity", [])), title="Recent Activity", border_style="magenta"), + Panel( + _render_activity_table(payload.get("recent_activity", [])), title="Recent Activity", border_style="magenta" + ), ) @@ -2992,27 +3150,93 @@ def _render_agent_detail(entry: dict, *, activity: list[dict]) -> Group: overview.add_row("Template", _agent_template_label(entry), "Output", _agent_output_label(entry)) overview.add_row("Mode", str(entry.get("mode") or "-"), "Presence", str(entry.get("presence") or "-")) overview.add_row("Reply", str(entry.get("reply") or "-"), "Confidence", str(entry.get("confidence") or "-")) - overview.add_row("Asset Class", str(entry.get("asset_class") or "-"), "Intake", str(entry.get("intake_model") or "-")) - overview.add_row("Trigger", str((entry.get("trigger_sources") or [None])[0] or "-"), "Return", str((entry.get("return_paths") or [None])[0] or "-")) - overview.add_row("Telemetry", str(entry.get("telemetry_shape") or "-"), "Worker", str(entry.get("worker_model") or "-")) - overview.add_row("Attestation", str(entry.get("attestation_state") or "-"), "Approval", str(entry.get("approval_state") or "-")) - overview.add_row("Acting As", str(entry.get("acting_agent_name") or "-"), "Identity", str(entry.get("identity_status") or "-")) - overview.add_row("Environment", str(entry.get("environment_label") or entry.get("base_url") or "-"), "Env Status", str(entry.get("environment_status") or "-")) - overview.add_row("Current Space", str(entry.get("active_space_name") or entry.get("active_space_id") or "-"), "Space Status", str(entry.get("space_status") or "-")) - overview.add_row("Default Space", str(entry.get("default_space_name") or entry.get("default_space_id") or "-"), "Allowed Spaces", str(entry.get("allowed_space_count") or 0)) - overview.add_row("Install", str(entry.get("install_id") or "-"), "Runtime Instance", str(entry.get("runtime_instance_id") or "-")) + overview.add_row( + "Asset Class", str(entry.get("asset_class") or "-"), "Intake", str(entry.get("intake_model") or "-") + ) + overview.add_row( + "Trigger", + str((entry.get("trigger_sources") or [None])[0] or "-"), + "Return", + str((entry.get("return_paths") or [None])[0] or "-"), + ) + overview.add_row( + "Telemetry", str(entry.get("telemetry_shape") or "-"), "Worker", str(entry.get("worker_model") or "-") + ) + overview.add_row( + "Attestation", str(entry.get("attestation_state") or "-"), "Approval", str(entry.get("approval_state") or "-") + ) + overview.add_row( + "Acting As", str(entry.get("acting_agent_name") or "-"), "Identity", str(entry.get("identity_status") or "-") + ) + overview.add_row( + "Environment", + str(entry.get("environment_label") or entry.get("base_url") or "-"), + "Env Status", + str(entry.get("environment_status") or "-"), + ) + overview.add_row( + "Current Space", + str(entry.get("active_space_name") or entry.get("active_space_id") or "-"), + "Space Status", + str(entry.get("space_status") or "-"), + ) + overview.add_row( + "Default Space", + str(entry.get("default_space_name") or entry.get("default_space_id") or "-"), + "Allowed Spaces", + str(entry.get("allowed_space_count") or 0), + ) + overview.add_row( + "Install", str(entry.get("install_id") or "-"), "Runtime Instance", str(entry.get("runtime_instance_id") or "-") + ) overview.add_row("Reachability", _reachability_copy(entry), "Reason", str(entry.get("confidence_reason") or "-")) - overview.add_row("Desired", str(entry.get("desired_state") or "-"), "Effective", str(entry.get("effective_state") or "-")) - overview.add_row("Connected", "yes" if entry.get("connected") else "no", "Queue", str(entry.get("backlog_depth") or 0)) - overview.add_row("Seen", _format_age(entry.get("last_seen_age_seconds")), "Reconnect", _format_age(entry.get("reconnect_backoff_seconds"))) - overview.add_row("Processed", str(entry.get("processed_count") or 0), "Dropped", str(entry.get("dropped_count") or 0)) - overview.add_row("Last Work", _format_timestamp(entry.get("last_work_received_at")), "Completed", _format_timestamp(entry.get("last_work_completed_at"))) - overview.add_row("Phase", str(entry.get("current_status") or "-"), "Activity", str(entry.get("current_activity") or "-")) + overview.add_row( + "Desired", str(entry.get("desired_state") or "-"), "Effective", str(entry.get("effective_state") or "-") + ) + overview.add_row( + "Connected", "yes" if entry.get("connected") else "no", "Queue", str(entry.get("backlog_depth") or 0) + ) + overview.add_row( + "Seen", + _format_age(entry.get("last_seen_age_seconds")), + "Reconnect", + _format_age(entry.get("reconnect_backoff_seconds")), + ) + overview.add_row( + "Processed", str(entry.get("processed_count") or 0), "Dropped", str(entry.get("dropped_count") or 0) + ) + overview.add_row( + "Last Work", + _format_timestamp(entry.get("last_work_received_at")), + "Completed", + _format_timestamp(entry.get("last_work_completed_at")), + ) + overview.add_row( + "Phase", str(entry.get("current_status") or "-"), "Activity", str(entry.get("current_activity") or "-") + ) overview.add_row("Tool", str(entry.get("current_tool") or "-"), "Adapter", str(entry.get("runtime_type") or "-")) - overview.add_row("Cred Source", str(entry.get("credential_source") or "-"), "Space", str(entry.get("space_id") or "-")) - overview.add_row("Agent ID", str(entry.get("agent_id") or "-"), "Last Reply", str(entry.get("last_reply_preview") or "-")) - overview.add_row("Last Error", str(entry.get("last_error") or "-"), "Confidence Detail", str(entry.get("confidence_detail") or "-")) - overview.add_row("Doctor", str(entry.get("last_successful_doctor_at") or "-"), "Doctor Status", str((entry.get("last_doctor_result") or {}).get("status") if isinstance(entry.get("last_doctor_result"), dict) else "-")) + overview.add_row( + "Cred Source", str(entry.get("credential_source") or "-"), "Space", str(entry.get("space_id") or "-") + ) + overview.add_row( + "Agent ID", str(entry.get("agent_id") or "-"), "Last Reply", str(entry.get("last_reply_preview") or "-") + ) + overview.add_row( + "Last Error", + str(entry.get("last_error") or "-"), + "Confidence Detail", + str(entry.get("confidence_detail") or "-"), + ) + overview.add_row( + "Doctor", + str(entry.get("last_successful_doctor_at") or "-"), + "Doctor Status", + str( + (entry.get("last_doctor_result") or {}).get("status") + if isinstance(entry.get("last_doctor_result"), dict) + else "-" + ), + ) paths = Table.grid(expand=True, padding=(0, 2)) paths.add_column(style="bold") @@ -3031,8 +3255,12 @@ def _render_agent_detail(entry: dict, *, activity: list[dict]) -> Group: @app.command("login") def login( - token: str = typer.Option(None, "--token", "-t", help="User PAT (prompted or reused from axctl login when omitted)"), - base_url: str = typer.Option(None, "--url", "-u", help="API base URL (defaults to existing axctl login or paxai.app)"), + token: str = typer.Option( + None, "--token", "-t", help="User PAT (prompted or reused from axctl login when omitted)" + ), + base_url: str = typer.Option( + None, "--url", "-u", help="API base URL (defaults to existing axctl login or paxai.app)" + ), space_id: str = typer.Option(None, "--space-id", "-s", help="Optional default space for managed agents"), as_json: bool = JSON_OPTION, ): @@ -3105,7 +3333,9 @@ def login( registry.setdefault("gateway", {}) registry["gateway"]["session_connected"] = True save_gateway_registry(registry) - record_gateway_activity("gateway_login", username=me.get("username"), base_url=resolved_base_url, space_id=selected_space) + record_gateway_activity( + "gateway_login", username=me.get("username"), base_url=resolved_base_url, space_id=selected_space + ) result = { "session_path": str(path), @@ -3160,8 +3390,23 @@ def status(as_json: bool = JSON_OPTION): ) if payload["agents"]: print_table( - ["Agent", "Type", "Mode", "Presence", "Output", "Confidence", "Acting As", "Current Space", "Seen", "Backlog", "Reason"], - [{**agent, "type": _agent_type_label(agent), "output": _agent_output_label(agent)} for agent in payload["agents"]], + [ + "Agent", + "Type", + "Mode", + "Presence", + "Output", + "Confidence", + "Acting As", + "Current Space", + "Seen", + "Backlog", + "Reason", + ], + [ + {**agent, "type": _agent_type_label(agent), "output": _agent_output_label(agent)} + for agent in payload["agents"] + ], keys=[ "name", "type", @@ -3249,12 +3494,7 @@ def _gateway_cli_argv(*args: str) -> list[str]: resolved = shutil.which("ax") or shutil.which("axctl") if resolved: return [resolved, *args] - command = ( - "import sys; " - "from ax_cli.main import main; " - "sys.argv = ['ax'] + sys.argv[1:]; " - "main()" - ) + command = "import sys; from ax_cli.main import main; sys.argv = ['ax'] + sys.argv[1:]; main()" return [sys.executable, "-c", command, *args] @@ -3312,7 +3552,7 @@ def _wait_for_ui_ready(process: subprocess.Popen[bytes], *, host: str, port: int return False -def _terminate_pids(pids: list[int], *, timeout: float = 3.0) -> tuple[list[int], list[int]]: +def _terminate_pids(pids: list[int], *, timeout: float = 8.0) -> tuple[list[int], list[int]]: requested: list[int] = [] forced: list[int] = [] for pid in sorted(set(pids)): @@ -3406,7 +3646,9 @@ def start( daemon_started = True else: detail = _tail_log_lines(daemon_log_path()) - err_console.print(f"[red]Failed to start Gateway daemon.[/red] {detail or 'Check gateway.log for details.'}") + err_console.print( + f"[red]Failed to start Gateway daemon.[/red] {detail or 'Check gateway.log for details.'}" + ) raise typer.Exit(1) else: daemon_note = "Gateway is not logged in yet; the UI can still start in disconnected mode." @@ -3649,8 +3891,12 @@ def deny_approval( @agents_app.command("add") def add_agent( name: str = typer.Argument(..., help="Managed agent name"), - template_id: str = typer.Option(None, "--template", help="Agent template: echo_test | ollama | hermes | claude_code_channel"), - runtime_type: str = typer.Option(None, "--type", help="Advanced/internal runtime backend: echo | exec | inbox"), + template_id: str = typer.Option( + None, "--template", help="Agent template: echo_test | ollama | hermes | sentinel_cli | claude_code_channel" + ), + runtime_type: str = typer.Option( + None, "--type", help="Advanced/internal runtime backend: echo | exec | hermes_sentinel | sentinel_cli | inbox" + ), exec_cmd: str = typer.Option(None, "--exec", help="Advanced override for exec-based templates"), workdir: str = typer.Option(None, "--workdir", help="Advanced working directory override"), ollama_model: str = typer.Option(None, "--ollama-model", help="Ollama model override for the Ollama template"), @@ -3697,7 +3943,11 @@ def add_agent( def update_agent( name: str = typer.Argument(..., help="Managed agent name"), template_id: str = typer.Option(None, "--template", help="Replace the agent template"), - runtime_type: str = typer.Option(None, "--type", help="Advanced/internal runtime backend override: echo | exec | inbox"), + runtime_type: str = typer.Option( + None, + "--type", + help="Advanced/internal runtime backend override: echo | exec | hermes_sentinel | sentinel_cli | inbox", + ), exec_cmd: str = typer.Option(None, "--exec", help="Advanced override for exec-based templates"), workdir: str = typer.Option(None, "--workdir", help="Advanced working directory override"), ollama_model: str = typer.Option(None, "--ollama-model", help="Ollama model override for the Ollama template"), diff --git a/ax_cli/gateway.py b/ax_cli/gateway.py index 15c6782..20625c5 100644 --- a/ax_cli/gateway.py +++ b/ax_cli/gateway.py @@ -17,6 +17,7 @@ import queue import re import shlex +import signal import subprocess import threading import time @@ -76,17 +77,50 @@ _CONTROLLED_WORK_STATES = {"idle", "queued", "working", "blocked"} _CONTROLLED_REPLY_MODES = {"interactive", "background", "summary_only", "silent"} _CONTROLLED_TELEMETRY_LEVELS = {"rich", "basic", "silent"} -_CONTROLLED_ASSET_CLASSES = {"interactive_agent", "background_worker", "scheduled_job", "alert_listener", "service_proxy"} -_CONTROLLED_INTAKE_MODELS = {"live_listener", "launch_on_send", "queue_accept", "queue_drain", "scheduled_run", "event_triggered", "manual_only"} -_CONTROLLED_TRIGGER_SOURCES = {"direct_message", "queued_job", "scheduled_invocation", "external_alert", "manual_trigger", "tool_call"} +_CONTROLLED_ASSET_CLASSES = { + "interactive_agent", + "background_worker", + "scheduled_job", + "alert_listener", + "service_proxy", +} +_CONTROLLED_INTAKE_MODELS = { + "live_listener", + "launch_on_send", + "queue_accept", + "queue_drain", + "scheduled_run", + "event_triggered", + "manual_only", +} +_CONTROLLED_TRIGGER_SOURCES = { + "direct_message", + "queued_job", + "scheduled_invocation", + "external_alert", + "manual_trigger", + "tool_call", +} _CONTROLLED_RETURN_PATHS = {"inline_reply", "sender_inbox", "summary_post", "task_update", "event_log", "silent"} _CONTROLLED_TELEMETRY_SHAPES = {"rich", "basic", "heartbeat_only", "opaque"} _CONTROLLED_WORKER_MODELS = {"queue_drain"} _CONTROLLED_ATTESTATION_STATES = {"verified", "drifted", "unknown", "blocked"} _CONTROLLED_APPROVAL_STATES = {"not_required", "pending", "approved", "rejected"} -_CONTROLLED_IDENTITY_STATUSES = {"verified", "unknown_identity", "credential_mismatch", "fallback_blocked", "bootstrap_only", "blocked"} +_CONTROLLED_IDENTITY_STATUSES = { + "verified", + "unknown_identity", + "credential_mismatch", + "fallback_blocked", + "bootstrap_only", + "blocked", +} _CONTROLLED_SPACE_STATUSES = {"active_allowed", "active_not_allowed", "no_active_space", "unknown"} -_CONTROLLED_ENVIRONMENT_STATUSES = {"environment_allowed", "environment_mismatch", "environment_unknown", "environment_blocked"} +_CONTROLLED_ENVIRONMENT_STATUSES = { + "environment_allowed", + "environment_mismatch", + "environment_unknown", + "environment_blocked", +} _CONTROLLED_ACTIVE_SPACE_SOURCES = {"explicit_request", "gateway_binding", "visible_default", "none"} _CONTROLLED_MODES = {"LIVE", "ON-DEMAND", "INBOX"} _CONTROLLED_PRESENCE = {"IDLE", "QUEUED", "WORKING", "BLOCKED", "STALE", "OFFLINE", "ERROR"} @@ -119,7 +153,16 @@ "unknown", "other", } -_WORKING_STATUSES = {"accepted", "started", "processing", "thinking", "tool_call", "tool_started", "streaming", "working"} +_WORKING_STATUSES = { + "accepted", + "started", + "processing", + "thinking", + "tool_call", + "tool_started", + "streaming", + "working", +} _BLOCKED_STATUSES = {"rate_limited"} @@ -227,6 +270,12 @@ def _template_operator_defaults(template_id: str | None, runtime_type: object) - "reply_mode": "interactive", "telemetry_level": "rich", }, + "sentinel_cli": { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "telemetry_level": "rich", + }, "claude_code_channel": { "placement": "attached", "activation": "attach_only", @@ -253,6 +302,18 @@ def _template_operator_defaults(template_id: str | None, runtime_type: object) - "reply_mode": "interactive", "telemetry_level": "basic", }, + "hermes_sentinel": { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "telemetry_level": "rich", + }, + "sentinel_cli": { + "placement": "hosted", + "activation": "persistent", + "reply_mode": "interactive", + "telemetry_level": "rich", + }, "inbox": { "placement": "mailbox", "activation": "queue_worker", @@ -260,7 +321,9 @@ def _template_operator_defaults(template_id: str | None, runtime_type: object) - "telemetry_level": "basic", }, } - return dict(defaults_by_template.get(template_key) or defaults_by_runtime.get(runtime_key) or defaults_by_runtime["exec"]) + return dict( + defaults_by_template.get(template_key) or defaults_by_runtime.get(runtime_key) or defaults_by_runtime["exec"] + ) def _template_asset_defaults(template_id: str | None, runtime_type: object) -> dict[str, Any]: @@ -312,6 +375,21 @@ def _template_asset_defaults(template_id: str | None, runtime_type: object) -> d "capabilities": ["reply", "progress", "tool_events"], "constraints": ["requires-repo", "requires-provider-auth"], }, + "sentinel_cli": { + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "rich", + "worker_model": None, + "addressable": True, + "messageable": True, + "schedulable": False, + "externally_triggered": False, + "tags": ["local", "live-listener", "hosted-by-gateway", "sentinel-cli", "rich-telemetry"], + "capabilities": ["reply", "progress", "tool_events", "session_resume"], + "constraints": ["requires-cli-auth"], + }, "claude_code_channel": { "asset_class": "interactive_agent", "intake_model": "live_listener", @@ -360,9 +438,13 @@ def _template_asset_defaults(template_id: str | None, runtime_type: object) -> d "capabilities": ["reply"], "constraints": [], }, + "hermes_sentinel": defaults_by_template["hermes"], + "sentinel_cli": defaults_by_template["sentinel_cli"], "inbox": defaults_by_template["inbox"], } - resolved = defaults_by_template.get(template_key) or defaults_by_runtime.get(runtime_key) or defaults_by_runtime["exec"] + resolved = ( + defaults_by_template.get(template_key) or defaults_by_runtime.get(runtime_key) or defaults_by_runtime["exec"] + ) return { "asset_class": resolved["asset_class"], "intake_model": resolved["intake_model"], @@ -411,8 +493,12 @@ def _output_label(return_paths: list[str]) -> str: }.get(primary, "Reply") -def infer_asset_descriptor(snapshot: dict[str, Any], *, operator_profile: dict[str, str] | None = None) -> dict[str, Any]: - defaults = _template_asset_defaults(str(snapshot.get("template_id") or "").strip() or None, snapshot.get("runtime_type")) +def infer_asset_descriptor( + snapshot: dict[str, Any], *, operator_profile: dict[str, str] | None = None +) -> dict[str, Any]: + defaults = _template_asset_defaults( + str(snapshot.get("template_id") or "").strip() or None, snapshot.get("runtime_type") + ) overrides = _override_fields(snapshot, domain="asset") telemetry_fallback = defaults["telemetry_shape"] if operator_profile: @@ -424,20 +510,28 @@ def infer_asset_descriptor(snapshot: dict[str, Any], *, operator_profile: dict[s asset_class = defaults["asset_class"] if "asset_class" in overrides: - asset_class = _normalized_controlled(snapshot.get("asset_class"), _CONTROLLED_ASSET_CLASSES, fallback=defaults["asset_class"]) + asset_class = _normalized_controlled( + snapshot.get("asset_class"), _CONTROLLED_ASSET_CLASSES, fallback=defaults["asset_class"] + ) intake_model = defaults["intake_model"] if "intake_model" in overrides: - intake_model = _normalized_controlled(snapshot.get("intake_model"), _CONTROLLED_INTAKE_MODELS, fallback=defaults["intake_model"]) + intake_model = _normalized_controlled( + snapshot.get("intake_model"), _CONTROLLED_INTAKE_MODELS, fallback=defaults["intake_model"] + ) worker_model = defaults.get("worker_model") if "worker_model" in overrides: - worker_model = _normalized_optional_controlled(snapshot.get("worker_model"), _CONTROLLED_WORKER_MODELS) or defaults.get("worker_model") + worker_model = _normalized_optional_controlled( + snapshot.get("worker_model"), _CONTROLLED_WORKER_MODELS + ) or defaults.get("worker_model") trigger_sources = list(defaults["trigger_sources"]) if "trigger_sources" in overrides or "trigger_source" in overrides: trigger_sources = _normalized_controlled_list( - snapshot.get("trigger_sources") if snapshot.get("trigger_sources") is not None else snapshot.get("trigger_source"), + snapshot.get("trigger_sources") + if snapshot.get("trigger_sources") is not None + else snapshot.get("trigger_source"), _CONTROLLED_TRIGGER_SOURCES, fallback=defaults["trigger_sources"], ) @@ -471,9 +565,16 @@ def infer_asset_descriptor(snapshot: dict[str, Any], *, operator_profile: dict[s constraints = _normalized_string_list(snapshot.get("constraints"), fallback=defaults["constraints"]) descriptor = { - "asset_id": str(snapshot.get("asset_id") or snapshot.get("agent_id") or snapshot.get("name") or "").strip() or None, + "asset_id": str(snapshot.get("asset_id") or snapshot.get("agent_id") or snapshot.get("name") or "").strip() + or None, "gateway_id": str(snapshot.get("gateway_id") or "").strip() or None, - "display_name": str(snapshot.get("display_name") or snapshot.get("name") or snapshot.get("template_label") or snapshot.get("runtime_type") or "Managed Asset"), + "display_name": str( + snapshot.get("display_name") + or snapshot.get("name") + or snapshot.get("template_label") + or snapshot.get("runtime_type") + or "Managed Asset" + ), "asset_class": asset_class, "intake_model": intake_model, "worker_model": worker_model, @@ -489,7 +590,9 @@ def infer_asset_descriptor(snapshot: dict[str, Any], *, operator_profile: dict[s "schedulable": _bool_with_fallback(snapshot.get("schedulable"), fallback=defaults["schedulable"]) if "schedulable" in overrides else defaults["schedulable"], - "externally_triggered": _bool_with_fallback(snapshot.get("externally_triggered"), fallback=defaults["externally_triggered"]) + "externally_triggered": _bool_with_fallback( + snapshot.get("externally_triggered"), fallback=defaults["externally_triggered"] + ) if "externally_triggered" in overrides else defaults["externally_triggered"], "tags": tags, @@ -530,6 +633,8 @@ def add(path_value: object) -> None: if workdir_raw: workdir = Path(workdir_raw).expanduser() add(workdir.parent / "hermes-agent") + if str(workdir).startswith("/home/ax-agent/agents"): + add("/home/ax-agent/shared/repos/hermes-agent") add(Path.home() / "hermes-agent") return candidates @@ -537,7 +642,8 @@ def add(path_value: object) -> None: def hermes_setup_status(entry: dict[str, Any]) -> dict[str, Any]: template_id = str(entry.get("template_id") or "").strip().lower() - if template_id != "hermes": + runtime_type = str(entry.get("runtime_type") or "").strip().lower() + if template_id != "hermes" and runtime_type != "hermes_sentinel": return {"ready": True, "template_id": template_id} candidates = _hermes_repo_candidates(entry) @@ -558,8 +664,7 @@ def hermes_setup_status(entry: dict[str, Any]) -> dict[str, Any]: "expected_path": str(expected), "summary": f"Hermes checkout not found at {expected}.", "detail": ( - f"Hermes checkout not found at {expected}. " - "Set HERMES_REPO_PATH or clone hermes-agent to ~/hermes-agent." + f"Hermes checkout not found at {expected}. Set HERMES_REPO_PATH or clone hermes-agent to ~/hermes-agent." ), } @@ -677,16 +782,24 @@ def ollama_setup_status(*, preferred_model: str | None = None) -> dict[str, Any] def infer_operator_profile(snapshot: dict[str, Any]) -> dict[str, str]: - defaults = _template_operator_defaults(str(snapshot.get("template_id") or "").strip() or None, snapshot.get("runtime_type")) + defaults = _template_operator_defaults( + str(snapshot.get("template_id") or "").strip() or None, snapshot.get("runtime_type") + ) overrides = _override_fields(snapshot, domain="operator") return { - "placement": _normalized_controlled(snapshot.get("placement"), _CONTROLLED_PLACEMENTS, fallback=defaults["placement"]) + "placement": _normalized_controlled( + snapshot.get("placement"), _CONTROLLED_PLACEMENTS, fallback=defaults["placement"] + ) if "placement" in overrides else defaults["placement"], - "activation": _normalized_controlled(snapshot.get("activation"), _CONTROLLED_ACTIVATIONS, fallback=defaults["activation"]) + "activation": _normalized_controlled( + snapshot.get("activation"), _CONTROLLED_ACTIVATIONS, fallback=defaults["activation"] + ) if "activation" in overrides else defaults["activation"], - "reply_mode": _normalized_controlled(snapshot.get("reply_mode"), _CONTROLLED_REPLY_MODES, fallback=defaults["reply_mode"]) + "reply_mode": _normalized_controlled( + snapshot.get("reply_mode"), _CONTROLLED_REPLY_MODES, fallback=defaults["reply_mode"] + ) if "reply_mode" in overrides else defaults["reply_mode"], "telemetry_level": _normalized_controlled( @@ -724,10 +837,14 @@ def _derive_liveness(snapshot: dict[str, Any], *, raw_state: str, last_seen_age: def _derive_work_state(snapshot: dict[str, Any], *, liveness: str) -> str: - attestation_state = _normalized_optional_controlled(snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + attestation_state = _normalized_optional_controlled( + snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES + ) approval_state = _normalized_optional_controlled(snapshot.get("approval_state"), _CONTROLLED_APPROVAL_STATES) identity_status = _normalized_optional_controlled(snapshot.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) - environment_status = _normalized_optional_controlled(snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + environment_status = _normalized_optional_controlled( + snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES + ) space_status = _normalized_optional_controlled(snapshot.get("space_status"), _CONTROLLED_SPACE_STATUSES) if liveness == "setup_error": return "blocked" @@ -759,7 +876,9 @@ def _doctor_has_failed(snapshot: dict[str, Any]) -> bool: return True checks = result.get("checks") if isinstance(checks, list): - return any(isinstance(item, dict) and str(item.get("status") or "").strip().lower() == "failed" for item in checks) + return any( + isinstance(item, dict) and str(item.get("status") or "").strip().lower() == "failed" for item in checks + ) return False @@ -796,10 +915,14 @@ def _derive_reply(reply_mode: str) -> str: def _derive_reachability(*, snapshot: dict[str, Any], mode: str, liveness: str, activation: str) -> str: - attestation_state = _normalized_optional_controlled(snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + attestation_state = _normalized_optional_controlled( + snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES + ) approval_state = _normalized_optional_controlled(snapshot.get("approval_state"), _CONTROLLED_APPROVAL_STATES) identity_status = _normalized_optional_controlled(snapshot.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) - environment_status = _normalized_optional_controlled(snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + environment_status = _normalized_optional_controlled( + snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES + ) space_status = _normalized_optional_controlled(snapshot.get("space_status"), _CONTROLLED_SPACE_STATUSES) if liveness == "setup_error": return "unavailable" @@ -827,7 +950,11 @@ def _setup_error_detail(snapshot: dict[str, Any]) -> str: summary = _doctor_summary(snapshot) if summary: return summary - return str(snapshot.get("last_error") or snapshot.get("last_reply_preview") or "Setup must be fixed before Gateway can send work.") + return str( + snapshot.get("last_error") + or snapshot.get("last_reply_preview") + or "Setup must be fixed before Gateway can send work." + ) def _doctor_summary(snapshot: dict[str, Any]) -> str: @@ -839,7 +966,11 @@ def _doctor_summary(snapshot: dict[str, Any]) -> str: return summary checks = result.get("checks") if isinstance(checks, list): - failed = [str(item.get("name") or "").strip() for item in checks if isinstance(item, dict) and str(item.get("status") or "").strip().lower() == "failed"] + failed = [ + str(item.get("name") or "").strip() + for item in checks + if isinstance(item, dict) and str(item.get("status") or "").strip().lower() == "failed" + ] if failed: return f"Doctor failed: {', '.join(filter(None, failed))}." return "" @@ -852,27 +983,56 @@ def _derive_confidence( liveness: str, reachability: str, ) -> tuple[str, str, str]: - attestation_state = _normalized_optional_controlled(snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + attestation_state = _normalized_optional_controlled( + snapshot.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES + ) approval_state = _normalized_optional_controlled(snapshot.get("approval_state"), _CONTROLLED_APPROVAL_STATES) - governance_reason = _normalized_optional_controlled(snapshot.get("confidence_reason"), _CONTROLLED_CONFIDENCE_REASONS) - governance_detail = str(snapshot.get("confidence_detail") or "").strip() or "Gateway blocked this runtime until its binding is approved." + governance_reason = _normalized_optional_controlled( + snapshot.get("confidence_reason"), _CONTROLLED_CONFIDENCE_REASONS + ) + governance_detail = ( + str(snapshot.get("confidence_detail") or "").strip() + or "Gateway blocked this runtime until its binding is approved." + ) identity_status = _normalized_optional_controlled(snapshot.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) - environment_status = _normalized_optional_controlled(snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + environment_status = _normalized_optional_controlled( + snapshot.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES + ) space_status = _normalized_optional_controlled(snapshot.get("space_status"), _CONTROLLED_SPACE_STATUSES) if liveness == "setup_error": return ("BLOCKED", "setup_blocked", _setup_error_detail(snapshot)) if identity_status == "unknown_identity": - return ("BLOCKED", "identity_unbound", "Gateway does not have a bound acting identity for this asset in the requested environment.") + return ( + "BLOCKED", + "identity_unbound", + "Gateway does not have a bound acting identity for this asset in the requested environment.", + ) if identity_status in {"credential_mismatch", "fallback_blocked"}: - return ("BLOCKED", "identity_mismatch", "Gateway blocked a mismatched acting identity instead of borrowing another identity.") + return ( + "BLOCKED", + "identity_mismatch", + "Gateway blocked a mismatched acting identity instead of borrowing another identity.", + ) if identity_status == "bootstrap_only": - return ("BLOCKED", "bootstrap_only", "Gateway bootstrap credentials can only be used for setup, verification, or repair flows.") + return ( + "BLOCKED", + "bootstrap_only", + "Gateway bootstrap credentials can only be used for setup, verification, or repair flows.", + ) if environment_status == "environment_mismatch": - return ("BLOCKED", "environment_mismatch", "Requested environment does not match the bound Gateway environment for this asset.") + return ( + "BLOCKED", + "environment_mismatch", + "Requested environment does not match the bound Gateway environment for this asset.", + ) if environment_status == "environment_blocked": return ("BLOCKED", "environment_mismatch", "Gateway blocked this asset in the requested environment.") if space_status == "active_not_allowed": - return ("BLOCKED", "active_space_not_allowed", "The resolved target space is not allowed for this acting identity.") + return ( + "BLOCKED", + "active_space_not_allowed", + "The resolved target space is not allowed for this acting identity.", + ) if space_status == "no_active_space": return ("BLOCKED", "no_active_space", "Gateway does not have an active space selected for this asset.") if space_status == "unknown": @@ -931,7 +1091,7 @@ def _binding_type_for_entry(entry: dict[str, Any]) -> str: def _launch_spec_for_entry(entry: dict[str, Any]) -> dict[str, Any]: - return { + launch_spec = { "runtime_type": str(entry.get("runtime_type") or "").strip() or None, "template_id": str(entry.get("template_id") or "").strip() or None, "command": str(entry.get("exec_command") or "").strip() or None, @@ -939,6 +1099,16 @@ def _launch_spec_for_entry(entry: dict[str, Any]) -> dict[str, Any]: "ollama_model": str(entry.get("ollama_model") or "").strip() or None, "transport": str(entry.get("transport") or "").strip() or None, } + model = str( + entry.get("hermes_model") + or entry.get("sentinel_model") + or entry.get("runtime_model") + or entry.get("model") + or "" + ).strip() + if model: + launch_spec["model"] = model + return launch_spec def _payload_hash(payload: dict[str, Any]) -> str: @@ -1025,7 +1195,12 @@ def _fallback_allowed_spaces(entry: dict[str, Any], session: dict[str, Any] | No rows.append( { "space_id": default_id, - "name": str(entry.get("default_space_name") or entry.get("space_name") or session.get("space_name") or default_id), + "name": str( + entry.get("default_space_name") + or entry.get("space_name") + or session.get("space_name") + or default_id + ), "is_default": True, } ) @@ -1060,7 +1235,9 @@ def _binding_candidate_for_entry(entry: dict[str, Any], registry: dict[str, Any] "path": path, "launch_spec": launch_spec, "launch_spec_hash": _payload_hash(launch_spec), - "created_from": str(entry.get("created_from") or ("ax_template" if entry.get("template_id") else "custom_bridge")), + "created_from": str( + entry.get("created_from") or ("ax_template" if entry.get("template_id") else "custom_bridge") + ), "created_via": str(entry.get("created_via") or "cli"), "approved_state": str(entry.get("approved_state") or "approved"), "first_seen_at": str(entry.get("first_seen_at") or _now_iso()), @@ -1142,15 +1319,27 @@ def find_identity_binding( continue if gateway_id and str(binding.get("gateway_id") or "") != gateway_id: continue - if normalized_base_url and _normalized_base_url(((binding.get("environment") or {}) if isinstance(binding.get("environment"), dict) else {}).get("base_url")) != normalized_base_url: + if ( + normalized_base_url + and _normalized_base_url( + ((binding.get("environment") or {}) if isinstance(binding.get("environment"), dict) else {}).get( + "base_url" + ) + ) + != normalized_base_url + ): continue return binding return None -def _identity_bindings_for_asset(registry: dict[str, Any], asset_id: str, *, gateway_id: str | None = None) -> list[dict[str, Any]]: +def _identity_bindings_for_asset( + registry: dict[str, Any], asset_id: str, *, gateway_id: str | None = None +) -> list[dict[str, Any]]: _ensure_registry_lists(registry) - rows = [binding for binding in registry.get("identity_bindings", []) if str(binding.get("asset_id") or "") == asset_id] + rows = [ + binding for binding in registry.get("identity_bindings", []) if str(binding.get("asset_id") or "") == asset_id + ] if gateway_id: rows = [binding for binding in rows if str(binding.get("gateway_id") or "") == gateway_id] return rows @@ -1161,15 +1350,25 @@ def upsert_identity_binding(registry: dict[str, Any], binding: dict[str, Any]) - bindings = registry["identity_bindings"] target_id = str(binding.get("identity_binding_id") or "") target_install_id = str(binding.get("install_id") or "") - target_base_url = _normalized_base_url(((binding.get("environment") or {}) if isinstance(binding.get("environment"), dict) else {}).get("base_url")) + target_base_url = _normalized_base_url( + ((binding.get("environment") or {}) if isinstance(binding.get("environment"), dict) else {}).get("base_url") + ) for idx, existing in enumerate(bindings): - existing_base_url = _normalized_base_url(((existing.get("environment") or {}) if isinstance(existing.get("environment"), dict) else {}).get("base_url")) + existing_base_url = _normalized_base_url( + ((existing.get("environment") or {}) if isinstance(existing.get("environment"), dict) else {}).get( + "base_url" + ) + ) if target_id and str(existing.get("identity_binding_id") or "") == target_id: merged = dict(existing) merged.update(binding) bindings[idx] = merged return merged - if target_install_id and str(existing.get("install_id") or "") == target_install_id and existing_base_url == target_base_url: + if ( + target_install_id + and str(existing.get("install_id") or "") == target_install_id + and existing_base_url == target_base_url + ): merged = dict(existing) merged.update(binding) bindings[idx] = merged @@ -1245,23 +1444,33 @@ def ensure_gateway_identity_binding( allowed_spaces = fetched if not allowed_spaces: allowed_spaces = _fallback_allowed_spaces(entry, session=session) - default_space_id = str( - entry.get("default_space_id") - or ((existing or {}).get("default_space_id") if isinstance(existing, dict) else "") - or next((item.get("space_id") for item in allowed_spaces if bool(item.get("is_default"))), None) - or entry.get("space_id") - or (session or {}).get("space_id") - or "" - ).strip() or None - active_space_id = str( - entry.get("active_space_id") - or ((existing or {}).get("active_space_id") if isinstance(existing, dict) else "") - or entry.get("space_id") - or default_space_id - or "" - ).strip() or None - default_space_name = _space_name_from_cache(allowed_spaces, default_space_id) or str(entry.get("default_space_name") or entry.get("space_name") or default_space_id or "") - active_space_name = _space_name_from_cache(allowed_spaces, active_space_id) or str(entry.get("active_space_name") or entry.get("space_name") or active_space_id or "") + default_space_id = ( + str( + entry.get("default_space_id") + or ((existing or {}).get("default_space_id") if isinstance(existing, dict) else "") + or next((item.get("space_id") for item in allowed_spaces if bool(item.get("is_default"))), None) + or entry.get("space_id") + or (session or {}).get("space_id") + or "" + ).strip() + or None + ) + active_space_id = ( + str( + entry.get("active_space_id") + or ((existing or {}).get("active_space_id") if isinstance(existing, dict) else "") + or entry.get("space_id") + or default_space_id + or "" + ).strip() + or None + ) + default_space_name = _space_name_from_cache(allowed_spaces, default_space_id) or str( + entry.get("default_space_name") or entry.get("space_name") or default_space_id or "" + ) + active_space_name = _space_name_from_cache(allowed_spaces, active_space_id) or str( + entry.get("active_space_name") or entry.get("space_name") or active_space_id or "" + ) binding = { "identity_binding_id": str((existing or {}).get("identity_binding_id") or f"idbind_{str(uuid.uuid4())}"), "asset_id": asset_id, @@ -1283,8 +1492,15 @@ def ensure_gateway_identity_binding( ), "credential_ref": { "kind": "token_file" if str(entry.get("token_file") or "").strip() else "unknown", - "id": str((existing or {}).get("credential_ref", {}).get("id") if isinstance((existing or {}).get("credential_ref"), dict) else "") or f"cred_{str(entry.get('name') or asset_id or 'asset')}_{_environment_label_for_base_url(base_url)}", - "display": "Gateway-managed agent token" if str(entry.get("credential_source") or "gateway") == "gateway" else "Non-gateway credential", + "id": str( + (existing or {}).get("credential_ref", {}).get("id") + if isinstance((existing or {}).get("credential_ref"), dict) + else "" + ) + or f"cred_{str(entry.get('name') or asset_id or 'asset')}_{_environment_label_for_base_url(base_url)}", + "display": "Gateway-managed agent token" + if str(entry.get("credential_source") or "gateway") == "gateway" + else "Non-gateway credential", "path_redacted": _redacted_path(entry.get("token_file")), }, "active_space_id": active_space_id, @@ -1328,11 +1544,25 @@ def evaluate_identity_space_binding( asset_bindings = _identity_bindings_for_asset(registry, asset_id, gateway_id=gateway_id) if asset_id else [] fallback_binding = asset_bindings[0] if asset_bindings else None acting_identity = ( - (binding.get("acting_identity") if isinstance(binding, dict) and isinstance(binding.get("acting_identity"), dict) else None) - or (fallback_binding.get("acting_identity") if isinstance(fallback_binding, dict) and isinstance(fallback_binding.get("acting_identity"), dict) else None) + ( + binding.get("acting_identity") + if isinstance(binding, dict) and isinstance(binding.get("acting_identity"), dict) + else None + ) + or ( + fallback_binding.get("acting_identity") + if isinstance(fallback_binding, dict) and isinstance(fallback_binding.get("acting_identity"), dict) + else None + ) or {} ) - bound_base_url = _normalized_base_url(((binding.get("environment") or {}) if isinstance(binding, dict) and isinstance(binding.get("environment"), dict) else {}).get("base_url")) + bound_base_url = _normalized_base_url( + ( + (binding.get("environment") or {}) + if isinstance(binding, dict) and isinstance(binding.get("environment"), dict) + else {} + ).get("base_url") + ) environment_status = "environment_unknown" if binding: environment_status = "environment_allowed" @@ -1373,8 +1603,20 @@ def evaluate_identity_space_binding( active_space_source = "visible_default" default_space_id = str((binding or {}).get("default_space_id") or "").strip() or None - default_space_name = str((binding or {}).get("default_space_name") or _space_name_from_cache(allowed_spaces, default_space_id) or default_space_id or "").strip() or None - active_space_name = _space_name_from_cache(allowed_spaces, active_space_id) or str((binding or {}).get("active_space_name") or active_space_id or "").strip() or None + default_space_name = ( + str( + (binding or {}).get("default_space_name") + or _space_name_from_cache(allowed_spaces, default_space_id) + or default_space_id + or "" + ).strip() + or None + ) + active_space_name = ( + _space_name_from_cache(allowed_spaces, active_space_id) + or str((binding or {}).get("active_space_name") or active_space_id or "").strip() + or None + ) if not active_space_id: space_status = "no_active_space" @@ -1386,7 +1628,8 @@ def evaluate_identity_space_binding( space_status = "active_not_allowed" return { - "identity_binding_id": str((binding or {}).get("identity_binding_id") or entry.get("identity_binding_id") or "") or None, + "identity_binding_id": str((binding or {}).get("identity_binding_id") or entry.get("identity_binding_id") or "") + or None, "asset_id": asset_id or None, "gateway_id": gateway_id, "install_id": install_id, @@ -1407,7 +1650,9 @@ def evaluate_identity_space_binding( "space_status": space_status, "last_space_verification_at": str((binding or {}).get("last_verified_at") or ""), "identity_binding_state": str((binding or {}).get("binding_state") or "unbound"), - "credential_ref": dict((binding or {}).get("credential_ref") or {}) if isinstance((binding or {}).get("credential_ref"), dict) else None, + "credential_ref": dict((binding or {}).get("credential_ref") or {}) + if isinstance((binding or {}).get("credential_ref"), dict) + else None, } @@ -1479,7 +1724,9 @@ def _refresh_attestation_for_matching_entries( entry.update(evaluate_runtime_attestation(registry, entry)) -def approve_gateway_approval(approval_id: str, *, scope: str = "asset", decided_by: str | None = None) -> dict[str, Any]: +def approve_gateway_approval( + approval_id: str, *, scope: str = "asset", decided_by: str | None = None +) -> dict[str, Any]: normalized_scope = str(scope or "asset").strip().lower() if normalized_scope not in {"once", "asset", "gateway"}: raise ValueError("Approval scope must be one of: once, asset, gateway.") @@ -1487,7 +1734,9 @@ def approve_gateway_approval(approval_id: str, *, scope: str = "asset", decided_ approval = _find_approval_by_id(registry, approval_id) if approval is None: raise LookupError(f"Approval not found: {approval_id}") - candidate_binding = approval.get("candidate_binding") if isinstance(approval.get("candidate_binding"), dict) else None + candidate_binding = ( + approval.get("candidate_binding") if isinstance(approval.get("candidate_binding"), dict) else None + ) if not candidate_binding: raise ValueError("Approval is missing its candidate binding.") now = _now_iso() @@ -1562,6 +1811,7 @@ def ensure_local_asset_binding( *, created_via: str | None = None, auto_approve: bool = True, + replace_existing: bool = False, ) -> dict[str, Any]: _ensure_registry_lists(registry) gateway_id = _gateway_id_from_registry(registry) @@ -1570,11 +1820,35 @@ def ensure_local_asset_binding( if not install_id: install_id = str(uuid.uuid4()) entry["install_id"] = install_id - existing = find_binding(registry, install_id=install_id) or find_binding(registry, asset_id=asset_id, gateway_id=gateway_id) + existing = find_binding(registry, install_id=install_id) or find_binding( + registry, asset_id=asset_id, gateway_id=gateway_id + ) if existing: - entry.setdefault("install_id", str(existing.get("install_id") or install_id)) - return existing - candidate = _binding_candidate_for_entry({**entry, "created_via": created_via or entry.get("created_via")}, registry) + entry["install_id"] = str(existing.get("install_id") or install_id) + if not replace_existing: + return existing + candidate = _binding_candidate_for_entry( + {**entry, "created_via": created_via or entry.get("created_via")}, registry + ) + candidate["first_seen_at"] = str(existing.get("first_seen_at") or candidate.get("first_seen_at") or _now_iso()) + if auto_approve: + candidate["approved_state"] = "approved" + candidate["approved_at"] = _now_iso() + binding = upsert_binding(registry, candidate) + if str(existing.get("candidate_signature") or "") != str(binding.get("candidate_signature") or ""): + _record_governance_activity( + "asset_binding_updated", + entry=entry, + asset_id=asset_id, + install_id=entry["install_id"], + binding_type=binding.get("binding_type"), + gateway_id=gateway_id, + path=binding.get("path"), + ) + return binding + candidate = _binding_candidate_for_entry( + {**entry, "created_via": created_via or entry.get("created_via")}, registry + ) if auto_approve: candidate["approved_state"] = "approved" candidate["approved_at"] = _now_iso() @@ -1644,7 +1918,9 @@ def evaluate_runtime_attestation(registry: dict[str, Any], entry: dict[str, Any] candidate = _binding_candidate_for_entry(entry, registry) latest_approval = _find_approval_for_signature(registry, candidate["candidate_signature"]) - def blocked(reason: str, detail: str, *, approval: dict[str, Any] | None = None, state: str = "blocked") -> dict[str, Any]: + def blocked( + reason: str, detail: str, *, approval: dict[str, Any] | None = None, state: str = "blocked" + ) -> dict[str, Any]: return { "asset_id": asset_id or None, "gateway_id": gateway_id, @@ -1654,7 +1930,9 @@ def blocked(reason: str, detail: str, *, approval: dict[str, Any] | None = None, "runtime_instance_id": str(entry.get("runtime_instance_id") or "") or None, "attestation_state": state, "drift_reason": reason, - "approval_state": "rejected" if approval and _approval_status(approval) == "rejected" else ("pending" if approval and _approval_status(approval) == "pending" else "not_required"), + "approval_state": "rejected" + if approval and _approval_status(approval) == "rejected" + else ("pending" if approval and _approval_status(approval) == "pending" else "not_required"), "approval_id": approval.get("approval_id") if approval else None, "confidence_reason": reason, "confidence_detail": detail, @@ -1670,11 +1948,15 @@ def blocked(reason: str, detail: str, *, approval: dict[str, Any] | None = None, return blocked("asset_mismatch", "Runtime install is bound to a different asset id than the one it claimed.") if latest_approval and _approval_status(latest_approval) == "rejected": - return blocked("approval_denied", "A prior approval request for this runtime binding was denied.", approval=latest_approval) + return blocked( + "approval_denied", "A prior approval request for this runtime binding was denied.", approval=latest_approval + ) if not install_binding: if asset_bindings: - same_gateway = next((binding for binding in asset_bindings if str(binding.get("gateway_id") or "") == gateway_id), None) + same_gateway = next( + (binding for binding in asset_bindings if str(binding.get("gateway_id") or "") == gateway_id), None + ) if same_gateway is None: approval = latest_approval or _create_binding_approval( registry, @@ -1685,7 +1967,12 @@ def blocked(reason: str, detail: str, *, approval: dict[str, Any] | None = None, risk="high", approval_kind="new_gateway", ) - return blocked("new_gateway", "This asset is requesting access from a new Gateway and needs approval.", approval=approval, state="unknown") + return blocked( + "new_gateway", + "This asset is requesting access from a new Gateway and needs approval.", + approval=approval, + state="unknown", + ) approval = latest_approval or _create_binding_approval( registry, entry, @@ -1695,7 +1982,12 @@ def blocked(reason: str, detail: str, *, approval: dict[str, Any] | None = None, risk="medium", approval_kind="new_binding", ) - return blocked("approval_required", "Gateway needs approval before trusting this new asset binding.", approval=approval, state="unknown") + return blocked( + "approval_required", + "Gateway needs approval before trusting this new asset binding.", + approval=approval, + state="unknown", + ) binding = install_binding if str(binding.get("gateway_id") or "") != gateway_id: @@ -1708,7 +2000,12 @@ def blocked(reason: str, detail: str, *, approval: dict[str, Any] | None = None, risk="high", approval_kind="new_gateway", ) - return blocked("new_gateway", "This asset binding is tied to a different Gateway and needs approval.", approval=approval, state="unknown") + return blocked( + "new_gateway", + "This asset binding is tied to a different Gateway and needs approval.", + approval=approval, + state="unknown", + ) if str(binding.get("approved_state") or "approved").lower() == "rejected": return blocked("approval_denied", "This asset binding was previously rejected.") @@ -1793,11 +2090,10 @@ def annotate_runtime_health( resolved_registry = load_gateway_registry() except Exception: resolved_registry = None - if resolved_registry and ( - resolved_registry.get("identity_bindings") - or enriched.get("identity_binding_id") - ): - identity_space = evaluate_identity_space_binding(resolved_registry, enriched, explicit_space_id=explicit_space_id) + if resolved_registry and (resolved_registry.get("identity_bindings") or enriched.get("identity_binding_id")): + identity_space = evaluate_identity_space_binding( + resolved_registry, enriched, explicit_space_id=explicit_space_id + ) enriched.update(identity_space) last_seen_age = _age_seconds(enriched.get("last_seen_at"), now=now) last_error_age = _age_seconds(enriched.get("last_listener_error_at"), now=now) @@ -1822,7 +2118,9 @@ def annotate_runtime_health( mode = _derive_mode(profile) presence = _derive_presence(mode=mode, liveness=liveness, work_state=work_state) reply = _derive_reply(profile["reply_mode"]) - reachability = _derive_reachability(snapshot=enriched, mode=mode, liveness=liveness, activation=profile["activation"]) + reachability = _derive_reachability( + snapshot=enriched, mode=mode, liveness=liveness, activation=profile["activation"] + ) confidence, confidence_reason, confidence_detail = _derive_confidence( enriched, mode=mode, @@ -1831,8 +2129,12 @@ def annotate_runtime_health( ) enriched.update(profile) - enriched["asset_class"] = _normalized_controlled(asset_descriptor["asset_class"], _CONTROLLED_ASSET_CLASSES, fallback="interactive_agent") - enriched["intake_model"] = _normalized_controlled(asset_descriptor["intake_model"], _CONTROLLED_INTAKE_MODELS, fallback="launch_on_send") + enriched["asset_class"] = _normalized_controlled( + asset_descriptor["asset_class"], _CONTROLLED_ASSET_CLASSES, fallback="interactive_agent" + ) + enriched["intake_model"] = _normalized_controlled( + asset_descriptor["intake_model"], _CONTROLLED_INTAKE_MODELS, fallback="launch_on_send" + ) if asset_descriptor.get("worker_model"): enriched["worker_model"] = asset_descriptor["worker_model"] enriched["trigger_sources"] = list(asset_descriptor.get("trigger_sources") or []) @@ -1863,12 +2165,22 @@ def annotate_runtime_health( fallback="unknown", ) enriched["confidence_detail"] = str(confidence_detail or "").strip() or None - enriched["attestation_state"] = _normalized_optional_controlled(enriched.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) - enriched["approval_state"] = _normalized_optional_controlled(enriched.get("approval_state"), _CONTROLLED_APPROVAL_STATES) - enriched["identity_status"] = _normalized_optional_controlled(enriched.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) + enriched["attestation_state"] = _normalized_optional_controlled( + enriched.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES + ) + enriched["approval_state"] = _normalized_optional_controlled( + enriched.get("approval_state"), _CONTROLLED_APPROVAL_STATES + ) + enriched["identity_status"] = _normalized_optional_controlled( + enriched.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES + ) enriched["space_status"] = _normalized_optional_controlled(enriched.get("space_status"), _CONTROLLED_SPACE_STATUSES) - enriched["environment_status"] = _normalized_optional_controlled(enriched.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) - enriched["active_space_source"] = _normalized_optional_controlled(enriched.get("active_space_source"), _CONTROLLED_ACTIVE_SPACE_SOURCES) + enriched["environment_status"] = _normalized_optional_controlled( + enriched.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES + ) + enriched["active_space_source"] = _normalized_optional_controlled( + enriched.get("active_space_source"), _CONTROLLED_ACTIVE_SPACE_SOURCES + ) enriched["queue_capable"] = profile["placement"] == "mailbox" enriched["queue_depth"] = int(enriched.get("backlog_depth") or 0) enriched.setdefault("last_successful_doctor_at", None) @@ -1929,6 +2241,63 @@ def agent_token_path(name: str) -> Path: return agent_dir(name) / "token" +def agent_pending_queue_path(name: str) -> Path: + return agent_dir(name) / "pending.json" + + +def _default_pending_queue() -> dict[str, Any]: + return {"version": 1, "items": []} + + +def load_agent_pending_messages(name: str) -> list[dict[str, Any]]: + payload = _read_json(agent_pending_queue_path(name), default=_default_pending_queue()) + items = payload.get("items") + if not isinstance(items, list): + return [] + return [dict(item) for item in items if isinstance(item, dict)] + + +def save_agent_pending_messages(name: str, items: list[dict[str, Any]]) -> Path: + payload = { + "version": 1, + "items": [dict(item) for item in items if isinstance(item, dict)], + } + _write_json(agent_pending_queue_path(name), payload) + return agent_pending_queue_path(name) + + +def append_agent_pending_message(name: str, message: dict[str, Any]) -> list[dict[str, Any]]: + message_id = str(message.get("message_id") or message.get("id") or "").strip() + items = load_agent_pending_messages(name) + if any(str(item.get("message_id") or "").strip() == message_id for item in items): + return items + items.append( + { + "message_id": message_id, + "parent_id": str(message.get("parent_id") or "").strip() or None, + "conversation_id": str(message.get("conversation_id") or "").strip() or None, + "content": str(message.get("content") or ""), + "display_name": str( + message.get("display_name") or message.get("agent_name") or message.get("sender_name") or "" + ) + or None, + "created_at": str(message.get("created_at") or _now_iso()), + "queued_at": _now_iso(), + } + ) + save_agent_pending_messages(name, items) + return items + + +def remove_agent_pending_message(name: str, message_id: str | None) -> list[dict[str, Any]]: + target = str(message_id or "").strip() + if not target: + return load_agent_pending_messages(name) + items = [item for item in load_agent_pending_messages(name) if str(item.get("message_id") or "").strip() != target] + save_agent_pending_messages(name, items) + return items + + def _default_registry() -> dict[str, Any]: return { "version": 1, @@ -2428,6 +2797,283 @@ def _is_passive_runtime(runtime_type: object) -> bool: return str(runtime_type or "").lower() in {"inbox", "passive", "monitor"} +def _gateway_pickup_activity(runtime_type: object, backlog_depth: int) -> str: + if _is_passive_runtime(runtime_type): + if backlog_depth > 1: + return f"Queued in Gateway ({backlog_depth} pending)" + return "Queued in Gateway" + if backlog_depth > 1: + return f"Picked up by Gateway ({backlog_depth} pending)" + return "Picked up by Gateway" + + +def _is_sentinel_cli_runtime(runtime_type: object) -> bool: + return str(runtime_type or "").strip().lower() in {"sentinel_cli", "claude_cli", "codex_cli"} + + +def _is_hermes_sentinel_runtime(runtime_type: object) -> bool: + return str(runtime_type or "").strip().lower() in {"hermes_sentinel", "hermes_sdk"} + + +def _gateway_repo_root() -> Path: + return Path(__file__).resolve().parent.parent + + +def _agents_dir_for_entry(entry: dict[str, Any]) -> Path: + workdir = Path(str(entry.get("workdir") or "")).expanduser() if str(entry.get("workdir") or "").strip() else None + if workdir is not None: + return workdir.parent + return Path("/home/ax-agent/agents") + + +def _hermes_sentinel_script(entry: dict[str, Any]) -> Path: + configured = str(entry.get("sentinel_script") or entry.get("hermes_sentinel_script") or "").strip() + if configured: + return Path(configured).expanduser() + return _agents_dir_for_entry(entry) / "claude_agent_v2.py" + + +def _hermes_sentinel_python(entry: dict[str, Any]) -> str: + configured = str(entry.get("hermes_python") or entry.get("python") or "").strip() + if configured: + return configured + hermes_repo = str(entry.get("hermes_repo_path") or "").strip() + if hermes_repo: + candidate = Path(hermes_repo).expanduser() / ".venv" / "bin" / "python3" + if candidate.exists(): + return str(candidate) + default = Path("/home/ax-agent/shared/repos/hermes-agent/.venv/bin/python3") + if default.exists(): + return str(default) + return "python3" + + +def _hermes_sentinel_model(entry: dict[str, Any]) -> str: + for key in ("hermes_model", "sentinel_model", "runtime_model", "model"): + value = str(entry.get(key) or "").strip() + if value: + return value + return str(os.environ.get("AX_GATEWAY_HERMES_MODEL") or "codex:gpt-5.5") + + +def _hermes_sentinel_workdir(entry: dict[str, Any]) -> Path: + raw = str(entry.get("workdir") or "").strip() + if raw: + return Path(raw).expanduser() + return Path("/home/ax-agent/agents") / str(entry.get("name") or "agent") + + +def _build_hermes_sentinel_cmd(entry: dict[str, Any]) -> list[str]: + timeout = str(entry.get("timeout_seconds") or entry.get("timeout") or 600) + update_interval = str(entry.get("update_interval") or 2.0) + cmd = [ + _hermes_sentinel_python(entry), + "-u", + str(_hermes_sentinel_script(entry)), + "--agent", + str(entry.get("name") or ""), + "--workdir", + str(_hermes_sentinel_workdir(entry)), + "--timeout", + timeout, + "--update-interval", + update_interval, + "--runtime", + "hermes_sdk", + "--model", + _hermes_sentinel_model(entry), + ] + allowed_tools = str(entry.get("allowed_tools") or "").strip() + if allowed_tools: + cmd.extend(["--allowed-tools", allowed_tools]) + system_prompt = str(entry.get("system_prompt") or "").strip() + if system_prompt: + cmd.extend(["--system-prompt", system_prompt]) + if _bool_with_fallback(entry.get("disable_codex_mcp"), fallback=False): + cmd.append("--disable-codex-mcp") + return cmd + + +def _build_hermes_sentinel_env(entry: dict[str, Any]) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if k not in ENV_DENYLIST} + token_file = Path(str(entry.get("token_file") or "")).expanduser() + token = token_file.read_text().strip() if token_file.exists() else "" + workdir = _hermes_sentinel_workdir(entry) + agents_dir = _agents_dir_for_entry(entry) + hermes_repo = str(entry.get("hermes_repo_path") or "").strip() or "/home/ax-agent/shared/repos/hermes-agent" + repo_root = str(_gateway_repo_root()) + + env.update( + { + "AX_TOKEN": token, + "AX_BASE_URL": str(entry.get("base_url") or ""), + "AX_AGENT_NAME": str(entry.get("name") or ""), + "AX_AGENT_ID": str(entry.get("agent_id") or ""), + "AX_SPACE_ID": str(entry.get("space_id") or ""), + "AX_CONFIG_DIR": str(workdir / ".ax"), + "AX_PYTHON": _hermes_sentinel_python(entry), + "HERMES_MAX_ITERATIONS": str( + entry.get("hermes_max_iterations") or os.environ.get("HERMES_MAX_ITERATIONS") or 30 + ), + } + ) + env.setdefault("AGENT_RUNNER_API_KEY", "staging-dispatch-key") + env.setdefault("INTERNAL_DISPATCH_API_KEY", env["AGENT_RUNNER_API_KEY"]) + + python_paths = [str(agents_dir), hermes_repo, repo_root] + existing_pythonpath = env.get("PYTHONPATH") + if existing_pythonpath: + python_paths.append(existing_pythonpath) + env["PYTHONPATH"] = ":".join(path for path in python_paths if path) + + path_entries = [str(_gateway_repo_root() / ".venv" / "bin"), "/home/ax-agent/shared/repos/ax-cli/.venv/bin"] + if env.get("PATH"): + path_entries.append(env["PATH"]) + env["PATH"] = ":".join(path_entries) + return env + + +def _sentinel_runtime_name(entry: dict[str, Any]) -> str: + runtime_type = str(entry.get("runtime_type") or "").strip().lower() + configured = ( + str(entry.get("sentinel_runtime") or entry.get("runtime_backend") or entry.get("cli_runtime") or "") + .strip() + .lower() + ) + if configured in {"claude", "claude_cli"}: + return "claude" + if configured in {"codex", "codex_cli"}: + return "codex" + if runtime_type == "codex_cli": + return "codex" + return "claude" + + +def _sentinel_session_scope(entry: dict[str, Any]) -> str: + scope = str(entry.get("sentinel_session_scope") or entry.get("session_scope") or "agent").strip().lower() + return scope if scope in {"agent", "thread", "message"} else "agent" + + +def _sentinel_session_key(entry: dict[str, Any], data: dict[str, Any] | None, message_id: str) -> str: + scope = _sentinel_session_scope(entry) + if scope == "message": + return message_id or str(uuid.uuid4()) + if scope == "thread": + data = data or {} + return str(data.get("parent_id") or data.get("conversation_id") or message_id or "default") + return f"space:{entry.get('space_id') or 'unknown'}:agent:{entry.get('name') or 'unknown'}" + + +def _sentinel_model(entry: dict[str, Any], runtime_name: str) -> str | None: + runtime_specific_key = "codex_model" if runtime_name == "codex" else "claude_model" + for key in ("model", "sentinel_model", f"{runtime_name}_model", runtime_specific_key): + value = str(entry.get(key) or "").strip() + if value: + return value + return None + + +def _build_sentinel_claude_cmd(entry: dict[str, Any], session_id: str | None) -> list[str]: + cmd = [ + "claude", + "-p", + "--output-format", + "stream-json", + "--dangerously-skip-permissions", + "--add-dir", + str(entry.get("add_dir") or "/home/ax-agent/shared/repos"), + ] + if session_id: + cmd.extend(["--resume", session_id]) + model = _sentinel_model(entry, "claude") + if model: + cmd.extend(["--model", model]) + allowed_tools = str(entry.get("allowed_tools") or "").strip() + if allowed_tools: + cmd.extend(["--allowedTools", allowed_tools]) + system_prompt = str(entry.get("system_prompt") or "").strip() + if system_prompt: + cmd.extend(["--append-system-prompt", system_prompt]) + return cmd + + +def _build_sentinel_codex_cmd(entry: dict[str, Any], session_id: str | None) -> list[str]: + workdir = str(entry.get("workdir") or os.getcwd()) + if session_id: + cmd = [ + "codex", + "exec", + "resume", + session_id, + "--json", + "--dangerously-bypass-approvals-and-sandbox", + "-C", + workdir, + ] + else: + cmd = [ + "codex", + "exec", + "--json", + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "-C", + workdir, + ] + if _bool_with_fallback(entry.get("disable_codex_mcp"), fallback=True): + cmd.extend(["-c", "mcp_servers.ax-platform.enabled=false"]) + model = _sentinel_model(entry, "codex") + if model: + cmd.extend(["-m", model]) + return cmd + + +def _summarize_sentinel_command(command: str) -> str: + short = " ".join(command.split()) + if len(short) > 90: + short = short[:87] + "..." + + lowered = f" {short.lower()} " + if "apply_patch" in lowered: + return "Applying patch..." + if any(token in lowered for token in (" rg ", " grep ", " find ", " fd ", " glob ")): + return "Searching codebase..." + if any( + token in lowered + for token in (" sed -n", " cat ", " head ", " tail ", " ls ", " pwd ", " git status", " git diff") + ): + return "Reading files..." + if any(token in lowered for token in (" pytest", " npm test", " pnpm test", " uv run", " cargo test")): + return "Running tests..." + return f"Running: {short}..." + + +def _sentinel_tool_summary(tool_name: str, tool_input: dict[str, Any]) -> str: + lowered = tool_name.lower() + if lowered in {"read", "read_file"}: + path = str(tool_input.get("file_path") or tool_input.get("path") or "") + short = path.rsplit("/", 1)[-1] if "/" in path else path + return f"Reading {short}..." if short else "Reading file..." + if lowered in {"write", "write_file"}: + path = str(tool_input.get("file_path") or tool_input.get("path") or "") + short = path.rsplit("/", 1)[-1] if "/" in path else path + return f"Writing {short}..." if short else "Writing file..." + if lowered in {"edit", "edit_file", "patch"}: + path = str(tool_input.get("file_path") or tool_input.get("path") or "") + short = path.rsplit("/", 1)[-1] if "/" in path else path + return f"Editing {short}..." if short else "Editing file..." + if lowered in {"bash", "shell"}: + command = str(tool_input.get("command") or "")[:60] + return f"Running: {command}..." if command else "Running command..." + if lowered in {"grep", "search", "search_files"}: + pattern = str(tool_input.get("pattern") or "") + return f"Searching: {pattern}..." if pattern else "Searching..." + if lowered in {"glob", "glob_files"}: + pattern = str(tool_input.get("pattern") or "") + return f"Finding files: {pattern}..." if pattern else "Finding files..." + return f"Using {tool_name}..." + + class ManagedAgentRuntime: """Listener + worker pair for one managed agent.""" @@ -2448,10 +3094,13 @@ def __init__( self._reply_anchor_ids: set[str] = set() self._seen_ids: set[str] = set() self._completed_seen_ids: set[str] = set() + self._sentinel_sessions: dict[str, str] = {} self._state_lock = threading.Lock() self._stream_client = None self._send_client = None self._stream_response = None + self._supervised_process: subprocess.Popen | None = None + self._supervised_thread: threading.Thread | None = None self._state: dict[str, Any] = { "effective_state": "stopped", "runtime_instance_id": None, @@ -2535,9 +3184,41 @@ def _consume_completed_seen(self, message_id: str) -> bool: def snapshot(self) -> dict[str, Any]: with self._state_lock: + snapshot = dict(self._state) + if not _is_passive_runtime(self.entry.get("runtime_type")): + return snapshot + registry = load_gateway_registry() + stored = find_agent_entry(registry, self.name) or {} + pending_items = load_agent_pending_messages(self.name) + backlog_depth = len(pending_items) + merged = dict(snapshot) + for key in ( + "processed_count", + "last_work_completed_at", + "last_reply_message_id", + "last_reply_preview", + "last_received_message_id", + "last_work_received_at", + ): + if key in stored: + merged[key] = stored.get(key) + merged["backlog_depth"] = backlog_depth + merged["current_status"] = "queued" if backlog_depth > 0 else None + merged["current_activity"] = ( + _gateway_pickup_activity(self.entry.get("runtime_type"), backlog_depth)[:240] if backlog_depth > 0 else None + ) + with self._state_lock: + self._state.update(merged) return dict(self._state) def start(self) -> None: + runtime_type = str(self.entry.get("runtime_type") or "").lower() + if ( + _is_hermes_sentinel_runtime(runtime_type) + and self._supervised_process is not None + and self._supervised_process.poll() is None + ): + return if self._listener_thread and self._listener_thread.is_alive(): return self.stop_event.clear() @@ -2545,14 +3226,19 @@ def start(self) -> None: self._reply_anchor_ids = set() self._seen_ids = set() self._completed_seen_ids = set() + self._sentinel_sessions = {} + pending_items = load_agent_pending_messages(self.name) if _is_passive_runtime(runtime_type) else [] + backlog_depth = len(pending_items) runtime_instance_id = str(uuid.uuid4()) self.entry["runtime_instance_id"] = runtime_instance_id self._update_state( effective_state="starting", runtime_instance_id=runtime_instance_id, - backlog_depth=0, - current_status=None, - current_activity=None, + backlog_depth=backlog_depth, + current_status="queued" if backlog_depth > 0 and _is_passive_runtime(runtime_type) else None, + current_activity=_gateway_pickup_activity(runtime_type, backlog_depth) + if backlog_depth > 0 and _is_passive_runtime(runtime_type) + else None, current_tool=None, current_tool_call_id=None, last_error=None, @@ -2560,6 +3246,9 @@ def start(self) -> None: last_started_at=_now_iso(), reconnect_backoff_seconds=0, ) + if _is_hermes_sentinel_runtime(runtime_type): + self._start_hermes_sentinel_process(runtime_instance_id=runtime_instance_id) + return self._worker_thread = None if not _is_passive_runtime(self.entry.get("runtime_type")): self._worker_thread = threading.Thread( @@ -2589,7 +3278,8 @@ def stop(self, timeout: float = 5.0) -> None: self._stream_response.close() except Exception: pass - for thread in (self._listener_thread, self._worker_thread): + self._stop_hermes_sentinel_process(timeout=timeout) + for thread in (self._listener_thread, self._worker_thread, self._supervised_thread): if thread and thread.is_alive(): thread.join(timeout=timeout) for client in (self._stream_client, self._send_client): @@ -2614,6 +3304,140 @@ def stop(self, timeout: float = 5.0) -> None: record_gateway_activity("runtime_stopped", entry=self.entry) self._log("stopped") + def _hermes_sentinel_log_path(self) -> Path: + configured = str(self.entry.get("log_path") or "").strip() + if configured: + return Path(configured).expanduser() + return _hermes_sentinel_workdir(self.entry) / "gateway-hermes-sentinel.log" + + def _start_hermes_sentinel_process(self, *, runtime_instance_id: str) -> None: + workdir = _hermes_sentinel_workdir(self.entry) + script = _hermes_sentinel_script(self.entry) + token_file = Path(str(self.entry.get("token_file") or "")).expanduser() + if not script.exists(): + error = f"Hermes sentinel script not found: {script}" + self._update_state( + effective_state="error", current_status="error", current_activity=error, last_error=error + ) + record_gateway_activity("runtime_error", entry=self.entry, error=error) + return + if not token_file.exists() or not token_file.read_text().strip(): + error = f"Gateway-managed token file is missing or empty: {token_file}" + self._update_state( + effective_state="error", current_status="error", current_activity=error, last_error=error + ) + record_gateway_activity("runtime_error", entry=self.entry, error=error) + return + + workdir.mkdir(parents=True, exist_ok=True) + log_path = self._hermes_sentinel_log_path() + log_path.parent.mkdir(parents=True, exist_ok=True) + cmd = _build_hermes_sentinel_cmd(self.entry) + env = _build_hermes_sentinel_env(self.entry) + try: + log_handle = log_path.open("a", encoding="utf-8") + log_handle.write( + f"\n[{_now_iso()}] Gateway starting Hermes sentinel: {' '.join(shlex.quote(part) for part in cmd)}\n" + ) + log_handle.flush() + process = subprocess.Popen( + cmd, + stdout=log_handle, + stderr=subprocess.STDOUT, + text=True, + cwd=str(workdir), + env=env, + start_new_session=True, + ) + log_handle.close() + except Exception as exc: + error = f"Failed to start Hermes sentinel: {str(exc)[:360]}" + self._update_state( + effective_state="error", current_status="error", current_activity=error, last_error=error + ) + record_gateway_activity("runtime_error", entry=self.entry, error=error) + return + + self._supervised_process = process + self._update_state( + effective_state="running", + current_status=None, + current_activity="Hermes sentinel listener running", + current_tool=None, + current_tool_call_id=None, + last_error=None, + last_connected_at=_now_iso(), + last_seen_at=_now_iso(), + reconnect_backoff_seconds=0, + ) + record_gateway_activity( + "runtime_started", + entry=self.entry, + runtime_instance_id=runtime_instance_id, + pid=process.pid, + log_path=str(log_path), + supervised_runtime="hermes_sentinel", + ) + self._supervised_thread = threading.Thread( + target=self._monitor_hermes_sentinel_process, + daemon=True, + name=f"gw-hermes-sentinel-{self.name}", + ) + self._supervised_thread.start() + self._log(f"started hermes_sentinel pid={process.pid}") + + def _monitor_hermes_sentinel_process(self) -> None: + process = self._supervised_process + if process is None: + return + while not self.stop_event.wait(timeout=5.0): + returncode = process.poll() + if returncode is None: + self._update_state(effective_state="running", last_seen_at=_now_iso(), last_error=None) + continue + status = "stopped" if returncode == 0 else "error" + error = None if returncode == 0 else f"Hermes sentinel exited with code {returncode}" + self._update_state( + effective_state=status, + current_status=None if returncode == 0 else "error", + current_activity=None if returncode == 0 else error, + current_tool=None, + current_tool_call_id=None, + last_error=error, + last_seen_at=_now_iso(), + ) + record_gateway_activity( + "runtime_exited", + entry=self.entry, + pid=process.pid, + exit_code=returncode, + error=error, + ) + return + + def _stop_hermes_sentinel_process(self, *, timeout: float = 5.0) -> None: + process = self._supervised_process + self._supervised_process = None + if process is None or process.poll() is not None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=timeout) + except ProcessLookupError: + return + except Exception: + try: + process.terminate() + process.wait(timeout=timeout) + except Exception: + pass + def _publish_processing_status( self, message_id: str, @@ -2694,7 +3518,8 @@ def _record_tool_call(self, *, message_id: str, event: dict[str, Any]) -> None: tool_name=tool_name, tool_call_id=tool_call_id, space_id=self.space_id, - tool_action=str(event.get("tool_action") or event.get("tool_action_name") or event.get("command") or "") or None, + tool_action=str(event.get("tool_action") or event.get("tool_action_name") or event.get("command") or "") + or None, resource_uri=str(event.get("resource_uri") or "ui://gateway/tool-call"), arguments_hash=_hash_tool_arguments(arguments), kind=str(event.get("kind_name") or event.get("result_kind") or "gateway_runtime"), @@ -2802,7 +3627,9 @@ def _handle_exec_event(self, event: dict[str, Any], *, message_id: str) -> None: status = str(event.get("status") or "success").strip() metadata = self._processing_status_metadata(event) self._record_tool_call(message_id=message_id, event=event) - step_status = "tool_complete" if status.lower() in {"success", "completed", "ok", "tool_complete"} else "error" + step_status = ( + "tool_complete" if status.lower() in {"success", "completed", "ok", "tool_complete"} else "error" + ) self._update_state( current_status=None if step_status == "tool_complete" else step_status, current_activity=None, @@ -2841,12 +3668,261 @@ def _handle_exec_event(self, event: dict[str, Any], *, message_id: str) -> None: activity_message=activity or None, ) - def _handle_prompt(self, prompt: str, *, message_id: str) -> str: + def _sentinel_session_id(self, session_key: str) -> str | None: + with self._state_lock: + return self._sentinel_sessions.get(session_key) + + def _remember_sentinel_session(self, session_key: str, session_id: str | None) -> None: + if not session_id: + return + with self._state_lock: + self._sentinel_sessions[session_key] = session_id + + def _build_sentinel_cmd(self, runtime_name: str, session_id: str | None) -> list[str]: + command_override = str(self.entry.get("sentinel_command") or "").strip() + if command_override: + command = shlex.split(command_override) + if session_id: + command.extend(["--resume", session_id]) + return command + if runtime_name == "codex": + return _build_sentinel_codex_cmd(self.entry, session_id) + return _build_sentinel_claude_cmd(self.entry, session_id) + + def _handle_sentinel_cli_prompt(self, prompt: str, *, message_id: str, data: dict[str, Any] | None = None) -> str: + runtime_name = _sentinel_runtime_name(self.entry) + session_key = _sentinel_session_key(self.entry, data, message_id) + existing_session = self._sentinel_session_id(session_key) + cmd = self._build_sentinel_cmd(runtime_name, existing_session) + env = sanitize_exec_env(prompt, self.entry) + if message_id: + env["AX_GATEWAY_MESSAGE_ID"] = message_id + if self.space_id: + env["AX_GATEWAY_SPACE_ID"] = self.space_id + env["AX_GATEWAY_SENTINEL_SESSION_KEY"] = session_key + + start_activity = ( + f"Resuming {runtime_name} sentinel session" + if existing_session + else f"Starting {runtime_name} sentinel session" + ) + self._publish_processing_status(message_id, "thinking", activity=start_activity) + self._update_state(current_status="thinking", current_activity=start_activity[:240]) + record_gateway_activity( + "runtime_status", + entry=self.entry, + message_id=message_id, + status="thinking", + activity_message=start_activity, + ) + + try: + process = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + cwd=self.entry.get("workdir") or None, + env=env, + ) + except FileNotFoundError: + return f"(handler not found: {cmd[0]})" + + if process.stdin is not None: + try: + process.stdin.write(prompt) + process.stdin.close() + except Exception: + pass + + accumulated_text = "" + stderr_lines: list[str] = [] + new_session_id: str | None = None + last_activity_time = time.time() + exit_reason = "done" + timeout_seconds = int( + self.entry.get("timeout_seconds") or self.entry.get("timeout") or DEFAULT_HANDLER_TIMEOUT_SECONDS + ) + finished = threading.Event() + + def _consume_stderr() -> None: + if process.stderr is None: + return + for raw in process.stderr: + stderr_lines.append(raw) + + def _timeout_watchdog() -> None: + nonlocal exit_reason + while not finished.wait(timeout=5.0): + if time.time() - last_activity_time <= timeout_seconds: + continue + exit_reason = "timeout" + try: + process.kill() + except Exception: + pass + return + + stderr_thread = threading.Thread(target=_consume_stderr, daemon=True, name=f"gw-sentinel-stderr-{self.name}") + watchdog_thread = threading.Thread( + target=_timeout_watchdog, daemon=True, name=f"gw-sentinel-watchdog-{self.name}" + ) + stderr_thread.start() + watchdog_thread.start() + + try: + if process.stdout is not None: + for raw in process.stdout: + line = raw.strip() + if not line: + continue + last_activity_time = time.time() + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + + event_type = str(event.get("type") or "") + if runtime_name == "codex": + if event_type == "thread.started": + new_session_id = str(event.get("thread_id") or "") or new_session_id + elif event_type == "item.started": + item = event.get("item") if isinstance(event.get("item"), dict) else {} + if str(item.get("type") or "") != "agent_message": + self._handle_sentinel_tool_item(item, message_id=message_id, phase="start") + elif event_type == "item.completed": + item = event.get("item") if isinstance(event.get("item"), dict) else {} + item_type = str(item.get("type") or "") + if item_type == "agent_message": + text = str(item.get("text") or "").strip() + if text: + accumulated_text = text + else: + self._handle_sentinel_tool_item(item, message_id=message_id, phase="result") + continue + + if event_type == "assistant": + for block in event.get("message", {}).get("content", []): + if not isinstance(block, dict): + continue + block_type = str(block.get("type") or "") + if block_type == "text": + accumulated_text = str(block.get("text") or accumulated_text) + elif block_type == "tool_use": + self._handle_claude_tool_use(block, message_id=message_id) + elif event_type == "content_block_delta": + delta = event.get("delta") if isinstance(event.get("delta"), dict) else {} + if delta.get("type") == "text_delta": + accumulated_text += str(delta.get("text") or "") + elif event_type == "result": + result_text = str(event.get("result") or "").strip() + if result_text: + accumulated_text = result_text + new_session_id = str(event.get("session_id") or "") or new_session_id + except Exception as exc: + exit_reason = "crashed" + record_gateway_activity( + "runtime_error", + entry=self.entry, + message_id=message_id or None, + error=f"sentinel stream error: {str(exc)[:360]}", + ) + finally: + finished.set() + + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + stderr_thread.join(timeout=1.0) + + if process.returncode != 0 and exit_reason == "done": + exit_reason = "crashed" + self._remember_sentinel_session(session_key, new_session_id) + if new_session_id: + record_gateway_activity( + "runtime_session_saved", + entry=self.entry, + message_id=message_id, + session_key=session_key, + session_id=new_session_id[:24], + ) + + final = accumulated_text.strip() + stderr = "".join(stderr_lines).strip() + if exit_reason == "timeout": + return final or f"Timed out after {timeout_seconds}s with no output." + if exit_reason == "crashed": + if final: + return final + if stderr: + return f"Hit an error processing that.\n\n(stderr: {stderr[:400]})" + return "Hit an error processing that." + return final or "Completed with no text output." + + def _handle_sentinel_tool_item(self, item: dict[str, Any], *, message_id: str, phase: str) -> None: + item_type = str(item.get("type") or "tool").strip() or "tool" + tool_call_id = str(item.get("id") or item.get("call_id") or uuid.uuid4()) + if item_type == "command_execution": + command = str(item.get("command") or "").strip() + arguments = {"command": command} if command else None + initial_data: dict[str, Any] = {} + if item.get("aggregated_output"): + initial_data["output"] = str(item.get("aggregated_output"))[:4000] + if item.get("exit_code") is not None: + initial_data["exit_code"] = item.get("exit_code") + event = { + "kind": "tool_start" if phase == "start" else "tool_result", + "tool_name": "shell", + "tool_action": command or "command_execution", + "tool_call_id": tool_call_id, + "arguments": arguments, + "initial_data": initial_data or None, + "message": _summarize_sentinel_command(command) if command else "Running command...", + "status": "tool_call" + if phase == "start" + else ("tool_complete" if int(item.get("exit_code") or 0) == 0 else "error"), + } + else: + event = { + "kind": "tool_start" if phase == "start" else "tool_result", + "tool_name": item_type, + "tool_action": str(item.get("title") or item_type), + "tool_call_id": tool_call_id, + "initial_data": {"item": item}, + "message": f"Using {item_type}", + "status": "tool_call" if phase == "start" else "tool_complete", + } + self._handle_exec_event(event, message_id=message_id) + + def _handle_claude_tool_use(self, block: dict[str, Any], *, message_id: str) -> None: + tool_name = str(block.get("name") or "tool").strip() + tool_input = block.get("input") if isinstance(block.get("input"), dict) else {} + tool_call_id = str(block.get("id") or uuid.uuid4()) + event = { + "kind": "tool_start", + "tool_name": tool_name, + "tool_action": str(tool_input.get("command") or tool_name), + "tool_call_id": tool_call_id, + "arguments": tool_input, + "message": _sentinel_tool_summary(tool_name, tool_input), + "status": "tool_call", + } + self._handle_exec_event(event, message_id=message_id) + + def _handle_prompt(self, prompt: str, *, message_id: str, data: dict[str, Any] | None = None) -> str: runtime_type = str(self.entry.get("runtime_type") or "echo").lower() if runtime_type == "echo": return _echo_handler(prompt, self.entry) if runtime_type in {"inbox", "passive", "monitor"}: return "" + if _is_sentinel_cli_runtime(runtime_type): + return self._handle_sentinel_cli_prompt(prompt, message_id=message_id, data=data) if runtime_type in {"exec", "command"}: command = str(self.entry.get("exec_command") or "").strip() if not command: @@ -2904,7 +3980,9 @@ def _worker_loop(self) -> None: start_activity = "Composing echo reply" elif runtime_type in {"exec", "command"}: start_activity = "Preparing runtime" - if runtime_type in {"echo", "exec", "command"}: + elif _is_sentinel_cli_runtime(runtime_type): + start_activity = "Preparing sentinel runtime" + if runtime_type in {"echo", "exec", "command"} or _is_sentinel_cli_runtime(runtime_type): self._update_state(current_status=start_status, current_activity=start_activity[:240]) self._publish_processing_status(message_id, start_status, activity=start_activity) record_gateway_activity( @@ -2915,7 +3993,7 @@ def _worker_loop(self) -> None: activity_message=start_activity, ) try: - response_text = self._handle_prompt(prompt, message_id=message_id) + response_text = self._handle_prompt(prompt, message_id=message_id, data=data) if response_text and self._send_client: result = self._send_client.send_message( self.space_id, @@ -2939,7 +4017,9 @@ def _worker_loop(self) -> None: reply_preview=preview or None, ) runtime_type = str(self.entry.get("runtime_type") or "echo").lower() - bridge_already_closed = runtime_type in {"exec", "command"} and self._consume_completed_seen(message_id) + bridge_already_closed = ( + runtime_type in {"exec", "command"} or _is_sentinel_cli_runtime(runtime_type) + ) and self._consume_completed_seen(message_id) if message_id and not bridge_already_closed: self._publish_processing_status(message_id, "completed") self._bump("processed_count") @@ -3038,19 +4118,16 @@ def _listener_loop(self) -> None: last_received_message_id=message_id, ) record_gateway_activity("message_received", entry=self.entry, message_id=message_id) + runtime_type = str(self.entry.get("runtime_type") or "").lower() try: - self._queue.put_nowait(data) - backlog_depth = self._queue.qsize() - runtime_type = str(self.entry.get("runtime_type") or "").lower() + if _is_passive_runtime(runtime_type): + pending_items = append_agent_pending_message(self.name, data) + backlog_depth = len(pending_items) + else: + self._queue.put_nowait(data) + backlog_depth = self._queue.qsize() pickup_status = "queued" if _is_passive_runtime(runtime_type) else "started" - accepted_activity = "Queued in Gateway" - if not _is_passive_runtime(runtime_type): - accepted_activity = "Picked up by Gateway" - if backlog_depth > 1: - if _is_passive_runtime(runtime_type): - accepted_activity = f"Queued in Gateway ({backlog_depth} pending)" - else: - accepted_activity = f"Picked up by Gateway ({backlog_depth} pending)" + accepted_activity = _gateway_pickup_activity(runtime_type, backlog_depth) self._update_state( backlog_depth=backlog_depth, current_status=pickup_status, @@ -3095,6 +4172,20 @@ def _listener_loop(self) -> None: error="queue full", ) self._log("queue full; dropped message") + except Exception as exc: + self._update_state(last_error=str(exc)[:400]) + self._publish_processing_status( + message_id, + "error", + error_message=str(exc)[:400], + ) + record_gateway_activity( + "message_queue_error", + entry=self.entry, + message_id=message_id, + error=str(exc)[:400], + ) + self._log(f"queue error: {exc}") except Exception as exc: if self.stop_event.is_set(): break @@ -3109,7 +4200,9 @@ def _listener_loop(self) -> None: last_listener_error_at=_now_iso(), reconnect_backoff_seconds=int(backoff), ) - record_gateway_activity(event_name, entry=self.entry, error=error_text, reconnect_in_seconds=int(backoff)) + record_gateway_activity( + event_name, entry=self.entry, error=error_text, reconnect_in_seconds=int(backoff) + ) self._log(f"listener error: {error_text}") time.sleep(backoff) backoff = min(backoff * 2, 30.0) @@ -3156,10 +4249,14 @@ def stop(self) -> None: def _reconcile_runtime(self, entry: dict[str, Any]) -> None: name = str(entry.get("name") or "") desired_state = str(entry.get("desired_state") or "stopped").lower() - attestation_state = _normalized_optional_controlled(entry.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES) + attestation_state = _normalized_optional_controlled( + entry.get("attestation_state"), _CONTROLLED_ATTESTATION_STATES + ) approval_state = _normalized_optional_controlled(entry.get("approval_state"), _CONTROLLED_APPROVAL_STATES) identity_status = _normalized_optional_controlled(entry.get("identity_status"), _CONTROLLED_IDENTITY_STATUSES) - environment_status = _normalized_optional_controlled(entry.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES) + environment_status = _normalized_optional_controlled( + entry.get("environment_status"), _CONTROLLED_ENVIRONMENT_STATUSES + ) space_status = _normalized_optional_controlled(entry.get("space_status"), _CONTROLLED_SPACE_STATUSES) runtime = self._runtimes.get(name) hermes_status = hermes_setup_status(entry) @@ -3171,7 +4268,9 @@ def _reconcile_runtime(self, entry: dict[str, Any]) -> None: { "effective_state": "error", "runtime_instance_id": None, - "last_error": str(hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete."), + "last_error": str( + hermes_status.get("detail") or hermes_status.get("summary") or "Hermes setup is incomplete." + ), "current_status": None, "current_activity": str(hermes_status.get("summary") or "Hermes setup is incomplete."), "current_tool": None, @@ -3221,7 +4320,9 @@ def _reconcile_registry(self, registry: dict[str, Any], session: dict[str, Any]) entry["install_id"] = str(uuid.uuid4()) asset_id = _asset_id_for_entry(entry) - existing_binding = find_binding(registry, install_id=str(entry.get("install_id") or "").strip()) if asset_id else None + existing_binding = ( + find_binding(registry, install_id=str(entry.get("install_id") or "").strip()) if asset_id else None + ) if not existing_binding and asset_id and not _bindings_for_asset(registry, asset_id): ensure_local_asset_binding( registry, @@ -3331,6 +4432,15 @@ def run(self, *, once: bool = False) -> None: registry.setdefault("gateway", {}) registry["gateway"]["last_started_at"] = registry["gateway"].get("last_started_at") or _now_iso() record_gateway_activity("gateway_started", pid=os.getpid()) + previous_handlers: dict[signal.Signals, Any] = {} + + def _request_stop(_signum: int, _frame: Any) -> None: + self.stop() + + if threading.current_thread() is threading.main_thread(): + for sig in (signal.SIGINT, signal.SIGTERM): + previous_handlers[sig] = signal.getsignal(sig) + signal.signal(sig, _request_stop) try: while not self._stop.is_set(): registry = load_gateway_registry() @@ -3340,8 +4450,15 @@ def run(self, *, once: bool = False) -> None: break time.sleep(self.poll_interval) finally: - for runtime in list(self._runtimes.values()): - runtime.stop() + for sig, handler in previous_handlers.items(): + signal.signal(sig, handler) + runtimes = list(self._runtimes.values()) + for runtime in runtimes: + if _is_hermes_sentinel_runtime(runtime.entry.get("runtime_type")): + runtime.stop(timeout=2.0) + for runtime in runtimes: + if not _is_hermes_sentinel_runtime(runtime.entry.get("runtime_type")): + runtime.stop(timeout=1.0) final_registry = load_gateway_registry() final_gateway = final_registry.setdefault("gateway", {}) final_gateway.update( diff --git a/ax_cli/gateway_runtime_types.py b/ax_cli/gateway_runtime_types.py index 7fe5fb2..f789ea0 100644 --- a/ax_cli/gateway_runtime_types.py +++ b/ax_cli/gateway_runtime_types.py @@ -47,8 +47,7 @@ def runtime_type_catalog() -> dict[str, dict[str, Any]]: "id": "exec", "label": "Command Bridge", "description": ( - "Gateway-owned command execution for bridges and adapters that print " - "AX_GATEWAY_EVENT lines." + "Gateway-owned command execution for bridges and adapters that print AX_GATEWAY_EVENT lines." ), "kind": "exec", "passive": False, @@ -100,6 +99,85 @@ def runtime_type_catalog() -> dict[str, dict[str, Any]]: "tools": "Gateway can record tool usage when the bridge emits tool events.", }, }, + "hermes_sentinel": { + "id": "hermes_sentinel", + "label": "Hermes Sentinel", + "description": ( + "Gateway-supervised long-running Hermes sentinel using the original " + "claude_agent_v2.py listener semantics." + ), + "kind": "supervised_process", + "passive": False, + "requires": [], + "form_fields": [ + { + "name": "workdir", + "label": "Workdir", + "required": True, + "placeholder": "/home/ax-agent/agents/dev_sentinel", + }, + { + "name": "model", + "label": "Model", + "required": False, + "placeholder": "codex:gpt-5.5", + }, + ], + "examples": [ + { + "label": "Hermes dev sentinel", + "runtime_type": "hermes_sentinel", + "workdir": "/home/ax-agent/agents/dev_sentinel", + "note": "Gateway starts the old listener once and monitors it; Hermes owns session continuity.", + }, + ], + "signals": { + **_shared_signals(), + "activity": ( + "Gateway reports process liveness; the sentinel listener emits the same processing " + "and tool activity signals as the pre-Gateway setup." + ), + "tools": "Tool telemetry comes from claude_agent_v2.py/Hermes callbacks, not a one-shot bridge.", + }, + }, + "sentinel_cli": { + "id": "sentinel_cli", + "label": "Sentinel CLI", + "description": ( + "Gateway-owned listener with the original sentinel CLI runner semantics: " + "session resume, queueing, and parsed Claude/Codex tool activity." + ), + "kind": "builtin", + "passive": False, + "requires": [], + "form_fields": [ + { + "name": "workdir", + "label": "Workdir", + "required": False, + "placeholder": str(repo_root), + }, + { + "name": "model", + "label": "Model", + "required": False, + "placeholder": "opus or gpt-5.4", + }, + ], + "examples": [ + { + "label": "Claude sentinel", + "runtime_type": "sentinel_cli", + "workdir": str(repo_root), + "note": "Uses Claude CLI by default and resumes the same session for agent-level continuity.", + }, + ], + "signals": { + **_shared_signals(), + "activity": "Gateway parses Claude/Codex JSON streams and emits working, thinking, and tool phases.", + "tools": "Codex command events are recorded as tool calls; Claude tool-use blocks are surfaced as live tool activity.", + }, + }, "inbox": { "id": "inbox", "label": "Passive Inbox", @@ -130,14 +208,17 @@ def runtime_type_definition(runtime_type: str) -> dict[str, Any]: def runtime_type_list() -> list[dict[str, Any]]: catalog = runtime_type_catalog() - ordered_ids = ["echo", "exec", "inbox"] + ordered_ids = ["echo", "exec", "hermes_sentinel", "sentinel_cli", "inbox"] return [catalog[runtime_id] for runtime_id in ordered_ids if runtime_id in catalog] def agent_template_catalog() -> dict[str, dict[str, Any]]: repo_root = _repo_root() skill_path = _gateway_setup_skill_path() - runtime_signals = {key: runtime_type_definition(key)["signals"] for key in ("echo", "exec", "inbox")} + runtime_signals = { + key: runtime_type_definition(key)["signals"] + for key in ("echo", "exec", "hermes_sentinel", "sentinel_cli", "inbox") + } return { "echo_test": { "id": "echo_test", @@ -200,19 +281,19 @@ def agent_template_catalog() -> dict[str, dict[str, Any]]: }, "hermes": { "id": "hermes", - "label": "Hermes", - "description": "Local Hermes agent bridge with strong activity and tool telemetry.", - "availability": "setup_required", + "label": "Hermes Sentinel", + "description": "Long-running Hermes coding sentinel managed by Gateway.", + "availability": "ready", "launchable": True, - "runtime_type": "exec", + "runtime_type": "hermes_sentinel", "asset_class": "interactive_agent", "intake_model": "live_listener", "trigger_sources": ["direct_message"], "return_paths": ["inline_reply"], "telemetry_shape": "rich", "suggested_name": "hermes-bot", - "operator_summary": "Best path for a capable local agent with tool use and rich progress.", - "recommended_test_message": "Pause for 5 seconds, narrate activity as you go, and end with: Gateway test OK.", + "operator_summary": "Best path for a capable coding agent with continuity and rich progress.", + "recommended_test_message": "Remember the word cobalt, reply briefly, then I will ask you what word I gave you.", "what_you_need": [ "A local hermes-agent checkout, usually at ~/hermes-agent or via HERMES_REPO_PATH.", "Hermes auth or model credentials such as ~/.hermes/auth.json or provider env vars.", @@ -220,14 +301,44 @@ def agent_template_catalog() -> dict[str, dict[str, Any]]: "setup_skill": "gateway-agent-setup", "setup_skill_path": str(skill_path), "defaults": { - "runtime_type": "exec", - "exec_command": "python3 examples/hermes_sentinel/hermes_bridge.py", + "runtime_type": "hermes_sentinel", "workdir": str(repo_root), }, - "signals": runtime_signals["exec"], + "signals": runtime_signals["hermes_sentinel"], "advanced": { - "adapter_label": "Gateway command bridge", - "supports_command_override": True, + "adapter_label": "Gateway-supervised Hermes listener", + "supports_command_override": False, + }, + }, + "sentinel_cli": { + "id": "sentinel_cli", + "label": "Sentinel CLI", + "description": "Original aX sentinel runner pattern managed by Gateway.", + "availability": "ready", + "launchable": True, + "runtime_type": "sentinel_cli", + "asset_class": "interactive_agent", + "intake_model": "live_listener", + "trigger_sources": ["direct_message"], + "return_paths": ["inline_reply"], + "telemetry_shape": "rich", + "suggested_name": "dev-sentinel", + "operator_summary": "Best fit for long-lived coding sentinels that need session continuity and tool activity.", + "recommended_test_message": "Remember the word cobalt, reply briefly, then I will ask you what word I gave you.", + "what_you_need": [ + "Claude CLI or Codex CLI installed and authenticated on this machine.", + "A workdir containing the sentinel's local instructions.", + ], + "setup_skill": "gateway-agent-setup", + "setup_skill_path": str(skill_path), + "defaults": { + "runtime_type": "sentinel_cli", + "workdir": str(repo_root), + }, + "signals": runtime_signals["sentinel_cli"], + "advanced": { + "adapter_label": "Gateway sentinel CLI runner", + "supports_command_override": False, }, }, "claude_code_channel": { @@ -306,7 +417,7 @@ def agent_template_definition(template_id: str) -> dict[str, Any]: def agent_template_list(*, include_advanced: bool = False) -> list[dict[str, Any]]: catalog = agent_template_catalog() - ordered_ids = ["echo_test", "ollama", "hermes", "claude_code_channel", "inbox"] + ordered_ids = ["echo_test", "ollama", "hermes", "sentinel_cli", "claude_code_channel", "inbox"] templates = [catalog[template_id] for template_id in ordered_ids if template_id in catalog] if include_advanced: return templates diff --git a/docs/gateway-agent-runtimes.md b/docs/gateway-agent-runtimes.md new file mode 100644 index 0000000..587cf29 --- /dev/null +++ b/docs/gateway-agent-runtimes.md @@ -0,0 +1,198 @@ +# Gateway Agent Runtimes + +Gateway is the management plane for local agents. It should not force every +agent brain into a new runtime shape. + +The proven local setup before Gateway was: + +- Long-running sentinel listeners in `/home/ax-agent/agents`, launched by + scripts such as `start_hermes_sentinel.sh`. +- Hermes-backed coding agents using `claude_agent_v2.py --runtime hermes_sdk` + with Codex/OpenAI models. +- Claude Code sessions connected through `axctl channel` using agent-bound + profiles. +- Per-agent workdirs, notes, and local instructions under + `/home/ax-agent/agents//`. + +Gateway keeps those pieces, but moves operator management into one place: + +- Mint and store agent-bound credentials. +- Bind identity to device, workdir, runtime type, and launch spec. +- Start, stop, and observe local runtimes. +- Show liveness, queue state, activity, and tool signals. +- Provide a single CLI/UI for dev, staging, and production operators. + +## Current PR88 State + +PR88 has enough Gateway plumbing to register agents, mint tokens, show status, +queue passive inbox work, run simple built-in runtimes, and run command bridges +that emit `AX_GATEWAY_EVENT` progress lines. It also has the first +Gateway-supervised Hermes sentinel runtime, which preserves the old long-running +listener behavior instead of launching a new model process per message. + +Current useful modes: + +- `echo`: prove Gateway delivery and UI status. +- `inbox`: prove queueing and manual acknowledgement paths. +- `exec`: run probes or one-shot bridges that explicitly persist any state they + need. +- `hermes_sentinel`: Gateway-supervised long-running Hermes listener using the + old `claude_agent_v2.py --runtime hermes_sdk` behavior. + +Target modes still to implement: + +- `claude_code_channel`: Gateway-registered attached Claude Code channel with + health/liveness tracking around `axctl channel`. + +Use `hermes_sentinel` for coding sentinel QA. Avoid using a one-shot `exec` +bridge as proof that `dev_sentinel` is fixed. It can prove Gateway dispatch, +but not the session continuity that made the old sentinel setup useful. + +## Preferred Runtime Patterns + +### Hermes Sentinel + +Use Hermes for coding sentinels that need tool use, repo access, session +continuity, and rich activity. On this host, the preferred model family is the +Codex/OpenAI path, for example `codex:gpt-5.5` when available. + +The old working launcher shape is: + +```bash +/home/ax-agent/agents/start_hermes_sentinel.sh dev_sentinel \ + --runtime hermes_sdk \ + --model codex:gpt-5.5 +``` + +The Gateway-managed shape preserves that runtime behavior: + +```bash +ax gateway agents add dev_sentinel \ + --template hermes \ + --workdir /home/ax-agent/agents/dev_sentinel + +ax gateway agents start dev_sentinel +ax gateway agents show dev_sentinel +``` + +Gateway should supervise the long-running listener process. The listener still +owns the Hermes session, runtime plugin, message queue, and tool callbacks. The +Gateway owns the credentials, process lifecycle, binding verification, and +operator status. + +Do not treat the one-shot `examples/hermes_sentinel/hermes_bridge.py` demo as +the production sentinel pattern. It is useful for proving that a Gateway command +bridge can call Hermes, but it creates a fresh agent per message and does not +match the old sentinel continuity model. + +### Claude Code Channel + +Use Claude Code channels for agents backed by a Claude subscription. The channel +is an attached live session, not a headless per-message subprocess. + +The old working launcher shape is: + +```bash +cd /home/ax-agent/channel +eval "$(axctl profile env )" +exec axctl channel --agent --space-id +``` + +Claude Code then runs with the channel MCP server loaded: + +```bash +claude --strict-mcp-config \ + --mcp-config .mcp.json \ + --dangerously-load-development-channels server:ax-channel +``` + +The Gateway-managed shape should register and monitor this attached runtime: + +```bash +ax gateway agents add orion \ + --template claude_code_channel \ + --workdir /home/ax-agent/channel + +ax gateway agents show orion +``` + +Gateway should know which local Claude Code channel is attached, which +agent-bound token/profile it uses, and whether it is stale or healthy. The +channel remains responsible for delivering messages into Claude Code and for +emitting `working` and `completed` processing signals. + +### Command Bridge + +Use command bridges for simple adapters, demos, and smoke tests. + +```bash +ax gateway agents add echo-bot --type echo +ax gateway agents add probe \ + --type exec \ + --exec "python3 examples/gateway_probe/probe_bridge.py" +``` + +Command bridges are valuable for probes and simple integrations. They are not +the preferred shape for coding sentinels because a per-message command loses +important in-process state unless the bridge explicitly persists and resumes it. + +## Signal Contract + +Every inbound message should have a visible delivery signal before the final +reply. This is how operators know work did not disappear into a black hole. + +Minimum signals: + +- `picked_up` or `working`: the runtime received the message. +- `thinking`: the model/runtime started processing. +- `tool_call`: the runtime is using a tool, with a useful tool name or summary + when available. +- `completed`: the runtime finished and either replied or explicitly queued the + work. +- `error`: the runtime failed and the operator should inspect logs. + +Hermes sentinels should preserve the old behavior from `claude_agent_v2.py`: +tool callbacks update the activity bubble with real work, such as reading a +file, running a command, searching, or writing a note. + +Claude Code channels should at least emit delivery and completion. Richer tool +signals depend on what Claude Code exposes through the channel. + +## Migration From CLI-Managed Agents + +1. Inventory the old agent directory, launcher, workdir, model, and profile. +2. Register the agent in Gateway without changing its platform identity. +3. Mint or attach an agent-bound credential owned by Gateway. +4. Store the launch spec in Gateway: runtime family, workdir, command/profile, + and expected environment. +5. Start the same runtime through Gateway supervision. +6. Verify that the first inbound message gets a visible pickup/activity signal. +7. Verify continuity with a two-message memory test in the same thread. +8. Keep the old systemd/CLI launcher disabled once Gateway supervision is + stable, so only one listener receives each message. + +The important rule is one live receive path per agent. If the old CLI listener +and Gateway both listen for the same agent identity, messages can route through +different paths and create the stale/missing-context behavior seen during the +Gateway migration. + +## Dev Server Notes + +For `dev.paxai.app`, prefer building and testing against development agents +first. A good first continuity test is: + +```text +@dev_sentinel remember the word cobalt and reply briefly. +@dev_sentinel what word did I ask you to remember? +``` + +Expected result: + +- The original message shows Gateway pickup/working activity quickly. +- The runtime uses the Hermes session from the first turn on the second turn. +- The reply remembers `cobalt`. +- Tool activity appears when the agent reads files, writes notes, or runs + commands. + +If the second reply has no memory of the first, the agent is still being +cold-started per message or messages are reaching multiple receive paths. diff --git a/examples/hermes_sentinel/README.md b/examples/hermes_sentinel/README.md index 3e690c3..2093a8c 100644 --- a/examples/hermes_sentinel/README.md +++ b/examples/hermes_sentinel/README.md @@ -2,6 +2,16 @@ A minimal, runnable example of giving an aX agent a capable brain. +This example is intentionally a one-shot bridge. It is useful for proving that +Gateway or `ax listen --exec` can call Hermes and surface basic activity, but it +is not the preferred production sentinel shape. + +For Gateway-managed coding sentinels, use the long-running listener pattern +documented in [Gateway Agent Runtimes](../../docs/gateway-agent-runtimes.md). +That pattern preserves the old `claude_agent_v2.py` behavior: message queueing, +thread/session continuity, Hermes tool callbacks, and reliable processing +signals while Gateway owns credentials and lifecycle management. + Most aX agent examples (`examples/echo_agent.*`) are one-liners that prove the integration surface works. This example goes one step further: it wires [hermes-agent](https://github.com/madtank/hermes-agent) — diff --git a/tests/test_gateway_commands.py b/tests/test_gateway_commands.py index 6603568..b2abc77 100644 --- a/tests/test_gateway_commands.py +++ b/tests/test_gateway_commands.py @@ -898,6 +898,183 @@ def test_managed_exec_runtime_parses_gateway_progress_events(tmp_path, monkeypat assert "tool_finished" in events +def test_managed_sentinel_cli_runtime_resumes_agent_session(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + token_file = tmp_path / "token" + token_file.write_text("axp_a_agent.secret") + popen_calls = [] + + class _FakePipe: + def __init__(self, lines=None): + self.lines = list(lines or []) + self.writes = [] + + def __iter__(self): + return iter(self.lines) + + def write(self, text): + self.writes.append(text) + + def read(self): + return "" + + def close(self): + return None + + class _FakeProcess: + def __init__(self, cmd, **kwargs): + popen_calls.append(cmd) + self.stdin = _FakePipe() + self.stderr = _FakePipe() + self.returncode = 0 + if len(popen_calls) == 1: + self.stdout = _FakePipe( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-1"}), + json.dumps( + { + "type": "item.started", + "item": {"type": "command_execution", "id": "tool-1", "command": "pwd"}, + } + ), + json.dumps( + { + "type": "item.completed", + "item": { + "type": "command_execution", + "id": "tool-1", + "command": "pwd", + "exit_code": 0, + "aggregated_output": "/tmp", + }, + } + ), + json.dumps({"type": "item.completed", "item": {"type": "agent_message", "text": "remembered"}}), + ] + ) + else: + self.stdout = _FakePipe( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-1"}), + json.dumps({"type": "item.completed", "item": {"type": "agent_message", "text": "cobalt"}}), + ] + ) + + def wait(self, timeout=None): + return self.returncode + + def kill(self): + self.returncode = -9 + + monkeypatch.setattr(gateway_core.subprocess, "Popen", lambda cmd, **kwargs: _FakeProcess(cmd, **kwargs)) + shared = _SharedRuntimeClient({}) + runtime = gateway_core.ManagedAgentRuntime( + { + "name": "dev_sentinel", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "sentinel_cli", + "sentinel_runtime": "codex", + "workdir": str(tmp_path), + "token_file": str(token_file), + }, + client_factory=lambda **kwargs: shared, + ) + runtime._send_client = shared + + first = runtime._handle_prompt("remember cobalt", message_id="msg-1", data={"id": "msg-1"}) + second = runtime._handle_prompt("what word?", message_id="msg-2", data={"id": "msg-2"}) + + assert first == "remembered" + assert second == "cobalt" + assert "resume" not in popen_calls[0] + assert "resume" in popen_calls[1] + assert "thread-1" in popen_calls[1] + assert [row["status"] for row in shared.processing] == [ + "thinking", + "tool_call", + "tool_complete", + "thinking", + ] + assert shared.tool_calls[0]["tool_name"] == "shell" + assert shared.tool_calls[0]["message_id"] == "msg-1" + + +def test_managed_hermes_sentinel_runtime_supervises_long_running_listener(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + token_file = tmp_path / "token" + token_file.write_text("axp_a_agent.secret") + workdir = tmp_path / "agents" / "dev_sentinel" + workdir.mkdir(parents=True) + script = tmp_path / "agents" / "claude_agent_v2.py" + observed = tmp_path / "observed.json" + monkeypatch.setenv("TEST_HERMES_SENTINEL_OBSERVED", str(observed)) + script.write_text( + """ +import json +import os +import time + +path = os.environ["TEST_HERMES_SENTINEL_OBSERVED"] +with open(path, "w", encoding="utf-8") as handle: + json.dump( + { + "AX_TOKEN": os.environ.get("AX_TOKEN"), + "AX_BASE_URL": os.environ.get("AX_BASE_URL"), + "AX_AGENT_NAME": os.environ.get("AX_AGENT_NAME"), + "AX_AGENT_ID": os.environ.get("AX_AGENT_ID"), + "AX_SPACE_ID": os.environ.get("AX_SPACE_ID"), + "AX_CONFIG_DIR": os.environ.get("AX_CONFIG_DIR"), + }, + handle, + ) +while True: + time.sleep(1) +""".strip() + ) + hermes_repo = tmp_path / "hermes-agent" + hermes_repo.mkdir() + + runtime = gateway_core.ManagedAgentRuntime( + { + "name": "dev_sentinel", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://dev.paxai.app", + "runtime_type": "hermes_sentinel", + "template_id": "hermes", + "workdir": str(workdir), + "token_file": str(token_file), + "hermes_repo_path": str(hermes_repo), + "hermes_python": sys.executable, + "log_path": str(tmp_path / "hermes.log"), + } + ) + + runtime.start() + deadline = time.time() + 3.0 + while time.time() < deadline and not observed.exists(): + time.sleep(0.05) + snapshot = runtime.snapshot() + runtime.stop() + + assert observed.exists() + env = json.loads(observed.read_text()) + assert env["AX_TOKEN"] == "axp_a_agent.secret" + assert env["AX_BASE_URL"] == "https://dev.paxai.app" + assert env["AX_AGENT_NAME"] == "dev_sentinel" + assert env["AX_AGENT_ID"] == "agent-1" + assert env["AX_SPACE_ID"] == "space-1" + assert env["AX_CONFIG_DIR"] == str(workdir / ".ax") + assert snapshot["effective_state"] == "running" + assert snapshot["current_activity"] == "Hermes sentinel listener running" + + def test_managed_inbox_runtime_queues_message_without_reply(tmp_path, monkeypatch): config_dir = tmp_path / "config" config_dir.mkdir() @@ -937,12 +1114,73 @@ def test_managed_inbox_runtime_queues_message_without_reply(tmp_path, monkeypatc assert [row["status"] for row in shared.processing] == ["queued"] assert shared.processing[0]["activity"] == "Queued in Gateway" assert shared.processing[0]["detail"] == {"backlog_depth": 1, "pickup_state": "queued"} + pending = gateway_core.load_agent_pending_messages("inbox-bot") + assert pending == [ + { + "message_id": "msg-1", + "parent_id": None, + "conversation_id": None, + "content": "@inbox-bot hello there", + "display_name": None, + "created_at": pending[0]["created_at"], + "queued_at": pending[0]["queued_at"], + } + ] recent = gateway_core.load_recent_gateway_activity() events = [row["event"] for row in recent] assert "message_received" in events assert "message_queued" in events +def test_passive_runtime_snapshot_rehydrates_manual_queue_updates(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + token_file = tmp_path / "token" + token_file.write_text("axp_a_agent.secret") + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "inbox-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "inbox", + "token_file": str(token_file), + "backlog_depth": 0, + "current_status": None, + "current_activity": None, + "processed_count": 1, + "last_reply_message_id": "reply-1", + "last_reply_preview": "handled", + } + ] + gateway_core.save_gateway_registry(registry) + gateway_core.save_agent_pending_messages("inbox-bot", []) + + runtime = gateway_core.ManagedAgentRuntime( + { + "name": "inbox-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "inbox", + "token_file": str(token_file), + }, + client_factory=lambda **kwargs: _SharedRuntimeClient({}), + ) + runtime._update_state(backlog_depth=1, current_status="queued", current_activity="Queued in Gateway") + + snapshot = runtime.snapshot() + + assert snapshot["backlog_depth"] == 0 + assert snapshot["current_status"] is None + assert snapshot["current_activity"] is None + assert snapshot["processed_count"] == 1 + assert snapshot["last_reply_message_id"] == "reply-1" + assert snapshot["last_reply_preview"] == "handled" + + def test_annotate_runtime_health_marks_stale_after_missed_heartbeat(): old_seen = (datetime.now(timezone.utc) - timedelta(seconds=gateway_core.RUNTIME_STALE_AFTER_SECONDS + 5)).isoformat() @@ -1438,9 +1676,9 @@ def test_gateway_templates_command_json(): assert result.exit_code == 0, result.output payload = json.loads(result.stdout) ids = [item["id"] for item in payload["templates"]] - assert ids[:4] == ["echo_test", "ollama", "hermes", "claude_code_channel"] + assert ids[:5] == ["echo_test", "ollama", "hermes", "sentinel_cli", "claude_code_channel"] assert "inbox" not in ids - assert payload["count"] == 4 + assert payload["count"] == 5 ollama = next(item for item in payload["templates"] if item["id"] == "ollama") assert ollama["runtime_type"] == "exec" assert ollama["launchable"] is True @@ -1479,10 +1717,14 @@ def test_gateway_runtime_types_command_json(): assert result.exit_code == 0, result.output payload = json.loads(result.stdout) ids = [item["id"] for item in payload["runtime_types"]] - assert ids == ["echo", "exec", "inbox"] + assert ids == ["echo", "exec", "hermes_sentinel", "sentinel_cli", "inbox"] exec_type = next(item for item in payload["runtime_types"] if item["id"] == "exec") assert exec_type["signals"]["activity"] assert exec_type["examples"] + hermes_type = next(item for item in payload["runtime_types"] if item["id"] == "hermes_sentinel") + assert hermes_type["kind"] == "supervised_process" + sentinel_type = next(item for item in payload["runtime_types"] if item["id"] == "sentinel_cli") + assert sentinel_type["signals"]["tools"] def test_gateway_ui_handler_serves_status_and_agent_detail(monkeypatch, tmp_path): @@ -1549,15 +1791,15 @@ def test_gateway_ui_handler_serves_status_and_agent_detail(monkeypatch, tmp_path runtime_types = client.get("/api/runtime-types") assert runtime_types.status_code == 200 runtime_payload = runtime_types.json() - assert runtime_payload["count"] == 3 + assert runtime_payload["count"] == 5 assert runtime_payload["runtime_types"][1]["id"] == "exec" templates = client.get("/api/templates") assert templates.status_code == 200 template_payload = templates.json() assert template_payload["templates"][0]["id"] == "echo_test" - assert template_payload["templates"][3]["launchable"] is False - assert template_payload["count"] == 4 + assert template_payload["templates"][4]["launchable"] is False + assert template_payload["count"] == 5 detail = client.get("/api/agents/echo-bot") assert detail.status_code == 200 @@ -1726,6 +1968,13 @@ def test_gateway_agents_update_changes_template_and_workdir(monkeypatch, tmp_pat stored = gateway_core.load_gateway_registry()["agents"][0] assert stored["template_id"] == "ollama" assert stored["workdir"] == str(tmp_path) + registry_after = gateway_core.load_gateway_registry() + binding = registry_after["bindings"][0] + assert binding["launch_spec"]["runtime_type"] == "exec" + assert binding["launch_spec"]["workdir"] == str(tmp_path) + assert binding["path"] == str(tmp_path) + attestation = gateway_core.evaluate_runtime_attestation(registry_after, stored) + assert attestation["attestation_state"] == "verified" def test_gateway_agents_add_ollama_persists_model_override(monkeypatch, tmp_path): @@ -1907,6 +2156,84 @@ def test_gateway_agents_send_uses_managed_identity(monkeypatch, tmp_path): assert recent[-1]["event"] == "manual_message_sent" +def test_gateway_agents_send_acknowledges_pending_inbox_message(monkeypatch, tmp_path): + config_dir = tmp_path / "config" + monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir)) + gateway_core.save_gateway_session( + { + "token": "axp_u_test.token", + "base_url": "https://paxai.app", + "space_id": "space-1", + "username": "codex", + } + ) + token_file = tmp_path / "sender.token" + token_file.write_text("axp_a_agent.secret") + registry = gateway_core.load_gateway_registry() + registry["agents"] = [ + { + "name": "sender-bot", + "agent_id": "agent-1", + "space_id": "space-1", + "base_url": "https://paxai.app", + "runtime_type": "inbox", + "desired_state": "running", + "effective_state": "running", + "token_file": str(token_file), + "transport": "gateway", + "credential_source": "gateway", + "backlog_depth": 1, + "current_status": "queued", + "current_activity": "Queued in Gateway", + "last_received_message_id": "msg-queued-1", + "last_work_received_at": "2026-04-23T18:00:00+00:00", + } + ] + gateway_core.save_gateway_registry(registry) + gateway_core.save_agent_pending_messages( + "sender-bot", + [ + { + "message_id": "msg-queued-1", + "parent_id": None, + "conversation_id": "msg-queued-1", + "content": "@sender-bot hello there", + "display_name": "madtank", + "created_at": "2026-04-23T18:00:00+00:00", + "queued_at": "2026-04-23T18:00:01+00:00", + } + ], + ) + monkeypatch.setattr(gateway_cmd, "AxClient", _FakeManagedSendClient) + + result = runner.invoke( + app, + [ + "gateway", + "agents", + "send", + "sender-bot", + "handled", + "--parent-id", + "msg-queued-1", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["message"]["parent_id"] == "msg-queued-1" + assert gateway_core.load_agent_pending_messages("sender-bot") == [] + updated = gateway_core.find_agent_entry(gateway_core.load_gateway_registry(), "sender-bot") + assert updated["backlog_depth"] == 0 + assert updated["current_status"] is None + assert updated["current_activity"] is None + assert updated["processed_count"] == 1 + assert updated["last_reply_message_id"] == "msg-sent-1" + recent = gateway_core.load_recent_gateway_activity() + assert recent[-1]["event"] == "manual_queue_acknowledged" + + def test_gateway_agents_send_blocks_identity_mismatch(monkeypatch, tmp_path): config_dir = tmp_path / "config" monkeypatch.setenv("AX_CONFIG_DIR", str(config_dir))