diff --git a/pyproject.toml b/pyproject.toml index 903f54ae..a6d1d4f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -276,6 +276,7 @@ markers = [ "flaky_reason: baseline flakiness taxonomy stamp (kind + reason); see tests/e2e/baseline/flaky.py", "docker_build: builds/runs a real Docker image via subprocess (requires a local Docker daemon; skipped unless DOCKER_TESTS_ENABLED=true, including in CI, which has Docker but shouldn't pay this cost by default)", "sandbox: drives a Docker Sandbox via the sbx CLI (needs sbx + a nested-virtualization-capable host + live Band; skipped unless SANDBOX_TESTS_ENABLED=true — CI has neither the CLI nor nested virt)", + "vscode_chat: drives a real signed-in VS Code window via the code chat CLI (GUI + interactive Copilot sign-in; skipped unless VSCODE_CHAT_TESTS_ENABLED=true — can never run in CI)", ] [tool.uv] diff --git a/tests/conftest.py b/tests/conftest.py index 9c6acbb9..13eabbc3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -73,6 +73,7 @@ class CollectionGateSettings(BaseSettings): e2e_tests_enabled: bool = False # E2E_TESTS_ENABLED docker_tests_enabled: bool = False # DOCKER_TESTS_ENABLED sandbox_tests_enabled: bool = False # SANDBOX_TESTS_ENABLED + vscode_chat_tests_enabled: bool = False # VSCODE_CHAT_TESTS_ENABLED def pytest_ignore_collect(collection_path: Path) -> bool | None: @@ -89,35 +90,39 @@ def pytest_ignore_collect(collection_path: Path) -> bool | None: return None +# Opt-in suite gates: marker -> the CollectionGateSettings field that opens it. +# The env var IS the field name uppercased (the pydantic-settings contract), so +# the skip reason is derived — one row here is all a new gated suite needs. +GATED_MARKERS: dict[str, str] = { + "e2e": "e2e_tests_enabled", + "docker_build": "docker_tests_enabled", + "sandbox": "sandbox_tests_enabled", + "vscode_chat": "vscode_chat_tests_enabled", +} + + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: - """Skip e2e- and docker_build-marked tests unless explicitly enabled. - - tests/e2e/ gates itself through its own conftest; this covers - e2e-marked tests living elsewhere (e.g. the codex ACP protocol - tests), which spawn real backends and must never ride a normal - unit run. - - docker_build-marked tests shell out to a real `docker build`/`docker - run` — CI runners do have a Docker daemon (unlike the nested - virtualization sbx tests need), so a plain Docker-availability check - isn't enough to keep them off CI; they need the same explicit opt-in - as e2e tests. + """Skip gate-marked suites unless their env gate is explicitly enabled. + + tests/e2e/ gates itself through its own conftest; this covers marked + tests living elsewhere (e.g. the codex ACP protocol tests). These + suites spawn real backends, real `docker build`s, sbx microVMs, or a + live VS Code window — none may ride a normal unit run, and none can + rely on mere tool availability (CI runners do have Docker), so each + needs its explicit opt-in. """ gates = CollectionGateSettings() - skip_e2e = pytest.mark.skip(reason="set E2E_TESTS_ENABLED=true to run e2e tests") - skip_docker = pytest.mark.skip( - reason="set DOCKER_TESTS_ENABLED=true to run docker_build tests" - ) - skip_sandbox = pytest.mark.skip( - reason="set SANDBOX_TESTS_ENABLED=true to run sbx sandbox tests" - ) + closed = { + marker: pytest.mark.skip( + reason=f"set {field.upper()}=true to run {marker}-marked tests" + ) + for marker, field in GATED_MARKERS.items() + if not getattr(gates, field) + } for item in items: - if not gates.e2e_tests_enabled and item.get_closest_marker("e2e"): - item.add_marker(skip_e2e) - if not gates.docker_tests_enabled and item.get_closest_marker("docker_build"): - item.add_marker(skip_docker) - if not gates.sandbox_tests_enabled and item.get_closest_marker("sandbox"): - item.add_marker(skip_sandbox) + for marker, skip in closed.items(): + if item.get_closest_marker(marker): + item.add_marker(skip) @pytest.fixture(autouse=True) diff --git a/tests/e2e/vscode/.claude/skills/vscode-chat-e2e/SKILL.md b/tests/e2e/vscode/.claude/skills/vscode-chat-e2e/SKILL.md new file mode 100644 index 00000000..fe46d3ac --- /dev/null +++ b/tests/e2e/vscode/.claude/skills/vscode-chat-e2e/SKILL.md @@ -0,0 +1,79 @@ +--- +name: vscode-chat-e2e +description: Run the semi-manual Copilot-in-VS-Code E2E validation suite (tests/e2e/vscode) end to end — verify prerequisites, auto-install what can be installed (VS Code, band-mcp), start the live run, and coach the human through the few one-time dialogs only they can click (Copilot sign-in, folder trust, MCP/tool approvals). Use when asked to run, rerun, or validate the VS Code Copilot surface, or to produce its scorecard evidence. +--- + +# Copilot-in-VS-Code E2E run + +This suite is **inherently semi-manual** (see `tests/e2e/vscode/README.md`): it +drives a real, visible VS Code window with a human-signed-in Copilot. Your job +is to make the human's part as small as possible: check every prerequisite, +install whatever can be installed non-interactively, run the suite, and tell +the user exactly what to click **only** when there is no other way. + +## Phase 1 — prerequisites (install, don't ask, where possible) + +Check in this order; fix silently where a non-interactive fix exists. + +1. **VS Code + `code` CLI** — `code --version` and `code chat --help`. + - Missing entirely: `brew install --cask visual-studio-code` (macOS; the + cask links `code` onto PATH itself). + - App present but no CLI: link it — + `ln -sf "/Applications/Visual Studio Code.app/Contents/Resources/app/bin/code" /opt/homebrew/bin/code`. + - `chat` subcommand missing: VS Code too old (< 1.101) — + `brew upgrade --cask visual-studio-code`. +2. **Copilot Chat extension** — ships built into current VS Code; verify with + `code --install-extension GitHub.copilot-chat` (a "already installed" + response is success). +3. **band-mcp** — `uvx band-mcp --version` (uvx fetches it on first use; no + install step). Pass `BAND_MCP_COMMAND="uvx band-mcp"` to the run. +4. **Credentials** — `.env.test` must define `BAND_API_KEY_USER` (provisioning + + observer) and `ANTHROPIC_API_KEY` (judge). Missing keys cannot be + installed: stop and ask the user for them. +5. **Copilot sign-in** — cannot be automated (interactive GitHub OAuth, no + headless path). If unsure whether a login exists, just start Phase 2: a + signed-out Copilot shows "Sign in to use Copilot" in the chat panel — then + instruct: *open Chat (⌃⌘I), click "Sign in to use Copilot", finish the + browser OAuth, then say "signed in"* and rerun. + +## Phase 2 — the run + +```bash +VSCODE_CHAT_TESTS_ENABLED=true E2E_TESTS_ENABLED=true \ +BAND_MCP_COMMAND="uvx band-mcp" \ +VSCODE_CHAT_SCORECARD_JSON=artifacts/-copilot-vscode-scorecard.json \ +uv run pytest tests/e2e/vscode -v -s --no-cov +``` + +Run it in the background and monitor per-cell PASSED/FAILED lines. Kill any +leftover `band-mcp --transport sse` process before starting. Expect ~4 minutes +for 6 cells. + +## Phase 3 — coach the one-time dialogs (first run on a machine only) + +The workspace (`~/band-e2e/vscode-chat-workspace`) and band-mcp port (8631) +are stable, so every choice below is remembered — reruns are unattended. +When the VS Code window opens, tell the user to click, as dialogs appear: + +1. "Do you trust the authors…" → **Trust Folder & Continue** +2. MCP server `band` start/trust → **Allow/Trust** +3. Tool confirmations → dropdown next to Allow → **"Always allow…" (workspace)** + — once per tool (~4 band tools), never "this session" +4. "Allow edits to sensitive files" (shouldn't appear on the visible-path + workspace; if it does) → dropdown → **Always allow** + +Never tell the user to enable `chat.tools.global.autoApprove` ("YOLO mode") — +machine-wide security downgrade; the harness deliberately avoids it. + +## Phase 4 — verdict + evidence + +- Green: report `N passed`, point at the scorecard JSON + `.meta.json` + sidecar (versions evidence). Commit artifacts only when asked. +- A cell failing on reply content is usually live-model flakiness — rerun + once before investigating; two identical failures = real, read the + traceback (full failure section prints at session end). +- "Quota reached" badge in the window = Copilot premium requests exhausted; + turns will stall or degrade. Surface it to the user: paid seat, wait for + reset, or switch the chat model picker to an included model. +- Reap check: provisioned agents/rooms are auto-reaped (`BAND_E2E_AUTOCLEAN`); + leftover `band-mcp` processes should be killed if the run was interrupted. diff --git a/tests/e2e/vscode/README.md b/tests/e2e/vscode/README.md new file mode 100644 index 00000000..1fa600c7 --- /dev/null +++ b/tests/e2e/vscode/README.md @@ -0,0 +1,163 @@ +# Copilot-in-VS-Code validation suite (opt-in, dev-machine only) + +> ## ⚠️ This is inherently a SEMI-MANUAL test suite +> +> It drives a **real, visible VS Code window** with a **human-signed-in +> Copilot**. A person must be at the machine: sign in to Copilot once, and on +> the **first run** click through a handful of one-time dialogs (folder trust, +> MCP server allow, per-tool "Always allow"). Those choices are remembered, so +> **reruns are unattended** — but the GUI window, the interactive GitHub OAuth, +> and the possibility of a new confirmation dialog never go away. It can never +> run in CI and is not a hands-off matrix suite; treat every green run as +> "passed on a supervised dev machine", and record it via the scorecard +> artifact. + +Drives **GitHub Copilot Chat inside a real VS Code window** (agent mode) against +the live Band platform and validates the L0–L4 common bar. The harness: + +1. provisions a fresh Band agent identity, +2. runs **band-mcp** (SSE, loopback) holding that identity's key, +3. scaffolds a throwaway workspace whose `.vscode/mcp.json` points Copilot at it, +4. opens the workspace window once, then submits each turn via `code chat -m agent`, +5. asserts **Band-side only** (reply capture + LLM judge) — the VS Code UI is + never scraped. + +**This suite can never run in CI**: the Copilot extension authenticates through +interactive GitHub OAuth in a browser (no headless/service-account path), and a +GUI window must stay open. It is gated behind `VSCODE_CHAT_TESTS_ENABLED` +(marker `vscode_chat`), like the `DOCKER_TESTS_ENABLED` pattern. + +## Surface semantics (why the tests look like this) + +- **No inbound channel.** Copilot in VS Code cannot be pushed a Band message, + and band-mcp exposes **no message-read/history tool** (verified against + band-mcp 1.3.2). Every turn is driver-initiated: the prompt relays the room + message and pins the room `chat_id`. This mirrors real usage, where the + developer relays context into chat and Copilot posts via `band_send_message`. +- **Recall = platform memory tools.** Cross-session/cross-restart recall flows + through `band_store_memory` / `band_list_memories` (the server runs with + `--tools memory`), which are platform-persisted and session-independent. +- **No delivery acks.** band-mcp is REST-only, so the baseline + `wait_for_reply` barrier (delivery-status PROCESSED) never fires; the cells + wait on captured room messages instead. +- **L4 usage is N/A**: the surface exposes no per-turn usage/billing signal. + The scorecard carries the rationale as a fixed `na` row. + +## Prerequisites + +- **VS Code ≥ 1.102** with the `code` CLI on PATH (macOS: run + "Shell Command: Install 'code' command in PATH" from the Command Palette), + or set `CODE_COMMAND`. +- **GitHub Copilot Chat extension installed and signed in** (interactive; do + this once in the normal VS Code UI before running). +- **band-mcp ≥ 1.3.2** installed (`uv tool install band-mcp`), or set + `BAND_MCP_COMMAND` (e.g. `"uvx band-mcp"`). +- `.env.test` with `BAND_API_KEY_USER` (provisioning/observer) and + `ANTHROPIC_API_KEY` (the judge). + +## Running + +**Easiest**: use the directory-scoped Claude Code skill — `/vscode-chat-e2e` +(defined in `.claude/skills/vscode-chat-e2e/SKILL.md` next to this suite). It +checks prerequisites, installs what it can, runs the suite, and tells you +exactly what to click on a first run. + +Manual equivalent: + +```bash +VSCODE_CHAT_TESTS_ENABLED=true E2E_TESTS_ENABLED=true \ +VSCODE_CHAT_SCORECARD_JSON=artifacts/int-1110-copilot-vscode-scorecard.json \ +uv run pytest tests/e2e/vscode -v -s --no-cov +``` + +During the run a VS Code window opens on the scaffolded workspace. The harness +minimizes prompts, and after the first run reruns are unattended: + +- **Workspace trust**: VS Code asks once per folder ("Do you trust the + authors…") — click **Trust Folder & Continue**. The workspace path is + stable by default, so the choice is remembered for every rerun. (The + launch deliberately does *not* use `--disable-workspace-trust`: Copilot's + AI features require a trusted workspace, so that flag would only defer + the same dialog to chat time.) +- **MCP server trust**: VS Code asks once per server *configuration*. The + workspace path and the band-mcp port are stable by default, so `mcp.json` + is byte-identical across runs — allow the `band` server on the first run + and the remembered trust holds for every rerun. (Changing + `VSCODE_CHAT_WORKSPACE` or `BAND_MCP_PORT` re-triggers the prompt.) +- **Tool approvals**: on each tool's first-ever call, open the dropdown next + to Allow and pick **"Always allow…" (workspace)** — one click per tool + (~4 band tools), remembered afterwards. The workspace also sets + `chat.autoReply` so agent-side questions never stall a turn. +- Leave the window open and unfocused-but-alive until the run finishes. + +> **Security note:** the harness deliberately does **not** enable +> `chat.tools.global.autoApprove` ("YOLO mode") — that disables tool approval +> for every workspace on the machine and VS Code escalates it to a global +> consent dialog. Approvals stay per-tool and workspace-scoped. If you enable +> YOLO in your own user settings, that is a machine-wide security trade-off +> you own. + +Artifacts (when `VSCODE_CHAT_SCORECARD_JSON` is set): + +- `artifacts/int-1110-copilot-vscode-scorecard.json` — pass/fail/skip rows plus + the fixed L4 `na` row (baseline scorecard row schema, merge-compatible). +- `…-scorecard.json.meta.json` — environment evidence: OS, `code --version`, + Copilot extension versions, band-mcp version. + +## Environment knobs + +| Env var | Default | Purpose | +|---|---|---| +| `VSCODE_CHAT_TESTS_ENABLED` | `false` | The collection gate | +| `CODE_COMMAND` | `code` | VS Code CLI binary (may be a full path) | +| `BAND_MCP_COMMAND` | `band-mcp` | band-mcp launcher (e.g. `uvx band-mcp`) | +| `BAND_MCP_PORT` | `8631` | band-mcp SSE port (`0` = ephemeral; stable keeps MCP trust remembered) | +| `VSCODE_CHAT_WORKSPACE` | `~/band-e2e/vscode-chat-workspace` | Workspace dir VS Code opens (stable keeps MCP trust remembered) | +| `VSCODE_CHAT_SCORECARD_JSON` | empty | Scorecard output path (empty = don't emit) | +| `VSCODE_CHAT_TIMEOUT` | `300` | Seconds allowed per live turn | + +Band endpoints, credentials, autoclean/orphan-sweep policy, and the judge model +come from the baseline settings (`tests/e2e/baseline/settings.py`). + +## Cells + +| Test | Level | Proves | +|---|---|---| +| `test_participation_reply_round_trip` | L0 | Room message → Copilot → `band_send_message` reply with the echo token | +| `test_original_functions_retained` | L1 | Native function (workspace file) + platform tool (`band_get_participants`) in one turn | +| `test_multi_participant_echo_peer` | L0 | A peer agent's message drives a turn; the reply engages the peer | +| `test_recall_across_chat_sessions` | L2 | Fact stored via memory tools survives to a fresh chat session | +| `test_no_leak_between_rooms` | L2 | Room B's answer names room B's marker, never room A's | +| `test_restart_recall_and_function` | L3 | band-mcp restart between turns; recall + native function still work | +| *(no test)* `usage_accounting` | L4 | `na` scorecard row — no per-turn usage signal exposed | + +## Manual variants / known weak spots + +- **Fresh chat session** (`new_session=True`) is expressed as a prompt preamble — + the `code chat` CLI has no verified per-invocation new-session switch. For a + stronger variant, click **New Chat** in the window between the two turns of + `test_recall_across_chat_sessions` while it waits. +- **Full window restart** (quit VS Code between the turns of + `test_restart_recall_and_function`, reopen with `code `) is a + manual variant; the automated cell restarts the platform bridge (band-mcp) + instead, which is deterministic. + +## Troubleshooting + +- **Prompt lands in the wrong window**: `code chat` targets the window whose + workspace matches the CWD; keep only the harness's workspace window open, or + close other VS Code windows. +- **421 responses from band-mcp**: the server sets + `ALLOWED_HOSTS='["localhost:*","127.0.0.1:*"]'` itself; if you changed the + host/port wiring, keep the allowlist in sync. +- **Turn times out with no reply**: check the window — a pending trust/approval + dialog blocks the turn. Approve it; the run continues on the next cell. +- **`PreflightError`**: the `code` CLI is missing or predates the `chat` + subcommand — upgrade VS Code / fix `CODE_COMMAND`. + +## Future work + +`driver.PromptDriver` is a protocol: a `@vscode/test-electron` backend (pinned +VS Code download, persistent signed-in profile, programmatic +`workbench.action.chat.open`) can replace `CodeChatDriver` without touching the +cells. diff --git a/tests/e2e/vscode/__init__.py b/tests/e2e/vscode/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/vscode/conftest.py b/tests/e2e/vscode/conftest.py new file mode 100644 index 00000000..ed8d24f5 --- /dev/null +++ b/tests/e2e/vscode/conftest.py @@ -0,0 +1,240 @@ +"""Pytest wiring for the Copilot-in-VS-Code suite. + +Mirrors the baseline conftest's registration style: the shared platform/capture +fixtures are plain imports re-exported via ``__all__`` (``pytest_plugins`` is +not allowed in a non-root conftest), and the hooks here add only what this +suite needs — the gate, the session loop + per-turn timeout markers, and the +suite-local scorecard plugin. + +The suite drives a real signed-in VS Code window, so beyond the env gate it has +human prerequisites (see README.md): VS Code installed with the Copilot Chat +extension signed in, and the window the ``driver`` fixture opens left alone. +""" + +from __future__ import annotations + +import logging +import shlex +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest +from band_rest import AsyncRestClient + +from tests.e2e.baseline.fixtures.capture import judge, reply_capture +from tests.e2e.baseline.fixtures.platform import ( + baseline_run_id, + baseline_settings, + baseline_user_client, + baseline_ws, + orphan_sweep, + reap_leaked_agents, + resource_manager, + user_ops, +) +from tests.e2e.baseline.settings import BaselineSettings +from tests.e2e.baseline.toolkit.provisioning import ProvisionedAgent, ResourceManager +from tests.e2e.vscode.driver import ( + CodeChatDriver, + PreflightError, + capture_versions, + vscode_window, +) +from tests.e2e.vscode.rooms import SurfaceRoom +from tests.e2e.vscode.scorecard import VSCodeScorecard +from tests.e2e.vscode.server import BandMCPServer +from tests.e2e.vscode.settings import VSCodeChatSettings +from tests.e2e.vscode.workspace import scaffold_workspace +from tests.toolkit.timeouts import effective_timeout + +logger = logging.getLogger(__name__) + +# Re-exported fixtures (defined in the baseline fixtures package). +__all__ = [ + "band_mcp", + "baseline_run_id", + "baseline_settings", + "baseline_user_client", + "baseline_ws", + "copilot_identity", + "driver", + "judge", + "orphan_sweep", + "reap_leaked_agents", + "reply_capture", + "resource_manager", + "surface_room", + "user_ops", + "vscode_settings", + "vscode_workspace", +] + + +@pytest.fixture(scope="session") +def vscode_settings() -> VSCodeChatSettings: + return VSCodeChatSettings() + + +@pytest.fixture(scope="session") +async def copilot_identity( + baseline_settings: BaselineSettings, + baseline_user_client: AsyncRestClient, + baseline_run_id: str, +) -> AsyncGenerator[ProvisionedAgent, None]: + """The one Band agent identity Copilot acts as, held for the whole session. + + Session-scoped (its own manager, not the per-test ``resource_manager``) + because the identity's key is baked into the band-mcp subprocess env and + the workspace's MCP config at session start. + """ + resources = ResourceManager( + user_client=baseline_user_client, + settings=baseline_settings, + run_id=baseline_run_id, + ) + identity = await resources.provision_agent("copilot") + yield identity + if baseline_settings.run.autoclean: + await resources.reap_all() + + +@pytest.fixture(scope="session") +async def band_mcp( + vscode_settings: VSCodeChatSettings, + baseline_settings: BaselineSettings, + copilot_identity: ProvisionedAgent, +) -> AsyncGenerator[BandMCPServer, None]: + server = BandMCPServer( + shlex.split(vscode_settings.band_mcp_command), + agent_key=copilot_identity.api_key, + base_url=baseline_settings.endpoints.rest_url, + port=vscode_settings.band_mcp_port, + ) + await server.start() + yield server + await server.stop() + + +@pytest.fixture(scope="session") +def vscode_workspace( + vscode_settings: VSCodeChatSettings, band_mcp: BandMCPServer +) -> Path: + """The (persistent by default) workspace VS Code opens. + + A stable path + stable band-mcp port keep ``.vscode/mcp.json`` identical + across runs, so VS Code's remembered MCP-server trust holds and reruns + prompt for nothing. Cell marker files use per-run tokens, so leftovers + from earlier runs never collide. + """ + workspace = Path(vscode_settings.vscode_chat_workspace) + workspace.mkdir(parents=True, exist_ok=True) + scaffold_workspace(workspace, band_mcp.sse_url) + return workspace + + +@pytest.fixture(scope="session") +async def driver( + vscode_settings: VSCodeChatSettings, vscode_workspace: Path +) -> AsyncGenerator[CodeChatDriver, None]: + """One ready-to-drive VS Code window for the session (see ``vscode_window``).""" + try: + async with vscode_window( + shlex.split(vscode_settings.code_command), vscode_workspace + ) as chat_driver: + yield chat_driver + except PreflightError as error: + pytest.fail(str(error)) + + +@pytest.fixture +def surface_room( + resource_manager: ResourceManager, + user_ops, + reply_capture, + driver: CodeChatDriver, + copilot_identity: ProvisionedAgent, + vscode_settings: VSCodeChatSettings, +): + """Factory: open a provisioned room with the Copilot surface bound. + + ``async with surface_room("recall") as room:`` yields a ``SurfaceRoom`` + (see ``rooms.py``) — the cells' one intent object; all turn plumbing + (mentions, prompt relay, reply wait) lives behind it. + """ + + @asynccontextmanager + async def open_room( + label: str, *, participants: tuple[str, ...] = () + ) -> AsyncGenerator[SurfaceRoom, None]: + room_id = await resource_manager.provision_room( + title=f"e2e-vscode-{label}", + participants=[copilot_identity.id, *participants], + ) + async with reply_capture(room_id) as capture: + yield SurfaceRoom( + room_id=room_id, + capture=capture, + driver=driver, + identity=copilot_identity, + user_ops=user_ops, + resources=resource_manager, + turn_timeout=vscode_settings.vscode_chat_timeout, + ) + + return open_room + + +def pytest_configure(config: pytest.Config) -> None: + """Register the suite scorecard plugin for an enabled live run. + + Gated on the suite gate too, not just the artifact path: a lingering + VSCODE_CHAT_SCORECARD_JSON in the environment must not make an unrelated + (gate-off, all-skip) run shell out to ``code``/``band-mcp`` for version + evidence — a cold ``uvx band-mcp`` would stall collection on a download. + """ + settings = VSCodeChatSettings() + if not (settings.vscode_chat_tests_enabled and settings.vscode_chat_scorecard_json): + return + suite_prefix = ( + Path(__file__).parent.resolve().relative_to(config.rootpath).as_posix() + "/" + ) + plugin = VSCodeScorecard(settings.vscode_chat_scorecard_json, suite_prefix) + plugin.metadata = capture_versions( + shlex.split(settings.code_command), shlex.split(settings.band_mcp_command) + ) + config.pluginmanager.register(plugin) + + +def pytest_runtest_setup(item: pytest.Item) -> None: + """Self-gate (besides the root markers) and fail loud on missing credentials.""" + if not Path(item.path).is_relative_to(Path(__file__).parent): + return + if not VSCodeChatSettings().vscode_chat_tests_enabled: + pytest.skip("VSCODE_CHAT_TESTS_ENABLED is not true") + if not BaselineSettings().credentials.api_key_user: + pytest.fail("BAND_API_KEY_USER not set (VS Code chat suite enabled)") + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Put suite tests on the session loop and give them the live-turn timeout. + + Same two markers the baseline conftest applies to its own subtree: the + session-scoped WS/REST fixtures live on the session loop, and live turns + need far more than the 30s pyproject default. + """ + suite_dir = Path(__file__).parent + session_marker = pytest.mark.asyncio(loop_scope="session") + base = VSCodeChatSettings().vscode_chat_timeout + for item in items: + if not Path(item.path).is_relative_to(suite_dir): + continue + # Prepended (append=False) so this beats pytest-asyncio's bare + # auto-mode marker — get_closest_marker takes the first hit, and the + # bare marker's default function loop would strand the session-scoped + # WS/REST fixtures on another loop (subscribe timeouts, closed-loop + # teardown errors). + item.add_marker(session_marker, append=False) + timeout = effective_timeout(item, base) + if timeout is not None: + item.add_marker(pytest.mark.timeout(timeout), append=False) diff --git a/tests/e2e/vscode/driver.py b/tests/e2e/vscode/driver.py new file mode 100644 index 00000000..478ce655 --- /dev/null +++ b/tests/e2e/vscode/driver.py @@ -0,0 +1,205 @@ +"""Drive Copilot Chat in a real VS Code window through the ``code chat`` CLI. + +The surface has no inbound channel: Copilot in VS Code cannot be pushed a Band +room message (band-mcp exposes no message-read tool either), so every turn is +driver-initiated — the prompt relays the triggering room message and instructs +Copilot to act through the band MCP tools. This mirrors real usage, where the +developer relays context into chat and Copilot posts to Band. + +``PromptDriver`` is the seam: assertions never touch VS Code (they are all +Band-side), so a future ``@vscode/test-electron`` backend only has to submit +prompts to slot in. +""" + +from __future__ import annotations + +import asyncio +import logging +import platform +import shutil +import subprocess +from collections.abc import AsyncIterator, Callable +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Protocol + +from tests.e2e.vscode.workspace import MCP_SERVER_NAME + +logger = logging.getLogger(__name__) + +# `code chat` submits and returns; the window processes the turn asynchronously. +# This bounds only the CLI handoff, not the model turn (Band-side waits do that). +SUBMIT_TIMEOUT_S = 60 + +# The `code chat` CLI has no verified per-invocation new-session switch, so a +# fresh-session request is expressed in the prompt itself. Weaker than a real +# session reset — the runbook documents the manual "New Chat" variant. +FRESH_SESSION_PREAMBLE = ( + "Start from a clean slate: ignore everything discussed earlier in this " + "chat and rely only on this message and your tools." +) + + +class PreflightError(RuntimeError): + """The VS Code CLI is missing or too old to drive; message points at the runbook.""" + + +class PromptDriver(Protocol): + """Submit one agent-mode prompt to the surface under test.""" + + async def submit_prompt(self, prompt: str, *, new_session: bool = False) -> None: + """Send ``prompt`` to Copilot Chat; ``new_session`` requests a fresh context.""" + ... + + +def turn_prompt( + chat_id: str, + agent_name: str, + *, + sender_name: str, + message: str, + instruction: str, +) -> str: + """The one prompt shape every cell submits. + + Relays the room message (the surface cannot read it back) and pins the + room id, since every band tool call takes an explicit ``chat_id``. + """ + return ( + f"You are the Band platform agent '{agent_name}'. You participate in " + f"Band chat rooms through the MCP tools of the '{MCP_SERVER_NAME}' " + f"server (band_send_message, band_get_participants, band_store_memory, " + f"band_list_memories, ...). Every band tool call must pass " + f"chat_id='{chat_id}'.\n\n" + f"New message in the room from {sender_name}:\n" + f"---\n{message}\n---\n\n" + f"{instruction}\n" + f"Always deliver your answer into the room with band_send_message, " + f"mentioning {sender_name}." + ) + + +# Seconds after opening the workspace for the window, extension host, and MCP +# client to come up before the first prompt is submitted. +WINDOW_OPEN_GRACE_S = 20 + + +class CodeChatDriver: + """Submit prompts via ``code chat -m agent`` against the workspace's window. + + Construct through :func:`vscode_window`, which owns the ready-to-drive + lifecycle (preflight -> open -> readiness grace). + """ + + def __init__(self, code_command: list[str], workspace: Path) -> None: + self._code_command = code_command + self._workspace = workspace + + async def submit_prompt(self, prompt: str, *, new_session: bool = False) -> None: + if new_session: + prompt = f"{FRESH_SESSION_PREAMBLE}\n\n{prompt}" + await self._run("chat", "-m", "agent", prompt) + + async def _run(self, *args: str) -> None: + process = await asyncio.create_subprocess_exec( + *self._code_command, + *args, + cwd=self._workspace, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + try: + output, _ = await asyncio.wait_for( + process.communicate(), timeout=SUBMIT_TIMEOUT_S + ) + except TimeoutError: + # A hung CLI must not outlive the turn: left alone it could still + # deliver its prompt minutes later, into a rerun's session. + process.kill() + await process.wait() + raise RuntimeError( + f"code {args[0]} did not return within {SUBMIT_TIMEOUT_S}s" + ) from None + if output: + logger.debug("code %s: %s", args[0], output.decode(errors="replace")) + if process.returncode != 0: + raise RuntimeError( + f"code {args[0]} exited with {process.returncode}: " + f"{output.decode(errors='replace').strip()}" + ) + + +@asynccontextmanager +async def vscode_window( + code_command: list[str], workspace: Path +) -> AsyncIterator[CodeChatDriver]: + """A ready-to-drive VS Code window on ``workspace``. + + Entry owns the whole becoming-drivable flow: preflight the CLI (raises + :class:`PreflightError` with the runbook pointer), open the workspace + window, and wait out the startup grace so the extension host and MCP + client are up before the first prompt. Opened deliberately *without* + ``--disable-workspace-trust``: Copilot's AI features require a trusted + workspace, so an untrusted launch only defers the dialog to chat time — + the stable workspace path makes "Trust Folder & Continue" a one-time click. + + Exit is a no-op by contract: the window is human-owned (the human signed + in to Copilot there) and stays open across runs. + """ + preflight(code_command) + driver = CodeChatDriver(code_command, workspace) + await driver._run(str(workspace)) + await asyncio.sleep(WINDOW_OPEN_GRACE_S) + yield driver + + +Runner = Callable[[list[str]], str] + + +def _default_runner(command: list[str]) -> str: + return subprocess.run( + command, capture_output=True, text=True, timeout=SUBMIT_TIMEOUT_S, check=True + ).stdout + + +def preflight(code_command: list[str], *, run: Runner = _default_runner) -> None: + """Fail fast (with the runbook pointer) when the VS Code CLI cannot drive chat.""" + hint = "see tests/e2e/vscode/README.md for setup" + if shutil.which(code_command[0]) is None: + raise PreflightError(f"'{code_command[0]}' not found on PATH — {hint}") + try: + run([*code_command, "--version"]) + run([*code_command, "chat", "--help"]) + except (subprocess.SubprocessError, OSError) as error: + raise PreflightError( + f"VS Code CLI cannot drive chat ({error}) — {hint}" + ) from error + + +def capture_versions( + code_command: list[str], + band_mcp_command: list[str], + *, + run: Runner = _default_runner, +) -> dict[str, str]: + """The environment evidence the scorecard sidecar records for a live run.""" + + def capture(command: list[str]) -> str: + try: + return run(command).strip() + except (subprocess.SubprocessError, OSError) as error: + return f"unavailable ({error})" + + extensions = capture([*code_command, "--list-extensions", "--show-versions"]) + copilot_lines = [ + line for line in extensions.splitlines() if "copilot" in line.lower() + ] + # Current VS Code bundles Copilot Chat, which --list-extensions omits; its + # version is then pinned by the VS Code build recorded above. + copilot = "; ".join(copilot_lines) or "built-in (bundled with this VS Code build)" + return { + "os": platform.platform(), + "vscode": capture([*code_command, "--version"]).replace("\n", " "), + "copilot_extensions": copilot, + "band_mcp": capture([*band_mcp_command, "--version"]), + } diff --git a/tests/e2e/vscode/rooms.py b/tests/e2e/vscode/rooms.py new file mode 100644 index 00000000..cb732bfa --- /dev/null +++ b/tests/e2e/vscode/rooms.py @@ -0,0 +1,127 @@ +"""A Band room with the Copilot-in-VS-Code surface in it, driven turn by turn. + +``SurfaceRoom`` is the one intent object the cells speak through: *someone says +something in the room, the driver relays it to Copilot, the agent's reply comes +back Band-side*. It hides the turn plumbing (mention wiring, prompt templating, +the captured-message wait) so a cell reads as scenario prose. + +The reply wait is a predicate over captured room messages, not the baseline +``wait_for_reply`` barrier: that barrier keys on the recipient's delivery-status +PROCESSED ack, which only an SDK runtime emits — this surface posts through +band-mcp (REST only), so no delivery ack ever arrives. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from tests.e2e.baseline.toolkit.capture import ReplyCapture +from tests.e2e.baseline.toolkit.observations.replies import Replies +from tests.e2e.baseline.toolkit.provisioning import ProvisionedAgent, ResourceManager +from tests.e2e.baseline.toolkit.user_ops import UserOps +from tests.e2e.vscode.driver import CodeChatDriver, turn_prompt + + +def transcript(replies: Replies, *, peer_message: str = "") -> str: + """The judge's view of a turn: the labeled reply, with the peer's message + when the verdict is about engaging it (a bare reply reads as unjudgeable).""" + lines = [f"Peer agent posted: {peer_message}"] if peer_message else [] + lines += [f"Reply from the agent under test: {reply.content}" for reply in replies] + return "\n".join(lines) + + +@dataclass +class SurfaceRoom: + """One provisioned room, its reply capture, and the turn protocol bound.""" + + room_id: str + capture: ReplyCapture + driver: CodeChatDriver + identity: ProvisionedAgent + user_ops: UserOps + resources: ResourceManager + turn_timeout: float + + async def user_turn( + self, message: str, *, instruction: str, new_session: bool = False + ) -> Replies: + """The user posts ``message`` (mentioning the agent); return the reply.""" + await self.user_ops.send_message( + self.room_id, + message, + mention_id=self.identity.id, + mention_name=self.identity.name, + ) + return await self._agent_reply( + sender_name="the user", + message=message, + instruction=instruction, + new_session=new_session, + ) + + async def peer_turn( + self, peer: ProvisionedAgent, message: str, *, instruction: str + ) -> Replies: + """A peer agent posts ``message`` (mentioning the agent); return the reply.""" + await self.resources.peer(peer).send_message( + self.room_id, + message, + mention_id=self.identity.id, + mention_name=self.identity.name, + ) + return await self._agent_reply( + sender_name=peer.name, message=message, instruction=instruction + ) + + async def remember(self, announcement: str, *, record: str) -> None: + """Seed a platform memory: the user announces a fact, the agent must + store ``record`` via ``band_store_memory``, and a Band-side barrier + proves the record landed (separating a store failure from a retrieval + failure in the next turn). + + The shape is load-bearing, learned from captured failures: a fresh + session (otherwise the agent sees an earlier store in its chat context + and acks without storing) and a hard imperative (a soft "store X and + confirm" gets satisficed into a bare confirmation). + """ + await self.user_turn( + announcement, + instruction=( + f"Call band_store_memory now, exactly once, even if similar " + f"records already exist — content exactly: '{record}'. " + f"Then acknowledge it in the room." + ), + new_session=True, + ) + observation = await self.capture.memory(self.identity) + observation.stored.assert_stored(content=record) + + async def _agent_reply( + self, + *, + sender_name: str, + message: str, + instruction: str, + new_session: bool = False, + ) -> Replies: + """Relay the room message to Copilot and wait for the agent's reply.""" + mark = self.capture.messages.snapshot() + await self.driver.submit_prompt( + turn_prompt( + self.room_id, + self.identity.name, + sender_name=sender_name, + message=message, + instruction=instruction, + ), + new_session=new_session, + ) + + def replied(_messages: list) -> bool: + return bool(self._replies_since(mark)) + + await self.capture.wait_until(replied, deadline_s=self.turn_timeout) + return self._replies_since(mark) + + def _replies_since(self, mark: int) -> Replies: + return self.capture.messages.since(mark).from_sender(self.identity.id) diff --git a/tests/e2e/vscode/scorecard.py b/tests/e2e/vscode/scorecard.py new file mode 100644 index 00000000..78045a53 --- /dev/null +++ b/tests/e2e/vscode/scorecard.py @@ -0,0 +1,88 @@ +"""Scorecard emission for the VS Code Copilot surface. + +Reuses the baseline scorecard's row schema and writer so the artifact merges +with the lane scorecards, but records rows itself: the baseline collector only +counts ``[adapter]``-parametrized matrix cells, and this suite's tests are +bespoke (unparametrized, one fixed surface column). +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +import pytest + +from tests.e2e.baseline.scorecard import ScorecardRow, Status, write_json + +logger = logging.getLogger(__name__) + +# The scorecard column for this surface (a surface id, not a registered +# matrix adapter — deliberately absent from the baseline Adapter registry). +SURFACE_ID = "copilot_vscode" + +# L4 usage has no test function: the surface offers nothing to assert — Copilot +# Chat exposes no per-turn usage/billing signal to the harness and posts no +# Band-side usage events. The rationale ships as a scorecard row instead. +USAGE_NA_ROW = ScorecardRow( + test="tests/e2e/vscode/test_copilot_chat.py::usage_accounting", + adapter=SURFACE_ID, + status="na", + reason=( + "Copilot Chat in VS Code exposes no per-turn usage/billing signal to " + "the harness and emits no Band-side usage events" + ), +) + + +def outcome_status(report: pytest.TestReport) -> Status | None: + """The row status one setup/call report contributes (same semantics as the + baseline collector: skip anywhere = skip, failed setup or call = fail, + passing call = pass, passing setup = no verdict yet).""" + match (report.when, report.skipped, report.failed): + case ("setup" | "call", True, _): + return "skip" + case ("setup" | "call", _, True): + return "fail" + case ("call", _, _): + return "pass" + case _: # teardown reports and passing setups carry no verdict + return None + + +class VSCodeScorecard: + """Pytest plugin: one row per suite test plus the fixed L4 ``na`` row, + written with a ``.meta.json`` environment sidecar at session end.""" + + def __init__(self, path: str | Path, suite_prefix: str) -> None: + """``suite_prefix``: the rootdir-relative nodeid prefix of the suite + (e.g. ``"tests/e2e/vscode/"``). Nodeids are rootdir-relative regardless + of the invocation directory — a filesystem-path filter would resolve + against the CWD and silently drop every row when pytest runs from + outside the repo root.""" + self._path = Path(path) + self._suite_prefix = suite_prefix + self._rows: dict[str, ScorecardRow] = {} + self.metadata: dict[str, str] = {} + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + if not report.nodeid.startswith(self._suite_prefix): + return + status = outcome_status(report) + if status is None: + return + self._rows[report.nodeid] = ScorecardRow( + test=report.nodeid, adapter=SURFACE_ID, status=status + ) + + def scorecard(self) -> list[ScorecardRow]: + rows = dict(self._rows) + rows[USAGE_NA_ROW.test] = USAGE_NA_ROW + return sorted(rows.values(), key=lambda row: row.test) + + def pytest_sessionfinish(self) -> None: + write_json(self.scorecard(), self._path) + meta_path = self._path.with_suffix(self._path.suffix + ".meta.json") + meta_path.write_text(json.dumps(self.metadata, indent=2) + "\n") + logger.info("scorecard written to %s (+ %s)", self._path, meta_path.name) diff --git a/tests/e2e/vscode/server.py b/tests/e2e/vscode/server.py new file mode 100644 index 00000000..b50d66b0 --- /dev/null +++ b/tests/e2e/vscode/server.py @@ -0,0 +1,151 @@ +"""Run band-mcp (SSE) as a restartable subprocess for the suite. + +The server holds the provisioned Copilot agent identity: VS Code's MCP client +presents no credentials, so the agent key travels in the subprocess env. It is +restartable on a stable port because the workspace's ``mcp.json`` is written +once — the L3 cell restarts this process to prove the surface survives its +platform bridge going away and coming back. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import socket + +logger = logging.getLogger(__name__) + +# Without this allowlist band-mcp's DNS-rebinding protection answers 421 to +# loopback SSE clients (same value the copilot_docker examples use). +ALLOWED_HOSTS = '["localhost:*","127.0.0.1:*"]' + +# The memory tool group is opt-in and is the suite's recall path: band-mcp +# exposes no room-history tool, so cross-session recall flows through +# band_store_memory / band_list_memories. +TOOL_GROUPS = "memory" + +READY_TIMEOUT_S = 30.0 +STOP_TIMEOUT_S = 10.0 + + +def _reserve_port() -> int: + """Take one ephemeral loopback port from the OS, then release it for the server. + + The close→rebind gap is the standard reservation race — acceptable for a + single local server (mirrors ``parlant_server._reserve_two_ports``). + """ + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class BandMCPServer: + """Lifecycle of one band-mcp SSE subprocess bound to a stable loopback port.""" + + def __init__( + self, command: list[str], *, agent_key: str, base_url: str, port: int = 0 + ) -> None: + """``port=0`` reserves an ephemeral one; a fixed port keeps the workspace's + mcp.json stable across runs, so VS Code's remembered MCP-server trust + holds and reruns need no re-approval.""" + self._command = command + self._agent_key = agent_key + self._base_url = base_url + self._port = port or _reserve_port() + self._process: asyncio.subprocess.Process | None = None + + @property + def sse_url(self) -> str: + return f"http://127.0.0.1:{self._port}/sse" + + @property + def running(self) -> bool: + return self._process is not None and self._process.returncode is None + + async def start(self) -> None: + await self._reject_occupied_port() + env = dict(os.environ) | { + "BAND_AGENT_KEY": self._agent_key, + "BAND_BASE_URL": self._base_url, + "ALLOWED_HOSTS": ALLOWED_HOSTS, + } + self._process = await asyncio.create_subprocess_exec( + *self._command, + "--transport", + "sse", + "--host", + "127.0.0.1", + "--port", + str(self._port), + "--tools", + TOOL_GROUPS, + env=env, + ) + try: + await self._wait_ready() + except Exception: + # A failed startup must not orphan the child: the session fixture + # never reaches its yield, so its teardown would never stop() it. + await self.stop() + raise + logger.info("band-mcp serving on %s", self.sse_url) + + async def stop(self) -> None: + if self._process is None or self._process.returncode is not None: + self._process = None + return + self._process.terminate() + try: + await asyncio.wait_for(self._process.wait(), timeout=STOP_TIMEOUT_S) + except TimeoutError: + logger.warning("band-mcp did not terminate in time; killing") + self._process.kill() + await self._process.wait() + self._process = None + + async def restart(self) -> None: + """Stop and start on the same port — the workspace mcp.json stays valid.""" + await self.stop() + await self.start() + + async def _reject_occupied_port(self) -> None: + """Fail loud when something already listens on the port. + + The readiness probe is a bare TCP connect — a stale listener (e.g. a + band-mcp orphaned by a crashed run, holding an already-reaped agent + key) would answer it for a child that failed to bind, and the suite + would run against the wrong Band identity. + """ + try: + _, writer = await asyncio.open_connection("127.0.0.1", self._port) + except OSError: + return + writer.close() + await writer.wait_closed() + raise RuntimeError( + f"port {self._port} is already in use — a stale band-mcp from a " + f"previous run? Stop it (or set BAND_MCP_PORT) and rerun." + ) + + async def _wait_ready(self) -> None: + deadline = asyncio.get_running_loop().time() + READY_TIMEOUT_S + while True: + assert self._process is not None + if self._process.returncode is not None: + raise RuntimeError( + f"band-mcp exited during startup ({self._process.returncode})" + ) + try: + _, writer = await asyncio.open_connection("127.0.0.1", self._port) + except OSError: + if asyncio.get_running_loop().time() > deadline: + raise RuntimeError( + f"band-mcp not accepting connections on {self._port} " + f"after {READY_TIMEOUT_S}s" + ) from None + await asyncio.sleep(0.2) + else: + writer.close() + await writer.wait_closed() + return diff --git a/tests/e2e/vscode/settings.py b/tests/e2e/vscode/settings.py new file mode 100644 index 00000000..3de101bf --- /dev/null +++ b/tests/e2e/vscode/settings.py @@ -0,0 +1,40 @@ +"""Settings for the Copilot-in-VS-Code validation suite. + +Only the suite's own knobs live here. Band endpoints, credentials, and the +judge/model configuration keep coming from ``tests.e2e.baseline.settings`` +(whose module import already loads ``.env.test``) — never duplicated. +""" + +from __future__ import annotations + +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + +# Importing for the .env.test side effect keeps the two settings sources in +# lockstep: both read the same already-loaded environment. +import tests.e2e.baseline.settings # noqa: F401 + +# Defaults chosen for unattended reruns: VS Code remembers MCP-server trust per +# configuration, so a stable workspace path + stable port keep mcp.json +# byte-identical across runs — one trust approval ever, then no prompts. +# Deliberately NOT under a dot-directory: Copilot's edit guard classifies +# hidden-path files as "sensitive" and demands per-edit confirmation. +DEFAULT_WORKSPACE = str(Path.home() / "band-e2e" / "vscode-chat-workspace") +DEFAULT_BAND_MCP_PORT = 8631 + + +class VSCodeChatSettings(BaseSettings): + """Env knobs for driving a real signed-in VS Code window (field == env var).""" + + model_config = SettingsConfigDict( + extra="ignore", case_sensitive=False, env_ignore_empty=True + ) + + vscode_chat_tests_enabled: bool = False # VSCODE_CHAT_TESTS_ENABLED + code_command: str = "code" # CODE_COMMAND (VS Code CLI binary override) + band_mcp_command: str = "band-mcp" # BAND_MCP_COMMAND (e.g. "uvx band-mcp") + band_mcp_port: int = DEFAULT_BAND_MCP_PORT # BAND_MCP_PORT (0 = ephemeral) + vscode_chat_workspace: str = DEFAULT_WORKSPACE # VSCODE_CHAT_WORKSPACE + vscode_chat_scorecard_json: str = "" # VSCODE_CHAT_SCORECARD_JSON ("" = no emit) + vscode_chat_timeout: int = 300 # VSCODE_CHAT_TIMEOUT (seconds per live turn) diff --git a/tests/e2e/vscode/test_copilot_chat.py b/tests/e2e/vscode/test_copilot_chat.py new file mode 100644 index 00000000..5fff3a5b --- /dev/null +++ b/tests/e2e/vscode/test_copilot_chat.py @@ -0,0 +1,209 @@ +"""Live L0–L3 cells for the GitHub Copilot in VS Code surface. + +Every cell speaks through ``SurfaceRoom`` (``rooms.py``): someone says +something in the room, the driver relays it to Copilot Chat, and the agent's +reply is asserted **Band-side** — it must land in the right room, from the +provisioned identity, via ``band_send_message``. Tokens are high-entropy so a +reply containing one proves the round trip, not model invention. + +L4 usage has no cell here: the surface exposes no per-turn usage signal — the +scorecard carries the rationale (see ``scorecard.USAGE_NA_ROW``). +""" + +from __future__ import annotations + +import uuid +from pathlib import Path + +import pytest + +from tests.e2e.baseline.toolkit.provisioning import ResourceManager +from tests.e2e.vscode.rooms import transcript +from tests.e2e.vscode.server import BandMCPServer +from tests.e2e.vscode.workspace import workspace_marker_path + +pytestmark = [pytest.mark.vscode_chat, pytest.mark.e2e] + + +def _token() -> str: + return uuid.uuid4().hex[:6] + + +async def test_participation_reply_round_trip(surface_room) -> None: + """L0: a room message reaches Copilot and its reply lands back in the room.""" + token = _token() + async with surface_room("participation") as room: + replies = await room.user_turn( + f"Ping — the echo token is {token}.", + instruction="Reply to this message, repeating its echo token exactly.", + ) + replies.assert_contains_any([token]) + + +async def test_original_functions_retained( + surface_room, vscode_workspace: Path, judge +) -> None: + """L1: one turn exercises a native VS Code function (create a workspace + file) AND a platform tool (``band_get_participants``) — the surface keeps + its original capabilities while participating.""" + token = _token() + filename = f"notes-{token}.txt" + async with surface_room("functions") as room: + replies = await room.user_turn( + f"Please set up a note file for this task and tell me who is " + f"in this room. The file token is {token}.", + instruction=( + f"First create a file named '{filename}' in the workspace root " + f"containing exactly the token {token}. Then look up the room " + f"roster with band_get_participants and reply listing every " + f"participant's name and confirming the file was created." + ), + ) + + marker = workspace_marker_path(vscode_workspace, filename) + assert marker.exists(), f"Copilot did not create {filename} in the workspace" + assert token in marker.read_text() + + verdict = await judge( + criteria=( + "The reply lists the chat room's participants (at least the agent " + "itself or the human user) and confirms that the requested note " + "file was created." + ), + transcript=transcript(replies), + ) + assert verdict.passed, verdict.reasoning + + +async def test_multi_participant_echo_peer( + surface_room, resource_manager: ResourceManager, judge +) -> None: + """L0 multi-participant: a peer agent's message drives a turn and the reply + engages the peer — the surface participates beyond 1:1 user chats.""" + token = _token() + echo = await resource_manager.provision_agent("echo") + echo_message = f"ECHO: {token}" + + async with surface_room("peers", participants=(echo.id,)) as room: + replies = await room.peer_turn( + echo, + echo_message, + instruction=( + "Another agent posted this in the room. Reply addressing that " + "agent and repeat its echo token exactly." + ), + ) + + replies.assert_contains_any([token]) + verdict = await judge( + criteria=( + "The agent under test replied to the peer agent's echo message, " + "engaging with it (addressing the peer and/or repeating its token)." + ), + transcript=transcript(replies, peer_message=echo_message), + ) + assert verdict.passed, verdict.reasoning + + +@pytest.mark.timeout(extra=360) # two full live turns +async def test_recall_across_chat_sessions(surface_room) -> None: + """L2 persistence: a fact stored in turn 1 survives to a fresh chat session. + + band-mcp exposes no room-history tool, so cross-session recall flows + through the platform memory tools — the fact is stored room-keyed with + ``band_store_memory`` and must come back via ``band_list_memories`` in a + session that never saw it. + """ + fact = f"codename-{_token()}" + async with surface_room("recall") as room: + await room.remember( + f"For later reference: the project codename is {fact}.", + record=f"project codename for room {room.room_id}: {fact}", + ) + + replies = await room.user_turn( + "What is the project codename?", + instruction=( + "Recall the project codename from your Band memory using " + "band_list_memories (and band_get_memory if needed) — the " + "record for THIS room (match the chat_id above); the memory " + "store may hold unrelated records. Do not guess — reply " + "stating the codename exactly." + ), + new_session=True, + ) + replies.assert_contains_any([fact]) + + +@pytest.mark.timeout(extra=700) # three full live turns across two rooms +async def test_no_leak_between_rooms(surface_room) -> None: + """L2 isolation: room-scoped facts stay scoped — room B's answer names room + B's marker and never room A's. + + Each ``code chat`` invocation is a fresh chat session and band-mcp has no + history tool, so room context lives in agent-wide platform memory. The + seeds record each marker *keyed by its room id*; isolation is then the + retrieval staying scoped to the asking room — pulling the sibling room's + marker instead is exactly the leak this cell exists to catch. + """ + token_a, token_b = f"alpha-{_token()}", f"bravo-{_token()}" + + async def seed(room, token: str) -> None: + await room.remember( + f"The marker for THIS room is {token}.", + record=f"marker for room {room.room_id}: {token}", + ) + + async with surface_room("isolation-a") as room_a: + await seed(room_a, token_a) + async with surface_room("isolation-b") as room_b: + await seed(room_b, token_b) + + replies = await room_b.user_turn( + "Which marker belongs to THIS room?", + instruction=( + "Look up the stored markers with band_list_memories and reply " + "with the one recorded for THIS room (match the chat_id above). " + "Never mention markers recorded for other rooms." + ), + new_session=True, + ) + replies.assert_contains_any([token_b]) + replies.assert_contains_none([token_a]) + + +@pytest.mark.timeout(extra=360) # two live turns plus a bridge restart +async def test_restart_recall_and_function( + surface_room, band_mcp: BandMCPServer, vscode_workspace: Path +) -> None: + """L3: the platform bridge (band-mcp) restarts between turns and the surface + still recalls the stored fact AND keeps its native function (file creation). + + A full VS Code window restart stays a documented manual variant in the + README — the bridge restart covers the platform side deterministically. + """ + fact = f"phase-{_token()}" + filename = f"restart-{_token()}.txt" + async with surface_room("restart") as room: + await room.remember( + f"Remember across restarts: the deploy phase is {fact}.", + record=f"deploy phase for room {room.room_id}: {fact}", + ) + + await band_mcp.restart() + + replies = await room.user_turn( + "After the maintenance window: what is the deploy phase?", + instruction=( + f"Recall the deploy phase from your Band memory with " + f"band_list_memories — the record for THIS room (match the " + f"chat_id above); do not guess — and reply stating it. " + f"Also create a file named '{filename}' in the workspace root " + f"containing that phase, to confirm your editor tools still work." + ), + new_session=True, + ) + + replies.assert_contains_any([fact]) + marker = workspace_marker_path(vscode_workspace, filename) + assert marker.exists(), f"Copilot did not create {filename} after the restart" diff --git a/tests/e2e/vscode/workspace.py b/tests/e2e/vscode/workspace.py new file mode 100644 index 00000000..c891964c --- /dev/null +++ b/tests/e2e/vscode/workspace.py @@ -0,0 +1,51 @@ +"""Scaffold the throwaway VS Code workspace the suite opens. + +Pure functions (unit-tested in CI without VS Code): they only write the two +``.vscode`` config files Copilot Chat reads — the MCP server entry pointing at +the harness's band-mcp instance, and the chat settings that keep a driven run +from stalling on per-tool approval prompts. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +# Auto-answer agent questions so a driven turn never stalls on a chat-side +# question (Manage approvals docs). Deliberately NOT chat.tools.global.autoApprove: +# that is machine-wide "YOLO mode" — VS Code escalates it to a global consent +# dialog and it disables tool approval in every workspace on the host. Tool +# approvals instead rely on remembered per-tool "Always allow" choices in this +# (persistent) workspace — one click per tool, ever (see README). +AUTO_REPLY_SETTING = "chat.autoReply" + +# MCP support is on by default in current VS Code; setting it explicitly keeps +# the run independent of a user-profile override ("none"). This is the current +# key — the boolean chat.mcp.enabled it replaced only works via VS Code's +# settings-migration shim. +MCP_ACCESS_SETTING = "chat.mcp.access" +MCP_ACCESS_ALL = "all" + +# The one MCP server name the prompts refer to ("the band tools"). +MCP_SERVER_NAME = "band" + + +def scaffold_workspace(root: Path, sse_url: str) -> None: + """Write ``.vscode/mcp.json`` + ``.vscode/settings.json`` under ``root``.""" + vscode_dir = root / ".vscode" + vscode_dir.mkdir(parents=True, exist_ok=True) + + mcp_config = {"servers": {MCP_SERVER_NAME: {"type": "sse", "url": sse_url}}} + settings = {AUTO_REPLY_SETTING: True, MCP_ACCESS_SETTING: MCP_ACCESS_ALL} + + (vscode_dir / "mcp.json").write_text(json.dumps(mcp_config, indent=2) + "\n") + (vscode_dir / "settings.json").write_text(json.dumps(settings, indent=2) + "\n") + + +def workspace_marker_path(root: Path, name: str) -> Path: + """Where a cell expects Copilot to have created ``name`` inside the workspace. + + One definition so the prompt that asks for the file and the assertion that + checks it can never drift apart. + """ + return root / name diff --git a/tests/framework_conformance/test_vscode_chat_toolkit.py b/tests/framework_conformance/test_vscode_chat_toolkit.py new file mode 100644 index 00000000..a71c3183 --- /dev/null +++ b/tests/framework_conformance/test_vscode_chat_toolkit.py @@ -0,0 +1,228 @@ +"""Unit guards for the Copilot-in-VS-Code harness toolkit (``tests/e2e/vscode``). + +The live suite needs a signed-in VS Code window and can never run in CI, so +these tests are the only ones protecting its plumbing on every PR: the prompt +shape Copilot receives, the workspace files VS Code reads, the band-mcp +process lifecycle, the version evidence recorded with a run, and the scorecard +rows the run emits. +""" + +from __future__ import annotations + +import json +import socket +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +import pytest + +from tests.e2e.vscode import server as server_module +from tests.e2e.vscode.driver import ( + PreflightError, + capture_versions, + preflight, + turn_prompt, +) +from tests.e2e.vscode.scorecard import ( + SURFACE_ID, + USAGE_NA_ROW, + VSCodeScorecard, + outcome_status, +) +from tests.e2e.vscode.server import BandMCPServer +from tests.e2e.vscode.workspace import ( + AUTO_REPLY_SETTING, + MCP_ACCESS_ALL, + MCP_ACCESS_SETTING, + MCP_SERVER_NAME, + scaffold_workspace, +) + + +# --- turn_prompt: the one shape every cell submits ---------------------------------- + + +def test_turn_prompt_carries_room_message_and_tool_contract() -> None: + prompt = turn_prompt( + "room-123", + "band-agent", + sender_name="Alex", + message="the marker is X9", + instruction="Echo the marker back.", + ) + # The surface cannot read the room, so the prompt itself must carry the + # message, the room id every tool call needs, and the reply-tool contract. + assert "room-123" in prompt + assert "band-agent" in prompt + assert "the marker is X9" in prompt + assert "Echo the marker back." in prompt + assert "band_send_message" in prompt + assert MCP_SERVER_NAME in prompt + assert prompt.count("Alex") >= 2 # named as sender and as reply mention + + +# --- workspace scaffolding: what VS Code reads -------------------------------------- + + +def test_scaffold_workspace_writes_mcp_entry_and_auto_reply(tmp_path: Path) -> None: + sse_url = "http://127.0.0.1:8391/sse" + scaffold_workspace(tmp_path, sse_url) + + mcp = json.loads((tmp_path / ".vscode" / "mcp.json").read_text()) + # Assert the contract fields, not the whole dict — a new optional key in + # the scaffold must not break this guard. + server = mcp["servers"][MCP_SERVER_NAME] + assert server["type"] == "sse" + assert server["url"] == sse_url + + settings = json.loads((tmp_path / ".vscode" / "settings.json").read_text()) + assert settings[AUTO_REPLY_SETTING] is True + assert settings[MCP_ACCESS_SETTING] == MCP_ACCESS_ALL + + +# --- driver preflight + version evidence -------------------------------------------- + + +def test_preflight_rejects_missing_binary() -> None: + with pytest.raises(PreflightError, match="README"): + preflight(["definitely-not-a-real-code-binary"]) + + +def test_capture_versions_shape_with_stub_runner() -> None: + outputs = { + "code --version": "1.103.0\nabc123\narm64", + "code --list-extensions --show-versions": ( + "github.copilot@1.5.0\ngithub.copilot-chat@0.32.0\nms-python.python@2026.1" + ), + "band-mcp --version": "band-mcp 1.3.2", + } + + versions = capture_versions( + ["code"], ["band-mcp"], run=lambda cmd: outputs[" ".join(cmd)] + ) + + # Behavioral contract only (what the sidecar must convey), not join format: + # the VS Code version+build survive on one line, copilot extensions are + # kept while unrelated ones are filtered, band-mcp passes through. + assert "1.103.0" in versions["vscode"] and "abc123" in versions["vscode"] + assert "\n" not in versions["vscode"] + assert "github.copilot@1.5.0" in versions["copilot_extensions"] + assert "github.copilot-chat@0.32.0" in versions["copilot_extensions"] + assert "ms-python" not in versions["copilot_extensions"] + assert "band-mcp 1.3.2" in versions["band_mcp"] + assert versions["os"] + + +def test_capture_versions_records_failures_instead_of_raising() -> None: + def failing(cmd: list[str]) -> str: + raise OSError("boom") + + versions = capture_versions(["code"], ["band-mcp"], run=failing) + assert "unavailable" in versions["vscode"] + assert "unavailable" in versions["band_mcp"] + + +# --- band-mcp server lifecycle: startup failure modes -------------------------------- + + +def _server(command: list[str], port: int = 0) -> BandMCPServer: + return BandMCPServer(command, agent_key="key", base_url="http://band", port=port) + + +async def test_start_rejects_port_held_by_another_process() -> None: + """A stale listener must fail the run loud, not answer the readiness probe + for a child that could never bind (the suite would then talk to the stale + server's Band identity).""" + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + sock.listen(1) + server = _server([sys.executable], port=sock.getsockname()[1]) + with pytest.raises(RuntimeError, match="already in use"): + await server.start() + assert not server.running # no child was ever spawned + + +async def test_failed_startup_never_orphans_the_child( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A child that stays alive but never binds must be terminated when start() + gives up — the session fixture's teardown never runs on a startup failure, + so an orphan would squat the stable port across reruns.""" + monkeypatch.setattr(server_module, "READY_TIMEOUT_S", 0.5) + never_binds = [sys.executable, "-c", "import time; time.sleep(60)"] + server = _server(never_binds) + + with pytest.raises(RuntimeError, match="not accepting connections"): + await server.start() + + assert not server.running + + +# --- scorecard: outcome mapping + emitted artifact ---------------------------------- + + +def _report(when: str, outcome: str, nodeid: str) -> pytest.TestReport: + """A minimal stand-in carrying the only report fields the plugin reads.""" + return cast( + pytest.TestReport, + SimpleNamespace( + when=when, + skipped=outcome == "skipped", + failed=outcome == "failed", + passed=outcome == "passed", + nodeid=nodeid, + ), + ) + + +@pytest.mark.parametrize( + ("when", "outcome", "expected"), + [ + ("setup", "skipped", "skip"), + ("setup", "failed", "fail"), + ("setup", "passed", None), # no verdict until the call phase + ("call", "failed", "fail"), + ("call", "passed", "pass"), + ("teardown", "failed", None), # teardown never overrides the call + ], +) +def test_outcome_status_mapping(when: str, outcome: str, expected: str | None) -> None: + report = _report(when, outcome, "tests/e2e/vscode/test_x.py::t") + assert outcome_status(report) == expected + + +def test_scorecard_keeps_suite_rows_plus_fixed_na_row(tmp_path: Path) -> None: + """One row per suite test plus the fixed L4 N/A; foreign reports ignored. + + The filter is a nodeid prefix — nodeids stay rootdir-relative wherever + pytest is invoked from, unlike report file paths. + """ + nodeid = "tests/e2e/vscode/test_copilot_chat.py::test_participation" + + plugin = VSCodeScorecard(tmp_path / "scorecard.json", "tests/e2e/vscode/") + for report in ( + _report("setup", "passed", nodeid), + _report("call", "passed", nodeid), + _report("call", "passed", "tests/other.py::t"), + ): + plugin.pytest_runtest_logreport(report) + + rows = {row.test: row for row in plugin.scorecard()} + assert rows[nodeid].status == "pass" + assert rows[nodeid].adapter == SURFACE_ID + assert rows[USAGE_NA_ROW.test] == USAGE_NA_ROW + assert len(rows) == 2 # the foreign report contributed nothing + + +def test_sessionfinish_writes_scorecard_and_metadata_sidecar(tmp_path: Path) -> None: + out = tmp_path / "scorecard.json" + plugin = VSCodeScorecard(out, "tests/e2e/vscode/") + plugin.metadata = {"vscode": "1.103.0"} + + plugin.pytest_sessionfinish() + + assert json.loads(out.read_text()) # the rows artifact (N/A row at minimum) + meta = json.loads((tmp_path / "scorecard.json.meta.json").read_text()) + assert meta == {"vscode": "1.103.0"} diff --git a/tests/test_env_gates.py b/tests/test_env_gates.py index 70d2c685..8e8e7695 100644 --- a/tests/test_env_gates.py +++ b/tests/test_env_gates.py @@ -1,35 +1,53 @@ -"""Set-but-empty env vars must not kill settings construction. +"""Guards for the opt-in suite gates (``tests/conftest.py``'s ``GATED_MARKERS``). -Some CI wrappers export gates as empty (``CI=``, ``E2E_TESTS_ENABLED=``). -Without ``env_ignore_empty=True`` pydantic-settings tries to parse ``""`` as -the field's bool/int type and raises a ValidationError — for the collection -gates that happens *inside a pytest collection hook*, killing the entire run. -These guards pin the empty-means-unset behavior for both settings surfaces. +Two failure modes are pinned here because both would bite *inside a pytest +collection hook*, killing entire runs instead of failing one test: + +* a set-but-empty gate var (``CI=``, as some CI wrappers export) must read as + disabled, not raise a ValidationError — the ``env_ignore_empty`` behavior; +* the gate table must stay consistent with what it references: every field it + names must exist on ``CollectionGateSettings``, and every marker it names + must be registered in pyproject (a drift means a gated suite either warns as + unknown or never skips). """ from __future__ import annotations +import tomllib + import pytest -from tests.conftest import CollectionGateSettings +from tests.conftest import GATED_MARKERS, CollectionGateSettings from tests.e2e.baseline.settings import BaselineSettings +from tests.paths import REPO_ROOT + + +def registered_pytest_markers() -> set[str]: + """The marker names registered in pyproject's ``[tool.pytest.ini_options]``.""" + pyproject = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) + entries: list[str] = pyproject["tool"]["pytest"]["ini_options"]["markers"] + return {entry.split(":", 1)[0].strip() for entry in entries} def test_empty_collection_gate_vars_read_as_disabled( monkeypatch: pytest.MonkeyPatch, ) -> None: - for var in ( - "CI", - "E2E_TESTS_ENABLED", - "DOCKER_TESTS_ENABLED", - "SANDBOX_TESTS_ENABLED", - ): - monkeypatch.setenv(var, "") + gate_fields = ["ci", *GATED_MARKERS.values()] + for field in gate_fields: + monkeypatch.setenv(field.upper(), "") gates = CollectionGateSettings() - assert gates.ci is False - assert gates.e2e_tests_enabled is False - assert gates.docker_tests_enabled is False - assert gates.sandbox_tests_enabled is False + for field in gate_fields: + assert getattr(gates, field) is False, field + + +def test_every_gated_marker_opens_with_a_real_settings_field() -> None: + gates = CollectionGateSettings() + for field in GATED_MARKERS.values(): + assert hasattr(gates, field), field + + +def test_every_gated_marker_is_registered_in_pyproject() -> None: + assert set(GATED_MARKERS) <= registered_pytest_markers() def test_empty_baseline_vars_fall_back_to_defaults(