From 11716d1a24db66e4ff42cd900d52eed5045cd16b Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Thu, 23 Jul 2026 11:57:27 +0300 Subject: [PATCH 1/7] =?UTF-8?q?feat(e2e):=20opt-in=20Copilot-in-VS-Code=20?= =?UTF-8?q?validation=20suite=20=E2=80=94=20toolkit=20+=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the harness plumbing for validating the GitHub Copilot in VS Code surface against the L0-L4 bar: a code-chat CLI driver (PromptDriver seam for a future test-electron backend), restartable band-mcp SSE server holding the provisioned agent identity, workspace scaffolding (.vscode/mcp.json + auto-approve settings), and a suite-local scorecard collector reusing the baseline row schema. Gated behind VSCODE_CHAT_TESTS_ENABLED (vscode_chat marker) since the surface needs a GUI window and interactive Copilot sign-in — never runs in CI. Pure parts are unit-tested in framework_conformance on every PR. Co-Authored-By: Claude Fable 5 Signed-off-by: Alexander Zaikman --- pyproject.toml | 1 + tests/conftest.py | 8 + tests/e2e/vscode/__init__.py | 0 tests/e2e/vscode/driver.py | 163 ++++++++++++++++ tests/e2e/vscode/scorecard.py | 83 ++++++++ tests/e2e/vscode/server.py | 116 +++++++++++ tests/e2e/vscode/settings.py | 28 +++ tests/e2e/vscode/workspace.py | 47 +++++ .../test_vscode_chat_toolkit.py | 184 ++++++++++++++++++ tests/test_env_gates.py | 2 + 10 files changed, 632 insertions(+) create mode 100644 tests/e2e/vscode/__init__.py create mode 100644 tests/e2e/vscode/driver.py create mode 100644 tests/e2e/vscode/scorecard.py create mode 100644 tests/e2e/vscode/server.py create mode 100644 tests/e2e/vscode/settings.py create mode 100644 tests/e2e/vscode/workspace.py create mode 100644 tests/framework_conformance/test_vscode_chat_toolkit.py diff --git a/pyproject.toml b/pyproject.toml index 49977ce90..f64788c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -275,6 +275,7 @@ markers = [ "flaky_reason: baseline flakiness taxonomy stamp (kind + reason); see tests/e2e/baseline/flaky.py", "docker_build: builds/runs a real Docker image via subprocess (requires a local Docker daemon; skipped unless DOCKER_TESTS_ENABLED=true, including in CI, which has Docker but shouldn't pay this cost by default)", "sandbox: drives a Docker Sandbox via the sbx CLI (needs sbx + a nested-virtualization-capable host + live Band; skipped unless SANDBOX_TESTS_ENABLED=true — CI has neither the CLI nor nested virt)", + "vscode_chat: drives a real signed-in VS Code window via the code chat CLI (GUI + interactive Copilot sign-in; skipped unless VSCODE_CHAT_TESTS_ENABLED=true — can never run in CI)", ] [tool.uv] diff --git a/tests/conftest.py b/tests/conftest.py index 9c6acbb99..ed46a3d3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -73,6 +73,7 @@ class CollectionGateSettings(BaseSettings): e2e_tests_enabled: bool = False # E2E_TESTS_ENABLED docker_tests_enabled: bool = False # DOCKER_TESTS_ENABLED sandbox_tests_enabled: bool = False # SANDBOX_TESTS_ENABLED + vscode_chat_tests_enabled: bool = False # VSCODE_CHAT_TESTS_ENABLED def pytest_ignore_collect(collection_path: Path) -> bool | None: @@ -111,6 +112,9 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: skip_sandbox = pytest.mark.skip( reason="set SANDBOX_TESTS_ENABLED=true to run sbx sandbox tests" ) + skip_vscode_chat = pytest.mark.skip( + reason="set VSCODE_CHAT_TESTS_ENABLED=true to run VS Code Copilot chat tests" + ) for item in items: if not gates.e2e_tests_enabled and item.get_closest_marker("e2e"): item.add_marker(skip_e2e) @@ -118,6 +122,10 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: item.add_marker(skip_docker) if not gates.sandbox_tests_enabled and item.get_closest_marker("sandbox"): item.add_marker(skip_sandbox) + if not gates.vscode_chat_tests_enabled and item.get_closest_marker( + "vscode_chat" + ): + item.add_marker(skip_vscode_chat) @pytest.fixture(autouse=True) diff --git a/tests/e2e/vscode/__init__.py b/tests/e2e/vscode/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/e2e/vscode/driver.py b/tests/e2e/vscode/driver.py new file mode 100644 index 000000000..a34c26ddc --- /dev/null +++ b/tests/e2e/vscode/driver.py @@ -0,0 +1,163 @@ +"""Drive Copilot Chat in a real VS Code window through the ``code chat`` CLI. + +The surface has no inbound channel: Copilot in VS Code cannot be pushed a Band +room message (band-mcp exposes no message-read tool either), so every turn is +driver-initiated — the prompt relays the triggering room message and instructs +Copilot to act through the band MCP tools. This mirrors real usage, where the +developer relays context into chat and Copilot posts to Band. + +``PromptDriver`` is the seam: assertions never touch VS Code (they are all +Band-side), so a future ``@vscode/test-electron`` backend only has to submit +prompts to slot in. +""" + +from __future__ import annotations + +import asyncio +import logging +import platform +import shutil +import subprocess +from collections.abc import Callable +from pathlib import Path +from typing import Protocol + +from tests.e2e.vscode.workspace import MCP_SERVER_NAME + +logger = logging.getLogger(__name__) + +# `code chat` submits and returns; the window processes the turn asynchronously. +# This bounds only the CLI handoff, not the model turn (Band-side waits do that). +SUBMIT_TIMEOUT_S = 60 + +# The `code chat` CLI has no verified per-invocation new-session switch, so a +# fresh-session request is expressed in the prompt itself. Weaker than a real +# session reset — the runbook documents the manual "New Chat" variant. +FRESH_SESSION_PREAMBLE = ( + "Start from a clean slate: ignore everything discussed earlier in this " + "chat and rely only on this message and your tools." +) + + +class PreflightError(RuntimeError): + """The VS Code CLI is missing or too old to drive; message points at the runbook.""" + + +class PromptDriver(Protocol): + """Submit one agent-mode prompt to the surface under test.""" + + async def submit_prompt(self, prompt: str, *, new_session: bool = False) -> None: + """Send ``prompt`` to Copilot Chat; ``new_session`` requests a fresh context.""" + ... + + +def turn_prompt( + chat_id: str, + agent_name: str, + *, + sender_name: str, + message: str, + instruction: str, +) -> str: + """The one prompt shape every cell submits. + + Relays the room message (the surface cannot read it back) and pins the + room id, since every band tool call takes an explicit ``chat_id``. + """ + return ( + f"You are the Band platform agent '{agent_name}'. You participate in " + f"Band chat rooms through the MCP tools of the '{MCP_SERVER_NAME}' " + f"server (band_send_message, band_get_participants, band_store_memory, " + f"band_list_memories, ...). Every band tool call must pass " + f"chat_id='{chat_id}'.\n\n" + f"New message in the room from {sender_name}:\n" + f"---\n{message}\n---\n\n" + f"{instruction}\n" + f"Always deliver your answer into the room with band_send_message, " + f"mentioning {sender_name}." + ) + + +class CodeChatDriver: + """Submit prompts via ``code chat -m agent`` against the workspace's window.""" + + def __init__(self, code_command: list[str], workspace: Path) -> None: + self._code_command = code_command + self._workspace = workspace + + async def open_window(self) -> None: + """Open the scaffolded workspace so ``code chat`` targets its window.""" + await self._run(str(self._workspace)) + + async def submit_prompt(self, prompt: str, *, new_session: bool = False) -> None: + if new_session: + prompt = f"{FRESH_SESSION_PREAMBLE}\n\n{prompt}" + await self._run("chat", "-m", "agent", prompt) + + async def _run(self, *args: str) -> None: + process = await asyncio.create_subprocess_exec( + *self._code_command, + *args, + cwd=self._workspace, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + output, _ = await asyncio.wait_for( + process.communicate(), timeout=SUBMIT_TIMEOUT_S + ) + if output: + logger.debug("code %s: %s", args[0], output.decode(errors="replace")) + if process.returncode != 0: + raise RuntimeError( + f"code {args[0]} exited with {process.returncode}: " + f"{output.decode(errors='replace').strip()}" + ) + + +Runner = Callable[[list[str]], str] + + +def _default_runner(command: list[str]) -> str: + return subprocess.run( + command, capture_output=True, text=True, timeout=SUBMIT_TIMEOUT_S, check=True + ).stdout + + +def preflight(code_command: list[str], *, run: Runner = _default_runner) -> None: + """Fail fast (with the runbook pointer) when the VS Code CLI cannot drive chat.""" + hint = "see tests/e2e/vscode/README.md for setup" + if shutil.which(code_command[0]) is None: + raise PreflightError(f"'{code_command[0]}' not found on PATH — {hint}") + try: + run([*code_command, "--version"]) + run([*code_command, "chat", "--help"]) + except (subprocess.SubprocessError, OSError) as error: + raise PreflightError( + f"VS Code CLI cannot drive chat ({error}) — {hint}" + ) from error + + +def capture_versions( + code_command: list[str], + band_mcp_command: list[str], + *, + run: Runner = _default_runner, +) -> dict[str, str]: + """The environment evidence the scorecard sidecar records for a live run.""" + + def capture(command: list[str]) -> str: + try: + return run(command).strip() + except (subprocess.SubprocessError, OSError) as error: + return f"unavailable ({error})" + + extensions = capture([*code_command, "--list-extensions", "--show-versions"]) + copilot_lines = [ + line for line in extensions.splitlines() if "copilot" in line.lower() + ] + return { + "os": platform.platform(), + "vscode": capture([*code_command, "--version"]).replace("\n", " "), + "copilot_extensions": "; ".join(copilot_lines) or extensions, + "band_mcp": capture([*band_mcp_command, "--version"]), + } diff --git a/tests/e2e/vscode/scorecard.py b/tests/e2e/vscode/scorecard.py new file mode 100644 index 000000000..610add5dd --- /dev/null +++ b/tests/e2e/vscode/scorecard.py @@ -0,0 +1,83 @@ +"""Scorecard emission for the VS Code Copilot surface. + +Reuses the baseline scorecard's row schema and writer so the artifact merges +with the lane scorecards, but records rows itself: the baseline collector only +counts ``[adapter]``-parametrized matrix cells, and this suite's tests are +bespoke (unparametrized, one fixed surface column). +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +import pytest + +from tests.e2e.baseline.scorecard import ScorecardRow, Status, write_json + +logger = logging.getLogger(__name__) + +# The scorecard column for this surface (a surface id, not a registered +# matrix adapter — deliberately absent from the baseline Adapter registry). +SURFACE_ID = "copilot_vscode" + +# L4 usage has no test function: the surface offers nothing to assert — Copilot +# Chat exposes no per-turn usage/billing signal to the harness and posts no +# Band-side usage events. The rationale ships as a scorecard row instead. +USAGE_NA_ROW = ScorecardRow( + test="tests/e2e/vscode/test_copilot_chat.py::usage_accounting", + adapter=SURFACE_ID, + status="na", + reason=( + "Copilot Chat in VS Code exposes no per-turn usage/billing signal to " + "the harness and emits no Band-side usage events" + ), +) + + +def outcome_status(report: pytest.TestReport) -> Status | None: + """The row status one setup/call report contributes (same semantics as the + baseline collector: skip anywhere = skip, failed setup or call = fail, + passing call = pass, passing setup = no verdict yet).""" + if report.when not in ("setup", "call"): + return None + if report.skipped: + return "skip" + if report.failed: + return "fail" + if report.when == "call": + return "pass" + return None + + +class VSCodeScorecard: + """Pytest plugin: one row per suite test plus the fixed L4 ``na`` row, + written with a ``.meta.json`` environment sidecar at session end.""" + + def __init__(self, path: str | Path, suite_dir: Path) -> None: + self._path = Path(path) + self._suite_dir = suite_dir + self._rows: dict[str, ScorecardRow] = {} + self.metadata: dict[str, str] = {} + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + if not Path(report.fspath).resolve().is_relative_to(self._suite_dir): + return + status = outcome_status(report) + if status is None: + return + self._rows[report.nodeid] = ScorecardRow( + test=report.nodeid, adapter=SURFACE_ID, status=status + ) + + def scorecard(self) -> list[ScorecardRow]: + rows = dict(self._rows) + rows[USAGE_NA_ROW.test] = USAGE_NA_ROW + return sorted(rows.values(), key=lambda row: row.test) + + def pytest_sessionfinish(self) -> None: + write_json(self.scorecard(), self._path) + meta_path = self._path.with_suffix(self._path.suffix + ".meta.json") + meta_path.write_text(json.dumps(self.metadata, indent=2) + "\n") + logger.info("scorecard written to %s (+ %s)", self._path, meta_path.name) diff --git a/tests/e2e/vscode/server.py b/tests/e2e/vscode/server.py new file mode 100644 index 000000000..ce2654451 --- /dev/null +++ b/tests/e2e/vscode/server.py @@ -0,0 +1,116 @@ +"""Run band-mcp (SSE) as a restartable subprocess for the suite. + +The server holds the provisioned Copilot agent identity: VS Code's MCP client +presents no credentials, so the agent key travels in the subprocess env. It is +restartable on a stable port because the workspace's ``mcp.json`` is written +once — the L3 cell restarts this process to prove the surface survives its +platform bridge going away and coming back. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import socket + +logger = logging.getLogger(__name__) + +# Without this allowlist band-mcp's DNS-rebinding protection answers 421 to +# loopback SSE clients (same value the copilot_docker examples use). +ALLOWED_HOSTS = '["localhost:*","127.0.0.1:*"]' + +# The memory tool group is opt-in and is the suite's recall path: band-mcp +# exposes no room-history tool, so cross-session recall flows through +# band_store_memory / band_list_memories. +TOOL_GROUPS = "memory" + +READY_TIMEOUT_S = 30.0 +STOP_TIMEOUT_S = 10.0 + + +def _reserve_port() -> int: + """Take one ephemeral loopback port from the OS, then release it for the server. + + The close→rebind gap is the standard reservation race — acceptable for a + single local server (mirrors ``parlant_server._reserve_two_ports``). + """ + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class BandMCPServer: + """Lifecycle of one band-mcp SSE subprocess bound to a stable loopback port.""" + + def __init__(self, command: list[str], *, agent_key: str, base_url: str) -> None: + self._command = command + self._agent_key = agent_key + self._base_url = base_url + self._port = _reserve_port() + self._process: asyncio.subprocess.Process | None = None + + @property + def sse_url(self) -> str: + return f"http://127.0.0.1:{self._port}/sse" + + async def start(self) -> None: + env = dict(os.environ) | { + "BAND_AGENT_KEY": self._agent_key, + "BAND_BASE_URL": self._base_url, + "ALLOWED_HOSTS": ALLOWED_HOSTS, + } + self._process = await asyncio.create_subprocess_exec( + *self._command, + "--transport", + "sse", + "--host", + "127.0.0.1", + "--port", + str(self._port), + "--tools", + TOOL_GROUPS, + env=env, + ) + await self._wait_ready() + logger.info("band-mcp serving on %s", self.sse_url) + + async def stop(self) -> None: + if self._process is None or self._process.returncode is not None: + self._process = None + return + self._process.terminate() + try: + await asyncio.wait_for(self._process.wait(), timeout=STOP_TIMEOUT_S) + except TimeoutError: + logger.warning("band-mcp did not terminate in time; killing") + self._process.kill() + await self._process.wait() + self._process = None + + async def restart(self) -> None: + """Stop and start on the same port — the workspace mcp.json stays valid.""" + await self.stop() + await self.start() + + async def _wait_ready(self) -> None: + deadline = asyncio.get_running_loop().time() + READY_TIMEOUT_S + while True: + assert self._process is not None + if self._process.returncode is not None: + raise RuntimeError( + f"band-mcp exited during startup ({self._process.returncode})" + ) + try: + _, writer = await asyncio.open_connection("127.0.0.1", self._port) + except OSError: + if asyncio.get_running_loop().time() > deadline: + raise RuntimeError( + f"band-mcp not accepting connections on {self._port} " + f"after {READY_TIMEOUT_S}s" + ) from None + await asyncio.sleep(0.2) + else: + writer.close() + await writer.wait_closed() + return diff --git a/tests/e2e/vscode/settings.py b/tests/e2e/vscode/settings.py new file mode 100644 index 000000000..54eb5996b --- /dev/null +++ b/tests/e2e/vscode/settings.py @@ -0,0 +1,28 @@ +"""Settings for the Copilot-in-VS-Code validation suite. + +Only the suite's own knobs live here. Band endpoints, credentials, and the +judge/model configuration keep coming from ``tests.e2e.baseline.settings`` +(whose module import already loads ``.env.test``) — never duplicated. +""" + +from __future__ import annotations + +from pydantic_settings import BaseSettings, SettingsConfigDict + +# Importing for the .env.test side effect keeps the two settings sources in +# lockstep: both read the same already-loaded environment. +import tests.e2e.baseline.settings # noqa: F401 + + +class VSCodeChatSettings(BaseSettings): + """Env knobs for driving a real signed-in VS Code window (field == env var).""" + + model_config = SettingsConfigDict( + extra="ignore", case_sensitive=False, env_ignore_empty=True + ) + + vscode_chat_tests_enabled: bool = False # VSCODE_CHAT_TESTS_ENABLED + code_command: str = "code" # CODE_COMMAND (VS Code CLI binary override) + band_mcp_command: str = "band-mcp" # BAND_MCP_COMMAND (e.g. "uvx band-mcp") + scorecard_json: str = "" # VSCODE_CHAT_SCORECARD_JSON (empty = don't emit) + turn_timeout: int = 300 # VSCODE_CHAT_TIMEOUT (seconds per live turn) diff --git a/tests/e2e/vscode/workspace.py b/tests/e2e/vscode/workspace.py new file mode 100644 index 000000000..b506b5146 --- /dev/null +++ b/tests/e2e/vscode/workspace.py @@ -0,0 +1,47 @@ +"""Scaffold the throwaway VS Code workspace the suite opens. + +Pure functions (unit-tested in CI without VS Code): they only write the two +``.vscode`` config files Copilot Chat reads — the MCP server entry pointing at +the harness's band-mcp instance, and the chat settings that keep a driven run +from stalling on per-tool approval prompts. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +# VS Code's tool auto-approval knob (Manage approvals docs, VS Code 1.103+). +# Security trade-off is deliberate and bounded: the workspace is a throwaway +# temp dir and the only MCP server is the harness's own band-mcp. If the +# installed VS Code rejects the key (or scopes it user-level only), the runbook +# fallback is one manual "Always allow" click per tool on the first turn. +AUTO_APPROVE_SETTING = "chat.tools.global.autoApprove" + +# MCP support is on by default in current VS Code; setting it explicitly keeps +# the run independent of a user-profile override. Unknown keys are ignored. +MCP_ENABLED_SETTING = "chat.mcp.enabled" + +# The one MCP server name the prompts refer to ("the band tools"). +MCP_SERVER_NAME = "band" + + +def scaffold_workspace(root: Path, sse_url: str) -> None: + """Write ``.vscode/mcp.json`` + ``.vscode/settings.json`` under ``root``.""" + vscode_dir = root / ".vscode" + vscode_dir.mkdir(parents=True, exist_ok=True) + + mcp_config = {"servers": {MCP_SERVER_NAME: {"type": "sse", "url": sse_url}}} + settings = {AUTO_APPROVE_SETTING: True, MCP_ENABLED_SETTING: True} + + (vscode_dir / "mcp.json").write_text(json.dumps(mcp_config, indent=2) + "\n") + (vscode_dir / "settings.json").write_text(json.dumps(settings, indent=2) + "\n") + + +def workspace_marker_path(root: Path, name: str) -> Path: + """Where a cell expects Copilot to have created ``name`` inside the workspace. + + One definition so the prompt that asks for the file and the assertion that + checks it can never drift apart. + """ + return root / name diff --git a/tests/framework_conformance/test_vscode_chat_toolkit.py b/tests/framework_conformance/test_vscode_chat_toolkit.py new file mode 100644 index 000000000..6ba6aa9e6 --- /dev/null +++ b/tests/framework_conformance/test_vscode_chat_toolkit.py @@ -0,0 +1,184 @@ +"""Unit guards for the Copilot-in-VS-Code harness toolkit (``tests/e2e/vscode``). + +The live suite needs a signed-in VS Code window and can never run in CI, so +these pure-function tests are the only ones protecting its plumbing on every +PR: the prompt shape Copilot receives, the workspace files VS Code reads, the +version evidence recorded with a run, and the scorecard rows the run emits. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.e2e.vscode.driver import ( + FRESH_SESSION_PREAMBLE, + PreflightError, + capture_versions, + preflight, + turn_prompt, +) +from tests.e2e.vscode.scorecard import ( + SURFACE_ID, + USAGE_NA_ROW, + VSCodeScorecard, + outcome_status, +) +from tests.e2e.vscode.workspace import ( + AUTO_APPROVE_SETTING, + MCP_SERVER_NAME, + scaffold_workspace, + workspace_marker_path, +) + + +# --- turn_prompt: the one shape every cell submits ---------------------------------- + + +def test_turn_prompt_carries_room_message_and_tool_contract() -> None: + prompt = turn_prompt( + "room-123", + "band-agent", + sender_name="Alex", + message="the marker is X9", + instruction="Echo the marker back.", + ) + # The surface cannot read the room, so the prompt itself must carry the + # message, the room id every tool call needs, and the reply-tool contract. + assert "room-123" in prompt + assert "band-agent" in prompt + assert "the marker is X9" in prompt + assert "Echo the marker back." in prompt + assert "band_send_message" in prompt + assert MCP_SERVER_NAME in prompt + assert prompt.count("Alex") >= 2 # named as sender and as reply mention + + +# --- workspace scaffolding: what VS Code reads -------------------------------------- + + +def test_scaffold_workspace_writes_mcp_entry_and_auto_approve(tmp_path: Path) -> None: + sse_url = "http://127.0.0.1:8391/sse" + scaffold_workspace(tmp_path, sse_url) + + mcp = json.loads((tmp_path / ".vscode" / "mcp.json").read_text()) + assert mcp["servers"][MCP_SERVER_NAME] == {"type": "sse", "url": sse_url} + + settings = json.loads((tmp_path / ".vscode" / "settings.json").read_text()) + assert settings[AUTO_APPROVE_SETTING] is True + + +def test_workspace_marker_path_stays_inside_workspace(tmp_path: Path) -> None: + marker = workspace_marker_path(tmp_path, "notes-abc.txt") + assert marker.parent == tmp_path + + +# --- driver preflight + version evidence -------------------------------------------- + + +def test_preflight_rejects_missing_binary() -> None: + with pytest.raises(PreflightError, match="README"): + preflight(["definitely-not-a-real-code-binary"]) + + +def test_capture_versions_shape_with_stub_runner() -> None: + outputs = { + "code --version": "1.103.0\nabc123\narm64", + "code --list-extensions --show-versions": ( + "github.copilot@1.5.0\ngithub.copilot-chat@0.32.0\nms-python.python@2026.1" + ), + "band-mcp --version": "band-mcp 1.3.2", + } + + versions = capture_versions( + ["code"], ["band-mcp"], run=lambda cmd: outputs[" ".join(cmd)] + ) + + assert versions["vscode"] == "1.103.0 abc123 arm64" + assert versions["copilot_extensions"] == ( + "github.copilot@1.5.0; github.copilot-chat@0.32.0" + ) + assert versions["band_mcp"] == "band-mcp 1.3.2" + assert versions["os"] + + +def test_capture_versions_records_failures_instead_of_raising() -> None: + def failing(cmd: list[str]) -> str: + raise OSError("boom") + + versions = capture_versions(["code"], ["band-mcp"], run=failing) + assert "unavailable" in versions["vscode"] + assert "unavailable" in versions["band_mcp"] + + +def test_fresh_session_preamble_is_prependable_text() -> None: + # The CLI has no verified new-session switch; the preamble is the fallback + # and must stay a plain instruction line (no templating placeholders). + assert "{" not in FRESH_SESSION_PREAMBLE + + +# --- scorecard: outcome mapping + emitted artifact ---------------------------------- + + +def _report(when: str, outcome: str, nodeid: str, fspath: str) -> SimpleNamespace: + return SimpleNamespace( + when=when, + skipped=outcome == "skipped", + failed=outcome == "failed", + passed=outcome == "passed", + nodeid=nodeid, + fspath=fspath, + ) + + +@pytest.mark.parametrize( + ("when", "outcome", "expected"), + [ + ("setup", "skipped", "skip"), + ("setup", "failed", "fail"), + ("setup", "passed", None), # no verdict until the call phase + ("call", "failed", "fail"), + ("call", "passed", "pass"), + ("teardown", "failed", None), # teardown never overrides the call + ], +) +def test_outcome_status_mapping(when: str, outcome: str, expected: str | None) -> None: + report = _report(when, outcome, "tests/e2e/vscode/test_x.py::t", "x") + assert outcome_status(report) == expected # type: ignore[arg-type] + + +def test_usage_na_row_carries_surface_and_reason() -> None: + assert USAGE_NA_ROW.adapter == SURFACE_ID + assert USAGE_NA_ROW.status == "na" + assert USAGE_NA_ROW.reason + + +def test_scorecard_writes_rows_metadata_and_fixed_na_row(tmp_path: Path) -> None: + suite_dir = tmp_path / "suite" + suite_dir.mkdir() + test_file = suite_dir / "test_copilot_chat.py" + test_file.touch() + out = tmp_path / "scorecard.json" + + plugin = VSCodeScorecard(out, suite_dir) + plugin.metadata = {"vscode": "1.103.0"} + nodeid = "tests/e2e/vscode/test_copilot_chat.py::test_participation" + plugin.pytest_runtest_logreport(_report("setup", "passed", nodeid, str(test_file))) # type: ignore[arg-type] + plugin.pytest_runtest_logreport(_report("call", "passed", nodeid, str(test_file))) # type: ignore[arg-type] + # A report from outside the suite dir must not add a row. + plugin.pytest_runtest_logreport( + _report("call", "passed", "tests/other.py::t", str(tmp_path / "other.py")) # type: ignore[arg-type] + ) + plugin.pytest_sessionfinish() + + rows = {row["test"]: row for row in json.loads(out.read_text())} + assert rows[nodeid]["status"] == "pass" + assert rows[nodeid]["adapter"] == SURFACE_ID + assert rows[USAGE_NA_ROW.test]["status"] == "na" + assert len(rows) == 2 + + meta = json.loads((tmp_path / "scorecard.json.meta.json").read_text()) + assert meta == {"vscode": "1.103.0"} diff --git a/tests/test_env_gates.py b/tests/test_env_gates.py index 70d2c685c..062f008ce 100644 --- a/tests/test_env_gates.py +++ b/tests/test_env_gates.py @@ -23,6 +23,7 @@ def test_empty_collection_gate_vars_read_as_disabled( "E2E_TESTS_ENABLED", "DOCKER_TESTS_ENABLED", "SANDBOX_TESTS_ENABLED", + "VSCODE_CHAT_TESTS_ENABLED", ): monkeypatch.setenv(var, "") gates = CollectionGateSettings() @@ -30,6 +31,7 @@ def test_empty_collection_gate_vars_read_as_disabled( assert gates.e2e_tests_enabled is False assert gates.docker_tests_enabled is False assert gates.sandbox_tests_enabled is False + assert gates.vscode_chat_tests_enabled is False def test_empty_baseline_vars_fall_back_to_defaults( From 8bf45ded7c8f2a887b2804e556f12e775d29e9c6 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Thu, 23 Jul 2026 12:02:32 +0300 Subject: [PATCH 2/7] feat(e2e): Copilot-in-VS-Code live cells, suite conftest, and runbook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six live cells map the surface to the bar: L0 participation (echo round-trip through band_send_message), L1 retained functions (native workspace file + band_get_participants in one turn), L0 multi-participant (peer-driven turn), L2 persistence (memory tools across a fresh chat session), L2 isolation (per-room markers, leak asserted absent), L3 restart (band-mcp bridge restart between turns). L4 usage ships as the scorecard's fixed N/A row — the surface exposes no per-turn usage signal. Turns are driver-initiated with the room message relayed in the prompt: the surface has no inbound channel and band-mcp exposes no message-read tool. Replies are awaited over captured room messages rather than the baseline delivery-status barrier, which only SDK runtimes ack. Co-Authored-By: Claude Fable 5 Signed-off-by: Alexander Zaikman --- tests/e2e/vscode/README.md | 130 ++++++++ tests/e2e/vscode/conftest.py | 183 +++++++++++ tests/e2e/vscode/test_copilot_chat.py | 424 ++++++++++++++++++++++++++ 3 files changed, 737 insertions(+) create mode 100644 tests/e2e/vscode/README.md create mode 100644 tests/e2e/vscode/conftest.py create mode 100644 tests/e2e/vscode/test_copilot_chat.py diff --git a/tests/e2e/vscode/README.md b/tests/e2e/vscode/README.md new file mode 100644 index 000000000..62a42f799 --- /dev/null +++ b/tests/e2e/vscode/README.md @@ -0,0 +1,130 @@ +# Copilot-in-VS-Code validation suite (opt-in, dev-machine only) + +Drives **GitHub Copilot Chat inside a real VS Code window** (agent mode) against +the live Band platform and validates the L0–L4 common bar. The harness: + +1. provisions a fresh Band agent identity, +2. runs **band-mcp** (SSE, loopback) holding that identity's key, +3. scaffolds a throwaway workspace whose `.vscode/mcp.json` points Copilot at it, +4. opens the workspace window once, then submits each turn via `code chat -m agent`, +5. asserts **Band-side only** (reply capture + LLM judge) — the VS Code UI is + never scraped. + +**This suite can never run in CI**: the Copilot extension authenticates through +interactive GitHub OAuth in a browser (no headless/service-account path), and a +GUI window must stay open. It is gated behind `VSCODE_CHAT_TESTS_ENABLED` +(marker `vscode_chat`), like the `DOCKER_TESTS_ENABLED` pattern. + +## Surface semantics (why the tests look like this) + +- **No inbound channel.** Copilot in VS Code cannot be pushed a Band message, + and band-mcp exposes **no message-read/history tool** (verified against + band-mcp 1.3.2). Every turn is driver-initiated: the prompt relays the room + message and pins the room `chat_id`. This mirrors real usage, where the + developer relays context into chat and Copilot posts via `band_send_message`. +- **Recall = platform memory tools.** Cross-session/cross-restart recall flows + through `band_store_memory` / `band_list_memories` (the server runs with + `--tools memory`), which are platform-persisted and session-independent. +- **No delivery acks.** band-mcp is REST-only, so the baseline + `wait_for_reply` barrier (delivery-status PROCESSED) never fires; the cells + wait on captured room messages instead. +- **L4 usage is N/A**: the surface exposes no per-turn usage/billing signal. + The scorecard carries the rationale as a fixed `na` row. + +## Prerequisites + +- **VS Code ≥ 1.102** with the `code` CLI on PATH (macOS: run + "Shell Command: Install 'code' command in PATH" from the Command Palette), + or set `CODE_COMMAND`. +- **GitHub Copilot Chat extension installed and signed in** (interactive; do + this once in the normal VS Code UI before running). +- **band-mcp ≥ 1.3.2** installed (`uv tool install band-mcp`), or set + `BAND_MCP_COMMAND` (e.g. `"uvx band-mcp"`). +- `.env.test` with `BAND_API_KEY_USER` (provisioning/observer) and + `ANTHROPIC_API_KEY` (the judge). + +## Running + +```bash +VSCODE_CHAT_TESTS_ENABLED=true E2E_TESTS_ENABLED=true \ +VSCODE_CHAT_SCORECARD_JSON=artifacts/int-1110-copilot-vscode-scorecard.json \ +uv run pytest tests/e2e/vscode -v -s --no-cov +``` + +During the run a VS Code window opens on the scaffolded temp workspace: + +- **Accept workspace trust** when prompted (first open). +- If VS Code prompts to **start/trust the `band` MCP server**, allow it. +- If individual tool calls still ask for approval (the workspace sets + `chat.tools.global.autoApprove`, but the installed VS Code may scope that + setting user-level only), pick **"Always allow"** on the first occurrence of + each tool — subsequent turns run unattended. +- Leave the window open and unfocused-but-alive until the run finishes. + +> **Security note:** `chat.tools.global.autoApprove` disables per-tool +> confirmation prompts. The workspace is a throwaway temp directory and the +> only configured MCP server is the harness's own band-mcp instance, which +> bounds the exposure — do not reuse this settings file in a real workspace. + +Artifacts (when `VSCODE_CHAT_SCORECARD_JSON` is set): + +- `artifacts/int-1110-copilot-vscode-scorecard.json` — pass/fail/skip rows plus + the fixed L4 `na` row (baseline scorecard row schema, merge-compatible). +- `…-scorecard.json.meta.json` — environment evidence: OS, `code --version`, + Copilot extension versions, band-mcp version. + +## Environment knobs + +| Env var | Default | Purpose | +|---|---|---| +| `VSCODE_CHAT_TESTS_ENABLED` | `false` | The collection gate | +| `CODE_COMMAND` | `code` | VS Code CLI binary (may be a full path) | +| `BAND_MCP_COMMAND` | `band-mcp` | band-mcp launcher (e.g. `uvx band-mcp`) | +| `VSCODE_CHAT_SCORECARD_JSON` | empty | Scorecard output path (empty = don't emit) | +| `VSCODE_CHAT_TIMEOUT` | `300` | Seconds allowed per live turn | + +Band endpoints, credentials, autoclean/orphan-sweep policy, and the judge model +come from the baseline settings (`tests/e2e/baseline/settings.py`). + +## Cells + +| Test | Level | Proves | +|---|---|---| +| `test_participation_reply_round_trip` | L0 | Room message → Copilot → `band_send_message` reply with the echo token | +| `test_original_functions_retained` | L1 | Native function (workspace file) + platform tool (`band_get_participants`) in one turn | +| `test_multi_participant_echo_peer` | L0 | A peer agent's message drives a turn; the reply engages the peer | +| `test_recall_across_chat_sessions` | L2 | Fact stored via memory tools survives to a fresh chat session | +| `test_no_leak_between_rooms` | L2 | Room B's answer names room B's marker, never room A's | +| `test_restart_recall_and_function` | L3 | band-mcp restart between turns; recall + native function still work | +| *(no test)* `usage_accounting` | L4 | `na` scorecard row — no per-turn usage signal exposed | + +## Manual variants / known weak spots + +- **Fresh chat session** (`new_session=True`) is expressed as a prompt preamble — + the `code chat` CLI has no verified per-invocation new-session switch. For a + stronger variant, click **New Chat** in the window between the two turns of + `test_recall_across_chat_sessions` while it waits. +- **Full window restart** (quit VS Code between the turns of + `test_restart_recall_and_function`, reopen with `code `) is a + manual variant; the automated cell restarts the platform bridge (band-mcp) + instead, which is deterministic. + +## Troubleshooting + +- **Prompt lands in the wrong window**: `code chat` targets the window whose + workspace matches the CWD; keep only the harness's workspace window open, or + close other VS Code windows. +- **421 responses from band-mcp**: the server sets + `ALLOWED_HOSTS='["localhost:*","127.0.0.1:*"]'` itself; if you changed the + host/port wiring, keep the allowlist in sync. +- **Turn times out with no reply**: check the window — a pending trust/approval + dialog blocks the turn. Approve it; the run continues on the next cell. +- **`PreflightError`**: the `code` CLI is missing or predates the `chat` + subcommand — upgrade VS Code / fix `CODE_COMMAND`. + +## Future work + +`driver.PromptDriver` is a protocol: a `@vscode/test-electron` backend (pinned +VS Code download, persistent signed-in profile, programmatic +`workbench.action.chat.open`) can replace `CodeChatDriver` without touching the +cells. diff --git a/tests/e2e/vscode/conftest.py b/tests/e2e/vscode/conftest.py new file mode 100644 index 000000000..2d2f0724f --- /dev/null +++ b/tests/e2e/vscode/conftest.py @@ -0,0 +1,183 @@ +"""Pytest wiring for the Copilot-in-VS-Code suite. + +Mirrors the baseline conftest's registration style: the shared platform/capture +fixtures are plain imports re-exported via ``__all__`` (``pytest_plugins`` is +not allowed in a non-root conftest), and the hooks here add only what this +suite needs — the gate, the session loop + per-turn timeout markers, and the +suite-local scorecard plugin. + +The suite drives a real signed-in VS Code window, so beyond the env gate it has +human prerequisites (see README.md): VS Code installed with the Copilot Chat +extension signed in, and the window the ``driver`` fixture opens left alone. +""" + +from __future__ import annotations + +import asyncio +import logging +import shlex +from collections.abc import AsyncGenerator +from pathlib import Path + +import pytest +from band_rest import AsyncRestClient + +from tests.e2e.baseline.fixtures.capture import judge, reply_capture +from tests.e2e.baseline.fixtures.platform import ( + baseline_run_id, + baseline_settings, + baseline_user_client, + baseline_ws, + orphan_sweep, + reap_leaked_agents, + resource_manager, + user_ops, +) +from tests.e2e.baseline.settings import BaselineSettings +from tests.e2e.baseline.toolkit.provisioning import ProvisionedAgent, ResourceManager +from tests.e2e.vscode.driver import ( + CodeChatDriver, + PreflightError, + capture_versions, + preflight, +) +from tests.e2e.vscode.scorecard import VSCodeScorecard +from tests.e2e.vscode.server import BandMCPServer +from tests.e2e.vscode.settings import VSCodeChatSettings +from tests.e2e.vscode.workspace import scaffold_workspace +from tests.toolkit.timeouts import effective_timeout + +logger = logging.getLogger(__name__) + +# Re-exported fixtures (defined in the baseline fixtures package). +__all__ = [ + "band_mcp", + "baseline_run_id", + "baseline_settings", + "baseline_user_client", + "baseline_ws", + "copilot_identity", + "driver", + "judge", + "orphan_sweep", + "reap_leaked_agents", + "reply_capture", + "resource_manager", + "user_ops", + "vscode_settings", + "vscode_workspace", +] + +# Seconds after `code ` for the window, extension host, and MCP +# client to come up before the first prompt is submitted. +WINDOW_OPEN_GRACE_S = 20 + + +@pytest.fixture(scope="session") +def vscode_settings() -> VSCodeChatSettings: + return VSCodeChatSettings() + + +@pytest.fixture(scope="session") +async def copilot_identity( + baseline_settings: BaselineSettings, + baseline_user_client: AsyncRestClient, + baseline_run_id: str, +) -> AsyncGenerator[ProvisionedAgent, None]: + """The one Band agent identity Copilot acts as, held for the whole session. + + Session-scoped (its own manager, not the per-test ``resource_manager``) + because the identity's key is baked into the band-mcp subprocess env and + the workspace's MCP config at session start. + """ + resources = ResourceManager( + user_client=baseline_user_client, + settings=baseline_settings, + run_id=baseline_run_id, + ) + identity = await resources.provision_agent("copilot") + yield identity + if baseline_settings.run.autoclean: + await resources.reap_all() + + +@pytest.fixture(scope="session") +async def band_mcp( + vscode_settings: VSCodeChatSettings, + baseline_settings: BaselineSettings, + copilot_identity: ProvisionedAgent, +) -> AsyncGenerator[BandMCPServer, None]: + server = BandMCPServer( + shlex.split(vscode_settings.band_mcp_command), + agent_key=copilot_identity.api_key, + base_url=baseline_settings.endpoints.rest_url, + ) + await server.start() + yield server + await server.stop() + + +@pytest.fixture(scope="session") +def vscode_workspace( + tmp_path_factory: pytest.TempPathFactory, band_mcp: BandMCPServer +) -> Path: + workspace = tmp_path_factory.mktemp("vscode-chat-ws") + scaffold_workspace(workspace, band_mcp.sse_url) + return workspace + + +@pytest.fixture(scope="session") +async def driver( + vscode_settings: VSCodeChatSettings, vscode_workspace: Path +) -> CodeChatDriver: + """Preflight the VS Code CLI, open the workspace window once, yield the driver.""" + code_command = shlex.split(vscode_settings.code_command) + try: + preflight(code_command) + except PreflightError as error: + pytest.fail(str(error)) + chat_driver = CodeChatDriver(code_command, vscode_workspace) + await chat_driver.open_window() + await asyncio.sleep(WINDOW_OPEN_GRACE_S) + return chat_driver + + +def pytest_configure(config: pytest.Config) -> None: + """Register the suite scorecard plugin when emission is enabled.""" + settings = VSCodeChatSettings() + if not settings.scorecard_json: + return + plugin = VSCodeScorecard(settings.scorecard_json, Path(__file__).parent.resolve()) + plugin.metadata = capture_versions( + shlex.split(settings.code_command), shlex.split(settings.band_mcp_command) + ) + config.pluginmanager.register(plugin) + + +def pytest_runtest_setup(item: pytest.Item) -> None: + """Self-gate (besides the root markers) and fail loud on missing credentials.""" + if not Path(item.path).is_relative_to(Path(__file__).parent): + return + if not VSCodeChatSettings().vscode_chat_tests_enabled: + pytest.skip("VSCODE_CHAT_TESTS_ENABLED is not true") + if not BaselineSettings().credentials.api_key_user: + pytest.fail("BAND_API_KEY_USER not set (VS Code chat suite enabled)") + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Put suite tests on the session loop and give them the live-turn timeout. + + Same two markers the baseline conftest applies to its own subtree: the + session-scoped WS/REST fixtures live on the session loop, and live turns + need far more than the 30s pyproject default. + """ + suite_dir = Path(__file__).parent + session_marker = pytest.mark.asyncio(loop_scope="session") + base = VSCodeChatSettings().turn_timeout + for item in items: + if not Path(item.path).is_relative_to(suite_dir): + continue + item.add_marker(session_marker) + timeout = effective_timeout(item, base) + if timeout is not None: + item.add_marker(pytest.mark.timeout(timeout), append=False) diff --git a/tests/e2e/vscode/test_copilot_chat.py b/tests/e2e/vscode/test_copilot_chat.py new file mode 100644 index 000000000..54b7130b2 --- /dev/null +++ b/tests/e2e/vscode/test_copilot_chat.py @@ -0,0 +1,424 @@ +"""Live L0–L3 cells for the GitHub Copilot in VS Code surface. + +Every turn is driver-initiated (see ``driver.turn_prompt``): the room message +is posted first, then the prompt relays it to Copilot Chat with the room id +and the band-tool contract. Assertions are all Band-side — the reply must land +in the right room, from the provisioned identity, via ``band_send_message``. + +The reply wait is ``wait_until`` over captured messages, not the baseline +``wait_for_reply``: that barrier keys on the recipient's delivery-status +PROCESSED ack, which only an SDK runtime emits — this surface posts through +band-mcp (REST only), so no delivery ack ever arrives. + +L4 usage has no cell here: the surface exposes no per-turn usage signal — the +scorecard carries the rationale (see ``scorecard.USAGE_NA_ROW``). +""" + +from __future__ import annotations + +import uuid +from pathlib import Path + +import pytest + +from tests.e2e.baseline.settings import BaselineSettings +from tests.e2e.baseline.toolkit.capture import CaptureFactory, ReplyCapture +from tests.e2e.baseline.toolkit.observations.replies import Replies +from tests.e2e.baseline.toolkit.provisioning import ProvisionedAgent, ResourceManager +from tests.e2e.baseline.toolkit.user_ops import UserOps +from tests.e2e.vscode.driver import CodeChatDriver, turn_prompt +from tests.e2e.vscode.server import BandMCPServer +from tests.e2e.vscode.settings import VSCodeChatSettings +from tests.e2e.vscode.workspace import workspace_marker_path + +pytestmark = [pytest.mark.vscode_chat, pytest.mark.e2e] + + +def _token() -> str: + return uuid.uuid4().hex[:6] + + +async def run_turn( + capture: ReplyCapture, + driver: CodeChatDriver, + *, + room_id: str, + identity: ProvisionedAgent, + sender_name: str, + message: str, + instruction: str, + deadline_s: float, + new_session: bool = False, +) -> Replies: + """Submit one relayed-message prompt and wait for the identity's room reply.""" + mark = capture.messages.snapshot() + await driver.submit_prompt( + turn_prompt( + room_id, + identity.name, + sender_name=sender_name, + message=message, + instruction=instruction, + ), + new_session=new_session, + ) + + def replied(_messages: list) -> bool: + return bool(capture.messages.since(mark).from_sender(identity.id)) + + await capture.wait_until(replied, deadline_s=deadline_s) + return capture.messages.since(mark).from_sender(identity.id) + + +async def test_participation_reply_round_trip( + vscode_settings: VSCodeChatSettings, + copilot_identity: ProvisionedAgent, + resource_manager: ResourceManager, + user_ops: UserOps, + reply_capture: CaptureFactory, + driver: CodeChatDriver, +) -> None: + """L0: a room message reaches Copilot and its reply lands back in the room. + + The token is high-entropy, so the reply containing it proves the relayed + message flowed through the turn and back via ``band_send_message`` — not + model invention. + """ + token = _token() + room_id = await resource_manager.provision_room( + title="e2e-vscode-participation", participants=[copilot_identity.id] + ) + async with reply_capture(room_id) as capture: + text = f"Ping — the echo token is {token}." + await user_ops.send_message( + room_id, + text, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + replies = await run_turn( + capture, + driver, + room_id=room_id, + identity=copilot_identity, + sender_name="the user", + message=text, + instruction="Reply to this message, repeating its echo token exactly.", + deadline_s=vscode_settings.turn_timeout, + ) + replies.assert_contains_any([token]) + + +async def test_original_functions_retained( + vscode_settings: VSCodeChatSettings, + copilot_identity: ProvisionedAgent, + resource_manager: ResourceManager, + user_ops: UserOps, + reply_capture: CaptureFactory, + driver: CodeChatDriver, + vscode_workspace: Path, + judge, +) -> None: + """L1: one turn exercises a native VS Code function (create a workspace + file) AND a platform tool (``band_get_participants``) — the surface keeps + its original capabilities while participating.""" + token = _token() + filename = f"notes-{token}.txt" + room_id = await resource_manager.provision_room( + title="e2e-vscode-functions", participants=[copilot_identity.id] + ) + async with reply_capture(room_id) as capture: + text = ( + f"Please set up a note file for this task and tell me who is " + f"in this room. The file token is {token}." + ) + await user_ops.send_message( + room_id, + text, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + replies = await run_turn( + capture, + driver, + room_id=room_id, + identity=copilot_identity, + sender_name="the user", + message=text, + instruction=( + f"First create a file named '{filename}' in the workspace root " + f"containing exactly the token {token}. Then look up the room " + f"roster with band_get_participants and reply listing every " + f"participant's name and confirming the file was created." + ), + deadline_s=vscode_settings.turn_timeout, + ) + + marker = workspace_marker_path(vscode_workspace, filename) + assert marker.exists(), f"Copilot did not create {filename} in the workspace" + assert token in marker.read_text() + + verdict = await judge( + criteria=( + "The reply lists the chat room's participants (at least the agent " + "itself or the human user) and confirms that the requested note " + "file was created." + ), + transcript="\n".join(reply.content for reply in replies), + ) + assert verdict.passed, verdict.reasoning + + +async def test_multi_participant_echo_peer( + vscode_settings: VSCodeChatSettings, + copilot_identity: ProvisionedAgent, + resource_manager: ResourceManager, + reply_capture: CaptureFactory, + driver: CodeChatDriver, + judge, +) -> None: + """L0 multi-participant: a peer agent's message drives a turn and the reply + engages the peer — the surface participates beyond 1:1 user chats.""" + token = _token() + echo = await resource_manager.provision_agent("echo") + room_id = await resource_manager.provision_room( + title="e2e-vscode-peers", participants=[copilot_identity.id, echo.id] + ) + async with reply_capture(room_id) as capture: + text = f"ECHO: {token}" + await resource_manager.peer(echo).send_message( + room_id, + text, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + replies = await run_turn( + capture, + driver, + room_id=room_id, + identity=copilot_identity, + sender_name=echo.name, + message=text, + instruction=( + "Another agent posted this in the room. Reply addressing that " + "agent and repeat its echo token exactly." + ), + deadline_s=vscode_settings.turn_timeout, + ) + + replies.assert_contains_any([token]) + verdict = await judge( + criteria="The reply engages with the other agent's echo message.", + transcript="\n".join(reply.content for reply in replies), + ) + assert verdict.passed, verdict.reasoning + + +@pytest.mark.timeout(extra=360) # two full live turns +async def test_recall_across_chat_sessions( + vscode_settings: VSCodeChatSettings, + copilot_identity: ProvisionedAgent, + resource_manager: ResourceManager, + user_ops: UserOps, + reply_capture: CaptureFactory, + driver: CodeChatDriver, +) -> None: + """L2 persistence: a fact stored in turn 1 survives to a fresh chat session. + + band-mcp exposes no room-history tool, so cross-session recall flows + through the platform memory tools — the fact is stored with + ``band_store_memory`` and must come back via ``band_list_memories`` in a + session that never saw it. + """ + fact = f"codename-{_token()}" + room_id = await resource_manager.provision_room( + title="e2e-vscode-recall", participants=[copilot_identity.id] + ) + async with reply_capture(room_id) as capture: + seed = f"For later reference: the project codename is {fact}." + await user_ops.send_message( + room_id, + seed, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + await run_turn( + capture, + driver, + room_id=room_id, + identity=copilot_identity, + sender_name="the user", + message=seed, + instruction=( + "Store the project codename in your Band memory with " + "band_store_memory, then confirm in the room that it is saved." + ), + deadline_s=vscode_settings.turn_timeout, + ) + + ask = "What is the project codename?" + await user_ops.send_message( + room_id, + ask, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + replies = await run_turn( + capture, + driver, + room_id=room_id, + identity=copilot_identity, + sender_name="the user", + message=ask, + instruction=( + "Recall the project codename from your Band memory using " + "band_list_memories (and band_get_memory if needed) — do not " + "guess — and reply stating it exactly." + ), + deadline_s=vscode_settings.turn_timeout, + new_session=True, + ) + replies.assert_contains_any([fact]) + + +@pytest.mark.timeout(extra=700) # three full live turns across two rooms +async def test_no_leak_between_rooms( + vscode_settings: VSCodeChatSettings, + copilot_identity: ProvisionedAgent, + resource_manager: ResourceManager, + user_ops: UserOps, + reply_capture: CaptureFactory, + driver: CodeChatDriver, +) -> None: + """L2 isolation: replies stay scoped to their room — room B's answer names + room B's token and never room A's.""" + token_a, token_b = f"alpha-{_token()}", f"bravo-{_token()}" + room_a = await resource_manager.provision_room( + title="e2e-vscode-isolation-a", participants=[copilot_identity.id] + ) + room_b = await resource_manager.provision_room( + title="e2e-vscode-isolation-b", participants=[copilot_identity.id] + ) + + async def seed(room_id: str, capture: ReplyCapture, token: str) -> None: + text = f"The marker for THIS room is {token}. Acknowledge it." + await user_ops.send_message( + room_id, + text, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + await run_turn( + capture, + driver, + room_id=room_id, + identity=copilot_identity, + sender_name="the user", + message=text, + instruction="Acknowledge this room's marker by repeating it.", + deadline_s=vscode_settings.turn_timeout, + ) + + async with reply_capture(room_a) as capture_a: + await seed(room_a, capture_a, token_a) + async with reply_capture(room_b) as capture_b: + await seed(room_b, capture_b, token_b) + + ask = "Which marker belongs to THIS room?" + await user_ops.send_message( + room_b, + ask, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + replies = await run_turn( + capture_b, + driver, + room_id=room_b, + identity=copilot_identity, + sender_name="the user", + message=ask, + instruction=( + "Reply with the marker that was stated in this room only. " + "Never mention markers from any other room or conversation." + ), + deadline_s=vscode_settings.turn_timeout, + ) + replies.assert_contains_any([token_b]) + replies.assert_contains_none([token_a]) + + +@pytest.mark.timeout(extra=360) # two live turns plus a bridge restart +async def test_restart_recall_and_function( + vscode_settings: VSCodeChatSettings, + baseline_settings: BaselineSettings, + copilot_identity: ProvisionedAgent, + resource_manager: ResourceManager, + user_ops: UserOps, + reply_capture: CaptureFactory, + driver: CodeChatDriver, + band_mcp: BandMCPServer, + vscode_workspace: Path, +) -> None: + """L3: the platform bridge (band-mcp) restarts between turns and the surface + still recalls the stored fact AND keeps its native function (file creation). + + A full VS Code window restart stays a documented manual variant in the + README — the bridge restart covers the platform side deterministically. + """ + fact = f"phase-{_token()}" + filename = f"restart-{_token()}.txt" + room_id = await resource_manager.provision_room( + title="e2e-vscode-restart", participants=[copilot_identity.id] + ) + async with reply_capture(room_id) as capture: + seed = f"Remember across restarts: the deploy phase is {fact}." + await user_ops.send_message( + room_id, + seed, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + await run_turn( + capture, + driver, + room_id=room_id, + identity=copilot_identity, + sender_name="the user", + message=seed, + instruction=( + "Store the deploy phase in your Band memory with " + "band_store_memory, then confirm in the room." + ), + deadline_s=vscode_settings.turn_timeout, + ) + + await band_mcp.restart() + + ask = "After the maintenance window: what is the deploy phase?" + await user_ops.send_message( + room_id, + ask, + mention_id=copilot_identity.id, + mention_name=copilot_identity.name, + ) + replies = await run_turn( + capture, + driver, + room_id=room_id, + identity=copilot_identity, + sender_name="the user", + message=ask, + instruction=( + f"Recall the deploy phase from your Band memory with " + f"band_list_memories — do not guess — and reply stating it. " + f"Also create a file named '{filename}' in the workspace root " + f"containing that phase, to confirm your editor tools still work." + ), + deadline_s=vscode_settings.turn_timeout, + new_session=True, + ) + + replies.assert_contains_any([fact]) + marker = workspace_marker_path(vscode_workspace, filename) + assert marker.exists(), f"Copilot did not create {filename} after the restart" From 826e104f65a106ecb11bac558950bd4d00bdfa82 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Thu, 23 Jul 2026 12:46:03 +0300 Subject: [PATCH 3/7] =?UTF-8?q?feat(e2e):=20live-green=20VS=20Code=20Copil?= =?UTF-8?q?ot=20suite=20=E2=80=94=20evidence,=20skill,=20unattended=20reru?= =?UTF-8?q?ns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live validation is green: 6/6 cells pass unattended (L0 participation + multi-participant, L1 native+platform functions, L2 memory recall and room isolation, L3 bridge restart), with the L4 usage N/A rationale on the scorecard. Artifacts recorded (scorecard + environment sidecar: macOS, VS Code 1.130.0, bundled Copilot Chat, band-mcp 1.3.2). Hardening from the live runs: - session-loop marker prepended (append=False) so pytest-asyncio's bare auto-mode marker cannot strand tests on a function loop away from the session-scoped WS/REST fixtures (subscribe timeouts, closed-loop teardowns) - settings field names now equal their env vars (the pydantic-settings contract) — VSCODE_CHAT_SCORECARD_JSON/VSCODE_CHAT_TIMEOUT were silently unread - stable workspace (~/band-e2e, visible path — hidden dirs trip Copilot's sensitive-file edit guard) + fixed band-mcp port keep mcp.json byte-identical, so folder trust, MCP trust, and per-tool approvals are one-time; reruns are fully unattended - chat.autoReply instead of chat.tools.global.autoApprove — the latter is machine-wide YOLO mode and triggers a global consent escalation - memory cells store room-keyed records, verify after store, and add a Band-side stored-memory barrier so a failure distinguishes the store path from the retrieval path from the prompt - judge transcripts label peer message vs agent reply (a bare reply reads as unjudgeable echo) - directory-scoped Claude skill (tests/e2e/vscode/.claude/skills) runs the whole flow: prereq checks, auto-install, dialog coaching - README leads with the semi-manual nature of the suite Co-Authored-By: Claude Fable 5 Signed-off-by: Alexander Zaikman --- .../.claude/skills/vscode-chat-e2e/SKILL.md | 79 +++++++++++++++++++ tests/e2e/vscode/README.md | 53 ++++++++++--- tests/e2e/vscode/conftest.py | 28 +++++-- tests/e2e/vscode/driver.py | 13 ++- tests/e2e/vscode/server.py | 9 ++- tests/e2e/vscode/settings.py | 16 +++- tests/e2e/vscode/test_copilot_chat.py | 74 ++++++++++++----- tests/e2e/vscode/workspace.py | 15 ++-- .../test_vscode_chat_toolkit.py | 6 +- 9 files changed, 238 insertions(+), 55 deletions(-) create mode 100644 tests/e2e/vscode/.claude/skills/vscode-chat-e2e/SKILL.md diff --git a/tests/e2e/vscode/.claude/skills/vscode-chat-e2e/SKILL.md b/tests/e2e/vscode/.claude/skills/vscode-chat-e2e/SKILL.md new file mode 100644 index 000000000..fe46d3ac2 --- /dev/null +++ b/tests/e2e/vscode/.claude/skills/vscode-chat-e2e/SKILL.md @@ -0,0 +1,79 @@ +--- +name: vscode-chat-e2e +description: Run the semi-manual Copilot-in-VS-Code E2E validation suite (tests/e2e/vscode) end to end — verify prerequisites, auto-install what can be installed (VS Code, band-mcp), start the live run, and coach the human through the few one-time dialogs only they can click (Copilot sign-in, folder trust, MCP/tool approvals). Use when asked to run, rerun, or validate the VS Code Copilot surface, or to produce its scorecard evidence. +--- + +# Copilot-in-VS-Code E2E run + +This suite is **inherently semi-manual** (see `tests/e2e/vscode/README.md`): it +drives a real, visible VS Code window with a human-signed-in Copilot. Your job +is to make the human's part as small as possible: check every prerequisite, +install whatever can be installed non-interactively, run the suite, and tell +the user exactly what to click **only** when there is no other way. + +## Phase 1 — prerequisites (install, don't ask, where possible) + +Check in this order; fix silently where a non-interactive fix exists. + +1. **VS Code + `code` CLI** — `code --version` and `code chat --help`. + - Missing entirely: `brew install --cask visual-studio-code` (macOS; the + cask links `code` onto PATH itself). + - App present but no CLI: link it — + `ln -sf "/Applications/Visual Studio Code.app/Contents/Resources/app/bin/code" /opt/homebrew/bin/code`. + - `chat` subcommand missing: VS Code too old (< 1.101) — + `brew upgrade --cask visual-studio-code`. +2. **Copilot Chat extension** — ships built into current VS Code; verify with + `code --install-extension GitHub.copilot-chat` (a "already installed" + response is success). +3. **band-mcp** — `uvx band-mcp --version` (uvx fetches it on first use; no + install step). Pass `BAND_MCP_COMMAND="uvx band-mcp"` to the run. +4. **Credentials** — `.env.test` must define `BAND_API_KEY_USER` (provisioning + + observer) and `ANTHROPIC_API_KEY` (judge). Missing keys cannot be + installed: stop and ask the user for them. +5. **Copilot sign-in** — cannot be automated (interactive GitHub OAuth, no + headless path). If unsure whether a login exists, just start Phase 2: a + signed-out Copilot shows "Sign in to use Copilot" in the chat panel — then + instruct: *open Chat (⌃⌘I), click "Sign in to use Copilot", finish the + browser OAuth, then say "signed in"* and rerun. + +## Phase 2 — the run + +```bash +VSCODE_CHAT_TESTS_ENABLED=true E2E_TESTS_ENABLED=true \ +BAND_MCP_COMMAND="uvx band-mcp" \ +VSCODE_CHAT_SCORECARD_JSON=artifacts/-copilot-vscode-scorecard.json \ +uv run pytest tests/e2e/vscode -v -s --no-cov +``` + +Run it in the background and monitor per-cell PASSED/FAILED lines. Kill any +leftover `band-mcp --transport sse` process before starting. Expect ~4 minutes +for 6 cells. + +## Phase 3 — coach the one-time dialogs (first run on a machine only) + +The workspace (`~/band-e2e/vscode-chat-workspace`) and band-mcp port (8631) +are stable, so every choice below is remembered — reruns are unattended. +When the VS Code window opens, tell the user to click, as dialogs appear: + +1. "Do you trust the authors…" → **Trust Folder & Continue** +2. MCP server `band` start/trust → **Allow/Trust** +3. Tool confirmations → dropdown next to Allow → **"Always allow…" (workspace)** + — once per tool (~4 band tools), never "this session" +4. "Allow edits to sensitive files" (shouldn't appear on the visible-path + workspace; if it does) → dropdown → **Always allow** + +Never tell the user to enable `chat.tools.global.autoApprove` ("YOLO mode") — +machine-wide security downgrade; the harness deliberately avoids it. + +## Phase 4 — verdict + evidence + +- Green: report `N passed`, point at the scorecard JSON + `.meta.json` + sidecar (versions evidence). Commit artifacts only when asked. +- A cell failing on reply content is usually live-model flakiness — rerun + once before investigating; two identical failures = real, read the + traceback (full failure section prints at session end). +- "Quota reached" badge in the window = Copilot premium requests exhausted; + turns will stall or degrade. Surface it to the user: paid seat, wait for + reset, or switch the chat model picker to an included model. +- Reap check: provisioned agents/rooms are auto-reaped (`BAND_E2E_AUTOCLEAN`); + leftover `band-mcp` processes should be killed if the run was interrupted. diff --git a/tests/e2e/vscode/README.md b/tests/e2e/vscode/README.md index 62a42f799..89a2815be 100644 --- a/tests/e2e/vscode/README.md +++ b/tests/e2e/vscode/README.md @@ -1,5 +1,17 @@ # Copilot-in-VS-Code validation suite (opt-in, dev-machine only) +> ## ⚠️ This is inherently a SEMI-MANUAL test suite +> +> It drives a **real, visible VS Code window** with a **human-signed-in +> Copilot**. A person must be at the machine: sign in to Copilot once, and on +> the **first run** click through a handful of one-time dialogs (folder trust, +> MCP server allow, per-tool "Always allow"). Those choices are remembered, so +> **reruns are unattended** — but the GUI window, the interactive GitHub OAuth, +> and the possibility of a new confirmation dialog never go away. It can never +> run in CI and is not a hands-off matrix suite; treat every green run as +> "passed on a supervised dev machine", and record it via the scorecard +> artifact. + Drives **GitHub Copilot Chat inside a real VS Code window** (agent mode) against the live Band platform and validates the L0–L4 common bar. The harness: @@ -45,26 +57,41 @@ GUI window must stay open. It is gated behind `VSCODE_CHAT_TESTS_ENABLED` ## Running +**Easiest**: use the directory-scoped Claude Code skill — `/vscode-chat-e2e` +(defined in `.claude/skills/vscode-chat-e2e/SKILL.md` next to this suite). It +checks prerequisites, installs what it can, runs the suite, and tells you +exactly what to click on a first run. + +Manual equivalent: + ```bash VSCODE_CHAT_TESTS_ENABLED=true E2E_TESTS_ENABLED=true \ VSCODE_CHAT_SCORECARD_JSON=artifacts/int-1110-copilot-vscode-scorecard.json \ uv run pytest tests/e2e/vscode -v -s --no-cov ``` -During the run a VS Code window opens on the scaffolded temp workspace: - -- **Accept workspace trust** when prompted (first open). -- If VS Code prompts to **start/trust the `band` MCP server**, allow it. -- If individual tool calls still ask for approval (the workspace sets - `chat.tools.global.autoApprove`, but the installed VS Code may scope that - setting user-level only), pick **"Always allow"** on the first occurrence of - each tool — subsequent turns run unattended. +During the run a VS Code window opens on the scaffolded workspace. The harness +minimizes prompts, and after the first run reruns are unattended: + +- **Workspace trust**: suppressed — the window is launched with + `--disable-workspace-trust` (scoped to that launch, not a global setting). +- **MCP server trust**: VS Code asks once per server *configuration*. The + workspace path and the band-mcp port are stable by default, so `mcp.json` + is byte-identical across runs — allow the `band` server on the first run + and the remembered trust holds for every rerun. (Changing + `VSCODE_CHAT_WORKSPACE` or `BAND_MCP_PORT` re-triggers the prompt.) +- **Tool approvals**: on each tool's first-ever call, open the dropdown next + to Allow and pick **"Always allow…" (workspace)** — one click per tool + (~4 band tools), remembered afterwards. The workspace also sets + `chat.autoReply` so agent-side questions never stall a turn. - Leave the window open and unfocused-but-alive until the run finishes. -> **Security note:** `chat.tools.global.autoApprove` disables per-tool -> confirmation prompts. The workspace is a throwaway temp directory and the -> only configured MCP server is the harness's own band-mcp instance, which -> bounds the exposure — do not reuse this settings file in a real workspace. +> **Security note:** the harness deliberately does **not** enable +> `chat.tools.global.autoApprove` ("YOLO mode") — that disables tool approval +> for every workspace on the machine and VS Code escalates it to a global +> consent dialog. Approvals stay per-tool and workspace-scoped. If you enable +> YOLO in your own user settings, that is a machine-wide security trade-off +> you own. Artifacts (when `VSCODE_CHAT_SCORECARD_JSON` is set): @@ -80,6 +107,8 @@ Artifacts (when `VSCODE_CHAT_SCORECARD_JSON` is set): | `VSCODE_CHAT_TESTS_ENABLED` | `false` | The collection gate | | `CODE_COMMAND` | `code` | VS Code CLI binary (may be a full path) | | `BAND_MCP_COMMAND` | `band-mcp` | band-mcp launcher (e.g. `uvx band-mcp`) | +| `BAND_MCP_PORT` | `8631` | band-mcp SSE port (`0` = ephemeral; stable keeps MCP trust remembered) | +| `VSCODE_CHAT_WORKSPACE` | `~/band-e2e/vscode-chat-workspace` | Workspace dir VS Code opens (stable keeps MCP trust remembered) | | `VSCODE_CHAT_SCORECARD_JSON` | empty | Scorecard output path (empty = don't emit) | | `VSCODE_CHAT_TIMEOUT` | `300` | Seconds allowed per live turn | diff --git a/tests/e2e/vscode/conftest.py b/tests/e2e/vscode/conftest.py index 2d2f0724f..3ce6a86ee 100644 --- a/tests/e2e/vscode/conftest.py +++ b/tests/e2e/vscode/conftest.py @@ -111,6 +111,7 @@ async def band_mcp( shlex.split(vscode_settings.band_mcp_command), agent_key=copilot_identity.api_key, base_url=baseline_settings.endpoints.rest_url, + port=vscode_settings.band_mcp_port, ) await server.start() yield server @@ -119,9 +120,17 @@ async def band_mcp( @pytest.fixture(scope="session") def vscode_workspace( - tmp_path_factory: pytest.TempPathFactory, band_mcp: BandMCPServer + vscode_settings: VSCodeChatSettings, band_mcp: BandMCPServer ) -> Path: - workspace = tmp_path_factory.mktemp("vscode-chat-ws") + """The (persistent by default) workspace VS Code opens. + + A stable path + stable band-mcp port keep ``.vscode/mcp.json`` identical + across runs, so VS Code's remembered MCP-server trust holds and reruns + prompt for nothing. Cell marker files use per-run tokens, so leftovers + from earlier runs never collide. + """ + workspace = Path(vscode_settings.vscode_chat_workspace) + workspace.mkdir(parents=True, exist_ok=True) scaffold_workspace(workspace, band_mcp.sse_url) return workspace @@ -145,9 +154,11 @@ async def driver( def pytest_configure(config: pytest.Config) -> None: """Register the suite scorecard plugin when emission is enabled.""" settings = VSCodeChatSettings() - if not settings.scorecard_json: + if not settings.vscode_chat_scorecard_json: return - plugin = VSCodeScorecard(settings.scorecard_json, Path(__file__).parent.resolve()) + plugin = VSCodeScorecard( + settings.vscode_chat_scorecard_json, Path(__file__).parent.resolve() + ) plugin.metadata = capture_versions( shlex.split(settings.code_command), shlex.split(settings.band_mcp_command) ) @@ -173,11 +184,16 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: """ suite_dir = Path(__file__).parent session_marker = pytest.mark.asyncio(loop_scope="session") - base = VSCodeChatSettings().turn_timeout + base = VSCodeChatSettings().vscode_chat_timeout for item in items: if not Path(item.path).is_relative_to(suite_dir): continue - item.add_marker(session_marker) + # Prepended (append=False) so this beats pytest-asyncio's bare + # auto-mode marker — get_closest_marker takes the first hit, and the + # bare marker's default function loop would strand the session-scoped + # WS/REST fixtures on another loop (subscribe timeouts, closed-loop + # teardown errors). + item.add_marker(session_marker, append=False) timeout = effective_timeout(item, base) if timeout is not None: item.add_marker(pytest.mark.timeout(timeout), append=False) diff --git a/tests/e2e/vscode/driver.py b/tests/e2e/vscode/driver.py index a34c26ddc..7730de422 100644 --- a/tests/e2e/vscode/driver.py +++ b/tests/e2e/vscode/driver.py @@ -86,7 +86,13 @@ def __init__(self, code_command: list[str], workspace: Path) -> None: self._workspace = workspace async def open_window(self) -> None: - """Open the scaffolded workspace so ``code chat`` targets its window.""" + """Open the scaffolded workspace so ``code chat`` targets its window. + + Deliberately without ``--disable-workspace-trust``: Copilot's AI + features require a *trusted* workspace, so an untrusted launch just + defers the dialog to chat time. The workspace path is stable — one + "Trust Folder & Continue" click is remembered for every rerun. + """ await self._run(str(self._workspace)) async def submit_prompt(self, prompt: str, *, new_session: bool = False) -> None: @@ -155,9 +161,12 @@ def capture(command: list[str]) -> str: copilot_lines = [ line for line in extensions.splitlines() if "copilot" in line.lower() ] + # Current VS Code bundles Copilot Chat, which --list-extensions omits; its + # version is then pinned by the VS Code build recorded above. + copilot = "; ".join(copilot_lines) or "built-in (bundled with this VS Code build)" return { "os": platform.platform(), "vscode": capture([*code_command, "--version"]).replace("\n", " "), - "copilot_extensions": "; ".join(copilot_lines) or extensions, + "copilot_extensions": copilot, "band_mcp": capture([*band_mcp_command, "--version"]), } diff --git a/tests/e2e/vscode/server.py b/tests/e2e/vscode/server.py index ce2654451..86f43aec3 100644 --- a/tests/e2e/vscode/server.py +++ b/tests/e2e/vscode/server.py @@ -43,11 +43,16 @@ def _reserve_port() -> int: class BandMCPServer: """Lifecycle of one band-mcp SSE subprocess bound to a stable loopback port.""" - def __init__(self, command: list[str], *, agent_key: str, base_url: str) -> None: + def __init__( + self, command: list[str], *, agent_key: str, base_url: str, port: int = 0 + ) -> None: + """``port=0`` reserves an ephemeral one; a fixed port keeps the workspace's + mcp.json stable across runs, so VS Code's remembered MCP-server trust + holds and reruns need no re-approval.""" self._command = command self._agent_key = agent_key self._base_url = base_url - self._port = _reserve_port() + self._port = port or _reserve_port() self._process: asyncio.subprocess.Process | None = None @property diff --git a/tests/e2e/vscode/settings.py b/tests/e2e/vscode/settings.py index 54eb5996b..3de101bf7 100644 --- a/tests/e2e/vscode/settings.py +++ b/tests/e2e/vscode/settings.py @@ -7,12 +7,22 @@ from __future__ import annotations +from pathlib import Path + from pydantic_settings import BaseSettings, SettingsConfigDict # Importing for the .env.test side effect keeps the two settings sources in # lockstep: both read the same already-loaded environment. import tests.e2e.baseline.settings # noqa: F401 +# Defaults chosen for unattended reruns: VS Code remembers MCP-server trust per +# configuration, so a stable workspace path + stable port keep mcp.json +# byte-identical across runs — one trust approval ever, then no prompts. +# Deliberately NOT under a dot-directory: Copilot's edit guard classifies +# hidden-path files as "sensitive" and demands per-edit confirmation. +DEFAULT_WORKSPACE = str(Path.home() / "band-e2e" / "vscode-chat-workspace") +DEFAULT_BAND_MCP_PORT = 8631 + class VSCodeChatSettings(BaseSettings): """Env knobs for driving a real signed-in VS Code window (field == env var).""" @@ -24,5 +34,7 @@ class VSCodeChatSettings(BaseSettings): vscode_chat_tests_enabled: bool = False # VSCODE_CHAT_TESTS_ENABLED code_command: str = "code" # CODE_COMMAND (VS Code CLI binary override) band_mcp_command: str = "band-mcp" # BAND_MCP_COMMAND (e.g. "uvx band-mcp") - scorecard_json: str = "" # VSCODE_CHAT_SCORECARD_JSON (empty = don't emit) - turn_timeout: int = 300 # VSCODE_CHAT_TIMEOUT (seconds per live turn) + band_mcp_port: int = DEFAULT_BAND_MCP_PORT # BAND_MCP_PORT (0 = ephemeral) + vscode_chat_workspace: str = DEFAULT_WORKSPACE # VSCODE_CHAT_WORKSPACE + vscode_chat_scorecard_json: str = "" # VSCODE_CHAT_SCORECARD_JSON ("" = no emit) + vscode_chat_timeout: int = 300 # VSCODE_CHAT_TIMEOUT (seconds per live turn) diff --git a/tests/e2e/vscode/test_copilot_chat.py b/tests/e2e/vscode/test_copilot_chat.py index 54b7130b2..e0f3f2d47 100644 --- a/tests/e2e/vscode/test_copilot_chat.py +++ b/tests/e2e/vscode/test_copilot_chat.py @@ -104,7 +104,7 @@ async def test_participation_reply_round_trip( sender_name="the user", message=text, instruction="Reply to this message, repeating its echo token exactly.", - deadline_s=vscode_settings.turn_timeout, + deadline_s=vscode_settings.vscode_chat_timeout, ) replies.assert_contains_any([token]) @@ -151,7 +151,7 @@ async def test_original_functions_retained( f"roster with band_get_participants and reply listing every " f"participant's name and confirming the file was created." ), - deadline_s=vscode_settings.turn_timeout, + deadline_s=vscode_settings.vscode_chat_timeout, ) marker = workspace_marker_path(vscode_workspace, filename) @@ -203,13 +203,21 @@ async def test_multi_participant_echo_peer( "Another agent posted this in the room. Reply addressing that " "agent and repeat its echo token exactly." ), - deadline_s=vscode_settings.turn_timeout, + deadline_s=vscode_settings.vscode_chat_timeout, ) replies.assert_contains_any([token]) + # Label both sides for the judge: the reply alone reads like an echo + # message itself and is unjudgeable without the peer's message. + transcript = f"Peer agent {echo.name} posted: {text}\n" + "\n".join( + f"Reply from the agent under test: {reply.content}" for reply in replies + ) verdict = await judge( - criteria="The reply engages with the other agent's echo message.", - transcript="\n".join(reply.content for reply in replies), + criteria=( + "The agent under test replied to the peer agent's echo message, " + "engaging with it (addressing the peer and/or repeating its token)." + ), + transcript=transcript, ) assert verdict.passed, verdict.reasoning @@ -250,11 +258,16 @@ async def test_recall_across_chat_sessions( sender_name="the user", message=seed, instruction=( - "Store the project codename in your Band memory with " - "band_store_memory, then confirm in the room that it is saved." + f"Store the project codename in your Band memory with " + f"band_store_memory, recording which room it belongs to — " + f"content like 'project codename for room {room_id}: {fact}'. " + f"Then confirm in the room that it is saved." ), - deadline_s=vscode_settings.turn_timeout, + deadline_s=vscode_settings.vscode_chat_timeout, ) + # Store-side barrier: separates a store failure (band-mcp/tool call + # never landed a record) from a retrieval failure in the next turn. + (await capture.memory(copilot_identity)).stored.assert_stored(content=fact) ask = "What is the project codename?" await user_ops.send_message( @@ -272,10 +285,12 @@ async def test_recall_across_chat_sessions( message=ask, instruction=( "Recall the project codename from your Band memory using " - "band_list_memories (and band_get_memory if needed) — do not " - "guess — and reply stating it exactly." + "band_list_memories (and band_get_memory if needed) — the " + "record for THIS room (match the chat_id above); the memory " + "store may hold unrelated records. Do not guess — reply " + "stating the codename exactly." ), - deadline_s=vscode_settings.turn_timeout, + deadline_s=vscode_settings.vscode_chat_timeout, new_session=True, ) replies.assert_contains_any([fact]) @@ -290,8 +305,15 @@ async def test_no_leak_between_rooms( reply_capture: CaptureFactory, driver: CodeChatDriver, ) -> None: - """L2 isolation: replies stay scoped to their room — room B's answer names - room B's token and never room A's.""" + """L2 isolation: room-scoped facts stay scoped — room B's answer names room + B's marker and never room A's. + + Each ``code chat`` invocation is a fresh chat session and band-mcp has no + history tool, so room context lives in agent-wide platform memory. The + seeds record each marker *keyed by its room id*; isolation is then the + retrieval staying scoped to the asking room — pulling the sibling room's + marker instead is exactly the leak this cell exists to catch. + """ token_a, token_b = f"alpha-{_token()}", f"bravo-{_token()}" room_a = await resource_manager.provision_room( title="e2e-vscode-isolation-a", participants=[copilot_identity.id] @@ -301,7 +323,7 @@ async def test_no_leak_between_rooms( ) async def seed(room_id: str, capture: ReplyCapture, token: str) -> None: - text = f"The marker for THIS room is {token}. Acknowledge it." + text = f"The marker for THIS room is {token}." await user_ops.send_message( room_id, text, @@ -315,9 +337,17 @@ async def seed(room_id: str, capture: ReplyCapture, token: str) -> None: identity=copilot_identity, sender_name="the user", message=text, - instruction="Acknowledge this room's marker by repeating it.", - deadline_s=vscode_settings.turn_timeout, + instruction=( + f"Store this room's marker with band_store_memory, recording " + f"which room it belongs to — content like 'marker for room " + f"{room_id}: {token}'. Then verify with band_list_memories " + f"that the record exists (store it again if not), and only " + f"then acknowledge it in the room." + ), + deadline_s=vscode_settings.vscode_chat_timeout, ) + # Store-side barrier (see test_recall_across_chat_sessions). + (await capture.memory(copilot_identity)).stored.assert_stored(content=token) async with reply_capture(room_a) as capture_a: await seed(room_a, capture_a, token_a) @@ -339,10 +369,12 @@ async def seed(room_id: str, capture: ReplyCapture, token: str) -> None: sender_name="the user", message=ask, instruction=( - "Reply with the marker that was stated in this room only. " - "Never mention markers from any other room or conversation." + "Look up the stored markers with band_list_memories and reply " + "with the one recorded for THIS room (match the chat_id above). " + "Never mention markers recorded for other rooms." ), - deadline_s=vscode_settings.turn_timeout, + deadline_s=vscode_settings.vscode_chat_timeout, + new_session=True, ) replies.assert_contains_any([token_b]) replies.assert_contains_none([token_a]) @@ -390,7 +422,7 @@ async def test_restart_recall_and_function( "Store the deploy phase in your Band memory with " "band_store_memory, then confirm in the room." ), - deadline_s=vscode_settings.turn_timeout, + deadline_s=vscode_settings.vscode_chat_timeout, ) await band_mcp.restart() @@ -415,7 +447,7 @@ async def test_restart_recall_and_function( f"Also create a file named '{filename}' in the workspace root " f"containing that phase, to confirm your editor tools still work." ), - deadline_s=vscode_settings.turn_timeout, + deadline_s=vscode_settings.vscode_chat_timeout, new_session=True, ) diff --git a/tests/e2e/vscode/workspace.py b/tests/e2e/vscode/workspace.py index b506b5146..552ef77e8 100644 --- a/tests/e2e/vscode/workspace.py +++ b/tests/e2e/vscode/workspace.py @@ -11,12 +11,13 @@ import json from pathlib import Path -# VS Code's tool auto-approval knob (Manage approvals docs, VS Code 1.103+). -# Security trade-off is deliberate and bounded: the workspace is a throwaway -# temp dir and the only MCP server is the harness's own band-mcp. If the -# installed VS Code rejects the key (or scopes it user-level only), the runbook -# fallback is one manual "Always allow" click per tool on the first turn. -AUTO_APPROVE_SETTING = "chat.tools.global.autoApprove" +# Auto-answer agent questions so a driven turn never stalls on a chat-side +# question (Manage approvals docs). Deliberately NOT chat.tools.global.autoApprove: +# that is machine-wide "YOLO mode" — VS Code escalates it to a global consent +# dialog and it disables tool approval in every workspace on the host. Tool +# approvals instead rely on remembered per-tool "Always allow" choices in this +# (persistent) workspace — one click per tool, ever (see README). +AUTO_REPLY_SETTING = "chat.autoReply" # MCP support is on by default in current VS Code; setting it explicitly keeps # the run independent of a user-profile override. Unknown keys are ignored. @@ -32,7 +33,7 @@ def scaffold_workspace(root: Path, sse_url: str) -> None: vscode_dir.mkdir(parents=True, exist_ok=True) mcp_config = {"servers": {MCP_SERVER_NAME: {"type": "sse", "url": sse_url}}} - settings = {AUTO_APPROVE_SETTING: True, MCP_ENABLED_SETTING: True} + settings = {AUTO_REPLY_SETTING: True, MCP_ENABLED_SETTING: True} (vscode_dir / "mcp.json").write_text(json.dumps(mcp_config, indent=2) + "\n") (vscode_dir / "settings.json").write_text(json.dumps(settings, indent=2) + "\n") diff --git a/tests/framework_conformance/test_vscode_chat_toolkit.py b/tests/framework_conformance/test_vscode_chat_toolkit.py index 6ba6aa9e6..4785e31b3 100644 --- a/tests/framework_conformance/test_vscode_chat_toolkit.py +++ b/tests/framework_conformance/test_vscode_chat_toolkit.py @@ -28,7 +28,7 @@ outcome_status, ) from tests.e2e.vscode.workspace import ( - AUTO_APPROVE_SETTING, + AUTO_REPLY_SETTING, MCP_SERVER_NAME, scaffold_workspace, workspace_marker_path, @@ -60,7 +60,7 @@ def test_turn_prompt_carries_room_message_and_tool_contract() -> None: # --- workspace scaffolding: what VS Code reads -------------------------------------- -def test_scaffold_workspace_writes_mcp_entry_and_auto_approve(tmp_path: Path) -> None: +def test_scaffold_workspace_writes_mcp_entry_and_auto_reply(tmp_path: Path) -> None: sse_url = "http://127.0.0.1:8391/sse" scaffold_workspace(tmp_path, sse_url) @@ -68,7 +68,7 @@ def test_scaffold_workspace_writes_mcp_entry_and_auto_approve(tmp_path: Path) -> assert mcp["servers"][MCP_SERVER_NAME] == {"type": "sse", "url": sse_url} settings = json.loads((tmp_path / ".vscode" / "settings.json").read_text()) - assert settings[AUTO_APPROVE_SETTING] is True + assert settings[AUTO_REPLY_SETTING] is True def test_workspace_marker_path_stays_inside_workspace(tmp_path: Path) -> None: From 23a50a6c4004dc487d43ac2dc79155b1efffd2d0 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Thu, 23 Jul 2026 13:15:15 +0300 Subject: [PATCH 4/7] =?UTF-8?q?refactor(e2e):=20intent-oriented=20VS=20Cod?= =?UTF-8?q?e=20suite=20=E2=80=94=20SurfaceRoom,=20vscode=5Fwindow,=20gate?= =?UTF-8?q?=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review pass over the suite for plumbing: - SurfaceRoom (rooms.py) is the cells' one intent object: user_turn / peer_turn / remember hide the mention wiring, prompt relay, and captured-reply wait, so a cell reads as scenario prose. remember() encodes the store-seed shape learned from captured failures — fresh session (a continued chat sees an earlier store and acks without storing) plus a hard imperative (a soft 'store and confirm' gets satisficed into a bare confirmation) — with a Band-side stored-record barrier separating store failures from retrieval failures. - vscode_window() async context manager owns the become-drivable flow (preflight -> open workspace -> startup grace); exit is a documented no-op — the window is human-owned. - Root conftest gate branches collapse into one GATED_MARKERS table; env var and skip reason derive from the settings field name, and tests/test_env_gates.py guards the table against both the settings fields and pyproject's marker registration. - Scorecard outcome mapping is a match/case over the report tuple; conformance tests assert typed rows and behavioral contracts (no exact-dict or join-format assertions). Live: full suite 6/6 unattended after the refactor; scorecard artifact refreshed. Co-Authored-By: Claude Fable 5 Signed-off-by: Alexander Zaikman --- tests/conftest.py | 61 ++- tests/e2e/vscode/conftest.py | 62 ++- tests/e2e/vscode/driver.py | 48 ++- tests/e2e/vscode/rooms.py | 127 +++++++ tests/e2e/vscode/scorecard.py | 18 +- tests/e2e/vscode/test_copilot_chat.py | 359 +++--------------- .../test_vscode_chat_toolkit.py | 86 +++-- tests/test_env_gates.py | 56 ++- 8 files changed, 394 insertions(+), 423 deletions(-) create mode 100644 tests/e2e/vscode/rooms.py diff --git a/tests/conftest.py b/tests/conftest.py index ed46a3d3a..13eabbc37 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,42 +90,39 @@ def pytest_ignore_collect(collection_path: Path) -> bool | None: return None +# Opt-in suite gates: marker -> the CollectionGateSettings field that opens it. +# The env var IS the field name uppercased (the pydantic-settings contract), so +# the skip reason is derived — one row here is all a new gated suite needs. +GATED_MARKERS: dict[str, str] = { + "e2e": "e2e_tests_enabled", + "docker_build": "docker_tests_enabled", + "sandbox": "sandbox_tests_enabled", + "vscode_chat": "vscode_chat_tests_enabled", +} + + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: - """Skip e2e- and docker_build-marked tests unless explicitly enabled. - - tests/e2e/ gates itself through its own conftest; this covers - e2e-marked tests living elsewhere (e.g. the codex ACP protocol - tests), which spawn real backends and must never ride a normal - unit run. - - docker_build-marked tests shell out to a real `docker build`/`docker - run` — CI runners do have a Docker daemon (unlike the nested - virtualization sbx tests need), so a plain Docker-availability check - isn't enough to keep them off CI; they need the same explicit opt-in - as e2e tests. + """Skip gate-marked suites unless their env gate is explicitly enabled. + + tests/e2e/ gates itself through its own conftest; this covers marked + tests living elsewhere (e.g. the codex ACP protocol tests). These + suites spawn real backends, real `docker build`s, sbx microVMs, or a + live VS Code window — none may ride a normal unit run, and none can + rely on mere tool availability (CI runners do have Docker), so each + needs its explicit opt-in. """ gates = CollectionGateSettings() - skip_e2e = pytest.mark.skip(reason="set E2E_TESTS_ENABLED=true to run e2e tests") - skip_docker = pytest.mark.skip( - reason="set DOCKER_TESTS_ENABLED=true to run docker_build tests" - ) - skip_sandbox = pytest.mark.skip( - reason="set SANDBOX_TESTS_ENABLED=true to run sbx sandbox tests" - ) - skip_vscode_chat = pytest.mark.skip( - reason="set VSCODE_CHAT_TESTS_ENABLED=true to run VS Code Copilot chat tests" - ) + closed = { + marker: pytest.mark.skip( + reason=f"set {field.upper()}=true to run {marker}-marked tests" + ) + for marker, field in GATED_MARKERS.items() + if not getattr(gates, field) + } for item in items: - if not gates.e2e_tests_enabled and item.get_closest_marker("e2e"): - item.add_marker(skip_e2e) - if not gates.docker_tests_enabled and item.get_closest_marker("docker_build"): - item.add_marker(skip_docker) - if not gates.sandbox_tests_enabled and item.get_closest_marker("sandbox"): - item.add_marker(skip_sandbox) - if not gates.vscode_chat_tests_enabled and item.get_closest_marker( - "vscode_chat" - ): - item.add_marker(skip_vscode_chat) + for marker, skip in closed.items(): + if item.get_closest_marker(marker): + item.add_marker(skip) @pytest.fixture(autouse=True) diff --git a/tests/e2e/vscode/conftest.py b/tests/e2e/vscode/conftest.py index 3ce6a86ee..1b15398ab 100644 --- a/tests/e2e/vscode/conftest.py +++ b/tests/e2e/vscode/conftest.py @@ -13,10 +13,10 @@ from __future__ import annotations -import asyncio import logging import shlex from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager from pathlib import Path import pytest @@ -39,8 +39,9 @@ CodeChatDriver, PreflightError, capture_versions, - preflight, + vscode_window, ) +from tests.e2e.vscode.rooms import SurfaceRoom from tests.e2e.vscode.scorecard import VSCodeScorecard from tests.e2e.vscode.server import BandMCPServer from tests.e2e.vscode.settings import VSCodeChatSettings @@ -63,15 +64,12 @@ "reap_leaked_agents", "reply_capture", "resource_manager", + "surface_room", "user_ops", "vscode_settings", "vscode_workspace", ] -# Seconds after `code ` for the window, extension host, and MCP -# client to come up before the first prompt is submitted. -WINDOW_OPEN_GRACE_S = 20 - @pytest.fixture(scope="session") def vscode_settings() -> VSCodeChatSettings: @@ -138,17 +136,53 @@ def vscode_workspace( @pytest.fixture(scope="session") async def driver( vscode_settings: VSCodeChatSettings, vscode_workspace: Path -) -> CodeChatDriver: - """Preflight the VS Code CLI, open the workspace window once, yield the driver.""" - code_command = shlex.split(vscode_settings.code_command) +) -> AsyncGenerator[CodeChatDriver, None]: + """One ready-to-drive VS Code window for the session (see ``vscode_window``).""" try: - preflight(code_command) + async with vscode_window( + shlex.split(vscode_settings.code_command), vscode_workspace + ) as chat_driver: + yield chat_driver except PreflightError as error: pytest.fail(str(error)) - chat_driver = CodeChatDriver(code_command, vscode_workspace) - await chat_driver.open_window() - await asyncio.sleep(WINDOW_OPEN_GRACE_S) - return chat_driver + + +@pytest.fixture +def surface_room( + resource_manager: ResourceManager, + user_ops, + reply_capture, + driver: CodeChatDriver, + copilot_identity: ProvisionedAgent, + vscode_settings: VSCodeChatSettings, +): + """Factory: open a provisioned room with the Copilot surface bound. + + ``async with surface_room("recall") as room:`` yields a ``SurfaceRoom`` + (see ``rooms.py``) — the cells' one intent object; all turn plumbing + (mentions, prompt relay, reply wait) lives behind it. + """ + + @asynccontextmanager + async def open_room( + label: str, *, participants: tuple[str, ...] = () + ) -> AsyncGenerator[SurfaceRoom, None]: + room_id = await resource_manager.provision_room( + title=f"e2e-vscode-{label}", + participants=[copilot_identity.id, *participants], + ) + async with reply_capture(room_id) as capture: + yield SurfaceRoom( + room_id=room_id, + capture=capture, + driver=driver, + identity=copilot_identity, + user_ops=user_ops, + resources=resource_manager, + turn_timeout=vscode_settings.vscode_chat_timeout, + ) + + return open_room def pytest_configure(config: pytest.Config) -> None: diff --git a/tests/e2e/vscode/driver.py b/tests/e2e/vscode/driver.py index 7730de422..24d12de10 100644 --- a/tests/e2e/vscode/driver.py +++ b/tests/e2e/vscode/driver.py @@ -18,7 +18,8 @@ import platform import shutil import subprocess -from collections.abc import Callable +from collections.abc import AsyncIterator, Callable +from contextlib import asynccontextmanager from pathlib import Path from typing import Protocol @@ -78,23 +79,22 @@ def turn_prompt( ) +# Seconds after opening the workspace for the window, extension host, and MCP +# client to come up before the first prompt is submitted. +WINDOW_OPEN_GRACE_S = 20 + + class CodeChatDriver: - """Submit prompts via ``code chat -m agent`` against the workspace's window.""" + """Submit prompts via ``code chat -m agent`` against the workspace's window. + + Construct through :func:`vscode_window`, which owns the ready-to-drive + lifecycle (preflight -> open -> readiness grace). + """ def __init__(self, code_command: list[str], workspace: Path) -> None: self._code_command = code_command self._workspace = workspace - async def open_window(self) -> None: - """Open the scaffolded workspace so ``code chat`` targets its window. - - Deliberately without ``--disable-workspace-trust``: Copilot's AI - features require a *trusted* workspace, so an untrusted launch just - defers the dialog to chat time. The workspace path is stable — one - "Trust Folder & Continue" click is remembered for every rerun. - """ - await self._run(str(self._workspace)) - async def submit_prompt(self, prompt: str, *, new_session: bool = False) -> None: if new_session: prompt = f"{FRESH_SESSION_PREAMBLE}\n\n{prompt}" @@ -120,6 +120,30 @@ async def _run(self, *args: str) -> None: ) +@asynccontextmanager +async def vscode_window( + code_command: list[str], workspace: Path +) -> AsyncIterator[CodeChatDriver]: + """A ready-to-drive VS Code window on ``workspace``. + + Entry owns the whole becoming-drivable flow: preflight the CLI (raises + :class:`PreflightError` with the runbook pointer), open the workspace + window, and wait out the startup grace so the extension host and MCP + client are up before the first prompt. Opened deliberately *without* + ``--disable-workspace-trust``: Copilot's AI features require a trusted + workspace, so an untrusted launch only defers the dialog to chat time — + the stable workspace path makes "Trust Folder & Continue" a one-time click. + + Exit is a no-op by contract: the window is human-owned (the human signed + in to Copilot there) and stays open across runs. + """ + preflight(code_command) + driver = CodeChatDriver(code_command, workspace) + await driver._run(str(workspace)) + await asyncio.sleep(WINDOW_OPEN_GRACE_S) + yield driver + + Runner = Callable[[list[str]], str] diff --git a/tests/e2e/vscode/rooms.py b/tests/e2e/vscode/rooms.py new file mode 100644 index 000000000..cb732bfa9 --- /dev/null +++ b/tests/e2e/vscode/rooms.py @@ -0,0 +1,127 @@ +"""A Band room with the Copilot-in-VS-Code surface in it, driven turn by turn. + +``SurfaceRoom`` is the one intent object the cells speak through: *someone says +something in the room, the driver relays it to Copilot, the agent's reply comes +back Band-side*. It hides the turn plumbing (mention wiring, prompt templating, +the captured-message wait) so a cell reads as scenario prose. + +The reply wait is a predicate over captured room messages, not the baseline +``wait_for_reply`` barrier: that barrier keys on the recipient's delivery-status +PROCESSED ack, which only an SDK runtime emits — this surface posts through +band-mcp (REST only), so no delivery ack ever arrives. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from tests.e2e.baseline.toolkit.capture import ReplyCapture +from tests.e2e.baseline.toolkit.observations.replies import Replies +from tests.e2e.baseline.toolkit.provisioning import ProvisionedAgent, ResourceManager +from tests.e2e.baseline.toolkit.user_ops import UserOps +from tests.e2e.vscode.driver import CodeChatDriver, turn_prompt + + +def transcript(replies: Replies, *, peer_message: str = "") -> str: + """The judge's view of a turn: the labeled reply, with the peer's message + when the verdict is about engaging it (a bare reply reads as unjudgeable).""" + lines = [f"Peer agent posted: {peer_message}"] if peer_message else [] + lines += [f"Reply from the agent under test: {reply.content}" for reply in replies] + return "\n".join(lines) + + +@dataclass +class SurfaceRoom: + """One provisioned room, its reply capture, and the turn protocol bound.""" + + room_id: str + capture: ReplyCapture + driver: CodeChatDriver + identity: ProvisionedAgent + user_ops: UserOps + resources: ResourceManager + turn_timeout: float + + async def user_turn( + self, message: str, *, instruction: str, new_session: bool = False + ) -> Replies: + """The user posts ``message`` (mentioning the agent); return the reply.""" + await self.user_ops.send_message( + self.room_id, + message, + mention_id=self.identity.id, + mention_name=self.identity.name, + ) + return await self._agent_reply( + sender_name="the user", + message=message, + instruction=instruction, + new_session=new_session, + ) + + async def peer_turn( + self, peer: ProvisionedAgent, message: str, *, instruction: str + ) -> Replies: + """A peer agent posts ``message`` (mentioning the agent); return the reply.""" + await self.resources.peer(peer).send_message( + self.room_id, + message, + mention_id=self.identity.id, + mention_name=self.identity.name, + ) + return await self._agent_reply( + sender_name=peer.name, message=message, instruction=instruction + ) + + async def remember(self, announcement: str, *, record: str) -> None: + """Seed a platform memory: the user announces a fact, the agent must + store ``record`` via ``band_store_memory``, and a Band-side barrier + proves the record landed (separating a store failure from a retrieval + failure in the next turn). + + The shape is load-bearing, learned from captured failures: a fresh + session (otherwise the agent sees an earlier store in its chat context + and acks without storing) and a hard imperative (a soft "store X and + confirm" gets satisficed into a bare confirmation). + """ + await self.user_turn( + announcement, + instruction=( + f"Call band_store_memory now, exactly once, even if similar " + f"records already exist — content exactly: '{record}'. " + f"Then acknowledge it in the room." + ), + new_session=True, + ) + observation = await self.capture.memory(self.identity) + observation.stored.assert_stored(content=record) + + async def _agent_reply( + self, + *, + sender_name: str, + message: str, + instruction: str, + new_session: bool = False, + ) -> Replies: + """Relay the room message to Copilot and wait for the agent's reply.""" + mark = self.capture.messages.snapshot() + await self.driver.submit_prompt( + turn_prompt( + self.room_id, + self.identity.name, + sender_name=sender_name, + message=message, + instruction=instruction, + ), + new_session=new_session, + ) + + def replied(_messages: list) -> bool: + return bool(self._replies_since(mark)) + + await self.capture.wait_until(replied, deadline_s=self.turn_timeout) + return self._replies_since(mark) + + def _replies_since(self, mark: int) -> Replies: + return self.capture.messages.since(mark).from_sender(self.identity.id) diff --git a/tests/e2e/vscode/scorecard.py b/tests/e2e/vscode/scorecard.py index 610add5dd..242a1ccb4 100644 --- a/tests/e2e/vscode/scorecard.py +++ b/tests/e2e/vscode/scorecard.py @@ -40,15 +40,15 @@ def outcome_status(report: pytest.TestReport) -> Status | None: """The row status one setup/call report contributes (same semantics as the baseline collector: skip anywhere = skip, failed setup or call = fail, passing call = pass, passing setup = no verdict yet).""" - if report.when not in ("setup", "call"): - return None - if report.skipped: - return "skip" - if report.failed: - return "fail" - if report.when == "call": - return "pass" - return None + match (report.when, report.skipped, report.failed): + case ("setup" | "call", True, _): + return "skip" + case ("setup" | "call", _, True): + return "fail" + case ("call", _, _): + return "pass" + case _: # teardown reports and passing setups carry no verdict + return None class VSCodeScorecard: diff --git a/tests/e2e/vscode/test_copilot_chat.py b/tests/e2e/vscode/test_copilot_chat.py index e0f3f2d47..5fff3a5be 100644 --- a/tests/e2e/vscode/test_copilot_chat.py +++ b/tests/e2e/vscode/test_copilot_chat.py @@ -1,14 +1,10 @@ """Live L0–L3 cells for the GitHub Copilot in VS Code surface. -Every turn is driver-initiated (see ``driver.turn_prompt``): the room message -is posted first, then the prompt relays it to Copilot Chat with the room id -and the band-tool contract. Assertions are all Band-side — the reply must land -in the right room, from the provisioned identity, via ``band_send_message``. - -The reply wait is ``wait_until`` over captured messages, not the baseline -``wait_for_reply``: that barrier keys on the recipient's delivery-status -PROCESSED ack, which only an SDK runtime emits — this surface posts through -band-mcp (REST only), so no delivery ack ever arrives. +Every cell speaks through ``SurfaceRoom`` (``rooms.py``): someone says +something in the room, the driver relays it to Copilot Chat, and the agent's +reply is asserted **Band-side** — it must land in the right room, from the +provisioned identity, via ``band_send_message``. Tokens are high-entropy so a +reply containing one proves the round trip, not model invention. L4 usage has no cell here: the surface exposes no per-turn usage signal — the scorecard carries the rationale (see ``scorecard.USAGE_NA_ROW``). @@ -21,14 +17,9 @@ import pytest -from tests.e2e.baseline.settings import BaselineSettings -from tests.e2e.baseline.toolkit.capture import CaptureFactory, ReplyCapture -from tests.e2e.baseline.toolkit.observations.replies import Replies -from tests.e2e.baseline.toolkit.provisioning import ProvisionedAgent, ResourceManager -from tests.e2e.baseline.toolkit.user_ops import UserOps -from tests.e2e.vscode.driver import CodeChatDriver, turn_prompt +from tests.e2e.baseline.toolkit.provisioning import ResourceManager +from tests.e2e.vscode.rooms import transcript from tests.e2e.vscode.server import BandMCPServer -from tests.e2e.vscode.settings import VSCodeChatSettings from tests.e2e.vscode.workspace import workspace_marker_path pytestmark = [pytest.mark.vscode_chat, pytest.mark.e2e] @@ -38,120 +29,35 @@ def _token() -> str: return uuid.uuid4().hex[:6] -async def run_turn( - capture: ReplyCapture, - driver: CodeChatDriver, - *, - room_id: str, - identity: ProvisionedAgent, - sender_name: str, - message: str, - instruction: str, - deadline_s: float, - new_session: bool = False, -) -> Replies: - """Submit one relayed-message prompt and wait for the identity's room reply.""" - mark = capture.messages.snapshot() - await driver.submit_prompt( - turn_prompt( - room_id, - identity.name, - sender_name=sender_name, - message=message, - instruction=instruction, - ), - new_session=new_session, - ) - - def replied(_messages: list) -> bool: - return bool(capture.messages.since(mark).from_sender(identity.id)) - - await capture.wait_until(replied, deadline_s=deadline_s) - return capture.messages.since(mark).from_sender(identity.id) - - -async def test_participation_reply_round_trip( - vscode_settings: VSCodeChatSettings, - copilot_identity: ProvisionedAgent, - resource_manager: ResourceManager, - user_ops: UserOps, - reply_capture: CaptureFactory, - driver: CodeChatDriver, -) -> None: - """L0: a room message reaches Copilot and its reply lands back in the room. - - The token is high-entropy, so the reply containing it proves the relayed - message flowed through the turn and back via ``band_send_message`` — not - model invention. - """ +async def test_participation_reply_round_trip(surface_room) -> None: + """L0: a room message reaches Copilot and its reply lands back in the room.""" token = _token() - room_id = await resource_manager.provision_room( - title="e2e-vscode-participation", participants=[copilot_identity.id] - ) - async with reply_capture(room_id) as capture: - text = f"Ping — the echo token is {token}." - await user_ops.send_message( - room_id, - text, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, - ) - replies = await run_turn( - capture, - driver, - room_id=room_id, - identity=copilot_identity, - sender_name="the user", - message=text, + async with surface_room("participation") as room: + replies = await room.user_turn( + f"Ping — the echo token is {token}.", instruction="Reply to this message, repeating its echo token exactly.", - deadline_s=vscode_settings.vscode_chat_timeout, ) replies.assert_contains_any([token]) async def test_original_functions_retained( - vscode_settings: VSCodeChatSettings, - copilot_identity: ProvisionedAgent, - resource_manager: ResourceManager, - user_ops: UserOps, - reply_capture: CaptureFactory, - driver: CodeChatDriver, - vscode_workspace: Path, - judge, + surface_room, vscode_workspace: Path, judge ) -> None: """L1: one turn exercises a native VS Code function (create a workspace file) AND a platform tool (``band_get_participants``) — the surface keeps its original capabilities while participating.""" token = _token() filename = f"notes-{token}.txt" - room_id = await resource_manager.provision_room( - title="e2e-vscode-functions", participants=[copilot_identity.id] - ) - async with reply_capture(room_id) as capture: - text = ( + async with surface_room("functions") as room: + replies = await room.user_turn( f"Please set up a note file for this task and tell me who is " - f"in this room. The file token is {token}." - ) - await user_ops.send_message( - room_id, - text, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, - ) - replies = await run_turn( - capture, - driver, - room_id=room_id, - identity=copilot_identity, - sender_name="the user", - message=text, + f"in this room. The file token is {token}.", instruction=( f"First create a file named '{filename}' in the workspace root " f"containing exactly the token {token}. Then look up the room " f"roster with band_get_participants and reply listing every " f"participant's name and confirming the file was created." ), - deadline_s=vscode_settings.vscode_chat_timeout, ) marker = workspace_marker_path(vscode_workspace, filename) @@ -164,125 +70,59 @@ async def test_original_functions_retained( "itself or the human user) and confirms that the requested note " "file was created." ), - transcript="\n".join(reply.content for reply in replies), + transcript=transcript(replies), ) assert verdict.passed, verdict.reasoning async def test_multi_participant_echo_peer( - vscode_settings: VSCodeChatSettings, - copilot_identity: ProvisionedAgent, - resource_manager: ResourceManager, - reply_capture: CaptureFactory, - driver: CodeChatDriver, - judge, + surface_room, resource_manager: ResourceManager, judge ) -> None: """L0 multi-participant: a peer agent's message drives a turn and the reply engages the peer — the surface participates beyond 1:1 user chats.""" token = _token() echo = await resource_manager.provision_agent("echo") - room_id = await resource_manager.provision_room( - title="e2e-vscode-peers", participants=[copilot_identity.id, echo.id] - ) - async with reply_capture(room_id) as capture: - text = f"ECHO: {token}" - await resource_manager.peer(echo).send_message( - room_id, - text, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, - ) - replies = await run_turn( - capture, - driver, - room_id=room_id, - identity=copilot_identity, - sender_name=echo.name, - message=text, + echo_message = f"ECHO: {token}" + + async with surface_room("peers", participants=(echo.id,)) as room: + replies = await room.peer_turn( + echo, + echo_message, instruction=( "Another agent posted this in the room. Reply addressing that " "agent and repeat its echo token exactly." ), - deadline_s=vscode_settings.vscode_chat_timeout, ) replies.assert_contains_any([token]) - # Label both sides for the judge: the reply alone reads like an echo - # message itself and is unjudgeable without the peer's message. - transcript = f"Peer agent {echo.name} posted: {text}\n" + "\n".join( - f"Reply from the agent under test: {reply.content}" for reply in replies - ) verdict = await judge( criteria=( "The agent under test replied to the peer agent's echo message, " "engaging with it (addressing the peer and/or repeating its token)." ), - transcript=transcript, + transcript=transcript(replies, peer_message=echo_message), ) assert verdict.passed, verdict.reasoning @pytest.mark.timeout(extra=360) # two full live turns -async def test_recall_across_chat_sessions( - vscode_settings: VSCodeChatSettings, - copilot_identity: ProvisionedAgent, - resource_manager: ResourceManager, - user_ops: UserOps, - reply_capture: CaptureFactory, - driver: CodeChatDriver, -) -> None: +async def test_recall_across_chat_sessions(surface_room) -> None: """L2 persistence: a fact stored in turn 1 survives to a fresh chat session. band-mcp exposes no room-history tool, so cross-session recall flows - through the platform memory tools — the fact is stored with + through the platform memory tools — the fact is stored room-keyed with ``band_store_memory`` and must come back via ``band_list_memories`` in a session that never saw it. """ fact = f"codename-{_token()}" - room_id = await resource_manager.provision_room( - title="e2e-vscode-recall", participants=[copilot_identity.id] - ) - async with reply_capture(room_id) as capture: - seed = f"For later reference: the project codename is {fact}." - await user_ops.send_message( - room_id, - seed, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, - ) - await run_turn( - capture, - driver, - room_id=room_id, - identity=copilot_identity, - sender_name="the user", - message=seed, - instruction=( - f"Store the project codename in your Band memory with " - f"band_store_memory, recording which room it belongs to — " - f"content like 'project codename for room {room_id}: {fact}'. " - f"Then confirm in the room that it is saved." - ), - deadline_s=vscode_settings.vscode_chat_timeout, - ) - # Store-side barrier: separates a store failure (band-mcp/tool call - # never landed a record) from a retrieval failure in the next turn. - (await capture.memory(copilot_identity)).stored.assert_stored(content=fact) - - ask = "What is the project codename?" - await user_ops.send_message( - room_id, - ask, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, + async with surface_room("recall") as room: + await room.remember( + f"For later reference: the project codename is {fact}.", + record=f"project codename for room {room.room_id}: {fact}", ) - replies = await run_turn( - capture, - driver, - room_id=room_id, - identity=copilot_identity, - sender_name="the user", - message=ask, + + replies = await room.user_turn( + "What is the project codename?", instruction=( "Recall the project codename from your Band memory using " "band_list_memories (and band_get_memory if needed) — the " @@ -290,21 +130,13 @@ async def test_recall_across_chat_sessions( "store may hold unrelated records. Do not guess — reply " "stating the codename exactly." ), - deadline_s=vscode_settings.vscode_chat_timeout, new_session=True, ) replies.assert_contains_any([fact]) @pytest.mark.timeout(extra=700) # three full live turns across two rooms -async def test_no_leak_between_rooms( - vscode_settings: VSCodeChatSettings, - copilot_identity: ProvisionedAgent, - resource_manager: ResourceManager, - user_ops: UserOps, - reply_capture: CaptureFactory, - driver: CodeChatDriver, -) -> None: +async def test_no_leak_between_rooms(surface_room) -> None: """L2 isolation: room-scoped facts stay scoped — room B's answer names room B's marker and never room A's. @@ -315,65 +147,25 @@ async def test_no_leak_between_rooms( marker instead is exactly the leak this cell exists to catch. """ token_a, token_b = f"alpha-{_token()}", f"bravo-{_token()}" - room_a = await resource_manager.provision_room( - title="e2e-vscode-isolation-a", participants=[copilot_identity.id] - ) - room_b = await resource_manager.provision_room( - title="e2e-vscode-isolation-b", participants=[copilot_identity.id] - ) - async def seed(room_id: str, capture: ReplyCapture, token: str) -> None: - text = f"The marker for THIS room is {token}." - await user_ops.send_message( - room_id, - text, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, - ) - await run_turn( - capture, - driver, - room_id=room_id, - identity=copilot_identity, - sender_name="the user", - message=text, - instruction=( - f"Store this room's marker with band_store_memory, recording " - f"which room it belongs to — content like 'marker for room " - f"{room_id}: {token}'. Then verify with band_list_memories " - f"that the record exists (store it again if not), and only " - f"then acknowledge it in the room." - ), - deadline_s=vscode_settings.vscode_chat_timeout, - ) - # Store-side barrier (see test_recall_across_chat_sessions). - (await capture.memory(copilot_identity)).stored.assert_stored(content=token) - - async with reply_capture(room_a) as capture_a: - await seed(room_a, capture_a, token_a) - async with reply_capture(room_b) as capture_b: - await seed(room_b, capture_b, token_b) - - ask = "Which marker belongs to THIS room?" - await user_ops.send_message( - room_b, - ask, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, + async def seed(room, token: str) -> None: + await room.remember( + f"The marker for THIS room is {token}.", + record=f"marker for room {room.room_id}: {token}", ) - replies = await run_turn( - capture_b, - driver, - room_id=room_b, - identity=copilot_identity, - sender_name="the user", - message=ask, + + async with surface_room("isolation-a") as room_a: + await seed(room_a, token_a) + async with surface_room("isolation-b") as room_b: + await seed(room_b, token_b) + + replies = await room_b.user_turn( + "Which marker belongs to THIS room?", instruction=( "Look up the stored markers with band_list_memories and reply " "with the one recorded for THIS room (match the chat_id above). " "Never mention markers recorded for other rooms." ), - deadline_s=vscode_settings.vscode_chat_timeout, new_session=True, ) replies.assert_contains_any([token_b]) @@ -382,15 +174,7 @@ async def seed(room_id: str, capture: ReplyCapture, token: str) -> None: @pytest.mark.timeout(extra=360) # two live turns plus a bridge restart async def test_restart_recall_and_function( - vscode_settings: VSCodeChatSettings, - baseline_settings: BaselineSettings, - copilot_identity: ProvisionedAgent, - resource_manager: ResourceManager, - user_ops: UserOps, - reply_capture: CaptureFactory, - driver: CodeChatDriver, - band_mcp: BandMCPServer, - vscode_workspace: Path, + surface_room, band_mcp: BandMCPServer, vscode_workspace: Path ) -> None: """L3: the platform bridge (band-mcp) restarts between turns and the surface still recalls the stored fact AND keeps its native function (file creation). @@ -400,54 +184,23 @@ async def test_restart_recall_and_function( """ fact = f"phase-{_token()}" filename = f"restart-{_token()}.txt" - room_id = await resource_manager.provision_room( - title="e2e-vscode-restart", participants=[copilot_identity.id] - ) - async with reply_capture(room_id) as capture: - seed = f"Remember across restarts: the deploy phase is {fact}." - await user_ops.send_message( - room_id, - seed, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, - ) - await run_turn( - capture, - driver, - room_id=room_id, - identity=copilot_identity, - sender_name="the user", - message=seed, - instruction=( - "Store the deploy phase in your Band memory with " - "band_store_memory, then confirm in the room." - ), - deadline_s=vscode_settings.vscode_chat_timeout, + async with surface_room("restart") as room: + await room.remember( + f"Remember across restarts: the deploy phase is {fact}.", + record=f"deploy phase for room {room.room_id}: {fact}", ) await band_mcp.restart() - ask = "After the maintenance window: what is the deploy phase?" - await user_ops.send_message( - room_id, - ask, - mention_id=copilot_identity.id, - mention_name=copilot_identity.name, - ) - replies = await run_turn( - capture, - driver, - room_id=room_id, - identity=copilot_identity, - sender_name="the user", - message=ask, + replies = await room.user_turn( + "After the maintenance window: what is the deploy phase?", instruction=( f"Recall the deploy phase from your Band memory with " - f"band_list_memories — do not guess — and reply stating it. " + f"band_list_memories — the record for THIS room (match the " + f"chat_id above); do not guess — and reply stating it. " f"Also create a file named '{filename}' in the workspace root " f"containing that phase, to confirm your editor tools still work." ), - deadline_s=vscode_settings.vscode_chat_timeout, new_session=True, ) diff --git a/tests/framework_conformance/test_vscode_chat_toolkit.py b/tests/framework_conformance/test_vscode_chat_toolkit.py index 4785e31b3..00790875b 100644 --- a/tests/framework_conformance/test_vscode_chat_toolkit.py +++ b/tests/framework_conformance/test_vscode_chat_toolkit.py @@ -11,6 +11,7 @@ import json from pathlib import Path from types import SimpleNamespace +from typing import cast import pytest @@ -65,7 +66,11 @@ def test_scaffold_workspace_writes_mcp_entry_and_auto_reply(tmp_path: Path) -> N scaffold_workspace(tmp_path, sse_url) mcp = json.loads((tmp_path / ".vscode" / "mcp.json").read_text()) - assert mcp["servers"][MCP_SERVER_NAME] == {"type": "sse", "url": sse_url} + # Assert the contract fields, not the whole dict — a new optional key in + # the scaffold must not break this guard. + server = mcp["servers"][MCP_SERVER_NAME] + assert server["type"] == "sse" + assert server["url"] == sse_url settings = json.loads((tmp_path / ".vscode" / "settings.json").read_text()) assert settings[AUTO_REPLY_SETTING] is True @@ -97,11 +102,15 @@ def test_capture_versions_shape_with_stub_runner() -> None: ["code"], ["band-mcp"], run=lambda cmd: outputs[" ".join(cmd)] ) - assert versions["vscode"] == "1.103.0 abc123 arm64" - assert versions["copilot_extensions"] == ( - "github.copilot@1.5.0; github.copilot-chat@0.32.0" - ) - assert versions["band_mcp"] == "band-mcp 1.3.2" + # Behavioral contract only (what the sidecar must convey), not join format: + # the VS Code version+build survive on one line, copilot extensions are + # kept while unrelated ones are filtered, band-mcp passes through. + assert "1.103.0" in versions["vscode"] and "abc123" in versions["vscode"] + assert "\n" not in versions["vscode"] + assert "github.copilot@1.5.0" in versions["copilot_extensions"] + assert "github.copilot-chat@0.32.0" in versions["copilot_extensions"] + assert "ms-python" not in versions["copilot_extensions"] + assert "band-mcp 1.3.2" in versions["band_mcp"] assert versions["os"] @@ -123,14 +132,18 @@ def test_fresh_session_preamble_is_prependable_text() -> None: # --- scorecard: outcome mapping + emitted artifact ---------------------------------- -def _report(when: str, outcome: str, nodeid: str, fspath: str) -> SimpleNamespace: - return SimpleNamespace( - when=when, - skipped=outcome == "skipped", - failed=outcome == "failed", - passed=outcome == "passed", - nodeid=nodeid, - fspath=fspath, +def _report(when: str, outcome: str, nodeid: str, fspath: str) -> pytest.TestReport: + """A minimal stand-in carrying the only report fields the plugin reads.""" + return cast( + pytest.TestReport, + SimpleNamespace( + when=when, + skipped=outcome == "skipped", + failed=outcome == "failed", + passed=outcome == "passed", + nodeid=nodeid, + fspath=fspath, + ), ) @@ -147,7 +160,7 @@ def _report(when: str, outcome: str, nodeid: str, fspath: str) -> SimpleNamespac ) def test_outcome_status_mapping(when: str, outcome: str, expected: str | None) -> None: report = _report(when, outcome, "tests/e2e/vscode/test_x.py::t", "x") - assert outcome_status(report) == expected # type: ignore[arg-type] + assert outcome_status(report) == expected def test_usage_na_row_carries_surface_and_reason() -> None: @@ -156,29 +169,36 @@ def test_usage_na_row_carries_surface_and_reason() -> None: assert USAGE_NA_ROW.reason -def test_scorecard_writes_rows_metadata_and_fixed_na_row(tmp_path: Path) -> None: +def test_scorecard_keeps_suite_rows_plus_fixed_na_row(tmp_path: Path) -> None: + """One row per suite test plus the fixed L4 N/A; foreign reports ignored.""" suite_dir = tmp_path / "suite" suite_dir.mkdir() - test_file = suite_dir / "test_copilot_chat.py" - test_file.touch() - out = tmp_path / "scorecard.json" + suite_test = suite_dir / "test_copilot_chat.py" + suite_test.touch() + nodeid = "tests/e2e/vscode/test_copilot_chat.py::test_participation" + + plugin = VSCodeScorecard(tmp_path / "scorecard.json", suite_dir) + for report in ( + _report("setup", "passed", nodeid, str(suite_test)), + _report("call", "passed", nodeid, str(suite_test)), + _report("call", "passed", "tests/other.py::t", str(tmp_path / "other.py")), + ): + plugin.pytest_runtest_logreport(report) + + rows = {row.test: row for row in plugin.scorecard()} + assert rows[nodeid].status == "pass" + assert rows[nodeid].adapter == SURFACE_ID + assert rows[USAGE_NA_ROW.test] == USAGE_NA_ROW + assert len(rows) == 2 # the foreign report contributed nothing - plugin = VSCodeScorecard(out, suite_dir) + +def test_sessionfinish_writes_scorecard_and_metadata_sidecar(tmp_path: Path) -> None: + out = tmp_path / "scorecard.json" + plugin = VSCodeScorecard(out, tmp_path) plugin.metadata = {"vscode": "1.103.0"} - nodeid = "tests/e2e/vscode/test_copilot_chat.py::test_participation" - plugin.pytest_runtest_logreport(_report("setup", "passed", nodeid, str(test_file))) # type: ignore[arg-type] - plugin.pytest_runtest_logreport(_report("call", "passed", nodeid, str(test_file))) # type: ignore[arg-type] - # A report from outside the suite dir must not add a row. - plugin.pytest_runtest_logreport( - _report("call", "passed", "tests/other.py::t", str(tmp_path / "other.py")) # type: ignore[arg-type] - ) - plugin.pytest_sessionfinish() - rows = {row["test"]: row for row in json.loads(out.read_text())} - assert rows[nodeid]["status"] == "pass" - assert rows[nodeid]["adapter"] == SURFACE_ID - assert rows[USAGE_NA_ROW.test]["status"] == "na" - assert len(rows) == 2 + plugin.pytest_sessionfinish() + assert json.loads(out.read_text()) # the rows artifact (N/A row at minimum) meta = json.loads((tmp_path / "scorecard.json.meta.json").read_text()) assert meta == {"vscode": "1.103.0"} diff --git a/tests/test_env_gates.py b/tests/test_env_gates.py index 062f008ce..8e8e76959 100644 --- a/tests/test_env_gates.py +++ b/tests/test_env_gates.py @@ -1,37 +1,53 @@ -"""Set-but-empty env vars must not kill settings construction. +"""Guards for the opt-in suite gates (``tests/conftest.py``'s ``GATED_MARKERS``). -Some CI wrappers export gates as empty (``CI=``, ``E2E_TESTS_ENABLED=``). -Without ``env_ignore_empty=True`` pydantic-settings tries to parse ``""`` as -the field's bool/int type and raises a ValidationError — for the collection -gates that happens *inside a pytest collection hook*, killing the entire run. -These guards pin the empty-means-unset behavior for both settings surfaces. +Two failure modes are pinned here because both would bite *inside a pytest +collection hook*, killing entire runs instead of failing one test: + +* a set-but-empty gate var (``CI=``, as some CI wrappers export) must read as + disabled, not raise a ValidationError — the ``env_ignore_empty`` behavior; +* the gate table must stay consistent with what it references: every field it + names must exist on ``CollectionGateSettings``, and every marker it names + must be registered in pyproject (a drift means a gated suite either warns as + unknown or never skips). """ from __future__ import annotations +import tomllib + import pytest -from tests.conftest import CollectionGateSettings +from tests.conftest import GATED_MARKERS, CollectionGateSettings from tests.e2e.baseline.settings import BaselineSettings +from tests.paths import REPO_ROOT + + +def registered_pytest_markers() -> set[str]: + """The marker names registered in pyproject's ``[tool.pytest.ini_options]``.""" + pyproject = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) + entries: list[str] = pyproject["tool"]["pytest"]["ini_options"]["markers"] + return {entry.split(":", 1)[0].strip() for entry in entries} def test_empty_collection_gate_vars_read_as_disabled( monkeypatch: pytest.MonkeyPatch, ) -> None: - for var in ( - "CI", - "E2E_TESTS_ENABLED", - "DOCKER_TESTS_ENABLED", - "SANDBOX_TESTS_ENABLED", - "VSCODE_CHAT_TESTS_ENABLED", - ): - monkeypatch.setenv(var, "") + gate_fields = ["ci", *GATED_MARKERS.values()] + for field in gate_fields: + monkeypatch.setenv(field.upper(), "") gates = CollectionGateSettings() - assert gates.ci is False - assert gates.e2e_tests_enabled is False - assert gates.docker_tests_enabled is False - assert gates.sandbox_tests_enabled is False - assert gates.vscode_chat_tests_enabled is False + for field in gate_fields: + assert getattr(gates, field) is False, field + + +def test_every_gated_marker_opens_with_a_real_settings_field() -> None: + gates = CollectionGateSettings() + for field in GATED_MARKERS.values(): + assert hasattr(gates, field), field + + +def test_every_gated_marker_is_registered_in_pyproject() -> None: + assert set(GATED_MARKERS) <= registered_pytest_markers() def test_empty_baseline_vars_fall_back_to_defaults( From 329b0caf5a24afe299a05079f644a980acfdaddb Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Thu, 23 Jul 2026 13:16:04 +0300 Subject: [PATCH 5/7] test: drop tautological guards from the VS Code toolkit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three tests restated definitions instead of protecting behavior — the marker-path helper's own expression, a static preamble constant, and a frozen row's fields echoed back. The N/A row's presence in the emitted scorecard (the part that can actually break) stays covered by the scorecard-rows test. Co-Authored-By: Claude Fable 5 Signed-off-by: Alexander Zaikman --- .../test_vscode_chat_toolkit.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/framework_conformance/test_vscode_chat_toolkit.py b/tests/framework_conformance/test_vscode_chat_toolkit.py index 00790875b..ff52dfd48 100644 --- a/tests/framework_conformance/test_vscode_chat_toolkit.py +++ b/tests/framework_conformance/test_vscode_chat_toolkit.py @@ -16,7 +16,6 @@ import pytest from tests.e2e.vscode.driver import ( - FRESH_SESSION_PREAMBLE, PreflightError, capture_versions, preflight, @@ -32,7 +31,6 @@ AUTO_REPLY_SETTING, MCP_SERVER_NAME, scaffold_workspace, - workspace_marker_path, ) @@ -76,11 +74,6 @@ def test_scaffold_workspace_writes_mcp_entry_and_auto_reply(tmp_path: Path) -> N assert settings[AUTO_REPLY_SETTING] is True -def test_workspace_marker_path_stays_inside_workspace(tmp_path: Path) -> None: - marker = workspace_marker_path(tmp_path, "notes-abc.txt") - assert marker.parent == tmp_path - - # --- driver preflight + version evidence -------------------------------------------- @@ -123,12 +116,6 @@ def failing(cmd: list[str]) -> str: assert "unavailable" in versions["band_mcp"] -def test_fresh_session_preamble_is_prependable_text() -> None: - # The CLI has no verified new-session switch; the preamble is the fallback - # and must stay a plain instruction line (no templating placeholders). - assert "{" not in FRESH_SESSION_PREAMBLE - - # --- scorecard: outcome mapping + emitted artifact ---------------------------------- @@ -163,12 +150,6 @@ def test_outcome_status_mapping(when: str, outcome: str, expected: str | None) - assert outcome_status(report) == expected -def test_usage_na_row_carries_surface_and_reason() -> None: - assert USAGE_NA_ROW.adapter == SURFACE_ID - assert USAGE_NA_ROW.status == "na" - assert USAGE_NA_ROW.reason - - def test_scorecard_keeps_suite_rows_plus_fixed_na_row(tmp_path: Path) -> None: """One row per suite test plus the fixed L4 N/A; foreign reports ignored.""" suite_dir = tmp_path / "suite" From fb26e75d6eb4b29d9b9f5b03087c07d892f16c56 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Thu, 23 Jul 2026 13:23:11 +0300 Subject: [PATCH 6/7] fix(e2e): harden band-mcp startup and use current MCP access setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on the VS Code suite: - workspace: write chat.mcp.access="all" (current key) instead of the deprecated boolean chat.mcp.enabled, which only works via VS Code's settings-migration shim. - server: reject a port already held by another process before spawning — a stale listener would answer the bare-TCP readiness probe for a child that failed to bind, and the suite would run against the wrong Band identity. - server: terminate the child when startup times out — the session fixture never reaches its yield on a start() failure, so its teardown would never stop() the orphan squatting the stable port. Both server fixes carry regression tests that fail on the old code. Co-Authored-By: Claude Fable 5 --- tests/e2e/vscode/server.py | 32 +++++++++++- tests/e2e/vscode/workspace.py | 9 ++-- .../test_vscode_chat_toolkit.py | 50 +++++++++++++++++-- 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/tests/e2e/vscode/server.py b/tests/e2e/vscode/server.py index 86f43aec3..b50d66b06 100644 --- a/tests/e2e/vscode/server.py +++ b/tests/e2e/vscode/server.py @@ -59,7 +59,12 @@ def __init__( def sse_url(self) -> str: return f"http://127.0.0.1:{self._port}/sse" + @property + def running(self) -> bool: + return self._process is not None and self._process.returncode is None + async def start(self) -> None: + await self._reject_occupied_port() env = dict(os.environ) | { "BAND_AGENT_KEY": self._agent_key, "BAND_BASE_URL": self._base_url, @@ -77,7 +82,13 @@ async def start(self) -> None: TOOL_GROUPS, env=env, ) - await self._wait_ready() + try: + await self._wait_ready() + except Exception: + # A failed startup must not orphan the child: the session fixture + # never reaches its yield, so its teardown would never stop() it. + await self.stop() + raise logger.info("band-mcp serving on %s", self.sse_url) async def stop(self) -> None: @@ -98,6 +109,25 @@ async def restart(self) -> None: await self.stop() await self.start() + async def _reject_occupied_port(self) -> None: + """Fail loud when something already listens on the port. + + The readiness probe is a bare TCP connect — a stale listener (e.g. a + band-mcp orphaned by a crashed run, holding an already-reaped agent + key) would answer it for a child that failed to bind, and the suite + would run against the wrong Band identity. + """ + try: + _, writer = await asyncio.open_connection("127.0.0.1", self._port) + except OSError: + return + writer.close() + await writer.wait_closed() + raise RuntimeError( + f"port {self._port} is already in use — a stale band-mcp from a " + f"previous run? Stop it (or set BAND_MCP_PORT) and rerun." + ) + async def _wait_ready(self) -> None: deadline = asyncio.get_running_loop().time() + READY_TIMEOUT_S while True: diff --git a/tests/e2e/vscode/workspace.py b/tests/e2e/vscode/workspace.py index 552ef77e8..c891964c3 100644 --- a/tests/e2e/vscode/workspace.py +++ b/tests/e2e/vscode/workspace.py @@ -20,8 +20,11 @@ AUTO_REPLY_SETTING = "chat.autoReply" # MCP support is on by default in current VS Code; setting it explicitly keeps -# the run independent of a user-profile override. Unknown keys are ignored. -MCP_ENABLED_SETTING = "chat.mcp.enabled" +# the run independent of a user-profile override ("none"). This is the current +# key — the boolean chat.mcp.enabled it replaced only works via VS Code's +# settings-migration shim. +MCP_ACCESS_SETTING = "chat.mcp.access" +MCP_ACCESS_ALL = "all" # The one MCP server name the prompts refer to ("the band tools"). MCP_SERVER_NAME = "band" @@ -33,7 +36,7 @@ def scaffold_workspace(root: Path, sse_url: str) -> None: vscode_dir.mkdir(parents=True, exist_ok=True) mcp_config = {"servers": {MCP_SERVER_NAME: {"type": "sse", "url": sse_url}}} - settings = {AUTO_REPLY_SETTING: True, MCP_ENABLED_SETTING: True} + settings = {AUTO_REPLY_SETTING: True, MCP_ACCESS_SETTING: MCP_ACCESS_ALL} (vscode_dir / "mcp.json").write_text(json.dumps(mcp_config, indent=2) + "\n") (vscode_dir / "settings.json").write_text(json.dumps(settings, indent=2) + "\n") diff --git a/tests/framework_conformance/test_vscode_chat_toolkit.py b/tests/framework_conformance/test_vscode_chat_toolkit.py index ff52dfd48..7f4e48c96 100644 --- a/tests/framework_conformance/test_vscode_chat_toolkit.py +++ b/tests/framework_conformance/test_vscode_chat_toolkit.py @@ -1,20 +1,24 @@ """Unit guards for the Copilot-in-VS-Code harness toolkit (``tests/e2e/vscode``). The live suite needs a signed-in VS Code window and can never run in CI, so -these pure-function tests are the only ones protecting its plumbing on every -PR: the prompt shape Copilot receives, the workspace files VS Code reads, the -version evidence recorded with a run, and the scorecard rows the run emits. +these tests are the only ones protecting its plumbing on every PR: the prompt +shape Copilot receives, the workspace files VS Code reads, the band-mcp +process lifecycle, the version evidence recorded with a run, and the scorecard +rows the run emits. """ from __future__ import annotations import json +import socket +import sys from pathlib import Path from types import SimpleNamespace from typing import cast import pytest +from tests.e2e.vscode import server as server_module from tests.e2e.vscode.driver import ( PreflightError, capture_versions, @@ -27,8 +31,11 @@ VSCodeScorecard, outcome_status, ) +from tests.e2e.vscode.server import BandMCPServer from tests.e2e.vscode.workspace import ( AUTO_REPLY_SETTING, + MCP_ACCESS_ALL, + MCP_ACCESS_SETTING, MCP_SERVER_NAME, scaffold_workspace, ) @@ -72,6 +79,7 @@ def test_scaffold_workspace_writes_mcp_entry_and_auto_reply(tmp_path: Path) -> N settings = json.loads((tmp_path / ".vscode" / "settings.json").read_text()) assert settings[AUTO_REPLY_SETTING] is True + assert settings[MCP_ACCESS_SETTING] == MCP_ACCESS_ALL # --- driver preflight + version evidence -------------------------------------------- @@ -116,6 +124,42 @@ def failing(cmd: list[str]) -> str: assert "unavailable" in versions["band_mcp"] +# --- band-mcp server lifecycle: startup failure modes -------------------------------- + + +def _server(command: list[str], port: int = 0) -> BandMCPServer: + return BandMCPServer(command, agent_key="key", base_url="http://band", port=port) + + +async def test_start_rejects_port_held_by_another_process() -> None: + """A stale listener must fail the run loud, not answer the readiness probe + for a child that could never bind (the suite would then talk to the stale + server's Band identity).""" + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + sock.listen(1) + server = _server([sys.executable], port=sock.getsockname()[1]) + with pytest.raises(RuntimeError, match="already in use"): + await server.start() + assert not server.running # no child was ever spawned + + +async def test_failed_startup_never_orphans_the_child( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A child that stays alive but never binds must be terminated when start() + gives up — the session fixture's teardown never runs on a startup failure, + so an orphan would squat the stable port across reruns.""" + monkeypatch.setattr(server_module, "READY_TIMEOUT_S", 0.5) + never_binds = [sys.executable, "-c", "import time; time.sleep(60)"] + server = _server(never_binds) + + with pytest.raises(RuntimeError, match="not accepting connections"): + await server.start() + + assert not server.running + + # --- scorecard: outcome mapping + emitted artifact ---------------------------------- From 410c32047cde48221960a1c787566a401e266181 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Thu, 23 Jul 2026 13:28:51 +0300 Subject: [PATCH 7/7] fix(e2e): code-review hardening for the VS Code suite harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - driver: kill a hung `code chat` CLI on submit timeout — left alone it could still deliver its prompt minutes later, into a rerun's session. - scorecard: filter reports by rootdir-relative nodeid prefix instead of resolving report file paths against the CWD, which silently dropped every row when pytest ran from outside the repo root. - conftest: register the scorecard plugin (and its code/band-mcp version capture subprocesses) only when the suite gate is on — a lingering VSCODE_CHAT_SCORECARD_JSON must not stall an unrelated unit run on a cold uvx download mid-collection. - README: workspace-trust bullet claimed --disable-workspace-trust is used; the launch deliberately omits it (Copilot requires a trusted workspace) — document the real one-time Trust Folder click. Co-Authored-By: Claude Fable 5 --- tests/e2e/vscode/README.md | 8 ++++-- tests/e2e/vscode/conftest.py | 15 ++++++++--- tests/e2e/vscode/driver.py | 15 ++++++++--- tests/e2e/vscode/scorecard.py | 11 +++++--- .../test_vscode_chat_toolkit.py | 25 +++++++++---------- 5 files changed, 49 insertions(+), 25 deletions(-) diff --git a/tests/e2e/vscode/README.md b/tests/e2e/vscode/README.md index 89a2815be..1fa600c79 100644 --- a/tests/e2e/vscode/README.md +++ b/tests/e2e/vscode/README.md @@ -73,8 +73,12 @@ uv run pytest tests/e2e/vscode -v -s --no-cov During the run a VS Code window opens on the scaffolded workspace. The harness minimizes prompts, and after the first run reruns are unattended: -- **Workspace trust**: suppressed — the window is launched with - `--disable-workspace-trust` (scoped to that launch, not a global setting). +- **Workspace trust**: VS Code asks once per folder ("Do you trust the + authors…") — click **Trust Folder & Continue**. The workspace path is + stable by default, so the choice is remembered for every rerun. (The + launch deliberately does *not* use `--disable-workspace-trust`: Copilot's + AI features require a trusted workspace, so that flag would only defer + the same dialog to chat time.) - **MCP server trust**: VS Code asks once per server *configuration*. The workspace path and the band-mcp port are stable by default, so `mcp.json` is byte-identical across runs — allow the `band` server on the first run diff --git a/tests/e2e/vscode/conftest.py b/tests/e2e/vscode/conftest.py index 1b15398ab..ed8d24f5f 100644 --- a/tests/e2e/vscode/conftest.py +++ b/tests/e2e/vscode/conftest.py @@ -186,13 +186,20 @@ async def open_room( def pytest_configure(config: pytest.Config) -> None: - """Register the suite scorecard plugin when emission is enabled.""" + """Register the suite scorecard plugin for an enabled live run. + + Gated on the suite gate too, not just the artifact path: a lingering + VSCODE_CHAT_SCORECARD_JSON in the environment must not make an unrelated + (gate-off, all-skip) run shell out to ``code``/``band-mcp`` for version + evidence — a cold ``uvx band-mcp`` would stall collection on a download. + """ settings = VSCodeChatSettings() - if not settings.vscode_chat_scorecard_json: + if not (settings.vscode_chat_tests_enabled and settings.vscode_chat_scorecard_json): return - plugin = VSCodeScorecard( - settings.vscode_chat_scorecard_json, Path(__file__).parent.resolve() + suite_prefix = ( + Path(__file__).parent.resolve().relative_to(config.rootpath).as_posix() + "/" ) + plugin = VSCodeScorecard(settings.vscode_chat_scorecard_json, suite_prefix) plugin.metadata = capture_versions( shlex.split(settings.code_command), shlex.split(settings.band_mcp_command) ) diff --git a/tests/e2e/vscode/driver.py b/tests/e2e/vscode/driver.py index 24d12de10..478ce6558 100644 --- a/tests/e2e/vscode/driver.py +++ b/tests/e2e/vscode/driver.py @@ -108,9 +108,18 @@ async def _run(self, *args: str) -> None: stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, ) - output, _ = await asyncio.wait_for( - process.communicate(), timeout=SUBMIT_TIMEOUT_S - ) + try: + output, _ = await asyncio.wait_for( + process.communicate(), timeout=SUBMIT_TIMEOUT_S + ) + except TimeoutError: + # A hung CLI must not outlive the turn: left alone it could still + # deliver its prompt minutes later, into a rerun's session. + process.kill() + await process.wait() + raise RuntimeError( + f"code {args[0]} did not return within {SUBMIT_TIMEOUT_S}s" + ) from None if output: logger.debug("code %s: %s", args[0], output.decode(errors="replace")) if process.returncode != 0: diff --git a/tests/e2e/vscode/scorecard.py b/tests/e2e/vscode/scorecard.py index 242a1ccb4..78045a536 100644 --- a/tests/e2e/vscode/scorecard.py +++ b/tests/e2e/vscode/scorecard.py @@ -55,14 +55,19 @@ class VSCodeScorecard: """Pytest plugin: one row per suite test plus the fixed L4 ``na`` row, written with a ``.meta.json`` environment sidecar at session end.""" - def __init__(self, path: str | Path, suite_dir: Path) -> None: + def __init__(self, path: str | Path, suite_prefix: str) -> None: + """``suite_prefix``: the rootdir-relative nodeid prefix of the suite + (e.g. ``"tests/e2e/vscode/"``). Nodeids are rootdir-relative regardless + of the invocation directory — a filesystem-path filter would resolve + against the CWD and silently drop every row when pytest runs from + outside the repo root.""" self._path = Path(path) - self._suite_dir = suite_dir + self._suite_prefix = suite_prefix self._rows: dict[str, ScorecardRow] = {} self.metadata: dict[str, str] = {} def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: - if not Path(report.fspath).resolve().is_relative_to(self._suite_dir): + if not report.nodeid.startswith(self._suite_prefix): return status = outcome_status(report) if status is None: diff --git a/tests/framework_conformance/test_vscode_chat_toolkit.py b/tests/framework_conformance/test_vscode_chat_toolkit.py index 7f4e48c96..a71c31837 100644 --- a/tests/framework_conformance/test_vscode_chat_toolkit.py +++ b/tests/framework_conformance/test_vscode_chat_toolkit.py @@ -163,7 +163,7 @@ async def test_failed_startup_never_orphans_the_child( # --- scorecard: outcome mapping + emitted artifact ---------------------------------- -def _report(when: str, outcome: str, nodeid: str, fspath: str) -> pytest.TestReport: +def _report(when: str, outcome: str, nodeid: str) -> pytest.TestReport: """A minimal stand-in carrying the only report fields the plugin reads.""" return cast( pytest.TestReport, @@ -173,7 +173,6 @@ def _report(when: str, outcome: str, nodeid: str, fspath: str) -> pytest.TestRep failed=outcome == "failed", passed=outcome == "passed", nodeid=nodeid, - fspath=fspath, ), ) @@ -190,23 +189,23 @@ def _report(when: str, outcome: str, nodeid: str, fspath: str) -> pytest.TestRep ], ) def test_outcome_status_mapping(when: str, outcome: str, expected: str | None) -> None: - report = _report(when, outcome, "tests/e2e/vscode/test_x.py::t", "x") + report = _report(when, outcome, "tests/e2e/vscode/test_x.py::t") assert outcome_status(report) == expected def test_scorecard_keeps_suite_rows_plus_fixed_na_row(tmp_path: Path) -> None: - """One row per suite test plus the fixed L4 N/A; foreign reports ignored.""" - suite_dir = tmp_path / "suite" - suite_dir.mkdir() - suite_test = suite_dir / "test_copilot_chat.py" - suite_test.touch() + """One row per suite test plus the fixed L4 N/A; foreign reports ignored. + + The filter is a nodeid prefix — nodeids stay rootdir-relative wherever + pytest is invoked from, unlike report file paths. + """ nodeid = "tests/e2e/vscode/test_copilot_chat.py::test_participation" - plugin = VSCodeScorecard(tmp_path / "scorecard.json", suite_dir) + plugin = VSCodeScorecard(tmp_path / "scorecard.json", "tests/e2e/vscode/") for report in ( - _report("setup", "passed", nodeid, str(suite_test)), - _report("call", "passed", nodeid, str(suite_test)), - _report("call", "passed", "tests/other.py::t", str(tmp_path / "other.py")), + _report("setup", "passed", nodeid), + _report("call", "passed", nodeid), + _report("call", "passed", "tests/other.py::t"), ): plugin.pytest_runtest_logreport(report) @@ -219,7 +218,7 @@ def test_scorecard_keeps_suite_rows_plus_fixed_na_row(tmp_path: Path) -> None: def test_sessionfinish_writes_scorecard_and_metadata_sidecar(tmp_path: Path) -> None: out = tmp_path / "scorecard.json" - plugin = VSCodeScorecard(out, tmp_path) + plugin = VSCodeScorecard(out, "tests/e2e/vscode/") plugin.metadata = {"vscode": "1.103.0"} plugin.pytest_sessionfinish()