Skip to content

Harden Agent Host E2E and protocol integration tests #327421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
roblourens merged 2 commits into from
Jul 25, 2026
Merged

Harden Agent Host E2E and protocol integration tests #327421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ef4a43b
Harden Agent Host protocol integration tests
roblourens Jul 24, 2026
9550975
Address Agent Host test hardening feedback
roblourens Jul 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 2 additions & 17 deletions src/vs/platform/agentHost/test/node/e2e/harness/agentHostE2ETestHarness.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { execSync } from 'child_process';
import { mkdtempSync, rmSync } from 'fs';
import { homedir, tmpdir, userInfo } from 'os';
import { fileURLToPath } from 'url';
import { raceTimeout, timeout } from '../../../../../../base/common/async.js';
import { timeout } from '../../../../../../base/common/async.js';
import { join } from '../../../../../../base/common/path.js';
import { removeAnsiEscapeCodes } from '../../../../../../base/common/strings.js';
import { URI } from '../../../../../../base/common/uri.js';
Expand All @@ -30,7 +30,7 @@ import {
import { CopilotCliConfigKey } from '../../../../common/copilotCliConfig.js';
import { CapiReplayMode } from './capiReplayProxy.js';
import {
getActionEnvelope, isActionNotification, IServerHandle, startRealServer, TestProtocolClient,
getActionEnvelope, isActionNotification, IServerHandle, startRealServer, stopServer, TestProtocolClient,
} from '../../serverIntegrationTestHelpers.js';
import { createProviderSession, dispatchTurn, dispatchTurnWithAttachments } from '../../providerIntegrationTestHelpers.js';
import { AgentHostUpdateSnapshotsEnvVar, AhpSnapshotScenario } from './ahpSnapshot.js';
Expand All @@ -45,7 +45,6 @@ import { AgentHostUpdateSnapshotsEnvVar, AhpSnapshotScenario } from './ahpSnapsh
const UPDATE_SNAPSHOTS = process.env[AgentHostUpdateSnapshotsEnvVar] === '1';
const RECORD = process.env['AGENT_HOST_REPLAY_RECORD'] === '1' || UPDATE_SNAPSHOTS;
const REPLAY_MODE: CapiReplayMode = RECORD ? 'record' : 'replay';
const SERVER_SHUTDOWN_TIMEOUT_MS = 30_000;

/**
* Upper bound on tests served by a single shared replay server before it is
Expand All @@ -58,20 +57,6 @@ const TEMP_DIR_CLEANUP_TIMEOUT_MS = 30_000;
export const REPLAY_PLACEHOLDER_TOKEN = 'replay-no-token';
export type AgentHostE2EModelTraffic = 'recorded' | 'none';

async function stopServer(server: IServerHandle | undefined): Promise<void> {
const serverProcess = server?.process;
if (!serverProcess || serverProcess.exitCode !== null || serverProcess.signalCode !== null) {
return;
}

const serverExit = new Promise<void>(resolve => serverProcess.once('exit', () => resolve()));
serverProcess.stdin?.end();
if (!await raceTimeout(serverExit.then(() => true), SERVER_SHUTDOWN_TIMEOUT_MS)) {
serverProcess.kill();
await serverExit;
}
}

export async function removeTempDirs(tempDirs: string[]): Promise<void> {
const pendingDirs = tempDirs.splice(0);
const errors = new Map<string, Error>();
Expand Down
7 changes: 4 additions & 3 deletions src/vs/platform/agentHost/test/node/protocol/agentHostServer.integrationTest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/

import { PROTOCOL_VERSION } from '../../../common/state/protocol/version/registry.js';
import { getAgentHostE2ETestTimeout, IServerHandle, startServer, TestProtocolClient } from '../serverIntegrationTestHelpers.js';
import { getAgentHostE2ETestTimeout, IServerHandle, startServer, stopServer, TestProtocolClient } from '../serverIntegrationTestHelpers.js';

suite('Agent Host Server', function () {

Expand All @@ -15,8 +15,9 @@ suite('Agent Host Server', function () {
server = await startServer({ quiet: false });
});

suiteTeardown(function () {
server.process.kill();
suiteTeardown(async function () {
this.timeout(getAgentHostE2ETestTimeout(20_000, 50_000));
await stopServer(server);
});

test('starts with production agent services registered', async function () {
Expand Down
6 changes: 4 additions & 2 deletions src/vs/platform/agentHost/test/node/protocol/clientTools.integrationTest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import {
IServerHandle,
isActionNotification,
startServer,
stopServer,
TestProtocolClient,
} from '../serverIntegrationTestHelpers.js';

Expand All @@ -40,8 +41,9 @@ suite('Protocol WebSocket — Client Tools', function () {
server = await startServer();
});

suiteTeardown(function () {
server.process.kill();
suiteTeardown(async function () {
this.timeout(getAgentHostE2ETestTimeout(20_000, 50_000));
await stopServer(server);
});

setup(async function () {
Expand Down
29 changes: 16 additions & 13 deletions src/vs/platform/agentHost/test/node/protocol/handshake.integrationTest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
type JsonRpcErrorResponse,
} from '../../../common/state/sessionProtocol.js';
import { ROOT_STATE_URI } from '../../../common/state/sessionState.js';
import { getAgentHostE2ETestTimeout, IServerHandle, nextSessionUri, startServer, TestProtocolClient } from '../serverIntegrationTestHelpers.js';
import { getAgentHostE2ETestTimeout, IServerHandle, nextSessionUri, startServer, stopServer, TestProtocolClient } from '../serverIntegrationTestHelpers.js';

suite('Protocol WebSocket — Handshake & Errors', function () {

Expand All @@ -23,8 +23,9 @@ suite('Protocol WebSocket — Handshake & Errors', function () {
server = await startServer();
});

suiteTeardown(function () {
server.process.kill();
suiteTeardown(async function () {
this.timeout(getAgentHostE2ETestTimeout(20_000, 50_000));
await stopServer(server);
});

setup(async function () {
Expand All @@ -38,7 +39,7 @@ suite('Protocol WebSocket — Handshake & Errors', function () {
});

test('handshake returns initialize response with protocol version', async function () {
this.timeout(5_000);
this.timeout(getAgentHostE2ETestTimeout(5_000, 20_000));

const result = await client.call<InitializeResult>('initialize', {
protocolVersions: [PROTOCOL_VERSION],
Expand All @@ -55,17 +56,19 @@ suite('Protocol WebSocket — Handshake & Errors', function () {
this.timeout(10_000);

const raw = new TestProtocolClient(server.port);
await raw.connect();

const responsePromise = raw.waitForRawMessage();
raw.sendRaw('this is not valid json{{{');
try {
await raw.connect();

const response = await responsePromise as JsonRpcErrorResponse;
assert.strictEqual(response.jsonrpc, '2.0');
assert.strictEqual(response.id, null);
assert.strictEqual(response.error.code, JSON_RPC_PARSE_ERROR);
const responsePromise = raw.waitForRawMessage();
raw.sendRaw('this is not valid json{{{');

raw.close();
const response = await responsePromise as JsonRpcErrorResponse;
assert.strictEqual(response.jsonrpc, '2.0');
assert.strictEqual(response.id, null);
assert.strictEqual(response.error.code, JSON_RPC_PARSE_ERROR);
} finally {
raw.close();
}
});

test('createSession with invalid provider does not crash server', async function () {
Expand Down
Loading
Loading