Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion test/e2e-scenario/framework-tests/e2e-clients.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ describe("E2E fixture clients", () => {

expect(runner.calls[0]).toEqual({
command: "openshell",
args: ["sandbox", "exec", "assistant", "--", "echo", "ok"],
args: ["sandbox", "exec", "--name", "assistant", "--", "echo", "ok"],
options: {
artifactName: "sandbox-exec-assistant",
},
Expand Down Expand Up @@ -192,6 +192,7 @@ describe("E2E fixture clients", () => {
expect(runner.calls[0]?.args).toEqual([
"sandbox",
"exec",
"--name",
"assistant",
"--",
"sh",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,27 @@ describe("live Vitest registry discovery support", () => {
]);
});

it("wires the provider-routed Model Router guard through live fixtures", () => {
const scenario = listScenarios().find(
(entry) => entry.id === "ubuntu-repo-cloud-openclaw-provider-routed",
);

expect(scenario).toBeTruthy();
expect(scenario!.environment?.onboarding).toBe("cloud-openclaw-provider-routed");
expect(scenario!.suiteIds).toEqual(["smoke", "model-router"]);
expect(liveScenarioSupport(scenario!)).toMatchObject({
supported: true,
reasons: [],
});
expect(buildLiveScenarioRunPlan(scenario!)).toEqual({
scenarioId: "ubuntu-repo-cloud-openclaw-provider-routed",
manifestPath: "test/e2e-scenario/manifests/openclaw-nvidia-provider-routed.yaml",
expectedStateId: "cloud-openclaw-ready",
suiteIds: ["smoke", "model-router"],
phases: ["environment", "onboarding", "state-validation", "runtime"],
});
});

it("keeps unsupported onboarding profiles skipped with a concrete reason", () => {
const scenario = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-hermes");

Expand Down
61 changes: 61 additions & 0 deletions test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,67 @@ describe("onboarding phase fixture", () => {
expect(runner.calls).toEqual([]);
});

it("runs provider-routed cloud OpenClaw onboarding with Model Router inputs", async () => {
const runner = new FakeRunner();
runner.enqueue(shellResult(0, "onboarded\n"));
const cleanup = new FakeCleanup();
const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret-token" });
const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets, cleanup);

const instance = await onboard.from(ready({ onboarding: "cloud-openclaw-provider-routed" }), {
sandboxName: "e2e-provider-routed",
});

expect(instance).toMatchObject({
onboarding: "cloud-openclaw-provider-routed",
sandboxName: "e2e-provider-routed",
agent: "openclaw",
provider: "nvidia",
providerEnv: "cloud",
});
expect(secrets.requiredCalls).toEqual(["NVIDIA_API_KEY"]);
expect(cleanup.calls[0]?.name).toBe("destroy NemoClaw sandbox e2e-provider-routed");
expect(runner.calls).toEqual([
{
command: "nemoclaw",
args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"],
options: {
artifactName: "onboard-cloud-openclaw-provider-routed",
env: expect.objectContaining({
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
NEMOCLAW_AGENT: "openclaw",
NEMOCLAW_POLICY_TIER: "open",
NEMOCLAW_PROVIDER: "routed",
NEMOCLAW_PROVIDER_KEY: "secret-token",
NEMOCLAW_SANDBOX_NAME: "e2e-provider-routed",
NVIDIA_API_KEY: "secret-token",
PATH: expect.any(String),
}),
redactionValues: ["secret-token"],
timeoutMs: 900_000,
},
},
]);
});

it("requires Docker for provider-routed cloud OpenClaw onboarding", async () => {
const onboard = new OnboardingPhaseFixture(
new HostCliClient(new FakeRunner()),
new FakeSecrets({ NVIDIA_API_KEY: "secret" }),
);

await expect(
onboard.from(
ready({
onboarding: "cloud-openclaw-provider-routed",
docker: { id: "docker-running", expectation: "required", available: false },
}),
),
).rejects.toThrow(
/cloud-openclaw-provider-routed onboarding requires an available Docker runtime/,
);
});

it("requires Docker for cloud OpenClaw onboarding", async () => {
const onboard = new OnboardingPhaseFixture(
new HostCliClient(new FakeRunner()),
Expand Down
123 changes: 122 additions & 1 deletion test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ describe("runtime phase fixture", () => {
args: [
"sandbox",
"exec",
"--name",
"e2e-ubuntu-repo-cloud-openclaw",
"--",
"curl",
Expand Down Expand Up @@ -170,6 +171,7 @@ describe("runtime phase fixture", () => {
expect(call?.args).toEqual([
"sandbox",
"exec",
"--name",
"e2e-ubuntu-repo-cloud-openclaw",
"--",
"curl",
Expand All @@ -183,7 +185,7 @@ describe("runtime phase fixture", () => {
"https://inference.local/v1/chat/completions",
]);
expect(call?.args).not.toContain("sh");
const payload = JSON.parse(call?.args[11] ?? "{}");
const payload = JSON.parse(call?.args[12] ?? "{}");
expect(payload).toEqual({
model: "default",
messages: [{ role: "user", content: "Reply with ok" }],
Expand All @@ -192,6 +194,124 @@ describe("runtime phase fixture", () => {
expect(call?.options?.artifactName).toBe("custom-chat");
});

it("checks host model-router health has a healthy endpoint", async () => {
const runner = new FakeRunner();
runner.enqueue(shellResult(0, JSON.stringify({ healthy_count: 1 })));

const result = await fixture(runner).expectModelRouterHealthyEndpoint({
apiKey: "provider-secret",
headers: ["X-Api-Key: header-secret"],
});

expect(result.endpoint).toBe("http://127.0.0.1:4000/health");
expect(runner.calls[0]).toEqual({
command: "curl",
args: [
"-fsS",
"--max-time",
"10",
"-H",
"X-Api-Key: header-secret",
"-H",
"Authorization: Bearer provider-secret",
"http://127.0.0.1:4000/health",
],
options: {
artifactName: "runtime-model-router-health",
redactionValues: expect.arrayContaining([
"X-Api-Key: header-secret",
"header-secret",
"provider-secret",
]),
timeoutMs: 60_000,
},
});
});

it("rejects host model-router health without a healthy endpoint", async () => {
const runner = new FakeRunner();
runner.enqueue(shellResult(0, JSON.stringify({ healthy_count: 0 })));

await expect(fixture(runner).expectModelRouterHealthyEndpoint()).rejects.toThrow(
"model-router health response reported no healthy endpoints",
);
});

it("checks provider-routed Model Router completion semantics", async () => {
const runner = new FakeRunner();
runner.enqueue(
shellResult(
0,
JSON.stringify({
model: "nvidia-routed",
choices: [{ message: { content: "PONG" } }],
}),
),
);

await fixture(runner).expectModelRouterProviderRoutedCompletion(instance());

const call = runner.calls[0];
expect(call?.command).toBe("openshell");
expect(call?.args).toEqual([
"sandbox",
"exec",
"--name",
"e2e-ubuntu-repo-cloud-openclaw",
"--",
"curl",
"-fsS",
"--max-time",
"90",
"-H",
"Content-Type: application/json",
"--data-raw",
expect.any(String),
"https://inference.local/v1/chat/completions",
]);
expect(JSON.parse(call?.args[12] ?? "{}")).toEqual({
model: "nvidia-routed",
messages: [
{
role: "user",
content: "Reply with exactly one word: PONG. Do not explain or include any other text.",
},
],
max_tokens: 256,
});
expect(call?.options?.artifactName).toBe("runtime-model-router-provider-routed-completion");
});

it("rejects provider-routed Model Router completions with the wrong model", async () => {
const runner = new FakeRunner();
runner.enqueue(
shellResult(
0,
JSON.stringify({ choices: [{ message: { content: "PONG" } }], model: "default" }),
),
);

await expect(
fixture(runner).expectModelRouterProviderRoutedCompletion(instance()),
).rejects.toThrow(
"model-router provider-routed completion response model was not provider-routed",
);
});

it("rejects provider-routed Model Router completions without PONG content", async () => {
const runner = new FakeRunner();
runner.enqueue(
shellResult(
0,
JSON.stringify({ choices: [{ message: { content: "hello" } }], model: "nvidia-routed" }),
),
);

await expect(
fixture(runner).expectModelRouterProviderRoutedCompletion(instance()),
).rejects.toThrow("model-router provider-routed completion response missing PONG content");
});

it("accepts configured status codes for auth-proxy and route-health checks", async () => {
const runner = new FakeRunner();
runner.enqueue(shellResult(0, "403"));
Expand All @@ -207,6 +327,7 @@ describe("runtime phase fixture", () => {
args: [
"sandbox",
"exec",
"--name",
"e2e-ubuntu-repo-cloud-openclaw",
"--",
"curl",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ describe("state-validation phase fixture", () => {
args: [
"sandbox",
"exec",
"--name",
"e2e-ubuntu-repo-cloud-openclaw",
"--",
"curl",
Expand Down
17 changes: 16 additions & 1 deletion test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ describe("live Vitest scenario matrix", () => {
it("builds the default live Vitest matrix from fixture-supported scenarios only", () => {
expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([
"ubuntu-repo-cloud-openclaw",
"ubuntu-repo-cloud-openclaw-provider-routed",
"ubuntu-repo-docker-post-reboot-recovery",
]);
expect(buildLiveScenarioMatrix()[0]).toMatchObject({
Expand All @@ -93,11 +94,24 @@ describe("live Vitest scenario matrix", () => {
supportReasons: [],
pendingRuntimeSuites: ["smoke", "inference", "credentials"],
});
expect(buildLiveScenarioMatrix()[1]).toMatchObject({
id: "ubuntu-repo-cloud-openclaw-provider-routed",
runner: "ubuntu-latest",
platform: "ubuntu-local",
install: "repo-current",
runtime: "docker-running",
onboarding: "cloud-openclaw-provider-routed",
expectedStateId: "cloud-openclaw-ready",
suites: ["smoke", "model-router"],
requiredSecrets: ["NVIDIA_API_KEY"],
supported: true,
supportReasons: [],
});
// Failing-test-first guard for #4423. Pinned in the matrix to
// confirm the lifecycle whitelist + post-reboot-recovery scenario
// are wired together; the actual RED/GREEN behavior is exercised
// by the live runner (gates on the fix landing in src/lib/).
expect(buildLiveScenarioMatrix()[1]).toMatchObject({
expect(buildLiveScenarioMatrix()[2]).toMatchObject({
id: "ubuntu-repo-docker-post-reboot-recovery",
runner: "ubuntu-latest",
platform: "ubuntu-local",
Expand Down Expand Up @@ -129,6 +143,7 @@ describe("live Vitest scenario matrix", () => {
const parsed = JSON.parse(lines[0]);
expect(parsed.map((entry: { id: string }) => entry.id)).toEqual([
"ubuntu-repo-cloud-openclaw",
"ubuntu-repo-cloud-openclaw-provider-routed",
"ubuntu-repo-docker-post-reboot-recovery",
]);
});
Expand Down
2 changes: 1 addition & 1 deletion test/e2e-scenario/framework/clients/sandbox.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ export class SandboxClient {
options: ShellProbeRunOptions = {},
): Promise<ShellProbeResult> {
validateSandboxName(name);
return this.openshell(["sandbox", "exec", name, "--", ...command], {
return this.openshell(["sandbox", "exec", "--name", name, "--", ...command], {
artifactName: `sandbox-exec-${name}`,
...options,
});
Expand Down
Loading
Loading