Skip to content
106 changes: 106 additions & 0 deletions src/lib/actions/sandbox/rebuild-shields-finally.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { createRequire } from "node:module";

import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest";

type RebuildSandbox = typeof import("../../../../dist/lib/actions/sandbox/rebuild")["rebuildSandbox"];

const requireDist = createRequire(import.meta.url);
const rebuildModulePath = "../../../../dist/lib/actions/sandbox/rebuild.js";

describe("rebuild shields relock guard", () => {
let rebuildSandbox: RebuildSandbox;
let spies: MockInstance[];
let errorSpy: MockInstance;
let logSpy: MockInstance;
let relockSpy: MockInstance;
const rebuildWindow = { relocked: false, wasLocked: true };

beforeEach(() => {
spies = [];
rebuildWindow.relocked = false;
delete require.cache[requireDist.resolve(rebuildModulePath)];

errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);

const gatewayDrift = requireDist("../../../../dist/lib/adapters/openshell/gateway-drift.js");
const gatewayRuntime = requireDist("../../../../dist/lib/gateway-runtime-action.js");
const sandboxList = requireDist("../../../../dist/lib/openshell-sandbox-list.js");
const resolve = requireDist("../../../../dist/lib/adapters/openshell/resolve.js");
const agentRuntime = requireDist("../../../../dist/lib/agent/runtime.js");
const onboardSession = requireDist("../../../../dist/lib/state/onboard-session.js");
const registry = requireDist("../../../../dist/lib/state/registry.js");
const sandboxState = requireDist("../../../../dist/lib/state/sandbox.js");
const sandboxSession = requireDist("../../../../dist/lib/state/sandbox-session.js");
const sandboxVersion = requireDist("../../../../dist/lib/sandbox/version.js");
const rebuildShields = requireDist("../../../../dist/lib/actions/sandbox/rebuild-shields.js");

relockSpy = vi
.spyOn(rebuildShields, "relockRebuildShieldsWindow")
.mockImplementation((...args: unknown[]) => {
const window = args[1] as typeof rebuildWindow;
window.relocked = true;
return true;
});

spies.push(
vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null),
vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null),
vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ recovered: false }),
vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({
result: { status: 0, output: "alpha Ready" },
}),
vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null),
vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null),
vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"),
vi.spyOn(onboardSession, "loadSession").mockReturnValue(null),
vi.spyOn(registry, "getSandbox").mockReturnValue({
name: "alpha",
provider: "ollama-local",
model: "nvidia/nemotron",
policies: [],
agent: null,
nimContainer: null,
} as never),
vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({
detected: false,
sessions: [],
}),
vi.spyOn(sandboxVersion, "checkAgentVersion").mockReturnValue({
expectedVersion: "0.1.0",
sandboxVersion: "0.0.1",
} as never),
vi.spyOn(rebuildShields, "openRebuildShieldsWindow").mockReturnValue(rebuildWindow),
relockSpy,
vi.spyOn(sandboxState, "backupSandboxState").mockImplementation(() => {
throw new Error("unexpected backup exception");
}),
);

({ rebuildSandbox } = requireDist(rebuildModulePath));
});

afterEach(() => {
for (const spy of spies) spy.mockRestore();
errorSpy.mockRestore();
logSpy.mockRestore();
delete require.cache[requireDist.resolve(rebuildModulePath)];
});

it("relocks shields when an unexpected exception escapes after auto-unlock", async () => {
await expect(rebuildSandbox("alpha", ["--yes"], { throwOnError: true })).rejects.toThrow(
"unexpected backup exception",
);

expect(relockSpy).toHaveBeenCalledWith(
"alpha",
rebuildWindow,
true,
expect.any(String),
);
expect(rebuildWindow.relocked).toBe(true);
});
});
86 changes: 86 additions & 0 deletions src/lib/actions/sandbox/rebuild-shields.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { G, R, RD as _RD, YW } from "../../cli/terminal-style";
import * as shields from "../../shields";

export interface RebuildShieldsWindow {
relocked: boolean;
wasLocked: boolean;
}

export function openRebuildShieldsWindow(
sandboxName: string,
cliName: string,
): RebuildShieldsWindow | null {
const window = {
relocked: false,
wasLocked: !shields.isShieldsDown(sandboxName),
};
if (!window.wasLocked) return window;

console.log("");
console.log(` ${YW}Shields are UP${R} — temporarily unlocking for rebuild backup...`);
try {
shields.shieldsDown(sandboxName, {
reason: "auto-unlock for rebuild",
skipTimer: true,
throwOnError: true,
});
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.error("");
console.error(` ${_RD}Failed to auto-unlock shields:${R} ${message}`);
console.error(" Sandbox is untouched — no data was lost.");
console.error(
` Run \`${cliName} ${sandboxName} shields down\` manually, then retry rebuild.`,
);
return null;
}
return window;
}

export function printRebuildShieldsRecovery(
sandboxName: string,
window: RebuildShieldsWindow,
cliName: string,
): void {
if (!window.wasLocked) return;
console.error(` 4. Restore shields lockdown:`);
console.error(` ${cliName} ${sandboxName} shields up`);
}

export function relockRebuildShieldsWindow(
sandboxName: string,
window: RebuildShieldsWindow,
sandboxStillExists: boolean,
cliName: string,
): boolean {
if (!window.wasLocked || window.relocked) return true;
if (!sandboxStillExists) {
console.warn("");
console.warn(
` ${YW}⚠${R} Cannot re-apply shields lockdown — sandbox no longer exists.`,
);
console.warn(
` After recovery, run \`${cliName} ${sandboxName} shields up\` to restore lockdown.`,
);
return false;
}

console.log("");
console.log(" Re-applying shields lockdown...");
try {
shields.shieldsUp(sandboxName, { throwOnError: true });
console.log(` ${G}✓${R} Shields restored to UP`);
window.relocked = true;
return true;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.error(` ${YW}⚠${R} Failed to re-apply shields lockdown: ${message}`);
console.error(
` Run \`${cliName} ${sandboxName} shields up\` manually to restore lockdown.`,
);
return false;
}
}
32 changes: 32 additions & 0 deletions src/lib/actions/sandbox/rebuild.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ import {
} from "../../state/sandbox-session";
import { removeSandboxRegistryEntry } from "./destroy";
import { executeSandboxCommand } from "./process-recovery";
import { openRebuildShieldsWindow, printRebuildShieldsRecovery, relockRebuildShieldsWindow } from "./rebuild-shields";

/**
* Emit timestamped rebuild diagnostics when verbose rebuild logging is enabled.
Expand Down Expand Up @@ -438,6 +439,20 @@ export async function rebuildSandbox(
}
}

const rebuildShieldsWindow = openRebuildShieldsWindow(sandboxName, CLI_NAME);
if (!rebuildShieldsWindow) return bail("Failed to auto-unlock shields.");

const relockShieldsIfNeeded = (sandboxStillExists: boolean): boolean =>
relockRebuildShieldsWindow(
sandboxName,
rebuildShieldsWindow,
sandboxStillExists,
CLI_NAME,
);

let sandboxStillExists = true;

try {
// Step 2: Backup
console.log(" Backing up sandbox state...");
log(`Agent type: ${sb.agent || "openclaw"}, stateDirs from manifest`);
Expand All @@ -456,13 +471,15 @@ export async function rebuildSandbox(
console.error(` Failed files: ${backup.failedFiles.join(", ")}`);
}
console.error(" Aborting rebuild to prevent data loss.");
relockShieldsIfNeeded(true);
bail("Failed to back up sandbox state.");
return;
}
const backupManifest = backup.manifest;
if (!backupManifest) {
console.error(" Failed to record backup metadata.");
console.error(" Aborting rebuild to prevent data loss.");
relockShieldsIfNeeded(true);
bail("Failed to record backup metadata.");
return;
}
Expand Down Expand Up @@ -515,9 +532,11 @@ export async function rebuildSandbox(
if (deleteResult.status !== 0 && !alreadyGone) {
console.error(" Failed to delete sandbox. Aborting rebuild.");
console.error(" State backup is preserved at: " + backupManifest.backupPath);
relockShieldsIfNeeded(true);
bail("Failed to delete sandbox.", deleteResult.status || 1);
return;
}
sandboxStillExists = false;
removeSandboxRegistryEntry(sandboxName);
log(
`Registry after remove: ${JSON.stringify(registry.listSandboxes().sandboxes.map((s: { name: string }) => s.name))}`,
Expand Down Expand Up @@ -683,6 +702,10 @@ export async function rebuildSandbox(
process.exit = _savedExit;
}

if (!onboardFailed) {
sandboxStillExists = true;
}

if (onboardFailed) {
// Clean up onboard's internal state that normally runs in
// process.once("exit") listeners — those never fire because we
Expand Down Expand Up @@ -715,7 +738,9 @@ export async function rebuildSandbox(
console.error(
` ${CLI_NAME} ${sandboxName} snapshot restore "${backupManifest.timestamp}"`,
);
printRebuildShieldsRecovery(sandboxName, rebuildShieldsWindow, CLI_NAME);
console.error("");
relockShieldsIfNeeded(false);
bail(
`Recreate failed (sandbox destroyed). Backup: ${backupManifest.backupPath}`,
onboardExitCode,
Expand Down Expand Up @@ -855,6 +880,8 @@ export async function rebuildSandbox(
});
log(`Registry updated: agentVersion=${agentDef.expectedVersion}`);

if (!relockShieldsIfNeeded(true)) return bail("Failed to re-apply shields lockdown.");

console.log("");
if (restore.success) {
console.log(` ${G}\u2713${R} Sandbox '${sandboxName}' rebuilt successfully`);
Expand All @@ -867,4 +894,9 @@ export async function rebuildSandbox(
);
console.log(` Backup available at: ${backupManifest.backupPath}`);
}
} finally {
if (!rebuildShieldsWindow.relocked) {
relockShieldsIfNeeded(sandboxStillExists);
}
}
}
Loading
Loading