fix(agent): harden recovery proxy env sourcing

.github/workflows/nightly-e2e.yaml

@@ -1004,8 +1004,7 @@ jobs:
       timeout_minutes: 30
       artifact_name: "issue-2478-crash-loop-recovery-install-log"
       artifact_path: "/tmp/nemoclaw-e2e-install.log"
-      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-2478"}'
-      nvidia_api_key: true
+      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_E2E_USE_COMPAT_MOCK":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-2478"}'
       github_token: true
     secrets: *nightly-e2e-default-secrets
   hermes-e2e:

src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts

@@ -31,6 +31,20 @@ function writeStub(dir: string, name: string, body: string) {
   return stub;
 }
 
+function removeTempDir(dir: string) {
+  fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+}
+
+function waitForPath(filePath: string, timeoutMs = 1000) {
+  const sleepView = new Int32Array(new SharedArrayBuffer(4));
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (fs.existsSync(filePath)) return true;
+    Atomics.wait(sleepView, 0, 0, 10);
+  }
+  return fs.existsSync(filePath);
+}
+
 const SHARED_PYTHON_STUB_BY_MODE = [
   'if [ "$1" = "-c" ]; then',
   "  exit 0",
@@ -114,7 +128,7 @@ describe("Hermes secret-boundary guard — guard snippet behaviour", () => {
           : "",
       };
     } finally {
-      fs.rmSync(tmp, { recursive: true, force: true });
+      removeTempDir(tmp);
     }
   }
 
@@ -207,7 +221,7 @@ describe("Hermes secret-boundary guard — full recovery script behaviour", () =
     writeStub(stubsDir, "pgrep", "exit 1");
     writeStub(stubsDir, "sleep", "exit 0");
     writeStub(stubsDir, "curl", 'printf "000"\nexit 0');
-    writeStub(stubsDir, "hermes", `: > ${JSON.stringify(hermesLaunchMarker)}\nexit 0`);
+    writeStub(stubsDir, "hermes", `: > ${JSON.stringify(hermesLaunchMarker)}\n/bin/sleep 5`);
   }
 
   function runRecovery(
@@ -289,7 +303,7 @@ describe("Hermes secret-boundary guard — full recovery script behaviour", () =
       expect(log).toContain("[SECURITY] Refusing Hermes startup");
       expect(log).toContain("TELEGRAM_BOT_TOKEN (line 2)");
     } finally {
-      fs.rmSync(harness.tmp, { recursive: true, force: true });
+      removeTempDir(harness.tmp);
     }
   });
 
@@ -326,7 +340,7 @@ describe("Hermes secret-boundary guard — full recovery script behaviour", () =
       expect(log).toContain("(line 2)");
       expect(log).not.toContain("1234567890:AAExample-RawSecretValueHere");
     } finally {
-      fs.rmSync(harness.tmp, { recursive: true, force: true });
+      removeTempDir(harness.tmp);
     }
   });
 
@@ -375,7 +389,7 @@ describe("Hermes secret-boundary guard — full recovery script behaviour", () =
       expect(log).toContain("TELEGRAM_BOT_TOKEN");
       expect(log).not.toContain("1234567890:AAExample-RawSecretValueHere");
     } finally {
-      fs.rmSync(harness.tmp, { recursive: true, force: true });
+      removeTempDir(harness.tmp);
     }
   });
 
@@ -443,11 +457,11 @@ describe("Hermes secret-boundary guard — full recovery script behaviour", () =
       expect(log).toContain("[SECURITY] Refusing Hermes startup because the process environment");
       expect(log).toContain("TELEGRAM_BOT_TOKEN");
     } finally {
-      fs.rmSync(harness.tmp, { recursive: true, force: true });
+      removeTempDir(harness.tmp);
     }
   });
 
-  it("refuses on runtime-env violation using the real validator against a proxy-env that exports a raw secret", () => {
+  it("does not import a raw secret from a metadata-safe proxy-env during runtime validation", () => {
     const harness = prepareRecoveryHarness("runtime-env-real");
     const envFile = path.join(harness.tmp, "hermes-dot-env");
     const proxyEnvFile = path.join(harness.tmp, "nemoclaw-proxy-env.sh");
@@ -460,8 +474,10 @@ describe("Hermes secret-boundary guard — full recovery script behaviour", () =
       "hermes",
       "validate-env-secret-boundary.py",
     );
-    // Clean .env so env-file passes, hostile proxy-env contributes the raw
-    // runtime-env secret that runtime-env validation must catch after sourcing.
+    // Clean .env so env-file passes. The hostile proxy-env used to contribute a
+    // raw runtime-env secret; recovery now rewrites that volatile shell file
+    // before sourcing it, so the runtime-env validator should never see the raw
+    // value.
     fs.writeFileSync(envFile, "API_SERVER_PORT=18642\n");
     fs.writeFileSync(
       proxyEnvFile,
@@ -481,15 +497,20 @@ describe("Hermes secret-boundary guard — full recovery script behaviour", () =
         envFilePath: envFile,
         proxyEnvPath: proxyEnvFile,
       });
-      expect(result.status).toBe(1);
-      expect(result.stdout).toContain("SECRET_BOUNDARY_REFUSED");
-      expect(fs.existsSync(harness.hermesLaunchMarker)).toBe(false);
+      expect(result.status).toBe(0);
+      expect(waitForPath(harness.hermesLaunchMarker)).toBe(true);
+      expect(result.stdout).not.toContain("SECRET_BOUNDARY_REFUSED");
+      expect(result.stderr).not.toContain("TELEGRAM_BOT_TOKEN");
+      const proxyEnv = fs.readFileSync(proxyEnvFile, "utf-8");
+      expect(proxyEnv).not.toContain("TELEGRAM_BOT_TOKEN");
+      expect(proxyEnv).toContain(harness.preloadTmpSafetyNet);
+      expect(proxyEnv).toContain(harness.preloadTmpCiao);
       const log = fs.readFileSync(harness.recoveryLogPath, "utf-8");
-      expect(log).toContain("[SECURITY] Refusing Hermes startup because the process environment");
-      expect(log).toContain("TELEGRAM_BOT_TOKEN");
+      expect(log).not.toContain("[SECURITY] Refusing Hermes startup");
+      expect(log).not.toContain("TELEGRAM_BOT_TOKEN");
       expect(log).not.toContain("1234567890:AAExample-RawSecretValueHere");
     } finally {
-      fs.rmSync(harness.tmp, { recursive: true, force: true });
+      removeTempDir(harness.tmp);
     }
   });
 });

src/lib/agent/runtime-hermes-secret-boundary-shape.test.ts

@@ -7,7 +7,7 @@
 // tests that actually execute the synthesised script live in
 // runtime-hermes-secret-boundary-behavioural.test.ts.
 
-import { describe, it, expect } from "vitest";
+import { describe, expect, it } from "vitest";
 import { HERMES_SECRET_BOUNDARY_VALIDATOR_PATH } from "../../../dist/lib/agent/hermes-recovery-boundary";
 import {
   buildHermesDashboardProcessRecoveryScript,
@@ -26,10 +26,10 @@ describe("Hermes secret-boundary guard — generated shell shape", () => {
     expect(script).toContain(`python3 '${VALIDATOR_PATH}' env-file /sandbox/.hermes/.env`);
   });
 
-  it("invokes the runtime-env validator after sourcing /tmp/nemoclaw-proxy-env.sh", () => {
+  it("invokes the runtime-env validator after sourcing the generated recovery env", () => {
     const script = buildRecoveryScript(hermesAgent, 8642);
     expect(script).not.toBeNull();
-    const proxyEnvIdx = script!.indexOf(". /tmp/nemoclaw-proxy-env.sh");
+    const proxyEnvIdx = script!.indexOf('. "$_NEMOCLAW_RECOVERY_SOURCE_ENV"');
     const runtimeGuardIdx = script!.indexOf(`python3 '${VALIDATOR_PATH}' runtime-env`);
     const launchIdx = script!.indexOf("nohup");
     expect(proxyEnvIdx).toBeGreaterThanOrEqual(0);
@@ -77,23 +77,31 @@ describe("Hermes secret-boundary guard — generated shell shape", () => {
     expect(script).not.toContain("SECRET_BOUNDARY_REFUSED");
   });
 
-  it("guards the dashboard-only recovery path with env-file before sourcing and runtime-env after", () => {
+  it("guards the dashboard-only recovery path with env-file before generated sourcing and runtime-env after", () => {
     const script = buildHermesDashboardProcessRecoveryScript({
       publicPort: 9119,
       internalPort: 19119,
       tuiEnabled: false,
     });
     const envFileIdx = script.indexOf(`python3 '${VALIDATOR_PATH}' env-file /sandbox/.hermes/.env`);
-    const proxyEnvIdx = script.indexOf(". /tmp/nemoclaw-proxy-env.sh");
+    const guardRecoveryIdx = script.indexOf("_nemoclaw_validate_recovery_proxy_env");
+    const proxyEnvIdx = script.indexOf('. "$_NEMOCLAW_RECOVERY_SOURCE_ENV"');
+    const bashrcIdx = script.indexOf("[ -f ~/.bashrc ] && . ~/.bashrc;");
     const runtimeIdx = script.indexOf(`python3 '${VALIDATOR_PATH}' runtime-env`);
     const launchIdx = script.indexOf('"$AGENT_BIN" dashboard');
     expect(envFileIdx).toBeGreaterThanOrEqual(0);
+    expect(guardRecoveryIdx).toBeGreaterThanOrEqual(0);
     expect(proxyEnvIdx).toBeGreaterThanOrEqual(0);
+    expect(bashrcIdx).toBeGreaterThanOrEqual(0);
     expect(runtimeIdx).toBeGreaterThanOrEqual(0);
     expect(launchIdx).toBeGreaterThanOrEqual(0);
     expect(envFileIdx).toBeLessThan(proxyEnvIdx);
+    expect(guardRecoveryIdx).toBeLessThan(proxyEnvIdx);
     expect(proxyEnvIdx).toBeLessThan(runtimeIdx);
+    expect(proxyEnvIdx).toBeLessThan(bashrcIdx);
+    expect(bashrcIdx).toBeLessThan(runtimeIdx);
     expect(runtimeIdx).toBeLessThan(launchIdx);
+    expect(script).not.toContain("if [ -r /tmp/nemoclaw-proxy-env.sh ]; then .");
     expect(script).toContain("SECRET_BOUNDARY_REFUSED");
   });
 

src/lib/agent/runtime-recovery-preload.test.ts

@@ -38,6 +38,7 @@ function makeHarness() {
     tmpSafetyNet: path.join(workDir, "nemoclaw-sandbox-safety-net.js"),
     tmpCiao: path.join(workDir, "nemoclaw-ciao-network-guard.js"),
     proxyEnv: path.join(workDir, "nemoclaw-proxy-env.sh"),
+    recoverySourceEnv: path.join(workDir, "nemoclaw-recovered-proxy-env.sh"),
     gatewayLog: path.join(workDir, "gateway.log"),
     hostileMarker: path.join(root, "hostile-proxy-env-sourced"),
   };
@@ -54,11 +55,12 @@ function rewriteRuntimePaths(script: string, paths: ReturnType<typeof makeHarnes
     .replaceAll(SAFETY_NET_GUARD.sourcePath, paths.sourceSafetyNet)
     .replaceAll(CIAO_GUARD.tmpPath, paths.tmpCiao)
     .replaceAll(CIAO_GUARD.sourcePath, paths.sourceCiao)
-    .replaceAll("/tmp/nemoclaw-proxy-env.sh", paths.proxyEnv);
+    .replaceAll("/tmp/nemoclaw-proxy-env.sh", paths.proxyEnv)
+    .replaceAll("/tmp/nemoclaw-recovered-proxy-env.sh", paths.recoverySourceEnv);
 }
 
 function runGuardRecovery(opts: {
-  proxyEnvContent?: string;
+  proxyEnvContent?: string | ((paths: ReturnType<typeof makeHarness>) => string);
   beforeScript?: (paths: ReturnType<typeof makeHarness>) => void;
   fakeRoot?: boolean;
   shell?: "bash" | "sh";
@@ -79,8 +81,10 @@ function runGuardRecovery(opts: {
   }
   opts.beforeScript?.(paths);
 
-  const proxyEnvContent = opts.proxyEnvContent
-    ? rewriteRuntimePaths(opts.proxyEnvContent, paths)
+  const rawProxyEnvContent =
+    typeof opts.proxyEnvContent === "function" ? opts.proxyEnvContent(paths) : opts.proxyEnvContent;
+  const proxyEnvContent = rawProxyEnvContent
+    ? rewriteRuntimePaths(rawProxyEnvContent, paths)
     : undefined;
   const writeProxyEnv = proxyEnvContent
     ? [
@@ -142,6 +146,7 @@ function runGuardRecovery(opts: {
       files: {
         gatewayLog: readIfExists(paths.gatewayLog) ?? "",
         proxyEnv: readIfExists(paths.proxyEnv),
+        recoverySourceEnv: readIfExists(paths.recoverySourceEnv),
         tmpSafetyNet: readIfExists(paths.tmpSafetyNet),
         tmpCiao: readIfExists(paths.tmpCiao),
         tmpSafetyNetMode: modeIfExists(paths.tmpSafetyNet),
@@ -208,7 +213,81 @@ describe("gateway recovery preload repair", () => {
     expect(nodeOptions.match(new RegExp(result.paths.tmpCiao, "g"))?.length).toBe(1);
   });
 
-  it("persists repairs for a metadata-safe proxy-env.sh that is missing one guard", () => {
+  it("rebuilds a metadata-safe proxy-env.sh without sourcing shell content", () => {
+    const result = runGuardRecovery({
+      proxyEnvContent: (paths) =>
+        [
+          `touch ${JSON.stringify(paths.hostileMarker)}`,
+          `export NODE_OPTIONS="--require ${SAFETY_NET_GUARD.tmpPath} --require=${CIAO_GUARD.tmpPath}"`,
+          "",
+        ].join("\n"),
+    });
+    expect(result.status).toBe(0);
+    expect(result.files.hostileProxyEnvSourced).toBe(false);
+    expect(result.files.proxyEnv).not.toContain("touch");
+    expect(result.files.proxyEnv).toContain(`--require ${result.paths.tmpSafetyNet}`);
+    expect(result.files.proxyEnv).toContain(`--require ${result.paths.tmpCiao}`);
+  });
+
+  it("preserves a trusted full proxy-env.sh while sourcing only a generated recovery copy", () => {
+    const result = runGuardRecovery({
+      fakeRoot: true,
+      proxyEnvContent: [
+        "# Proxy configuration (overrides narrow OpenShell defaults on connect)",
+        'export OPENCLAW_GATEWAY_URL="ws://127.0.0.1:18789"',
+        "export OPENCLAW_GATEWAY_TOKEN='trusted-token'",
+        "# nemoclaw-configure-guard begin",
+        "openclaw() {",
+        '  command openclaw "$@"',
+        "}",
+        "# nemoclaw-configure-guard end",
+        `export NODE_OPTIONS="\${NODE_OPTIONS:+$NODE_OPTIONS }--require ${SAFETY_NET_GUARD.tmpPath}"`,
+        'export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--require /tmp/nemoclaw-http-proxy-fix.js"',
+        `export NODE_OPTIONS="\${NODE_OPTIONS:+$NODE_OPTIONS }--require ${CIAO_GUARD.tmpPath}"`,
+        "# Tool cache redirects — keep transient tool state under /tmp",
+        "export npm_config_cache=/tmp/.npm-cache",
+        "",
+      ].join("\n"),
+    });
+    expect(result.status).toBe(0);
+    expect(result.files.proxyEnv).toContain("OPENCLAW_GATEWAY_TOKEN='trusted-token'");
+    expect(result.files.proxyEnv).toContain("openclaw() {");
+    expect(result.files.proxyEnv).toContain("/tmp/nemoclaw-http-proxy-fix.js");
+    expect(result.files.recoverySourceEnv).toContain("OPENCLAW_GATEWAY_TOKEN='trusted-token'");
+    expect(result.files.recoverySourceEnv).toContain("openclaw() {");
+    expect(result.files.recoverySourceEnv).toContain("/tmp/nemoclaw-http-proxy-fix.js");
+    const nodeOptions = result.stdout.match(/^NODE_OPTIONS=(.*)$/m)?.[1] ?? "";
+    expect(nodeOptions.match(new RegExp(result.paths.tmpSafetyNet, "g"))?.length).toBe(1);
+    expect(nodeOptions.match(new RegExp(result.paths.tmpCiao, "g"))?.length).toBe(1);
+  });
+
+  it("repairs an incomplete trusted proxy-env.sh without dropping full runtime entries", () => {
+    const result = runGuardRecovery({
+      fakeRoot: true,
+      proxyEnvContent: [
+        "# Proxy configuration (overrides narrow OpenShell defaults on connect)",
+        'export OPENCLAW_GATEWAY_URL="ws://127.0.0.1:18789"',
+        "export OPENCLAW_GATEWAY_TOKEN='trusted-token'",
+        "# nemoclaw-configure-guard begin",
+        "openclaw() {",
+        '  command openclaw "$@"',
+        "}",
+        "# nemoclaw-configure-guard end",
+        `export NODE_OPTIONS="\${NODE_OPTIONS:+$NODE_OPTIONS }--require ${SAFETY_NET_GUARD.tmpPath}"`,
+        'export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--require /tmp/nemoclaw-http-proxy-fix.js"',
+        "export npm_config_cache=/tmp/.npm-cache",
+        "",
+      ].join("\n"),
+    });
+    expect(result.status).toBe(0);
+    expect(result.files.proxyEnv).toContain("OPENCLAW_GATEWAY_TOKEN='trusted-token'");
+    expect(result.files.proxyEnv).toContain("openclaw() {");
+    expect(result.files.proxyEnv).toContain("/tmp/nemoclaw-http-proxy-fix.js");
+    expect(result.files.proxyEnv).toContain(`--require ${result.paths.tmpCiao}`);
+    expect(result.files.gatewayLog).toContain("proxy-env.sh incomplete");
+  });
+
+  it("rewrites a non-root metadata-safe proxy-env.sh that is missing one guard", () => {
     const result = runGuardRecovery({
       proxyEnvContent: `export NODE_OPTIONS="--require ${SAFETY_NET_GUARD.tmpPath}"\n`,
     });
@@ -218,7 +297,7 @@ describe("gateway recovery preload repair", () => {
     expect(result.files.proxyEnvMode).toBe(0o444);
     expect(result.files.proxyEnv).toContain(`--require ${result.paths.tmpSafetyNet}`);
     expect(result.files.proxyEnv).toContain(`--require ${result.paths.tmpCiao}`);
-    expect(result.files.gatewayLog).toContain("proxy-env.sh incomplete");
+    expect(result.files.gatewayLog).toContain("proxy-env.sh missing or unsafe");
   });
 
   it("rebuilds an unsafe proxy-env.sh without sourcing attacker-controlled content", () => {
@@ -297,6 +376,18 @@ describe("gateway recovery preload repair", () => {
     expect(result.files.gatewayLog).toContain("refusing preload install");
   });
 
+  it("refuses recovery when a trusted packaged preload source is group writable", () => {
+    const result = runGuardRecovery({
+      beforeScript(paths) {
+        fs.chmodSync(paths.sourceCiao, 0o664);
+      },
+    });
+    expect(result.status).toBe(17);
+    expect(result.stdout).toContain("GUARDS_MISSING");
+    expect(result.files.gatewayLog).toContain("trusted preload source");
+    expect(result.files.gatewayLog).toContain("unsafe mode=664");
+  });
+
   it("refuses recovery when a trusted packaged preload source is group writable in root mode", () => {
     const result = runGuardRecovery({
       fakeRoot: true,
@@ -330,7 +421,7 @@ describe("gateway recovery preload repair", () => {
       const validateIdx = script!.indexOf(
         "_nemoclaw_validate_recovery_proxy_env /tmp/nemoclaw-proxy-env.sh",
       );
-      const sourceIdx = script!.indexOf("then . /tmp/nemoclaw-proxy-env.sh");
+      const sourceIdx = script!.indexOf('then . "$_NEMOCLAW_RECOVERY_SOURCE_ENV"');
       expect(validateIdx).toBeGreaterThanOrEqual(0);
       expect(sourceIdx).toBeGreaterThanOrEqual(0);
       expect(validateIdx).toBeLessThan(sourceIdx);