fix(inference): retry ollama validation probes and docker runtime detection

src/lib/adapters/docker/index.ts

@@ -5,6 +5,7 @@ export * from "./run";
 export * from "./exec";
 export * from "./pull";
 export * from "./info";
+export * from "./runtime";
 export * from "./inspect";
 export * from "./image";
 export * from "./container";

src/lib/adapters/docker/runtime.test.ts

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("../../runner", () => ({
+  ROOT: "/repo/root",
+  run: vi.fn(),
+  runCapture: vi.fn(),
+}));
+
+import {
+  DOCKER_INFO_RUNTIME_PROBE_ATTEMPTS,
+  DOCKER_INFO_RUNTIME_PROBE_TIMEOUT_MS,
+  detectContainerRuntimeFromDockerInfo,
+} from "./runtime";
+
+describe("docker runtime detection", () => {
+  it("retries indeterminate docker info output before returning a runtime", () => {
+    const calls: unknown[] = [];
+    const outputs = ["", "", "Operating System: Docker Desktop"];
+
+    const runtime = detectContainerRuntimeFromDockerInfo({
+      dockerInfoImpl: (opts) => {
+        calls.push(opts);
+        return outputs.shift() ?? "";
+      },
+    });
+
+    expect(runtime).toBe("docker-desktop");
+    expect(calls).toHaveLength(DOCKER_INFO_RUNTIME_PROBE_ATTEMPTS);
+    expect(calls).toEqual(
+      Array.from({ length: DOCKER_INFO_RUNTIME_PROBE_ATTEMPTS }, () => ({
+        ignoreError: true,
+        timeout: DOCKER_INFO_RUNTIME_PROBE_TIMEOUT_MS,
+      })),
+    );
+  });
+
+  it("returns unknown after all attempts are indeterminate", () => {
+    const calls: unknown[] = [];
+
+    const runtime = detectContainerRuntimeFromDockerInfo({
+      attempts: 2,
+      dockerInfoImpl: (opts) => {
+        calls.push(opts);
+        return "";
+      },
+      timeoutMs: 1234,
+    });
+
+    expect(runtime).toBe("unknown");
+    expect(calls).toEqual([
+      { ignoreError: true, timeout: 1234 },
+      { ignoreError: true, timeout: 1234 },
+    ]);
+  });
+});

src/lib/adapters/docker/runtime.ts

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import type { ContainerRuntime } from "../../platform";
+import { inferContainerRuntime } from "../../platform";
+import { dockerInfo } from "./info";
+import type { DockerCaptureOptions } from "./run";
+
+export const DOCKER_INFO_RUNTIME_PROBE_ATTEMPTS = 3;
+export const DOCKER_INFO_RUNTIME_PROBE_TIMEOUT_MS = 5000;
+
+type DockerInfoProbe = (opts: DockerCaptureOptions) => string;
+
+export interface DetectContainerRuntimeOptions {
+  attempts?: number;
+  dockerInfoImpl?: DockerInfoProbe;
+  timeoutMs?: number;
+}
+
+export function detectContainerRuntimeFromDockerInfo(
+  opts: DetectContainerRuntimeOptions = {},
+): ContainerRuntime {
+  const attempts = Math.max(1, Math.floor(opts.attempts ?? DOCKER_INFO_RUNTIME_PROBE_ATTEMPTS));
+  const timeout = Math.max(1, Math.floor(opts.timeoutMs ?? DOCKER_INFO_RUNTIME_PROBE_TIMEOUT_MS));
+  const probe = opts.dockerInfoImpl ?? dockerInfo;
+
+  for (let attempt = 0; attempt < attempts; attempt++) {
+    const runtime = inferContainerRuntime(probe({ ignoreError: true, timeout }));
+    if (runtime !== "unknown") return runtime;
+  }
+
+  return "unknown";
+}

src/lib/adapters/http/probe.test.ts

@@ -186,6 +186,30 @@ describe("http-probe helpers", () => {
     expect(result.message).toContain("curl failed");
     expect(result.stderr).toContain("spawn ENOENT");
   });
+
+  it("reports spawnSync ETIMEDOUT as a timeout status", () => {
+    const result = runCurlProbe(["-sS", "https://example.test/models"], {
+      spawnSyncImpl: () => {
+        const error = Object.assign(new Error("spawnSync curl ETIMEDOUT"), {
+          code: "ETIMEDOUT",
+          errno: -60,
+        });
+        return {
+          pid: 1,
+          output: [],
+          stdout: "",
+          stderr: "",
+          status: null,
+          signal: null,
+          error,
+        };
+      },
+    });
+
+    expect(result.ok).toBe(false);
+    expect(result.curlStatus).toBe(-110);
+    expect(result.message).toContain("ETIMEDOUT");
+  });
 });
 
 describe("runChatCompletionsStreamingProbe", () => {
@@ -237,6 +261,33 @@ describe("runChatCompletionsStreamingProbe", () => {
     expect(result.curlStatus).toBe(28);
   });
 
+  it("reports chat streaming spawnSync ETIMEDOUT as a timeout status", () => {
+    const result = runChatCompletionsStreamingProbe(
+      ["-sS", "--max-time", "120", "https://example.test/v1/chat/completions"],
+      {
+        spawnSyncImpl: () => {
+          const error = Object.assign(new Error("spawnSync curl ETIMEDOUT"), {
+            code: "ETIMEDOUT",
+            errno: -60,
+          });
+          return {
+            pid: 1,
+            output: [],
+            stdout: "",
+            stderr: "",
+            status: null,
+            signal: null,
+            error,
+          };
+        },
+      },
+    );
+
+    expect(result.ok).toBe(false);
+    expect(result.curlStatus).toBe(-110);
+    expect(result.message).toContain("ETIMEDOUT");
+  });
+
   it("does not treat a lone DONE frame as successful streaming data", () => {
     const result = runChatCompletionsStreamingProbe(
       ["-sS", "--max-time", "120", "https://example.test/v1/chat/completions"],
@@ -390,6 +441,39 @@ describe("runStreamingEventProbe", () => {
     expect(result.message).toContain("Streaming probe failed");
   });
 
+  it("records normalized timeout status for responses streaming spawnSync ETIMEDOUT", () => {
+    withTraceFile((traceFile) => {
+      const result = runStreamingEventProbe(["-sS", "https://example.test/v1/responses"], {
+        spawnSyncImpl: () => {
+          const error = Object.assign(new Error("spawnSync curl ETIMEDOUT"), {
+            code: "ETIMEDOUT",
+            errno: -60,
+          });
+          return {
+            pid: 1,
+            output: [],
+            stdout: "",
+            stderr: "",
+            status: null,
+            signal: null,
+            error,
+          };
+        },
+      });
+
+      expect(result.ok).toBe(false);
+      flushTrace();
+      const artifact = JSON.parse(fs.readFileSync(traceFile, "utf8")) as TraceArtifact;
+      const span = artifact.resource_spans[0].scope_spans[0].spans.find(
+        (entry) => entry.name === "nemoclaw.inference.curl_streaming_event_probe",
+      );
+      expect(span?.events[0].attributes).toMatchObject({
+        ok: false,
+        curl_status: -110,
+      });
+    });
+  });
+
   it("cleans up temp files after probe", () => {
     let outputPath = "";
     runStreamingEventProbe(["-sS", "--max-time", "15", "https://example.test/v1/responses"], {

src/lib/adapters/http/probe.ts

@@ -103,6 +103,14 @@ function resolveCurlProcessTimeoutMs(argv: string[], opts: CurlProbeOptions): nu
   );
 }
 
+function normalizeSpawnErrorCode(error: unknown): number {
+  if (isErrnoException(error) && error.code === "ETIMEDOUT") return -110;
+  const rawErrorCode = isErrnoException(error)
+    ? (error.errno ?? error.code)
+    : undefined;
+  return typeof rawErrorCode === "number" ? rawErrorCode : 1;
+}
+
 function sanitizeCurlUrl(value: string): string {
   try {
     const url = new URL(value);
@@ -220,10 +228,7 @@ function runCurlProbeImpl(argv: string[], opts: CurlProbeOptions = {}): CurlProb
     );
     const body = fs.existsSync(bodyFile) ? fs.readFileSync(bodyFile, "utf8") : "";
     if (result.error) {
-      const rawErrorCode = isErrnoException(result.error)
-        ? (result.error.errno ?? result.error.code)
-        : undefined;
-      const errorCode = typeof rawErrorCode === "number" ? rawErrorCode : 1;
+      const errorCode = normalizeSpawnErrorCode(result.error);
       const errorMessage = compactText(
         `${result.error.message || String(result.error)} ${String(result.stderr || "")}`,
       );
@@ -335,10 +340,7 @@ function runChatCompletionsStreamingProbeImpl(
 
     const body = fs.existsSync(bodyFile) ? fs.readFileSync(bodyFile, "utf8") : "";
     if (result.error) {
-      const rawErrorCode = isErrnoException(result.error)
-        ? (result.error.errno ?? result.error.code)
-        : undefined;
-      const errorCode = typeof rawErrorCode === "number" ? rawErrorCode : 1;
+      const errorCode = normalizeSpawnErrorCode(result.error);
       const errorMessage = compactText(
         `${result.error.message || String(result.error)} ${String(result.stderr || "")}`,
       );
@@ -457,13 +459,14 @@ function runStreamingEventProbeImpl(
     if (result.error || (result.status !== null && result.status !== 0 && result.status !== 28)) {
       // curl exit 28 = timeout, which is expected — we cap with --max-time
       // and may still have collected enough events before the timeout.
+      const curlStatus = result.error ? normalizeSpawnErrorCode(result.error) : (result.status ?? 1);
       const detail = result.error
         ? String(result.error.message || result.error)
         : String(result.stderr || "");
       emitCurlResultTraceEvent({
         ok: false,
         missing_events_count: REQUIRED_STREAMING_EVENTS.length,
-        curl_status: result.status ?? 1,
+        curl_status: curlStatus,
       });
       return {
         ok: false,

src/lib/inference/local.ts

@@ -11,7 +11,6 @@ import os from "node:os";
 import nodePath from "node:path";
 import type { CurlProbeResult } from "../adapters/http/probe";
 import { runCurlProbe } from "../adapters/http/probe";
-import type { ContainerRuntime } from "../platform";
 import type { CaptureResult } from "../runner";
 import { buildSubprocessEnv } from "../subprocess-env";
 import {
@@ -39,12 +38,11 @@ import {
   SMALLEST_OLLAMA_MODEL_TAG,
 } from "./ollama-model-registry";
 
-const { containerCanReachHostLoopback, inferContainerRuntime, isWsl } = require("../platform");
-const { dockerInfo } = require("../adapters/docker/info");
+const { containerCanReachHostLoopback, isWsl } = require("../platform");
+const { detectContainerRuntimeFromDockerInfo } =
+  require("../adapters/docker/runtime") as typeof import("../adapters/docker/runtime");
 const { detectNvidiaPlatform } = require("./nim");
 
-const DOCKER_INFO_RUNTIME_PROBE_TIMEOUT_MS = 1500;
-
 /**
  * Port containers use to reach Ollama. Returns the raw Ollama port when the
  * container can reach the host's 127.0.0.1 directly (Docker Desktop on WSL),
@@ -54,9 +52,7 @@ const DOCKER_INFO_RUNTIME_PROBE_TIMEOUT_MS = 1500;
 let _ollamaContainerPort: number | null = null;
 export function getOllamaContainerPort(): number {
   if (_ollamaContainerPort !== null) return _ollamaContainerPort;
-  const runtime = inferContainerRuntime(
-    dockerInfo({ ignoreError: true, timeout: DOCKER_INFO_RUNTIME_PROBE_TIMEOUT_MS }),
-  ) as ContainerRuntime;
+  const runtime = detectContainerRuntimeFromDockerInfo();
   _ollamaContainerPort = containerCanReachHostLoopback(runtime) ? OLLAMA_PORT : OLLAMA_PROXY_PORT;
   return _ollamaContainerPort;
 }