fix(e2e): stabilize nightly recovery coverage

.github/workflows/nightly-e2e.yaml

@@ -26,7 +26,7 @@
 #                            through inference.local with a hermetic local mock (#2766).
 #   kimi-inference-compat-e2e
 #                            Validates Kimi K2.6 safe exec splitting through OpenClaw trajectories
-#                            with a hermetic OpenAI-compatible mock (#2620).
+#                            against public NVIDIA Endpoints, with a mock fallback (#2620).
 #   bedrock-runtime-compatible-anthropic-e2e
 #                            Validates the silent Bedrock Runtime custom Anthropic endpoint path
 #                            through a hermetic fake Bedrock Runtime host for OpenClaw and Hermes.
@@ -855,12 +855,42 @@ jobs:
 
       - name: Run Kimi inference compatibility E2E test
         env:
+          # Kimi uses the public NVIDIA Endpoints key intentionally. The script
+          # validates this nvapi-* key, then mirrors it only inside the process
+          # for the shared onboarding/provider-registration path.
+          NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }}
+          NEMOCLAW_PROVIDER: cloud
+          NEMOCLAW_MODEL: moonshotai/kimi-k2.6
+          NEMOCLAW_PREFERRED_API: openai-completions
+          NEMOCLAW_KIMI_USE_MOCK: "0"
           NEMOCLAW_NON_INTERACTIVE: "1"
           NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
           NEMOCLAW_SANDBOX_NAME: "e2e-kimi-compat"
           GITHUB_TOKEN: ${{ github.token }}
         run: bash test/e2e/test-kimi-inference-compat.sh
 
+      - name: Sanitize Kimi logs on failure
+        if: failure()
+        shell: bash
+        env:
+          NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          for file in \
+            /tmp/nemoclaw-e2e-kimi-inference-compat-onboard.log \
+            /tmp/nemoclaw-e2e-kimi-inference-compat-build.log \
+            /tmp/nemoclaw-e2e-kimi-inference-compat-agent.log; do
+            [ -f "$file" ] || continue
+            if [ -n "${NVIDIA_API_KEY:-}" ]; then
+              perl -0pi -e 's/\Q$ENV{NVIDIA_API_KEY}\E/[REDACTED_NVIDIA_API_KEY]/g' "$file"
+            fi
+            if [ -n "${GITHUB_TOKEN:-}" ]; then
+              perl -0pi -e 's/\Q$ENV{GITHUB_TOKEN}\E/[REDACTED_GITHUB_TOKEN]/g' "$file"
+            fi
+            perl -0pi -e 's/nvapi-[A-Za-z0-9._-]+/[REDACTED_NVIDIA_API_KEY]/g; s/gh[pousr]_[A-Za-z0-9_]+/[REDACTED_GITHUB_TOKEN]/g' "$file"
+          done
+
       - name: Upload onboard log on failure
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

ci/test-file-size-budget.json

@@ -8,7 +8,7 @@
     "test/channels-add-preset.test.ts": 1871,
     "test/generate-openclaw-config.test.ts": 1989,
     "test/install-preflight.test.ts": 4207,
-    "test/nemoclaw-start.test.ts": 5231,
+    "test/nemoclaw-start.test.ts": 5230,
     "test/onboard-messaging.test.ts": 2063,
     "test/onboard-selection.test.ts": 6891,
     "test/onboard.test.ts": 4774,

nemoclaw-blueprint/openclaw-plugins/kimi-inference-compat/index.js

@@ -9,11 +9,19 @@ function normalizeBaseUrl(value) {
   return String(value || "").trim().replace(/\/+$/, "");
 }
 
+const KIMI_K26_MODEL_ID = "moonshotai/kimi-k2.6";
+const MANAGED_KIMI_K26_MODEL_REF = `inference/${KIMI_K26_MODEL_ID}`;
+
+function isKimiModelId(value) {
+  const modelId = normalize(value);
+  return modelId === KIMI_K26_MODEL_ID || modelId === MANAGED_KIMI_K26_MODEL_REF;
+}
+
 function isManagedKimi(ctx) {
   const model = ctx && ctx.model ? ctx.model : {};
   return (
     normalize(ctx && ctx.provider) === "inference" &&
-    normalize(ctx && ctx.modelId) === "moonshotai/kimi-k2.6" &&
+    [ctx && ctx.modelId, model.id, model.name].some(isKimiModelId) &&
     normalize((ctx && ctx.modelApi) || model.api) === "openai-completions" &&
     normalizeBaseUrl(model.baseUrl) === "https://inference.local/v1"
   );

scripts/lib/openclaw_device_approval_policy.py

@@ -3,7 +3,10 @@
 
 """Shared OpenClaw device approval policy for NemoClaw sandbox helpers."""
 
+import json
 import os
+import re
+from pathlib import Path
 
 
 ALLOWED_CLIENTS = {"openclaw-control-ui"}
@@ -73,3 +76,155 @@ def gateway_approval_env(source_env=None):
     for key in GATEWAY_APPROVAL_ENV_KEYS:
         env.pop(key, None)
     return env
+
+
+def _norm(value):
+    return str(value or "").strip()
+
+
+def _scope_set(entry, key="scopes"):
+    if not isinstance(entry, dict):
+        return set()
+    return {_norm(scope) for scope in (entry.get(key) or []) if _norm(scope)}
+
+
+def _load_device_state(devices_dir, name):
+    try:
+        value = json.loads((devices_dir / name).read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+    return value if isinstance(value, dict) else {}
+
+
+def _save_device_state(devices_dir, name, value):
+    path = devices_dir / name
+    tmp = path.with_name(f".{path.name}.tmp")
+    with tmp.open("w", encoding="utf-8") as handle:
+        handle.write(json.dumps(value, indent=2, sort_keys=True) + "\n")
+        handle.flush()
+        os.fsync(handle.fileno())
+    os.replace(tmp, path)
+
+
+def _output_mentions_request_id(output, request_id):
+    request = _norm(request_id)
+    if not request:
+        return False
+    return bool(re.search(r"(?<![0-9A-Za-z_-])" + re.escape(request) + r"(?![0-9A-Za-z_-])", output or ""))
+
+
+def _is_scope_upgrade_approval_compat_failure(output):
+    text = _norm(output).lower()
+    return "scope upgrade pending approval" in text and (
+        "gatewayclientrequesterror" in text or "gateway" in text
+    )
+
+
+def recover_failed_scope_approval(request_id, state_dir=None, approve_output="", original_request=None):
+    """Repair a narrow OpenClaw 2026.5.x nonzero scope-upgrade approval state.
+
+    OpenClaw can apply, replace, or leave behind an allowlisted CLI/webchat
+    operator.write upgrade while returning a gateway-connect failure to the
+    caller. This helper only edits local OpenClaw device state when the pending
+    request and paired device are already present, the approve output matches
+    the known gateway scope-upgrade failure signature, the requested scopes are
+    limited to NemoClaw's allowlist, and the device already has
+    operator.pairing. It never grants operator.admin.
+    """
+
+    request_id = _norm(request_id)
+    if not request_id:
+        return None
+    devices_dir = Path(state_dir or os.environ.get("OPENCLAW_STATE_DIR") or "/sandbox/.openclaw") / "devices"
+    pending = _load_device_state(devices_dir, "pending.json")
+    paired = _load_device_state(devices_dir, "paired.json")
+
+    original_key = None
+    original = original_request if isinstance(original_request, dict) else None
+    for key, item in pending.items():
+        if isinstance(item, dict) and _norm(item.get("requestId")) == request_id:
+            original_key = key
+            original = item
+            break
+    if not isinstance(original, dict):
+        return None
+
+    requested = _scope_set(original) or _scope_set(original, "requestedScopes")
+    device_id = _norm(original.get("deviceId"))
+    paired_entry = paired.get(device_id) if device_id else None
+    paired_scopes = _scope_set(paired_entry or {}, "approvedScopes") | _scope_set(paired_entry or {})
+    allowed = {"operator.pairing", "operator.read", "operator.write"}
+    if (
+        not device_id
+        or not requested
+        or not requested.issubset(allowed)
+        or "operator.pairing" not in paired_scopes
+        or not isinstance(paired_entry, dict)
+    ):
+        return None
+
+    still_pending = original_key is not None
+    if not still_pending and requested.issubset(paired_scopes):
+        return {
+            "requestId": request_id,
+            "deviceId": device_id,
+            "approvedScopes": sorted(requested),
+            "compatibility": "openclaw-approve-applied-after-nonzero",
+        }
+
+    replacement_allowed = allowed | {"operator.admin"}
+    candidates = []
+    mentioned = []
+    for key, item in pending.items():
+        item_scopes = _scope_set(item) if isinstance(item, dict) else set()
+        if (
+            isinstance(item, dict)
+            and _norm(item.get("requestId")) != request_id
+            and _norm(item.get("deviceId")) == device_id
+            and "operator.admin" in item_scopes
+            and requested.issubset(item_scopes)
+            and item_scopes.issubset(replacement_allowed)
+        ):
+            candidates.append((key, item))
+            if _output_mentions_request_id(approve_output, item.get("requestId")):
+                mentioned.append((key, item))
+
+    recovery_key = None
+    compatibility = None
+    if len(mentioned) == 1:
+        recovery_key = mentioned[0][0]
+        compatibility = "openclaw-approve-recovered-replacement"
+    elif len(candidates) == 1 and not re.search(r"\brequestId\b|\brequest[-_ ]?id\b", approve_output or "", re.IGNORECASE):
+        recovery_key = candidates[0][0]
+        compatibility = "openclaw-approve-recovered-replacement"
+    elif still_pending and not candidates and _is_scope_upgrade_approval_compat_failure(approve_output):
+        recovery_key = original_key
+        compatibility = "openclaw-approve-recovered-original"
+    else:
+        return None
+
+    approved = set(paired_scopes) | requested
+    if "operator.write" in approved:
+        approved.add("operator.read")
+    if {"operator.read", "operator.write"} & approved:
+        approved.add("operator.pairing")
+    if not approved.issubset(allowed):
+        return None
+    approved_list = [scope for scope in ("operator.pairing", "operator.read", "operator.write") if scope in approved]
+    paired_entry["scopes"] = approved_list
+    paired_entry["approvedScopes"] = approved_list
+    token = paired_entry.get("tokens", {}).get("operator")
+    if isinstance(token, dict):
+        token["scopes"] = approved_list
+    pending.pop(request_id, None)
+    if recovery_key:
+        pending.pop(recovery_key, None)
+    paired[device_id] = paired_entry
+    _save_device_state(devices_dir, "pending.json", pending)
+    _save_device_state(devices_dir, "paired.json", paired)
+    return {
+        "requestId": request_id,
+        "deviceId": device_id,
+        "approvedScopes": approved_list,
+        "compatibility": compatibility,
+    }

scripts/nemoclaw-start.sh

@@ -1879,10 +1879,14 @@ def load_approval_policy(path):
         raise RuntimeError('approval policy helper could not be loaded')
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)
-    return module.approval_request_decision, module.gateway_approval_env
+    return (
+        module.approval_request_decision,
+        module.gateway_approval_env,
+        getattr(module, 'recover_failed_scope_approval', None),
+    )
 
 
-approval_request_decision, gateway_approval_env = load_approval_policy(APPROVAL_POLICY_FILE)
+approval_request_decision, gateway_approval_env, recover_failed_scope_approval = load_approval_policy(APPROVAL_POLICY_FILE)
 
 OPENCLAW = os.environ.get('OPENCLAW_BIN', 'openclaw')
 
@@ -2011,6 +2015,19 @@ while time.time() < DEADLINE:
                 HANDLED.add(request_id)
                 APPROVED += 1
                 print(f'[auto-pair] approved request={request_id} client={client_id} mode={client_mode}')
+            elif callable(recover_failed_scope_approval):
+                recovered = recover_failed_scope_approval(
+                    request_id,
+                    os.environ.get('OPENCLAW_STATE_DIR') or '/sandbox/.openclaw',
+                    aerr or aout or '',
+                    device,
+                )
+                if recovered:
+                    HANDLED.add(request_id)
+                    APPROVED += 1
+                    print(f'[auto-pair] recovered failed approve request={request_id} client={client_id} mode={client_mode}')
+                elif aout or aerr:
+                    print(f'[auto-pair] approve failed request={request_id}: {(aerr or aout)[:400]}')
             elif aout or aerr:
                 print(f'[auto-pair] approve failed request={request_id}: {(aerr or aout)[:400]}')
         time.sleep(SLOW_INTERVAL if SLOW_MODE else 1)
@@ -2400,11 +2417,22 @@ def output_mentions_request_id(value):
     request = norm(value)
     return bool(request and re.search(r"(?<![0-9A-Za-z_-])" + re.escape(request) + r"(?![0-9A-Za-z_-])", approve_output))
 
+def is_scope_upgrade_approval_compat_failure(output):
+    text = norm(output).lower()
+    return "scope upgrade pending approval" in text and (
+        "gatewayclientrequesterror" in text or "gateway" in text
+    )
+
 requested = scope_set(before)
 device_id = norm(before.get("deviceId"))
 pending = load("pending.json")
 paired = load("paired.json")
-still_pending = any(isinstance(item, dict) and item.get("requestId") == request_id for item in pending.values())
+original_pending_key = None
+for key, item in pending.items():
+    if isinstance(item, dict) and item.get("requestId") == request_id:
+        original_pending_key = key
+        break
+still_pending = original_pending_key is not None
 paired_entry = paired.get(device_id) if device_id else None
 paired_scopes = scope_set(paired_entry or {}, "approvedScopes") | scope_set(paired_entry or {})
 # Compatibility boundary: treat a nonzero approve as success only when OpenClaw
@@ -2421,7 +2449,7 @@ if request_id and requested and not still_pending and isinstance(paired_entry, d
 # replacing operator.write approvals with admin-shaped pending requests or
 # exposes a supported approval repair API.
 allowed = {"operator.pairing", "operator.read", "operator.write"}
-if not request_id or not device_id or not requested or not requested.issubset(allowed) or "operator.pairing" not in paired_scopes or still_pending:
+if not request_id or not device_id or not requested or not requested.issubset(allowed) or "operator.pairing" not in paired_scopes:
     raise SystemExit(1)
 replacement_allowed = allowed | {"operator.admin"}
 candidates = []
@@ -2433,10 +2461,14 @@ for key, item in pending.items():
         candidates.append((key, item))
         if output_mentions_request_id(item.get("requestId")):
             mentioned.append((key, item))
+compatibility = "openclaw-approve-recovered-replacement"
 if len(mentioned) == 1:
     replacement_key, replacement = mentioned[0]
 elif len(candidates) == 1 and not re.search(r"\brequestId\b|\brequest[-_ ]?id\b", approve_output, re.IGNORECASE):
     replacement_key, replacement = candidates[0]
+elif still_pending and not candidates and is_scope_upgrade_approval_compat_failure(approve_output):
+    replacement_key = original_pending_key
+    compatibility = "openclaw-approve-recovered-original"
 else:
     raise SystemExit(1)
 approved = set(paired_scopes) | requested
@@ -2457,7 +2489,7 @@ pending.pop(replacement_key, None)
 paired[device_id] = paired_entry
 save("pending.json", pending)
 save("paired.json", paired)
-print(json.dumps({"requestId": request_id, "deviceId": device_id, "approvedScopes": approved_list, "compatibility": "openclaw-approve-recovered-replacement"}, sort_keys=True))
+print(json.dumps({"requestId": request_id, "deviceId": device_id, "approvedScopes": approved_list, "compatibility": compatibility}, sort_keys=True))
 raise SystemExit(0)
 PYAPPROVEAFTER
         return 0