feat(webxr): XR input recording, replay, and hand/controller path trace

deps/cloudxr/webxr_client/src/RecorderComponent.tsx

@@ -0,0 +1,70 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * RecorderComponent.tsx - Null-rendering R3F component that drives the recorder each frame.
+ *
+ * Runs at priority -1001 so it executes before CloudXRComponent (-1000), ensuring
+ * beginFrame() advances the replay pointer before the CloudXR SDK reads the
+ * monkey-patched XRFrame methods.
+ */
+
+import { useFrame } from "@react-three/fiber";
+import { useRef } from "react";
+
+import type { XRInputRecorder } from "./xrInputRecorder";
+
+interface RecorderComponentProps {
+  recorder: XRInputRecorder;
+  /** Called approximately every 30 frames during recording with current frame count. */
+  onFrameRecord?: (count: number) => void;
+  /** True when the CloudXR session is in the Connected state — gates replay frame advancement. */
+  isConnected: boolean;
+}
+
+export function RecorderComponent({
+  recorder,
+  onFrameRecord,
+  isConnected,
+}: RecorderComponentProps) {
+  const tickRef = useRef(0);
+
+  useFrame((state) => {
+    const xrFrame = state.gl.xr.getFrame() as XRFrame | null;
+    if (!xrFrame) return;
+
+    // Only advance replay/record when the session is visible (avoids poisoning recordings
+    // with null poses on Quest sleep) AND CloudXR is Connected (avoids consuming replay
+    // frames during Connecting state before the server is ready).
+    const session = state.gl.xr.getSession() as XRSession | null;
+    const isVisible = session?.visibilityState === "visible";
+
+    recorder.setSceneRefSpace(state.gl.xr.getReferenceSpace());
+    recorder.beginFrame(xrFrame, isConnected && isVisible);
+
+    if (recorder.mode === "recording" && onFrameRecord) {
+      tickRef.current++;
+      if (tickRef.current % 30 === 0) {
+        onFrameRecord(recorder.recordedFrameCount);
+      }
+    } else {
+      tickRef.current = 0;
+    }
+  }, -1001);
+
+  return null;
+}

deps/cloudxr/webxr_client/src/RecorderContext.tsx

@@ -0,0 +1,176 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * RecorderContext.tsx - React context for XRInputRecorder state.
+ *
+ * Owns the recorder instance and the React-visible state (mode, saved
+ * recording, showTrace).  Heavy per-frame work (beginFrame) runs in
+ * RecorderComponent via useFrame; this context provides actions and state
+ * to descendant components.
+ */
+
+import React, {
+  createContext,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
+
+import { XRInputRecorder, type Recording } from "./xrInputRecorder";
+
+export interface RecorderContextValue {
+  recorder: XRInputRecorder;
+  mode: "idle" | "recording" | "replaying";
+  savedRecording: Recording | null;
+  recordedFrameCount: number;
+  startRecord: () => void;
+  stopRecord: () => void;
+  startReplay: () => void;
+  stopReplay: () => void;
+  onSaveRecording: () => void;
+  onLoadRecording: () => void;
+  setSavedRecording: (r: Recording) => void;
+  onFrameRecord: (count: number) => void;
+}
+
+const RecorderContext = createContext<RecorderContextValue | null>(null);
+
+export function RecorderProvider({ children }: { children: React.ReactNode }) {
+  const recorder = useMemo(() => new XRInputRecorder(), []);
+  const [mode, setMode] = useState<"idle" | "recording" | "replaying">("idle");
+  const [savedRecording, setSavedRecordingState] = useState<Recording | null>(
+    null,
+  );
+  const [recordedFrameCount, setRecordedFrameCount] = useState(0);
+  const fileInputRef = useRef<HTMLInputElement | null>(null);
+
+  const startRecord = useCallback(() => {
+    if (recorder.mode !== "idle") return;
+    recorder.startRecording();
+    setMode("recording");
+    setRecordedFrameCount(0);
+  }, [recorder]);
+
+  const stopRecord = useCallback(() => {
+    if (recorder.mode !== "recording") return;
+    recorder.stopRecording();
+    const rec = recorder.getRecording();
+    setSavedRecordingState(rec);
+    setRecordedFrameCount(rec.frames.length);
+    setMode("idle");
+  }, [recorder]);
+
+  const startReplay = useCallback(() => {
+    if (recorder.mode !== "idle" || !savedRecording) return;
+    recorder.startReplay(savedRecording, true);
+    setMode("replaying");
+  }, [recorder, savedRecording]);
+
+  const stopReplay = useCallback(() => {
+    if (recorder.mode !== "replaying") return;
+    recorder.stopReplay();
+    setMode("idle");
+  }, [recorder]);
+
+  const onSaveRecording = useCallback(() => {
+    if (!savedRecording) return;
+    const blob = new Blob([JSON.stringify(savedRecording)], {
+      type: "application/json",
+    });
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = "isaacteleop-input-recording.json";
+    a.click();
+    URL.revokeObjectURL(url);
+  }, [savedRecording]);
+
+  const setSavedRecording = useCallback((r: Recording) => {
+    setSavedRecordingState(r);
+    setRecordedFrameCount(r.frames.length);
+  }, []);
+
+  useEffect(() => {
+    const input = document.createElement("input");
+    input.type = "file";
+    input.accept = ".json,application/json";
+    input.style.display = "none";
+    input.addEventListener("change", (e: Event) => {
+      const file = (e.target as HTMLInputElement).files?.[0];
+      if (!file) return;
+      file
+        .text()
+        .then((text) => {
+          try {
+            const recording = XRInputRecorder.importJSON(text);
+            setSavedRecording(recording);
+          } catch (err) {
+            console.error("[Recorder] Failed to load recording:", err);
+          }
+        })
+        .catch((err) => {
+          console.error("[Recorder] Failed to read file:", err);
+        });
+      (e.target as HTMLInputElement).value = "";
+    });
+    document.body.appendChild(input);
+    fileInputRef.current = input;
+    return () => {
+      document.body.removeChild(input);
+      fileInputRef.current = null;
+    };
+  }, [setSavedRecording]);
+
+  const onLoadRecording = useCallback(() => {
+    fileInputRef.current?.click();
+  }, []);
+
+  const onFrameRecord = useCallback((count: number) => {
+    setRecordedFrameCount(count);
+  }, []);
+
+  const value: RecorderContextValue = {
+    recorder,
+    mode,
+    savedRecording,
+    recordedFrameCount,
+    startRecord,
+    stopRecord,
+    startReplay,
+    stopReplay,
+    onSaveRecording,
+    onLoadRecording,
+    setSavedRecording,
+    onFrameRecord,
+  };
+
+  return (
+    <RecorderContext.Provider value={value}>
+      {children}
+    </RecorderContext.Provider>
+  );
+}
+
+export function useRecorder(): RecorderContextValue {
+  const ctx = useContext(RecorderContext);
+  if (!ctx) throw new Error("useRecorder must be used inside RecorderProvider");
+  return ctx;
+}