diff --git a/deps/cloudxr/webxr_client/src/App.tsx b/deps/cloudxr/webxr_client/src/App.tsx index fdcc1cdb7..ea49a1fe0 100644 --- a/deps/cloudxr/webxr_client/src/App.tsx +++ b/deps/cloudxr/webxr_client/src/App.tsx @@ -30,57 +30,72 @@ * and disconnect when in XR mode. */ -import { checkCapabilities } from '@helpers/BrowserCapabilities'; -import { getDeviceProfile, resolveDeviceProfileId } from '@helpers/DeviceProfiles'; -import { loadIWERIfNeeded } from '@helpers/LoadIWER'; -import { overridePressureObserver } from '@helpers/overridePressureObserver'; -import { kPerformanceOptions } from '@helpers/PerformanceProfiles'; -import CloudXRComponent from '@helpers/react/CloudXRComponent'; -import { SimpleEnvironment } from '@helpers/react/SimpleEnvironment'; -import { getControlPanelPositionVector } from '@helpers/react/utils'; +import { checkCapabilities } from "@helpers/BrowserCapabilities"; import { - logImmersiveXRSessionToConsole -} from '@helpers/webxrModeDebugText'; -import { SuppressWebGLRendererWhenHeadless } from './SuppressWebGLRendererWhenHeadless'; + getDeviceProfile, + resolveDeviceProfileId, +} from "@helpers/DeviceProfiles"; +import { loadIWERIfNeeded } from "@helpers/LoadIWER"; +import { overridePressureObserver } from "@helpers/overridePressureObserver"; +import { kPerformanceOptions } from "@helpers/PerformanceProfiles"; +import CloudXRComponent from "@helpers/react/CloudXRComponent"; +import { SimpleEnvironment } from "@helpers/react/SimpleEnvironment"; +import { getControlPanelPositionVector } from "@helpers/react/utils"; +import { logImmersiveXRSessionToConsole } from "@helpers/webxrModeDebugText"; +import { SuppressWebGLRendererWhenHeadless } from "./SuppressWebGLRendererWhenHeadless"; import { DEFAULT_TELEOP_PATH, loadStoredTeleopPath, parseTeleopPathFromHash, saveStoredTeleopPath, -} from '@helpers/TeleopProjects'; -import * as CloudXR from '@nvidia/cloudxr'; -import { getResolutionValidationError } from '@nvidia/cloudxr'; -import { signal, computed } from '@preact/signals-react'; -import { Canvas } from '@react-three/fiber'; -import { setPreferredColorScheme } from '@react-three/uikit'; -import { XR, createXRStore, noEvents, PointerEvents, XROrigin, useXR } from '@react-three/xr'; -import type { XRDevice } from 'iwer'; -import { useState, useMemo, useEffect, useRef } from 'react'; - -import { v5 } from 'uuid'; -import { CloudXR2DUI, COUNTDOWN_STORAGE_KEY } from './CloudXR2DUI'; -import { readUrlParam } from './config/resolve'; -import CloudXR3DUI from './CloudXRUI'; -import { HeadsetControlChannel } from '@helpers/controlChannel'; +} from "@helpers/TeleopProjects"; +import * as CloudXR from "@nvidia/cloudxr"; +import { getResolutionValidationError } from "@nvidia/cloudxr"; +import { signal, computed } from "@preact/signals-react"; +import { Canvas } from "@react-three/fiber"; +import { setPreferredColorScheme } from "@react-three/uikit"; +import { + XR, + createXRStore, + noEvents, + PointerEvents, + XROrigin, + useXR, +} from "@react-three/xr"; +import type { XRDevice } from "iwer"; +import { useState, useMemo, useEffect, useRef } from "react"; + +import { v5 } from "uuid"; +import { CloudXR2DUI, COUNTDOWN_STORAGE_KEY } from "./CloudXR2DUI"; +import { readUrlParam } from "./config/resolve"; +import CloudXR3DUI from "./CloudXRUI"; +import { HeadsetControlChannel } from "@helpers/controlChannel"; +import { RecorderProvider, useRecorder } from "./RecorderContext"; +import { RecorderComponent } from "./RecorderComponent"; +import { TraceVisualization } from "./TraceVisualization"; // Performance metrics signals - raw numeric data, one per callback cadence. // Signals update their value without triggering React re-renders. // See: https://pmndrs.github.io/uikit/docs/advanced/performance const renderMetrics = signal<{ fps: number } | null>(null); -const streamingMetrics = signal<{ fps: number; latencyMs: number } | null>(null); +const streamingMetrics = signal<{ fps: number; latencyMs: number } | null>( + null, +); // Computed signals derive formatted text from raw data. // When renderMetrics.value changes, computed() automatically recalculates the text. // The @react-three/uikit Text component subscribes to these computed signals // and updates the displayed text directly in Three.js - bypassing React entirely. const renderFpsText = computed(() => - renderMetrics.value ? renderMetrics.value.fps.toFixed(1) : '-' + renderMetrics.value ? renderMetrics.value.fps.toFixed(1) : "-", ); const streamingFpsText = computed(() => - streamingMetrics.value ? streamingMetrics.value.fps.toFixed(1) : '-' + streamingMetrics.value ? streamingMetrics.value.fps.toFixed(1) : "-", ); const poseToRenderText = computed(() => - streamingMetrics.value ? `${streamingMetrics.value.latencyMs.toFixed(1)}ms` : '-' + streamingMetrics.value + ? `${streamingMetrics.value.latencyMs.toFixed(1)}ms` + : "-", ); const CONTROL_PANEL_LAYOUT = { @@ -92,49 +107,70 @@ const CONTROL_PANEL_LAYOUT = { // Override PressureObserver early to catch errors from buggy browser implementations overridePressureObserver(); -setPreferredColorScheme('dark'); +setPreferredColorScheme("dark"); +const TELEOP_CHANNEL_UUID: Uint8Array = v5( + "teleop_command", + v5.DNS, + new Uint8Array(16), +); -const TELEOP_CHANNEL_UUID: Uint8Array = v5('teleop_command', v5.DNS, new Uint8Array(16)); - -type AvailableChannel = CloudXR.Session['availableMessageChannels'][number]; +type AvailableChannel = CloudXR.Session["availableMessageChannels"][number]; function findChannelByUuid( channels: AvailableChannel[], - targetUuid: Uint8Array + targetUuid: Uint8Array, ): AvailableChannel | undefined { return channels.find( - ch => + (ch) => ch.uuid.length === targetUuid.length && - ch.uuid.every((b: number, i: number) => b === targetUuid[i]) + ch.uuid.every((b: number, i: number) => b === targetUuid[i]), ); } const START_TELEOP_COMMAND = { - type: 'teleop_command', + type: "teleop_command", message: { - command: 'start teleop', + command: "start teleop", }, } as const; /** When set with ``serverIP`` + ``port``, WebXR builds ``wss://{serverIP}:{port}/oob/v1/ws``. */ function isOobEnabled(searchParams: URLSearchParams): boolean { - const v = readUrlParam(searchParams, 'oobEnable'); - return v === '1' || v?.toLowerCase() === 'true'; + const v = readUrlParam(searchParams, "oobEnable"); + return v === "1" || v?.toLowerCase() === "true"; } -function buildOobHubWsUrlFromQuery(searchParams: URLSearchParams): string | null { +function buildOobHubWsUrlFromQuery( + searchParams: URLSearchParams, +): string | null { if (!isOobEnabled(searchParams)) return null; - const serverIP = readUrlParam(searchParams, 'serverIP')?.trim(); - const portStr = readUrlParam(searchParams, 'port')?.trim(); - if (!serverIP || portStr === undefined || portStr === '') return null; + const serverIP = readUrlParam(searchParams, "serverIP")?.trim(); + const portStr = readUrlParam(searchParams, "port")?.trim(); + if (!serverIP || portStr === undefined || portStr === "") return null; if (!/^\d{1,5}$/.test(portStr)) return null; const host = - serverIP.includes(':') && !serverIP.startsWith('[') ? `[${serverIP}]` : serverIP; + serverIP.includes(":") && !serverIP.startsWith("[") + ? `[${serverIP}]` + : serverIP; return `wss://${host}:${portStr}/oob/v1/ws`; } -function App() { +function AppContent() { + const { + recorder, + mode: recorderMode, + savedRecording, + recordedFrameCount, + startRecord, + stopRecord, + startReplay, + stopReplay, + onSaveRecording: saveRecording, + onLoadRecording: loadRecording, + onFrameRecord, + } = useRecorder(); + const COUNTDOWN_MAX_SECONDS = 9; // 2D UI management const [cloudXR2DUI, setCloudXR2DUI] = useState(null); @@ -146,15 +182,17 @@ function App() { // Connection state management const [isConnected, setIsConnected] = useState(false); // Session status management - const [sessionStatus, setSessionStatus] = useState('Disconnected'); + const [sessionStatus, setSessionStatus] = useState("Disconnected"); // Error message management - const [errorMessage, setErrorMessage] = useState(''); + const [errorMessage, setErrorMessage] = useState(""); // CloudXR session reference - const [cloudXRSession, setCloudXRSession] = useState(null); + const [cloudXRSession, setCloudXRSession] = useState( + null, + ); // XR mode state for UI visibility const [isXRMode, setIsXRMode] = useState(false); // Server address being used for connection - const [serverAddress, setServerAddress] = useState(''); + const [serverAddress, setServerAddress] = useState(""); // Teleop countdown and state const [isCountingDown, setIsCountingDown] = useState(false); const [countdownRemaining, setCountdownRemaining] = useState(0); @@ -186,19 +224,20 @@ function App() { // Note: React Three Fiber's emulation is disabled (emulate: false) to avoid conflicts useEffect(() => { const loadIWER = async () => { - const { supportsImmersive, iwerLoaded: wasIwerLoaded } = await loadIWERIfNeeded(); - if (!supportsImmersive) { - setErrorMessage('Immersive mode not supported'); - setIwerLoaded(false); - setCapabilitiesValid(false); - capabilitiesCheckedRef.current = false; // Reset check flag on failure - return; - } + const { supportsImmersive, iwerLoaded: wasIwerLoaded } = + await loadIWERIfNeeded(); + if (!supportsImmersive) { + setErrorMessage("Immersive mode not supported"); + setIwerLoaded(false); + setCapabilitiesValid(false); + capabilitiesCheckedRef.current = false; // Reset check flag on failure + return; + } // IWER loaded successfully, now we can proceed with capability checks - setIwerLoaded(true); + setIwerLoaded(true); // Store whether IWER was loaded for status message display later - if (wasIwerLoaded) { - sessionStorage.setItem('iwerWasLoaded', 'true'); + if (wasIwerLoaded) { + sessionStorage.setItem("iwerWasLoaded", "true"); } }; @@ -208,7 +247,10 @@ function App() { // Update button state when IWER fails and UI becomes ready useEffect(() => { if (cloudXR2DUI && !iwerLoaded && !capabilitiesValid) { - cloudXR2DUI.setStartButtonState(true, 'CONNECT (immersive mode not supported)'); + cloudXR2DUI.setStartButtonState( + true, + "CONNECT (immersive mode not supported)", + ); } }, [cloudXR2DUI, iwerLoaded, capabilitiesValid]); @@ -226,51 +268,62 @@ function App() { capabilitiesCheckedRef.current = true; // Disable button and show checking status - cloudXR2DUI.setStartButtonState(true, 'CONNECT (checking capabilities)'); + cloudXR2DUI.setStartButtonState(true, "CONNECT (checking capabilities)"); // Set by the IWER load effect above; passed to checkCapabilities to skip browser // version checks that don't apply when running under a desktop XR emulator. - const iwerWasLoaded = sessionStorage.getItem('iwerWasLoaded') === 'true'; - let result: { success: boolean; failures: string[]; warnings: string[] } = { - success: false, - failures: [], - warnings: [], - }; + const iwerWasLoaded = sessionStorage.getItem("iwerWasLoaded") === "true"; + let result: { success: boolean; failures: string[]; warnings: string[] } = + { + success: false, + failures: [], + warnings: [], + }; try { result = await checkCapabilities(iwerWasLoaded); } catch (error) { - cloudXR2DUI.showStatus(`Capability check error: ${error}`, 'error'); + cloudXR2DUI.showStatus(`Capability check error: ${error}`, "error"); setCapabilitiesValid(false); - cloudXR2DUI.setStartButtonState(true, 'CONNECT (capability check failed)'); + cloudXR2DUI.setStartButtonState( + true, + "CONNECT (capability check failed)", + ); capabilitiesCheckedRef.current = false; // Reset on error for potential retry return; } if (!result.success) { cloudXR2DUI.showStatus( - 'Browser does not meet required capabilities:\n' + result.failures.join('\n'), - 'error' + "Browser does not meet required capabilities:\n" + + result.failures.join("\n"), + "error", ); setCapabilitiesValid(false); - cloudXR2DUI.setStartButtonState(true, 'CONNECT (capability check failed)'); + cloudXR2DUI.setStartButtonState( + true, + "CONNECT (capability check failed)", + ); capabilitiesCheckedRef.current = false; // Reset on failure for potential retry return; } // Show final status message with IWER info if applicable if (result.warnings.length > 0) { - cloudXR2DUI.showStatus('Performance notice:\n' + result.warnings.join('\n'), 'info'); + cloudXR2DUI.showStatus( + "Performance notice:\n" + result.warnings.join("\n"), + "info", + ); } else if (iwerWasLoaded) { // Include IWER status in the final success message cloudXR2DUI.showStatus( - 'CloudXR.js SDK is supported.\nUsing IWER (Immersive Web Emulator Runtime) - Emulating Meta Quest 3.', - 'info' + "CloudXR.js SDK is supported.\nUsing IWER (Immersive Web Emulator Runtime) - Emulating Meta Quest 3.", + "info", ); } else { - cloudXR2DUI.showStatus('CloudXR.js SDK is supported.', 'success'); + cloudXR2DUI.showStatus("CloudXR.js SDK is supported.", "success"); } setCapabilitiesValid(true); - cloudXR2DUI.setStartButtonState(false, 'CONNECT'); + cloudXR2DUI.setStartButtonState(false, "CONNECT"); cloudXR2DUI.updateConnectButtonState(); }; @@ -283,15 +336,20 @@ function App() { // Derive the active device profile from the UI. This drives XR store defaults. // The UI can change these values, so we need to recompute when config changes. const deviceProfile = useMemo( - () => getDeviceProfile(resolveDeviceProfileId(cloudXR2DUI?.getConfiguration().deviceProfileId)), - [cloudXR2DUI, configVersion] + () => + getDeviceProfile( + resolveDeviceProfileId(cloudXR2DUI?.getConfiguration().deviceProfileId), + ), + [cloudXR2DUI, configVersion], ); const xrFoveation = - deviceProfile.web?.foveation ?? kPerformanceOptions.xrWebGLLayer_fixedFoveationLevel; + deviceProfile.web?.foveation ?? + kPerformanceOptions.xrWebGLLayer_fixedFoveationLevel; const xrFrameBufferScaling = deviceProfile.web?.frameBufferScaling ?? kPerformanceOptions.xrWebGLLayer_framebufferScaleFactor; - const hideControllerModel = cloudXR2DUI?.getConfiguration().hideControllerModel ?? false; + const hideControllerModel = + cloudXR2DUI?.getConfiguration().hideControllerModel ?? false; // XR store must be created after we know which device profile is active. // useMemo prevents re-creating the store for unrelated UI changes. @@ -303,7 +361,7 @@ function App() { frameBufferScaling: xrFrameBufferScaling, // Use local WebXR input profile assets only when bundled (optional build without assets) ...(process.env.WEBXR_ASSETS_VERSION && { - baseAssetPath: `${new URL('.', window.location.href).href}npm/@webxr-input-profiles/assets@${process.env.WEBXR_ASSETS_VERSION}/dist/profiles/`, + baseAssetPath: `${new URL(".", window.location.href).href}npm/@webxr-input-profiles/assets@${process.env.WEBXR_ASSETS_VERSION}/dist/profiles/`, }), hand: { model: false, // Disable hand models but keep pointer functionality @@ -326,7 +384,7 @@ function App() { offerSession: true, }), // hideControllerModel omitted: changing it must not recreate the store or the session would be lost - [xrFoveation, xrFrameBufferScaling] + [xrFoveation, xrFrameBufferScaling], ); // Apply controller model visibility when the option changes. store.setController() updates @@ -339,19 +397,20 @@ function App() { useEffect(() => { // Create and initialize the 2D UI manager. const ui = new CloudXR2DUI(() => { - setConfigVersion(v => v + 1); + setConfigVersion((v) => v + 1); }); // Teleop path: URL hash -> last-used (localStorage) -> DEFAULT_TELEOP_PATH. let resolvedPath = parseTeleopPathFromHash(window.location.hash); if (!resolvedPath) { resolvedPath = - parseTeleopPathFromHash(`#/${loadStoredTeleopPath() ?? ''}`) ?? DEFAULT_TELEOP_PATH; + parseTeleopPathFromHash(`#/${loadStoredTeleopPath() ?? ""}`) ?? + DEFAULT_TELEOP_PATH; } // Reflect canonical form (parse may have lowercased/truncated). `#/…` is a // fragment-relative URL so replaceState preserves path and search. const canonicalHash = `#/${resolvedPath}`; if (window.location.hash !== canonicalHash) { - window.history.replaceState(null, '', canonicalHash); + window.history.replaceState(null, "", canonicalHash); } saveStoredTeleopPath(resolvedPath); @@ -361,7 +420,7 @@ function App() { const config = ui.getConfiguration(); const resolutionError = getResolutionValidationError( config.perEyeWidth, - config.perEyeHeight + config.perEyeHeight, ); if (resolutionError) { ui.updateConnectButtonState(); @@ -370,31 +429,35 @@ function App() { // CloudXR2DUI.updateConfiguration already sets immersiveMode to 'vr' when headless is on. // Repeat the rule here so session entry stays correct even if config were stale or built // elsewhere; immersive-ar is wrong for headless (no passthrough blit path). - const immersiveMode: 'ar' | 'vr' = config.headless ? 'vr' : config.immersiveMode; - if (immersiveMode === 'ar') { + const immersiveMode: "ar" | "vr" = config.headless + ? "vr" + : config.immersiveMode; + if (immersiveMode === "ar") { await store.enterAR(); - } else if (immersiveMode === 'vr') { + } else if (immersiveMode === "vr") { await store.enterVR(); } else { - setErrorMessage('Unrecognized immersive mode'); + setErrorMessage("Unrecognized immersive mode"); } - store.setFrameRate((supportedFrameRates: ArrayLike): number | false => { - let frameRate = ui.getConfiguration().deviceFrameRate; - let found = false; - for (let i = 0; i < supportedFrameRates.length; ++i) { - if (supportedFrameRates[i] === frameRate) { - found = true; - break; + store.setFrameRate( + (supportedFrameRates: ArrayLike): number | false => { + let frameRate = ui.getConfiguration().deviceFrameRate; + let found = false; + for (let i = 0; i < supportedFrameRates.length; ++i) { + if (supportedFrameRates[i] === frameRate) { + found = true; + break; + } } - } - if (found) { - console.info('Changed frame rate to', frameRate); - return frameRate; - } else { - console.error('Failed to change frame rate to', frameRate); - return false; - } - }); + if (found) { + console.info("Changed frame rate to", frameRate); + return frameRate; + } else { + console.error("Failed to change frame rate to", frameRate); + return false; + } + }, + ); }; ui.setupConnectButtonHandler(doConnect, (error: Error) => { @@ -414,10 +477,17 @@ function App() { // Address-bar hash edits need a reload to re-run init. useEffect(() => { const onHashChange = () => window.location.reload(); - window.addEventListener('hashchange', onHashChange); - return () => window.removeEventListener('hashchange', onHashChange); + window.addEventListener("hashchange", onHashChange); + return () => window.removeEventListener("hashchange", onHashChange); }, []); + // Wire the 2D panel "Load Recording…" button to the recorder file picker. + useEffect(() => { + const btn = document.getElementById("loadRecordingBtn"); + btn?.addEventListener("click", loadRecording); + return () => btn?.removeEventListener("click", loadRecording); + }, [loadRecording]); + // Update HTML error message display when error state changes useEffect(() => { if (cloudXR2DUI) { @@ -434,7 +504,7 @@ function App() { const handleXRStateChange = () => { const xrState = store.getState(); - if (xrState.mode === 'immersive-ar' || xrState.mode === 'immersive-vr') { + if (xrState.mode === "immersive-ar" || xrState.mode === "immersive-vr") { // XR session is active setIsXRMode(true); @@ -442,11 +512,13 @@ function App() { const session = xrState.session; if (session) { const enabledFeatures = session.enabledFeatures || []; - const hasBodyTracking = enabledFeatures.includes('body-tracking'); + const hasBodyTracking = enabledFeatures.includes("body-tracking"); + console.warn( + `[Body Tracking] XR Session started. Body tracking enabled: ${hasBodyTracking}`, + ); console.warn( - `[Body Tracking] XR Session started. Body tracking enabled: ${hasBodyTracking}` + `[Body Tracking] Enabled features: ${enabledFeatures.join(", ")}`, ); - console.warn(`[Body Tracking] Enabled features: ${enabledFeatures.join(', ')}`); } // One dump per immersive session: session.mode is authoritative (immersive-vr vs immersive-ar). @@ -456,14 +528,14 @@ function App() { } if (cloudXR2DUI) { - cloudXR2DUI.setStartButtonState(true, 'CONNECT (XR session active)'); + cloudXR2DUI.setStartButtonState(true, "CONNECT (XR session active)"); } } else { immersiveSessionDumpLoggedRef.current = false; // XR session ended setIsXRMode(false); if (cloudXR2DUI) { - cloudXR2DUI.setStartButtonState(false, 'CONNECT'); + cloudXR2DUI.setStartButtonState(false, "CONNECT"); cloudXR2DUI.updateConnectButtonState(); } @@ -492,13 +564,16 @@ function App() { const handleStatusChange = (connected: boolean, status: string) => { setIsConnected(connected); setSessionStatus(status); - controlChannelRef.current?.sendStreamStatus(connected && status === 'Connected'); + controlChannelRef.current?.sendStreamStatus( + connected && status === "Connected", + ); // Reload on session end per mode; read live off the stable 2D UI to avoid a stale closure. const autoRefreshMode = cloudXR2DUI?.getConfiguration().autoRefreshMode; if ( - (status === 'Disconnected' && (autoRefreshMode === 'clean' || autoRefreshMode === 'any')) || - (status === 'Error' && autoRefreshMode === 'any') + (status === "Disconnected" && + (autoRefreshMode === "clean" || autoRefreshMode === "any")) || + (status === "Error" && autoRefreshMode === "any") ) { window.location.reload(); } @@ -510,7 +585,10 @@ function App() { }; // Streaming performance metrics callback handler - updates raw data signal - const handleStreamingPerformanceMetrics = (fps: number, latencyMs: number) => { + const handleStreamingPerformanceMetrics = ( + fps: number, + latencyMs: number, + ) => { streamingMetrics.value = { fps, latencyMs }; }; @@ -520,7 +598,7 @@ function App() { */ const sendMessage = async (message: any) => { if (!cloudXRSession) { - console.error('CloudXR session not available'); + console.error("CloudXR session not available"); return false; } @@ -528,42 +606,44 @@ function App() { const channels = cloudXRSession.availableMessageChannels; const channel = findChannelByUuid(channels, TELEOP_CHANNEL_UUID); if (channel) { - console.info(`Using teleop MessageChannel (${channels.length} channel(s) available)`); + console.info( + `Using teleop MessageChannel (${channels.length} channel(s) available)`, + ); try { const encoder = new TextEncoder(); const data = encoder.encode(JSON.stringify(message)); const success = channel.sendServerMessage(data); if (success) { - console.info('Message sent via MessageChannel:', message); + console.info("Message sent via MessageChannel:", message); } else { - console.error('Failed to send message via MessageChannel'); + console.error("Failed to send message via MessageChannel"); } return success; } catch (error) { - console.error('Error sending via MessageChannel:', error); + console.error("Error sending via MessageChannel:", error); return false; } } // Fallback to legacy API - console.info('Using legacy sendServerMessage API'); + console.info("Using legacy sendServerMessage API"); try { cloudXRSession.sendServerMessage(message); - console.info('Message sent via legacy API:', message); + console.info("Message sent via legacy API:", message); return true; } catch (error) { - console.error('Error sending via legacy API:', error); + console.error("Error sending via legacy API:", error); return false; } }; // UI Event Handlers const handleStartTeleop = async () => { - console.info('Start Teleop pressed'); + console.info("Start Teleop pressed"); if (!cloudXRSession) { - console.error('CloudXR session not available'); + console.error("CloudXR session not available"); return; } @@ -589,7 +669,7 @@ function App() { setCountdownRemaining(countdownDuration); countdownTimerRef.current = window.setInterval(() => { - setCountdownRemaining(prev => { + setCountdownRemaining((prev) => { if (prev <= 1) { // Countdown finished if (countdownTimerRef.current !== null) { @@ -599,7 +679,7 @@ function App() { setIsCountingDown(false); // Send start teleop command - sendMessage(START_TELEOP_COMMAND).then(success => { + sendMessage(START_TELEOP_COMMAND).then((success) => { if (success) { setIsTeleopRunning(true); } else { @@ -614,9 +694,8 @@ function App() { }, 1000); }; - const handleResetTeleop = async () => { - console.info('Reset Teleop pressed'); + console.info("Reset Teleop pressed"); // Cancel any active countdown if (countdownTimerRef.current !== null) { @@ -627,23 +706,23 @@ function App() { setCountdownRemaining(0); if (!cloudXRSession) { - console.error('CloudXR session not available'); + console.error("CloudXR session not available"); return; } // Send stop teleop command first const stopCommand = { - type: 'teleop_command', + type: "teleop_command", message: { - command: 'stop teleop', + command: "stop teleop", }, }; // Send reset teleop command const resetCommand = { - type: 'teleop_command', + type: "teleop_command", message: { - command: 'reset teleop', + command: "reset teleop", }, }; @@ -657,7 +736,7 @@ function App() { }; const handleDisconnect = () => { - console.info('Disconnect pressed'); + console.info("Disconnect pressed"); // Cleanup countdown state on disconnect if (countdownTimerRef.current !== null) { @@ -682,7 +761,7 @@ function App() { if (session) { session.end().catch((err: unknown) => { setErrorMessage( - `Failed to end XR session: ${err instanceof Error ? err.message : String(err)}` + `Failed to end XR session: ${err instanceof Error ? err.message : String(err)}`, ); }); } @@ -697,27 +776,30 @@ function App() { return; } - console.info('[Teleop] OOB control WebSocket:', hubWsUrl); + console.info("[Teleop] OOB control WebSocket:", hubWsUrl); const channel = new HeadsetControlChannel({ url: hubWsUrl, - token: readUrlParam(p, 'controlToken') ?? undefined, + token: readUrlParam(p, "controlToken") ?? undefined, onConfig: () => { // Config push handling deferred to phase 2. }, getMetricsSnapshot: () => { - const snapshots: Array<{ cadence: string; metrics: Record }> = []; + const snapshots: Array<{ + cadence: string; + metrics: Record; + }> = []; const rm = renderMetrics.value; if (rm) { snapshots.push({ - cadence: 'render', + cadence: "render", metrics: { [CloudXR.MetricsName.RenderFramerate]: rm.fps }, }); } const sm = streamingMetrics.value; if (sm) { snapshots.push({ - cadence: 'frame', + cadence: "frame", metrics: { [CloudXR.MetricsName.StreamingFramerate]: sm.fps, [CloudXR.MetricsName.PoseToRenderTime]: sm.latencyMs, @@ -739,57 +821,66 @@ function App() { // Countdown configuration handlers (0-5 seconds) const handleIncreaseCountdown = () => { if (isCountingDown) return; - setCountdownDuration(prev => Math.min(COUNTDOWN_MAX_SECONDS, prev + 1)); + setCountdownDuration((prev) => Math.min(COUNTDOWN_MAX_SECONDS, prev + 1)); }; const handleDecreaseCountdown = () => { if (isCountingDown) return; - setCountdownDuration(prev => Math.max(0, prev - 1)); + setCountdownDuration((prev) => Math.max(0, prev - 1)); }; // Memo config based on configVersion (manual dependency tracker incremented on config changes) // eslint-disable-next-line react-hooks/exhaustive-deps const config = useMemo( () => (cloudXR2DUI ? cloudXR2DUI.getConfiguration() : null), - [cloudXR2DUI, configVersion] + [cloudXR2DUI, configVersion], ); // Build ICE server config from URL params (set in USB-local mode by oob_teleop_env.py). // turnServer e.g. "turn:127.0.0.1:3478?transport=tcp", iceRelayOnly=1 forces relay-only ICE. - const iceServersConfig = useMemo(() => { + const iceServersConfig = useMemo< + CloudXR.SessionOptions["iceServers"] | undefined + >(() => { const p = new URLSearchParams(window.location.search); - const turnServer = readUrlParam(p, 'turnServer'); + const turnServer = readUrlParam(p, "turnServer"); if (!turnServer) return undefined; - const turnUsername = readUrlParam(p, 'turnUsername') ?? undefined; - const turnCredential = readUrlParam(p, 'turnCredential') ?? undefined; - const iceRelayOnly = readUrlParam(p, 'iceRelayOnly') === '1'; + const turnUsername = readUrlParam(p, "turnUsername") ?? undefined; + const turnCredential = readUrlParam(p, "turnCredential") ?? undefined; + const iceRelayOnly = readUrlParam(p, "iceRelayOnly") === "1"; return { - iceServers: [{ - urls: turnServer, - ...(turnUsername !== undefined && { username: turnUsername }), - ...(turnCredential !== undefined && { credential: turnCredential }), - }], - ...(iceRelayOnly && { iceTransportPolicy: 'relay' as RTCIceTransportPolicy }), + iceServers: [ + { + urls: turnServer, + ...(turnUsername !== undefined && { username: turnUsername }), + ...(turnCredential !== undefined && { credential: turnCredential }), + }, + ], + ...(iceRelayOnly && { + iceTransportPolicy: "relay" as RTCIceTransportPolicy, + }), }; }, []); // Calculate panel position from config and memoize it as the vector used in CloudXR3DUI. const controlPanelPositionVector = useMemo( () => - getControlPanelPositionVector(config?.controlPanelPosition ?? 'center', CONTROL_PANEL_LAYOUT), - [config?.controlPanelPosition] + getControlPanelPositionVector( + config?.controlPanelPosition ?? "center", + CONTROL_PANEL_LAYOUT, + ), + [config?.controlPanelPosition], ); // Sync XR mode state to body class for CSS styling useEffect(() => { if (isXRMode) { - document.body.classList.add('xr-mode'); + document.body.classList.add("xr-mode"); } else { - document.body.classList.remove('xr-mode'); + document.body.classList.remove("xr-mode"); } return () => { - document.body.classList.remove('xr-mode'); + document.body.classList.remove("xr-mode"); }; }, [isXRMode]); @@ -808,22 +899,26 @@ function App() { const channels = cloudXRSession.availableMessageChannels; if (channels.length > 0) { - console.info(`[MessageChannel] ${channels.length} channel(s) available:`); + console.info( + `[MessageChannel] ${channels.length} channel(s) available:`, + ); channels.forEach((ch, i) => { const uuidHex = Array.from(ch.uuid as Uint8Array) - .map((b: number) => b.toString(16).padStart(2, '0')) - .join(''); - console.info( - ` [${i}] uuid=${uuidHex} status=${ch.status}` - ); + .map((b: number) => b.toString(16).padStart(2, "0")) + .join(""); + console.info(` [${i}] uuid=${uuidHex} status=${ch.status}`); }); const channel = findChannelByUuid(channels, TELEOP_CHANNEL_UUID); if (!channel) { - console.info('[MessageChannel] Teleop channel not found yet, will retry...'); + console.info( + "[MessageChannel] Teleop channel not found yet, will retry...", + ); return; } - console.info('[MessageChannel] Found teleop channel, setting up receiver'); + console.info( + "[MessageChannel] Found teleop channel, setting up receiver", + ); receiverActive = true; const receiveMessages = async () => { @@ -831,25 +926,25 @@ function App() { try { const data = await channel.receiveMessage(); if (data === null) { - console.info('MessageChannel closed'); + console.info("MessageChannel closed"); break; } // Decode and handle message const decoder = new TextDecoder(); const messageText = decoder.decode(data); - console.info('Received message via MessageChannel:', messageText); + console.info("Received message via MessageChannel:", messageText); // Parse if JSON try { const message = JSON.parse(messageText); - console.info('Parsed message:', message); + console.info("Parsed message:", message); // Handle message here if needed } catch { - console.info('Non-JSON message:', messageText); + console.info("Non-JSON message:", messageText); } } catch (error) { - console.error('Error receiving message:', error); + console.error("Error receiving message:", error); break; } } @@ -878,10 +973,10 @@ function App() { { + onWheel={(e) => { e.preventDefault(); }} > @@ -909,12 +1006,21 @@ function App() { {cloudXR2DUI && config && ( <> + + { + onError={(error) => { if (cloudXR2DUI) { cloudXR2DUI.showError(error); } @@ -923,7 +1029,9 @@ function App() { onSessionReady={setCloudXRSession} onServerAddress={setServerAddress} onRenderPerformanceMetrics={handleRenderPerformanceMetrics} - onStreamingPerformanceMetrics={handleStreamingPerformanceMetrics} + onStreamingPerformanceMetrics={ + handleStreamingPerformanceMetrics + } headless={!!config.headless} /> {!config.headless && ( @@ -937,10 +1045,10 @@ function App() { sessionStatus={sessionStatus} playLabel={ isTeleopRunning - ? 'Running' + ? "Running" : isCountingDown ? `Starting in ${countdownRemaining} sec...` - : 'Play' + : "Play" } playInProgress={isCountingDown || isTeleopRunning} countdownSeconds={countdownDuration} @@ -952,6 +1060,16 @@ function App() { renderFpsText={renderFpsText} streamingFpsText={streamingFpsText} poseToRenderText={poseToRenderText} + showRecordingControls={config.showRecordingControls} + recorderMode={recorderMode} + hasSavedRecording={savedRecording !== null} + recordedFrameCount={recordedFrameCount} + onStartRecord={startRecord} + onStopRecord={stopRecord} + onStartReplay={startReplay} + onStopReplay={stopReplay} + onSaveRecording={saveRecording} + showTrace={config.showTrace ?? false} /> )} @@ -962,4 +1080,12 @@ function App() { ); } +function App() { + return ( + + + + ); +} + export default App; diff --git a/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx b/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx index 13619f38e..4b3b19288 100644 --- a/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx @@ -36,7 +36,7 @@ import { getDeviceProfile, resolveDeviceProfileId, type DeviceProfileId, -} from '@helpers/DeviceProfiles'; +} from "@helpers/DeviceProfiles"; import { type AutoRefreshMode, loadPerProject, @@ -44,13 +44,13 @@ import { parseControlPanelPosition, ReactUIConfig, savePerProject, -} from '@helpers/react/utils'; +} from "@helpers/react/utils"; import { DEFAULT_TELEOP_PATH, DROPDOWN_ENTRIES, getProjectBreadcrumb, getProjectSettings, -} from '@helpers/TeleopProjects'; +} from "@helpers/TeleopProjects"; import { CloudXRConfig, enableLocalStorage, @@ -58,9 +58,9 @@ import { getResolutionFromInputs, setSelectValueIfAvailable, setupCertificateAcceptanceLink, -} from '@helpers/utils'; -import { URL_PARAMS } from './config/params'; -import { seedsFromParams } from './config/resolve'; +} from "@helpers/utils"; +import { URL_PARAMS } from "./config/params"; +import { seedsFromParams } from "./config/resolve"; import { getGridValidationError, getGridValidationMessageForConnect, @@ -68,17 +68,23 @@ import { getResolutionValidationMessageForConnect, validateDepthReprojectionGrid, validatePerEyeResolution, -} from '@nvidia/cloudxr'; +} from "@nvidia/cloudxr"; /** Full config: CloudXR connection settings + React UI options. */ -type AppConfig = CloudXRConfig & ReactUIConfig; +type AppConfig = CloudXRConfig & + ReactUIConfig & { + /** Show rolling path-trace dots for hands/controllers in XR. */ + showTrace: boolean; + /** Show Record / Replay / Save buttons in the in-XR control panel. */ + showRecordingControls: boolean; + }; /** * localStorage key for the teleop-start countdown. Owned by the countdown feature in * App.tsx but defined here with the other storage keys so reset clears the same key the * feature writes. (Imported by App.tsx; App already depends on this module, so no cycle.) */ -export const COUNTDOWN_STORAGE_KEY = 'cxr.react.countdownSeconds'; +export const COUNTDOWN_STORAGE_KEY = "cxr.react.countdownSeconds"; /** * 2D UI Management for CloudXR React Example @@ -166,6 +172,10 @@ export class CloudXR2DUI { private mediaPortInput!: HTMLInputElement; /** Dropdown for controller model visibility (show / hide) */ private controllerModelVisibilitySelect!: HTMLSelectElement; + /** Dropdown to show/hide the rolling trace in XR */ + private showTraceInXRSelect!: HTMLSelectElement; + /** Dropdown to show/hide recording controls in the in-XR control panel */ + private showRecordingControlsSelect!: HTMLSelectElement; /** Skip client CloudXR `render` (headless: client blit off; tracking on) */ private headlessInput!: HTMLInputElement; /** When to reload the page after the XR session ends (never / clean / any) */ @@ -232,8 +242,10 @@ export class CloudXR2DUI { // clients: localStorage restores the values saved when the profile was picked, // which otherwise pins them forever. Safe because any manual edit of a // profile-bound field switches the persisted profile to 'custom'. - const persistedProfileId = resolveDeviceProfileId(this.deviceProfileSelect.value); - if (persistedProfileId !== 'custom') { + const persistedProfileId = resolveDeviceProfileId( + this.deviceProfileSelect.value, + ); + if (persistedProfileId !== "custom") { this.applyDeviceProfileToForm(persistedProfileId); this.persistProfileFieldsToLocalStorage(); } @@ -243,9 +255,12 @@ export class CloudXR2DUI { this.setupEventListeners(); this.restoreGroupExpandedState(); // Set initial display value - this.posePredictionFactorValue.textContent = this.posePredictionFactorInput.value; + this.posePredictionFactorValue.textContent = + this.posePredictionFactorInput.value; this.updateConfiguration(); - this.updateDeviceProfileWarning(resolveDeviceProfileId(this.deviceProfileSelect.value)); + this.updateDeviceProfileWarning( + resolveDeviceProfileId(this.deviceProfileSelect.value), + ); this.updateConnectButtonState(); this.initialized = true; } catch (error) { @@ -258,13 +273,13 @@ export class CloudXR2DUI { private applyTeleopPath(): void { const breadcrumb = getProjectBreadcrumb(this.teleopPath); this.teleopModeSubtitle.replaceChildren(); - this.teleopModeSubtitle.appendChild(document.createTextNode('for ')); + this.teleopModeSubtitle.appendChild(document.createTextNode("for ")); breadcrumb.forEach((label, i) => { if (i > 0) { // aria-hidden so screen readers skip the chevron glyph. - const sep = document.createElement('span'); - sep.setAttribute('aria-hidden', 'true'); - sep.textContent = ' \u203A '; + const sep = document.createElement("span"); + sep.setAttribute("aria-hidden", "true"); + sep.textContent = " \u203A "; this.teleopModeSubtitle.appendChild(sep); } this.teleopModeSubtitle.appendChild(document.createTextNode(label)); @@ -278,15 +293,17 @@ export class CloudXR2DUI { private applyPerProjectSettings(): void { const settings = getProjectSettings(this.teleopPath); const boolFromStorage = (raw: string) => - raw === 'true' ? true : raw === 'false' ? false : undefined; + raw === "true" ? true : raw === "false" ? false : undefined; const panelHidden = loadPerProject( - 'panelHiddenAtStart', this.teleopPath, + "panelHiddenAtStart", + this.teleopPath, boolFromStorage, settings.panelHiddenAtStart ?? false, ); this.panelHiddenAtStartSelect.value = String(panelHidden); const headless = loadPerProject( - 'headless', this.teleopPath, + "headless", + this.teleopPath, boolFromStorage, settings.headless ?? false, ); @@ -299,13 +316,13 @@ export class CloudXR2DUI { */ private applyHeadlessImmersiveDropdown(): void { if (this.headlessInput.checked) { - this.immersiveSelect.value = 'vr'; + this.immersiveSelect.value = "vr"; this.immersiveSelect.disabled = true; this.immersiveSelect.title = - 'Headless requires VR (immersive-vr); AR passthrough is not available in this mode.'; + "Headless requires VR (immersive-vr); AR passthrough is not available in this mode."; } else { this.immersiveSelect.disabled = false; - this.immersiveSelect.title = ''; + this.immersiveSelect.title = ""; } } @@ -316,7 +333,7 @@ export class CloudXR2DUI { * where hover is not reliable, so the marker itself must be self-describing. */ private decoratePerProjectLabels(): void { - const marker = ' (saved per teleop application)'; + const marker = " (saved per teleop application)"; for (const el of [this.panelHiddenAtStartSelect, this.headlessInput]) { const label = el.labels?.[0]; if (!label || label.textContent?.includes(marker)) continue; @@ -329,21 +346,24 @@ export class CloudXR2DUI { const select = this.teleopProjectSelect; select.replaceChildren(); - const INDENT = '\u00A0\u00A0\u00A0'; + const INDENT = "\u00A0\u00A0\u00A0"; const currentHash = `#/${this.teleopPath}`; // Static prompt when collapsed (the breadcrumb already shows the current path); // the active entry is suffixed and disabled so it reads as already selected. - const prompt = document.createElement('option'); - prompt.value = ''; - prompt.textContent = 'Change teleop application'; + const prompt = document.createElement("option"); + prompt.value = ""; + prompt.textContent = "Change teleop application"; select.appendChild(prompt); for (const entry of DROPDOWN_ENTRIES) { - const option = document.createElement('option'); + const option = document.createElement("option"); option.value = entry.hash; const isCurrent = entry.hash === currentHash; - option.textContent = INDENT.repeat(entry.depth) + entry.label + (isCurrent ? ' (current)' : ''); + option.textContent = + INDENT.repeat(entry.depth) + + entry.label + + (isCurrent ? " (current)" : ""); if (isCurrent) option.disabled = true; select.appendChild(option); } @@ -365,11 +385,11 @@ export class CloudXR2DUI { private applyDefaultDeviceProfileFromUserAgent(): void { let stored: string | null = null; try { - stored = localStorage.getItem('deviceProfile'); + stored = localStorage.getItem("deviceProfile"); } catch (_) {} if (stored != null) return; const detected = detectDeviceProfileId(); - if (detected === 'custom') return; + if (detected === "custom") return; this.deviceProfileSelect.value = detected; this.applyDeviceProfileToForm(detected); } @@ -392,13 +412,13 @@ export class CloudXR2DUI { | HTMLSelectElement | null; if (!el) continue; - if (field.kind === 'checked') { - (el as HTMLInputElement).checked = raw === 'true'; + if (field.kind === "checked") { + (el as HTMLInputElement).checked = raw === "true"; } else { el.value = raw; } } - if (seeds.has('headless')) { + if (seeds.has("headless")) { this.applyHeadlessImmersiveDropdown(); } } @@ -408,66 +428,107 @@ export class CloudXR2DUI { * Throws an error if any required element is not found */ private initializeElements(): void { - this.startButton = this.getElement('startButton'); - this.serverIpInput = this.getElement('serverIpInput'); - this.serverIpClearButton = this.getElement('serverIpClearButton'); - this.portInput = this.getElement('portInput'); - this.proxyUrlInput = this.getElement('proxyUrl'); - this.immersiveSelect = this.getElement('immersive'); - this.deviceFrameRateSelect = this.getElement('deviceFrameRate'); - this.maxStreamingBitrateMbpsSelect = - this.getElement('maxStreamingBitrateMbps'); - this.codecSelect = this.getElement('codec'); - this.perEyeWidthInput = this.getElement('perEyeWidth'); - this.perEyeHeightInput = this.getElement('perEyeHeight'); - this.reprojectionGridColsInput = this.getElement('reprojectionGridCols'); - this.reprojectionGridRowsInput = this.getElement('reprojectionGridRows'); + this.startButton = this.getElement("startButton"); + this.serverIpInput = this.getElement("serverIpInput"); + this.serverIpClearButton = this.getElement( + "serverIpClearButton", + ); + this.portInput = this.getElement("portInput"); + this.proxyUrlInput = this.getElement("proxyUrl"); + this.immersiveSelect = this.getElement("immersive"); + this.deviceFrameRateSelect = + this.getElement("deviceFrameRate"); + this.maxStreamingBitrateMbpsSelect = this.getElement( + "maxStreamingBitrateMbps", + ); + this.codecSelect = this.getElement("codec"); + this.perEyeWidthInput = this.getElement("perEyeWidth"); + this.perEyeHeightInput = this.getElement("perEyeHeight"); + this.reprojectionGridColsInput = this.getElement( + "reprojectionGridCols", + ); + this.reprojectionGridRowsInput = this.getElement( + "reprojectionGridRows", + ); this.resolutionWidthValidationMessage = document.getElementById( - 'resolutionWidthValidationMessage' + "resolutionWidthValidationMessage", ); this.resolutionHeightValidationMessage = document.getElementById( - 'resolutionHeightValidationMessage' + "resolutionHeightValidationMessage", ); this.reprojectionGridColsValidationMessage = document.getElementById( - 'reprojectionGridColsValidationMessage' + "reprojectionGridColsValidationMessage", ); this.reprojectionGridRowsValidationMessage = document.getElementById( - 'reprojectionGridRowsValidationMessage' + "reprojectionGridRowsValidationMessage", + ); + this.enablePoseSmoothingSelect = this.getElement( + "enablePoseSmoothing", + ); + this.posePredictionFactorInput = this.getElement( + "posePredictionFactor", + ); + this.posePredictionFactorValue = this.getElement( + "posePredictionFactorValue", + ); + this.enableTexSubImage2DSelect = this.getElement( + "enableTexSubImage2D", + ); + this.useQuestColorWorkaroundSelect = this.getElement( + "useQuestColorWorkaround", + ); + this.serverTypeSelect = this.getElement("serverType"); + this.deviceProfileSelect = + this.getElement("deviceProfile"); + this.panelHiddenAtStartSelect = + this.getElement("panelHiddenAtStart"); + this.referenceSpaceSelect = + this.getElement("referenceSpace"); + this.xrOffsetXInput = this.getElement("xrOffsetX"); + this.xrOffsetYInput = this.getElement("xrOffsetY"); + this.xrOffsetZInput = this.getElement("xrOffsetZ"); + this.controlPanelPositionSelect = this.getElement( + "controlPanelPosition", ); - this.enablePoseSmoothingSelect = this.getElement('enablePoseSmoothing'); - this.posePredictionFactorInput = this.getElement('posePredictionFactor'); - this.posePredictionFactorValue = this.getElement('posePredictionFactorValue'); - this.enableTexSubImage2DSelect = this.getElement('enableTexSubImage2D'); - this.useQuestColorWorkaroundSelect = - this.getElement('useQuestColorWorkaround'); - this.serverTypeSelect = this.getElement('serverType'); - this.deviceProfileSelect = this.getElement('deviceProfile'); - this.panelHiddenAtStartSelect = this.getElement('panelHiddenAtStart'); - this.referenceSpaceSelect = this.getElement('referenceSpace'); - this.xrOffsetXInput = this.getElement('xrOffsetX'); - this.xrOffsetYInput = this.getElement('xrOffsetY'); - this.xrOffsetZInput = this.getElement('xrOffsetZ'); - this.controlPanelPositionSelect = this.getElement('controlPanelPosition'); - this.proxyDefaultText = this.getElement('proxyDefaultText'); - this.deviceProfileWarning = this.getElement('deviceProfileWarning'); - this.errorMessageBox = this.getElement('errorMessageBox'); - this.errorMessageText = this.getElement('errorMessageText'); - this.validationMessageBox = this.getElement('validationMessageBox'); - this.validationMessageText = this.getElement('validationMessageText'); - this.certAcceptanceLink = this.getElement('certAcceptanceLink'); - this.certLink = this.getElement('certLink'); - this.mediaAddressInput = this.getElement('mediaAddress'); - this.mediaPortInput = this.getElement('mediaPort'); + this.proxyDefaultText = this.getElement("proxyDefaultText"); + this.deviceProfileWarning = this.getElement( + "deviceProfileWarning", + ); + this.errorMessageBox = this.getElement("errorMessageBox"); + this.errorMessageText = this.getElement("errorMessageText"); + this.validationMessageBox = this.getElement( + "validationMessageBox", + ); + this.validationMessageText = this.getElement( + "validationMessageText", + ); + this.certAcceptanceLink = + this.getElement("certAcceptanceLink"); + this.certLink = this.getElement("certLink"); + this.mediaAddressInput = this.getElement("mediaAddress"); + this.mediaPortInput = this.getElement("mediaPort"); this.controllerModelVisibilitySelect = this.getElement( - 'controllerModelVisibility' + "controllerModelVisibility", + ); + this.showTraceInXRSelect = + this.getElement("showTraceInXR"); + this.showRecordingControlsSelect = this.getElement( + "showRecordingControls", + ); + this.headlessInput = this.getElement("cloudxrHeadless"); + this.autoRefreshModeSelect = this.getElement( + "cloudxrAutoRefreshMode", + ); + this.teleopModeSubtitle = + this.getElement("teleopModeSubtitle"); + this.teleopProjectSelect = this.getElement( + "teleopProjectSelect", + ); + this.resetSettingsButton = this.getElement( + "resetSettingsButton", ); - this.headlessInput = this.getElement('cloudxrHeadless'); - this.autoRefreshModeSelect = this.getElement('cloudxrAutoRefreshMode'); - this.teleopModeSubtitle = this.getElement('teleopModeSubtitle'); - this.teleopProjectSelect = this.getElement('teleopProjectSelect'); - this.resetSettingsButton = this.getElement('resetSettingsButton'); // Optional: absent in trimmed builds; renderUrlParamsHelp() no-ops when null. - this.urlParamsHelpList = document.getElementById('urlParamsHelpList'); + this.urlParamsHelpList = document.getElementById("urlParamsHelpList"); } /** @@ -489,11 +550,16 @@ export class CloudXR2DUI { * @returns Default configuration object */ private getDefaultConfiguration(): AppConfig { - const useSecure = typeof window !== 'undefined' ? window.location.protocol === 'https:' : false; + const useSecure = + typeof window !== "undefined" + ? window.location.protocol === "https:" + : false; // Default port: HTTP → 49100, HTTPS without proxy → 48322, HTTPS with proxy → 443 const defaultPort = useSecure ? 48322 : 49100; return { - serverIP: (typeof window !== 'undefined' && window.location.hostname) || '127.0.0.1', + serverIP: + (typeof window !== "undefined" && window.location.hostname) || + "127.0.0.1", port: defaultPort, useSecureConnection: useSecure, perEyeWidth: 2048, @@ -504,21 +570,23 @@ export class CloudXR2DUI { // and the 'selected' options in index.html. deviceFrameRate: 72, maxStreamingBitrateMbps: 25, - codec: 'av1', - immersiveMode: 'ar', - deviceProfileId: 'custom', - serverType: 'manual', + codec: "av1", + immersiveMode: "ar", + deviceProfileId: "custom", + serverType: "manual", panelHiddenAtStart: false, - proxyUrl: '', - referenceSpaceType: 'auto', - controlPanelPosition: 'center', + proxyUrl: "", + referenceSpaceType: "auto", + controlPanelPosition: "center", enablePoseSmoothing: true, posePredictionFactor: 1.0, enableTexSubImage2D: false, useQuestColorWorkaround: false, hideControllerModel: false, + showTrace: false, + showRecordingControls: false, headless: false, - autoRefreshMode: 'clean', + autoRefreshMode: "clean", teleopPath: DEFAULT_TELEOP_PATH, }; } @@ -534,32 +602,43 @@ export class CloudXR2DUI { key: string; }> { return [ - { el: this.serverTypeSelect, key: 'serverType' }, - { el: this.serverIpInput, key: 'serverIp' }, - { el: this.portInput, key: 'port' }, - { el: this.perEyeWidthInput, key: 'perEyeWidth' }, - { el: this.perEyeHeightInput, key: 'perEyeHeight' }, - { el: this.reprojectionGridColsInput, key: 'reprojectionGridCols' }, - { el: this.reprojectionGridRowsInput, key: 'reprojectionGridRows' }, - { el: this.proxyUrlInput, key: 'proxyUrl' }, - { el: this.deviceFrameRateSelect, key: 'deviceFrameRate' }, - { el: this.maxStreamingBitrateMbpsSelect, key: 'maxStreamingBitrateMbps' }, - { el: this.codecSelect, key: 'codec' }, - { el: this.enablePoseSmoothingSelect, key: 'enablePoseSmoothing' }, - { el: this.posePredictionFactorInput, key: 'posePredictionFactor' }, - { el: this.enableTexSubImage2DSelect, key: 'enableTexSubImage2D' }, - { el: this.useQuestColorWorkaroundSelect, key: 'useQuestColorWorkaround' }, - { el: this.immersiveSelect, key: 'immersiveMode' }, - { el: this.deviceProfileSelect, key: 'deviceProfile' }, - { el: this.controlPanelPositionSelect, key: 'controlPanelPosition' }, - { el: this.referenceSpaceSelect, key: 'referenceSpace' }, - { el: this.xrOffsetXInput, key: 'xrOffsetX' }, - { el: this.xrOffsetYInput, key: 'xrOffsetY' }, - { el: this.xrOffsetZInput, key: 'xrOffsetZ' }, - { el: this.mediaAddressInput, key: 'mediaAddress' }, - { el: this.mediaPortInput, key: 'mediaPort' }, - { el: this.controllerModelVisibilitySelect, key: 'controllerModelVisibility' }, - { el: this.autoRefreshModeSelect, key: 'autoRefreshMode' }, + { el: this.serverTypeSelect, key: "serverType" }, + { el: this.serverIpInput, key: "serverIp" }, + { el: this.portInput, key: "port" }, + { el: this.perEyeWidthInput, key: "perEyeWidth" }, + { el: this.perEyeHeightInput, key: "perEyeHeight" }, + { el: this.reprojectionGridColsInput, key: "reprojectionGridCols" }, + { el: this.reprojectionGridRowsInput, key: "reprojectionGridRows" }, + { el: this.proxyUrlInput, key: "proxyUrl" }, + { el: this.deviceFrameRateSelect, key: "deviceFrameRate" }, + { + el: this.maxStreamingBitrateMbpsSelect, + key: "maxStreamingBitrateMbps", + }, + { el: this.codecSelect, key: "codec" }, + { el: this.enablePoseSmoothingSelect, key: "enablePoseSmoothing" }, + { el: this.posePredictionFactorInput, key: "posePredictionFactor" }, + { el: this.enableTexSubImage2DSelect, key: "enableTexSubImage2D" }, + { + el: this.useQuestColorWorkaroundSelect, + key: "useQuestColorWorkaround", + }, + { el: this.immersiveSelect, key: "immersiveMode" }, + { el: this.deviceProfileSelect, key: "deviceProfile" }, + { el: this.controlPanelPositionSelect, key: "controlPanelPosition" }, + { el: this.referenceSpaceSelect, key: "referenceSpace" }, + { el: this.xrOffsetXInput, key: "xrOffsetX" }, + { el: this.xrOffsetYInput, key: "xrOffsetY" }, + { el: this.xrOffsetZInput, key: "xrOffsetZ" }, + { el: this.mediaAddressInput, key: "mediaAddress" }, + { el: this.mediaPortInput, key: "mediaPort" }, + { + el: this.controllerModelVisibilitySelect, + key: "controllerModelVisibility", + }, + { el: this.showTraceInXRSelect, key: "showTraceInXR" }, + { el: this.showRecordingControlsSelect, key: "showRecordingControls" }, + { el: this.autoRefreshModeSelect, key: "autoRefreshMode" }, ]; } @@ -599,7 +678,7 @@ export class CloudXR2DUI { } } } catch (error) { - console.warn('Failed to clear stored settings:', error); + console.warn("Failed to clear stored settings:", error); } // applyUrlSeeds() runs after setupLocalStorage() on load, so a form-backed query @@ -615,7 +694,7 @@ export class CloudXR2DUI { url.searchParams.delete(param.url ?? param.key); } } - window.history.replaceState(null, '', url.toString()); + window.history.replaceState(null, "", url.toString()); window.location.reload(); } @@ -629,8 +708,8 @@ export class CloudXR2DUI { this.urlParamsHelpList.replaceChildren(); for (const param of URL_PARAMS) { if (!param.description) continue; - const li = document.createElement('li'); - const code = document.createElement('code'); + const li = document.createElement("li"); + const code = document.createElement("code"); code.textContent = param.url ?? param.key; li.appendChild(code); li.appendChild(document.createTextNode(` — ${param.description}`)); @@ -639,35 +718,40 @@ export class CloudXR2DUI { } /** localStorage key prefix for each collapsible advanced group's open/closed state. */ - private static readonly GROUP_STATE_PREFIX = 'cxr.group.'; + private static readonly GROUP_STATE_PREFIX = "cxr.group."; /** * Settings persisted per teleop application under `cxr.isaac.|` * (see {@link applyPerProjectSettings} and helpers/react/utils savePerProject). * Centralized so resetToDefaults clears exactly the keys the per-project handlers write. */ - private static readonly PER_PROJECT_SETTING_KEYS = ['panelHiddenAtStart', 'headless']; + private static readonly PER_PROJECT_SETTING_KEYS = [ + "panelHiddenAtStart", + "headless", + ]; /** * Restore each advanced group's expanded/collapsed state from localStorage and persist it on * toggle, so a user's "open" sections stay open across reloads. Keyed by the group's element id. */ private restoreGroupExpandedState(): void { - const groups = document.querySelectorAll('details.settings-group[id]'); + const groups = document.querySelectorAll( + "details.settings-group[id]", + ); for (const group of Array.from(groups)) { const key = `${CloudXR2DUI.GROUP_STATE_PREFIX}${group.id}`; try { const saved = localStorage.getItem(key); - if (saved === 'true') group.open = true; - else if (saved === 'false') group.open = false; + if (saved === "true") group.open = true; + else if (saved === "false") group.open = false; } catch (_) {} const handler = () => { try { localStorage.setItem(key, String(group.open)); } catch (_) {} }; - group.addEventListener('toggle', handler); - this.eventListeners.push({ element: group, event: 'toggle', handler }); + group.addEventListener("toggle", handler); + this.eventListeners.push({ element: group, event: "toggle", handler }); } } @@ -677,20 +761,22 @@ export class CloudXR2DUI { */ private setupProxyConfiguration(): void { // Update port placeholder based on protocol - if (window.location.protocol === 'https:') { - this.portInput.placeholder = 'Port (default: 48322, or 443 if proxy URL set)'; + if (window.location.protocol === "https:") { + this.portInput.placeholder = + "Port (default: 48322, or 443 if proxy URL set)"; } else { - this.portInput.placeholder = 'Port (default: 49100)'; + this.portInput.placeholder = "Port (default: 49100)"; } // Set default text and placeholder based on protocol - if (window.location.protocol === 'https:') { + if (window.location.protocol === "https:") { this.proxyDefaultText.textContent = - 'Optional: Leave empty for direct WSS connection, or provide URL for proxy routing (e.g., https://proxy.example.com/)'; - this.proxyUrlInput.placeholder = ''; + "Optional: Leave empty for direct WSS connection, or provide URL for proxy routing (e.g., https://proxy.example.com/)"; + this.proxyUrlInput.placeholder = ""; } else { - this.proxyDefaultText.textContent = 'Not needed for HTTP - uses direct WS connection'; - this.proxyUrlInput.placeholder = ''; + this.proxyDefaultText.textContent = + "Not needed for HTTP - uses direct WS connection"; + this.proxyUrlInput.placeholder = ""; } } @@ -707,106 +793,156 @@ export class CloudXR2DUI { }; // Helper function to add listeners and store them for cleanup - const addListener = (element: HTMLElement, event: string, handler: EventListener) => { + const addListener = ( + element: HTMLElement, + event: string, + handler: EventListener, + ) => { element.addEventListener(event, handler); this.eventListeners.push({ element, event, handler }); }; // Add event listeners for all form fields - addListener(this.serverTypeSelect, 'change', updateConfig); - addListener(this.serverIpInput, 'input', updateConfig); - addListener(this.serverIpInput, 'change', updateConfig); + addListener(this.serverTypeSelect, "change", updateConfig); + addListener(this.serverIpInput, "input", updateConfig); + addListener(this.serverIpInput, "change", updateConfig); // Show the clear ("x") button only while the server IP field has a value, // and clear the prefill (incl. localStorage) on click so the browser's // autocomplete dropdown of previously connected servers can show again. const updateServerIpClearButton = () => { - this.serverIpClearButton.classList.toggle('visible', this.serverIpInput.value.length > 0); + this.serverIpClearButton.classList.toggle( + "visible", + this.serverIpInput.value.length > 0, + ); }; - addListener(this.serverIpInput, 'input', updateServerIpClearButton); - addListener(this.serverIpClearButton, 'click', () => { - this.serverIpInput.value = ''; + addListener(this.serverIpInput, "input", updateServerIpClearButton); + addListener(this.serverIpClearButton, "click", () => { + this.serverIpInput.value = ""; // Update the live config + clear-button state directly; 'change' persists // the now-empty value via the enableLocalStorage handler. updateServerIpClearButton(); updateConfig(); - this.serverIpInput.dispatchEvent(new Event('change', { bubbles: true })); + this.serverIpInput.dispatchEvent(new Event("change", { bubbles: true })); this.serverIpInput.focus(); }); updateServerIpClearButton(); - addListener(this.portInput, 'input', updateConfig); - addListener(this.portInput, 'change', updateConfig); + addListener(this.portInput, "input", updateConfig); + addListener(this.portInput, "change", updateConfig); const updateResValidation = () => this.updateResolutionValidationMessage(); - addListener(this.perEyeWidthInput, 'input', onProfileLinkedChange); - addListener(this.perEyeWidthInput, 'change', onProfileLinkedChange); - addListener(this.perEyeWidthInput, 'blur', updateResValidation); - addListener(this.perEyeWidthInput, 'keyup', updateResValidation); - addListener(this.perEyeHeightInput, 'input', onProfileLinkedChange); - addListener(this.perEyeHeightInput, 'change', onProfileLinkedChange); - addListener(this.perEyeHeightInput, 'blur', updateResValidation); - addListener(this.perEyeHeightInput, 'keyup', updateResValidation); + addListener(this.perEyeWidthInput, "input", onProfileLinkedChange); + addListener(this.perEyeWidthInput, "change", onProfileLinkedChange); + addListener(this.perEyeWidthInput, "blur", updateResValidation); + addListener(this.perEyeWidthInput, "keyup", updateResValidation); + addListener(this.perEyeHeightInput, "input", onProfileLinkedChange); + addListener(this.perEyeHeightInput, "change", onProfileLinkedChange); + addListener(this.perEyeHeightInput, "blur", updateResValidation); + addListener(this.perEyeHeightInput, "keyup", updateResValidation); this.updateResolutionValidationMessage(); const updateGridValidation = () => this.updateGridValidationMessage(); - addListener(this.reprojectionGridColsInput, 'input', onProfileLinkedChange); - addListener(this.reprojectionGridColsInput, 'change', onProfileLinkedChange); - addListener(this.reprojectionGridColsInput, 'blur', updateGridValidation); - addListener(this.reprojectionGridColsInput, 'keyup', updateGridValidation); - addListener(this.reprojectionGridRowsInput, 'input', onProfileLinkedChange); - addListener(this.reprojectionGridRowsInput, 'change', onProfileLinkedChange); - addListener(this.reprojectionGridRowsInput, 'blur', updateGridValidation); - addListener(this.reprojectionGridRowsInput, 'keyup', updateGridValidation); + addListener(this.reprojectionGridColsInput, "input", onProfileLinkedChange); + addListener( + this.reprojectionGridColsInput, + "change", + onProfileLinkedChange, + ); + addListener(this.reprojectionGridColsInput, "blur", updateGridValidation); + addListener(this.reprojectionGridColsInput, "keyup", updateGridValidation); + addListener(this.reprojectionGridRowsInput, "input", onProfileLinkedChange); + addListener( + this.reprojectionGridRowsInput, + "change", + onProfileLinkedChange, + ); + addListener(this.reprojectionGridRowsInput, "blur", updateGridValidation); + addListener(this.reprojectionGridRowsInput, "keyup", updateGridValidation); this.updateGridValidationMessage(); - addListener(this.deviceFrameRateSelect, 'change', onProfileLinkedChange); - addListener(this.maxStreamingBitrateMbpsSelect, 'change', onProfileLinkedChange); - addListener(this.codecSelect, 'change', onProfileLinkedChange); - addListener(this.enablePoseSmoothingSelect, 'change', onProfileLinkedChange); - addListener(this.posePredictionFactorInput, 'change', onProfileLinkedChange); - addListener(this.posePredictionFactorInput, 'input', () => { + addListener(this.deviceFrameRateSelect, "change", onProfileLinkedChange); + addListener( + this.maxStreamingBitrateMbpsSelect, + "change", + onProfileLinkedChange, + ); + addListener(this.codecSelect, "change", onProfileLinkedChange); + addListener( + this.enablePoseSmoothingSelect, + "change", + onProfileLinkedChange, + ); + addListener( + this.posePredictionFactorInput, + "change", + onProfileLinkedChange, + ); + addListener(this.posePredictionFactorInput, "input", () => { this.setProfileToCustomIfNeeded(); - this.posePredictionFactorValue.textContent = this.posePredictionFactorInput.value; + this.posePredictionFactorValue.textContent = + this.posePredictionFactorInput.value; this.updateConfiguration(); }); - addListener(this.enableTexSubImage2DSelect, 'change', onProfileLinkedChange); - addListener(this.useQuestColorWorkaroundSelect, 'change', onProfileLinkedChange); - addListener(this.immersiveSelect, 'change', updateConfig); - addListener(this.panelHiddenAtStartSelect, 'change', () => { + addListener( + this.enableTexSubImage2DSelect, + "change", + onProfileLinkedChange, + ); + addListener( + this.useQuestColorWorkaroundSelect, + "change", + onProfileLinkedChange, + ); + addListener(this.immersiveSelect, "change", updateConfig); + addListener(this.panelHiddenAtStartSelect, "change", () => { // Pass the raw select value string through savePerProject; the matching // loadPerProject parses `'true'`/`'false'` back into a boolean. - savePerProject('panelHiddenAtStart', this.teleopPath, this.panelHiddenAtStartSelect.value); + savePerProject( + "panelHiddenAtStart", + this.teleopPath, + this.panelHiddenAtStartSelect.value, + ); updateConfig(); }); - addListener(this.referenceSpaceSelect, 'change', updateConfig); - addListener(this.xrOffsetXInput, 'input', updateConfig); - addListener(this.xrOffsetXInput, 'change', updateConfig); - addListener(this.xrOffsetYInput, 'input', updateConfig); - addListener(this.xrOffsetYInput, 'change', updateConfig); - addListener(this.xrOffsetZInput, 'input', updateConfig); - addListener(this.xrOffsetZInput, 'change', updateConfig); - addListener(this.controlPanelPositionSelect, 'change', updateConfig); - addListener(this.teleopProjectSelect, 'change', () => { + addListener(this.referenceSpaceSelect, "change", updateConfig); + addListener(this.xrOffsetXInput, "input", updateConfig); + addListener(this.xrOffsetXInput, "change", updateConfig); + addListener(this.xrOffsetYInput, "input", updateConfig); + addListener(this.xrOffsetYInput, "change", updateConfig); + addListener(this.xrOffsetZInput, "input", updateConfig); + addListener(this.xrOffsetZInput, "change", updateConfig); + addListener(this.controlPanelPositionSelect, "change", updateConfig); + addListener(this.teleopProjectSelect, "change", () => { const value = this.teleopProjectSelect.value; if (!value) return; // Reset to the prompt before navigating so if the reload is aborted the // control doesn't end up stuck on the just-picked value. this.teleopProjectSelect.selectedIndex = 0; - window.location.hash = value.replace(/^#/, ''); + window.location.hash = value.replace(/^#/, ""); }); - addListener(this.proxyUrlInput, 'input', updateConfig); - addListener(this.proxyUrlInput, 'change', updateConfig); - addListener(this.mediaAddressInput, 'input', updateConfig); - addListener(this.mediaAddressInput, 'change', updateConfig); - addListener(this.mediaPortInput, 'input', updateConfig); - addListener(this.mediaPortInput, 'change', updateConfig); - addListener(this.controllerModelVisibilitySelect, 'change', updateConfig); - addListener(this.headlessInput, 'change', () => { - savePerProject('headless', this.teleopPath, this.headlessInput.checked ? 'true' : 'false'); + addListener(this.proxyUrlInput, "input", updateConfig); + addListener(this.proxyUrlInput, "change", updateConfig); + addListener(this.mediaAddressInput, "input", updateConfig); + addListener(this.mediaAddressInput, "change", updateConfig); + addListener(this.mediaPortInput, "input", updateConfig); + addListener(this.mediaPortInput, "change", updateConfig); + addListener(this.controllerModelVisibilitySelect, "change", updateConfig); + addListener(this.showTraceInXRSelect, "change", updateConfig); + addListener(this.showRecordingControlsSelect, "change", updateConfig); + addListener(this.headlessInput, "change", () => { + savePerProject( + "headless", + this.teleopPath, + this.headlessInput.checked ? "true" : "false", + ); this.applyHeadlessImmersiveDropdown(); this.updateConfiguration(); }); - addListener(this.autoRefreshModeSelect, 'change', updateConfig); - - addListener(this.resetSettingsButton, 'click', () => { - if (window.confirm('Reset all settings to their defaults? This reloads the page.')) { + addListener(this.autoRefreshModeSelect, "change", updateConfig); + + addListener(this.resetSettingsButton, "click", () => { + if ( + window.confirm( + "Reset all settings to their defaults? This reloads the page.", + ) + ) { this.resetToDefaults(); } }); @@ -815,23 +951,27 @@ export class CloudXR2DUI { // negative. Each ± button (data-target = input id) flips its field's sign. Dispatch // 'change' so the existing offset listeners (updateConfiguration + localStorage) run. for (const btn of Array.from( - document.querySelectorAll('.input-sign-btn') + document.querySelectorAll(".input-sign-btn"), )) { const targetId = btn.dataset.target; if (!targetId) continue; - const input = document.getElementById(targetId) as HTMLInputElement | null; + const input = document.getElementById( + targetId, + ) as HTMLInputElement | null; if (!input) continue; - addListener(btn, 'click', () => { + addListener(btn, "click", () => { // Pure sign flip: a no-op on an empty/non-numeric field rather than inserting 0. const value = parseFloat(input.value); if (!Number.isFinite(value)) return; input.value = String(-value); - input.dispatchEvent(new Event('change', { bubbles: true })); + input.dispatchEvent(new Event("change", { bubbles: true })); }); } - addListener(this.deviceProfileSelect, 'change', () => { - this.applyDeviceProfileToForm(resolveDeviceProfileId(this.deviceProfileSelect.value)); + addListener(this.deviceProfileSelect, "change", () => { + this.applyDeviceProfileToForm( + resolveDeviceProfileId(this.deviceProfileSelect.value), + ); this.persistProfileFieldsToLocalStorage(); this.updateConfiguration(); }); @@ -842,7 +982,7 @@ export class CloudXR2DUI { this.portInput, this.proxyUrlInput, this.certAcceptanceLink, - this.certLink + this.certLink, ); } @@ -850,22 +990,22 @@ export class CloudXR2DUI { private updateResolutionValidationMessage(): void { const { w: wNum, h: hNum } = getResolutionFromInputs( this.perEyeWidthInput, - this.perEyeHeightInput + this.perEyeHeightInput, ); const { widthError, heightError } = validatePerEyeResolution(wNum, hNum); if (this.resolutionWidthValidationMessage) { - const showWidth = widthError ?? ''; + const showWidth = widthError ?? ""; this.resolutionWidthValidationMessage.textContent = showWidth; this.resolutionWidthValidationMessage.className = showWidth - ? 'config-text resolution-validation-error' - : 'config-text'; + ? "config-text resolution-validation-error" + : "config-text"; } if (this.resolutionHeightValidationMessage) { - const showHeight = heightError ?? ''; + const showHeight = heightError ?? ""; this.resolutionHeightValidationMessage.textContent = showHeight; this.resolutionHeightValidationMessage.className = showHeight - ? 'config-text resolution-validation-error' - : 'config-text'; + ? "config-text resolution-validation-error" + : "config-text"; } this.updateConnectButtonState(); } @@ -874,57 +1014,61 @@ export class CloudXR2DUI { private updateGridValidationMessage(): void { const { reprojectionGridCols, reprojectionGridRows } = getGridFromInputs( this.reprojectionGridColsInput, - this.reprojectionGridRowsInput - ); - const { reprojectionGridColsError, reprojectionGridRowsError } = validateDepthReprojectionGrid( - reprojectionGridCols, - reprojectionGridRows + this.reprojectionGridRowsInput, ); + const { reprojectionGridColsError, reprojectionGridRowsError } = + validateDepthReprojectionGrid(reprojectionGridCols, reprojectionGridRows); if (this.reprojectionGridColsValidationMessage) { - const showGridCols = reprojectionGridColsError ?? ''; + const showGridCols = reprojectionGridColsError ?? ""; this.reprojectionGridColsValidationMessage.textContent = showGridCols; this.reprojectionGridColsValidationMessage.className = showGridCols - ? 'config-text resolution-validation-error' - : 'config-text'; + ? "config-text resolution-validation-error" + : "config-text"; } if (this.reprojectionGridRowsValidationMessage) { - const showGridRows = reprojectionGridRowsError ?? ''; + const showGridRows = reprojectionGridRowsError ?? ""; this.reprojectionGridRowsValidationMessage.textContent = showGridRows; this.reprojectionGridRowsValidationMessage.className = showGridRows - ? 'config-text resolution-validation-error' - : 'config-text'; + ? "config-text resolution-validation-error" + : "config-text"; } this.updateConnectButtonState(); } /** Disable Connect button and show validation error when resolution invalid; enable when valid. */ public updateConnectButtonState(): void { - const { w, h } = getResolutionFromInputs(this.perEyeWidthInput, this.perEyeHeightInput); + const { w, h } = getResolutionFromInputs( + this.perEyeWidthInput, + this.perEyeHeightInput, + ); const { reprojectionGridCols, reprojectionGridRows } = getGridFromInputs( this.reprojectionGridColsInput, - this.reprojectionGridRowsInput + this.reprojectionGridRowsInput, ); const resolutionError = getResolutionValidationError(w, h); - const gridError = getGridValidationError(reprojectionGridCols, reprojectionGridRows); + const gridError = getGridValidationError( + reprojectionGridCols, + reprojectionGridRows, + ); const connectMessage = getResolutionValidationMessageForConnect(w, h); const gridConnectMessage = getGridValidationMessageForConnect( reprojectionGridCols, - reprojectionGridRows + reprojectionGridRows, ); const combinedConnectMessage = [connectMessage, gridConnectMessage] .filter(Boolean) - .join('\n'); + .join("\n"); if (combinedConnectMessage) { this.validationMessageText.textContent = combinedConnectMessage; - this.validationMessageBox.className = 'validation-message-box show'; + this.validationMessageBox.className = "validation-message-box show"; } else { - this.validationMessageText.textContent = ''; - this.validationMessageBox.className = 'validation-message-box'; + this.validationMessageText.textContent = ""; + this.validationMessageBox.className = "validation-message-box"; } // Only update button when idle (don't override "CONNECT (starting...)" or "CONNECT (XR session active)") - if (this.startButton && this.startButton.innerHTML === 'CONNECT') { + if (this.startButton && this.startButton.innerHTML === "CONNECT") { const shouldEnable = !resolutionError && !gridError; - this.setStartButtonState(!shouldEnable, 'CONNECT'); + this.setStartButtonState(!shouldEnable, "CONNECT"); } } @@ -945,14 +1089,15 @@ export class CloudXR2DUI { const { w: perEyeWidth, h: perEyeHeight } = getResolutionFromInputs( this.perEyeWidthInput, - this.perEyeHeightInput + this.perEyeHeightInput, ); const { reprojectionGridCols, reprojectionGridRows } = getGridFromInputs( this.reprojectionGridColsInput, - this.reprojectionGridRowsInput + this.reprojectionGridRowsInput, ); const newConfiguration: AppConfig = { - serverIP: this.serverIpInput.value || this.getDefaultConfiguration().serverIP, + serverIP: + this.serverIpInput.value || this.getDefaultConfiguration().serverIP, port: portValue || defaultPort, useSecureConnection: useSecure, perEyeWidth, @@ -966,24 +1111,32 @@ export class CloudXR2DUI { parseInt(this.maxStreamingBitrateMbpsSelect.value) || this.getDefaultConfiguration().maxStreamingBitrateMbps, codec: - (this.codecSelect.value as 'h264' | 'h265' | 'av1') || this.getDefaultConfiguration().codec, + (this.codecSelect.value as "h264" | "h265" | "av1") || + this.getDefaultConfiguration().codec, // Headless mode turns off the client's CloudXR frame blit but keeps tracking; the WebXR // session must be immersive-vr. immersive-ar uses passthrough semantics that do not match // that pipeline, so we ignore the AR/VR dropdown whenever headless is checked. immersiveMode: this.headlessInput.checked - ? 'vr' - : (this.immersiveSelect.value as 'ar' | 'vr') || + ? "vr" + : (this.immersiveSelect.value as "ar" | "vr") || this.getDefaultConfiguration().immersiveMode, deviceProfileId: resolveDeviceProfileId(this.deviceProfileSelect.value), - serverType: this.serverTypeSelect.value || this.getDefaultConfiguration().serverType, - proxyUrl: this.proxyUrlInput.value || this.getDefaultConfiguration().proxyUrl, + serverType: + this.serverTypeSelect.value || + this.getDefaultConfiguration().serverType, + proxyUrl: + this.proxyUrlInput.value || this.getDefaultConfiguration().proxyUrl, referenceSpaceType: - (this.referenceSpaceSelect.value as 'auto' | 'local-floor' | 'local' | 'viewer') || - this.getDefaultConfiguration().referenceSpaceType, - enablePoseSmoothing: this.enablePoseSmoothingSelect.value === 'true', + (this.referenceSpaceSelect.value as + | "auto" + | "local-floor" + | "local" + | "viewer") || this.getDefaultConfiguration().referenceSpaceType, + enablePoseSmoothing: this.enablePoseSmoothingSelect.value === "true", posePredictionFactor: parseFloat(this.posePredictionFactorInput.value), - enableTexSubImage2D: this.enableTexSubImage2DSelect.value === 'true', - useQuestColorWorkaround: this.useQuestColorWorkaroundSelect.value === 'true', + enableTexSubImage2D: this.enableTexSubImage2DSelect.value === "true", + useQuestColorWorkaround: + this.useQuestColorWorkaroundSelect.value === "true", // Convert cm from UI into meters for config (respect 0; if invalid, use 0) xrOffsetX: (() => { const v = parseFloat(this.xrOffsetXInput.value); @@ -999,7 +1152,7 @@ export class CloudXR2DUI { })(), controlPanelPosition: parseControlPanelPosition( this.controlPanelPositionSelect.value, - this.getDefaultConfiguration().controlPanelPosition ?? 'center' + this.getDefaultConfiguration().controlPanelPosition ?? "center", ), // Parse media address and port if provided mediaAddress: this.mediaAddressInput.value.trim() || undefined, @@ -1007,14 +1160,17 @@ export class CloudXR2DUI { const v = parseInt(this.mediaPortInput.value, 10); return !isNaN(v) ? v : undefined; })(), - hideControllerModel: this.controllerModelVisibilitySelect.value === 'hide', + hideControllerModel: + this.controllerModelVisibilitySelect.value === "hide", + showTrace: this.showTraceInXRSelect.value !== "false", + showRecordingControls: this.showRecordingControlsSelect.value === "true", // See immersiveMode above: when true, callers must start an immersive-vr WebXR session. headless: this.headlessInput.checked, autoRefreshMode: parseAutoRefreshMode( this.autoRefreshModeSelect.value, - this.getDefaultConfiguration().autoRefreshMode ?? 'clean' + this.getDefaultConfiguration().autoRefreshMode ?? "clean", ), - panelHiddenAtStart: this.panelHiddenAtStartSelect.value === 'true', + panelHiddenAtStart: this.panelHiddenAtStartSelect.value === "true", teleopPath: this.teleopPath, }; @@ -1037,7 +1193,7 @@ export class CloudXR2DUI { const cloudxr = profile.cloudxr; this.updateDeviceProfileWarning(profileId); - if (!cloudxr || profileId === 'custom') { + if (!cloudxr || profileId === "custom") { return; } @@ -1048,60 +1204,100 @@ export class CloudXR2DUI { this.perEyeHeightInput.value = String(cloudxr.perEyeHeight); } this.reprojectionGridColsInput.value = - cloudxr.reprojectionGridCols !== undefined ? String(cloudxr.reprojectionGridCols) : ''; + cloudxr.reprojectionGridCols !== undefined + ? String(cloudxr.reprojectionGridCols) + : ""; this.reprojectionGridRowsInput.value = - cloudxr.reprojectionGridRows !== undefined ? String(cloudxr.reprojectionGridRows) : ''; + cloudxr.reprojectionGridRows !== undefined + ? String(cloudxr.reprojectionGridRows) + : ""; if (cloudxr.deviceFrameRate !== undefined) { - setSelectValueIfAvailable(this.deviceFrameRateSelect, String(cloudxr.deviceFrameRate)); + setSelectValueIfAvailable( + this.deviceFrameRateSelect, + String(cloudxr.deviceFrameRate), + ); } if (cloudxr.maxStreamingBitrateKbps !== undefined) { const mbps = Math.round(cloudxr.maxStreamingBitrateKbps / 1000); - setSelectValueIfAvailable(this.maxStreamingBitrateMbpsSelect, String(mbps)); + setSelectValueIfAvailable( + this.maxStreamingBitrateMbpsSelect, + String(mbps), + ); } if (cloudxr.codec) { setSelectValueIfAvailable(this.codecSelect, cloudxr.codec); } if (cloudxr.enablePoseSmoothing !== undefined) { - this.enablePoseSmoothingSelect.value = String(cloudxr.enablePoseSmoothing); + this.enablePoseSmoothingSelect.value = String( + cloudxr.enablePoseSmoothing, + ); } if (cloudxr.posePredictionFactor !== undefined) { - this.posePredictionFactorInput.value = String(cloudxr.posePredictionFactor); - this.posePredictionFactorValue.textContent = this.posePredictionFactorInput.value; + this.posePredictionFactorInput.value = String( + cloudxr.posePredictionFactor, + ); + this.posePredictionFactorValue.textContent = + this.posePredictionFactorInput.value; } if (cloudxr.enableTexSubImage2D !== undefined) { - this.enableTexSubImage2DSelect.value = String(cloudxr.enableTexSubImage2D); + this.enableTexSubImage2DSelect.value = String( + cloudxr.enableTexSubImage2D, + ); } if (cloudxr.useQuestColorWorkaround !== undefined) { - this.useQuestColorWorkaroundSelect.value = String(cloudxr.useQuestColorWorkaround); + this.useQuestColorWorkaroundSelect.value = String( + cloudxr.useQuestColorWorkaround, + ); } } /** When user edits a profile-driven setting, switch device profile to Custom and persist. */ private setProfileToCustomIfNeeded(): void { - if (this.deviceProfileSelect.value === 'custom') return; - this.deviceProfileSelect.value = 'custom'; - this.updateDeviceProfileWarning('custom'); + if (this.deviceProfileSelect.value === "custom") return; + this.deviceProfileSelect.value = "custom"; + this.updateDeviceProfileWarning("custom"); try { - localStorage.setItem('deviceProfile', 'custom'); + localStorage.setItem("deviceProfile", "custom"); } catch (_) {} } /** Persist profile-driven form fields to localStorage so they are restored on load. */ private persistProfileFieldsToLocalStorage(): void { try { - localStorage.setItem('perEyeWidth', this.perEyeWidthInput.value); - localStorage.setItem('perEyeHeight', this.perEyeHeightInput.value); - localStorage.setItem('reprojectionGridCols', this.reprojectionGridColsInput.value); - localStorage.setItem('reprojectionGridRows', this.reprojectionGridRowsInput.value); - localStorage.setItem('deviceFrameRate', this.deviceFrameRateSelect.value); - localStorage.setItem('maxStreamingBitrateMbps', this.maxStreamingBitrateMbpsSelect.value); - localStorage.setItem('codec', this.codecSelect.value); - localStorage.setItem('enablePoseSmoothing', this.enablePoseSmoothingSelect.value); - localStorage.setItem('posePredictionFactor', this.posePredictionFactorInput.value); - localStorage.setItem('enableTexSubImage2D', this.enableTexSubImage2DSelect.value); - localStorage.setItem('useQuestColorWorkaround', this.useQuestColorWorkaroundSelect.value); + localStorage.setItem("perEyeWidth", this.perEyeWidthInput.value); + localStorage.setItem("perEyeHeight", this.perEyeHeightInput.value); + localStorage.setItem( + "reprojectionGridCols", + this.reprojectionGridColsInput.value, + ); + localStorage.setItem( + "reprojectionGridRows", + this.reprojectionGridRowsInput.value, + ); + localStorage.setItem("deviceFrameRate", this.deviceFrameRateSelect.value); + localStorage.setItem( + "maxStreamingBitrateMbps", + this.maxStreamingBitrateMbpsSelect.value, + ); + localStorage.setItem("codec", this.codecSelect.value); + localStorage.setItem( + "enablePoseSmoothing", + this.enablePoseSmoothingSelect.value, + ); + localStorage.setItem( + "posePredictionFactor", + this.posePredictionFactorInput.value, + ); + localStorage.setItem( + "enableTexSubImage2D", + this.enableTexSubImage2DSelect.value, + ); + localStorage.setItem( + "useQuestColorWorkaround", + this.useQuestColorWorkaroundSelect.value, + ); } catch (e) { - console.warn('Failed to persist profile fields to localStorage:', e); + console.warn("Failed to persist profile fields to localStorage:", e); } } @@ -1109,14 +1305,15 @@ export class CloudXR2DUI { if (!this.deviceProfileWarning) return; const profile = getDeviceProfile(profileId); const needsHttps = profile.connection?.httpsRequired === true; - const isHttp = window.location.protocol === 'http:'; + const isHttp = window.location.protocol === "http:"; if (needsHttps && isHttp) { - this.deviceProfileWarning.textContent = 'This device requires HTTPS mode.'; - this.deviceProfileWarning.style.display = 'block'; + this.deviceProfileWarning.textContent = + "This device requires HTTPS mode."; + this.deviceProfileWarning.style.display = "block"; } else { - this.deviceProfileWarning.style.display = 'none'; - this.deviceProfileWarning.textContent = ''; + this.deviceProfileWarning.style.display = "none"; + this.deviceProfileWarning.textContent = ""; } } @@ -1147,12 +1344,12 @@ export class CloudXR2DUI { */ public setupConnectButtonHandler( onConnect: () => Promise, - onError: (error: Error) => void + onError: (error: Error) => void, ): void { if (this.startButton) { // Remove any existing listener if (this.handleConnectClick) { - this.startButton.removeEventListener('click', this.handleConnectClick); + this.startButton.removeEventListener("click", this.handleConnectClick); } // Create new handler @@ -1163,28 +1360,31 @@ export class CloudXR2DUI { return; } const cfg = this.getConfiguration(); - const resolutionError = getResolutionValidationError(cfg.perEyeWidth, cfg.perEyeHeight); + const resolutionError = getResolutionValidationError( + cfg.perEyeWidth, + cfg.perEyeHeight, + ); const gridError = getGridValidationError( cfg.reprojectionGridCols, - cfg.reprojectionGridRows + cfg.reprojectionGridRows, ); if (resolutionError || gridError) { this.updateConnectButtonState(); return; } - this.setStartButtonState(true, 'CONNECT (starting XR session...)'); + this.setStartButtonState(true, "CONNECT (starting XR session...)"); try { await onConnect(); } catch (error) { - this.setStartButtonState(false, 'CONNECT'); + this.setStartButtonState(false, "CONNECT"); this.updateConnectButtonState(); onError(error as Error); } }; // Add the new listener - this.startButton.addEventListener('click', this.handleConnectClick); + this.startButton.addEventListener("click", this.handleConnectClick); } } @@ -1193,12 +1393,12 @@ export class CloudXR2DUI { * @param message - Message to display * @param type - Message type: 'success', 'error', or 'info' */ - public showStatus(message: string, type: 'success' | 'error' | 'info'): void { + public showStatus(message: string, type: "success" | "error" | "info"): void { if (this.errorMessageText && this.errorMessageBox) { this.errorMessageText.textContent = message; this.errorMessageBox.className = `error-message-box show ${type}`; } - console[type === 'error' ? 'error' : 'info'](message); + console[type === "error" ? "error" : "info"](message); } /** @@ -1206,7 +1406,7 @@ export class CloudXR2DUI { * @param message - Error message to display */ public showError(message: string): void { - this.showStatus(message, 'error'); + this.showStatus(message, "error"); } /** @@ -1214,7 +1414,7 @@ export class CloudXR2DUI { */ public hideError(): void { if (this.errorMessageBox) { - this.errorMessageBox.classList.remove('show'); + this.errorMessageBox.classList.remove("show"); } } @@ -1231,7 +1431,7 @@ export class CloudXR2DUI { // Remove CONNECT button listener if (this.startButton && this.handleConnectClick) { - this.startButton.removeEventListener('click', this.handleConnectClick); + this.startButton.removeEventListener("click", this.handleConnectClick); this.handleConnectClick = null; } diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index 4a6c82c7b..55284bd01 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -35,19 +35,26 @@ * back to the parent component through callback props. */ -import arrowLeftStartOnRectangleSvg from './icons/arrow-left-start-on-rectangle.svg'; -import arrowUturnLeftSvg from './icons/arrow-uturn-left.svg'; -import playCircleSvg from './icons/play-circle.svg'; -import { PerformanceCanvasImage } from '@helpers/react/PerformanceCanvasImage'; -import { useXRButton } from '@helpers/react/useXRButton'; -import { ReadonlySignal } from '@preact/signals-react'; -import { useFrame } from '@react-three/fiber'; -import { Handle, HandleTarget, HandleState } from '@react-three/handle'; -import { Container, Text, Image } from '@react-three/uikit'; -import { Button } from '@react-three/uikit-default'; -import React, { useRef, useState, useEffect } from 'react'; -import { Color, Group, Mesh, MeshStandardMaterial, Object3D, Vector3 } from 'three'; -import { damp } from 'three/src/math/MathUtils.js'; +import arrowLeftStartOnRectangleSvg from "./icons/arrow-left-start-on-rectangle.svg"; +import arrowUturnLeftSvg from "./icons/arrow-uturn-left.svg"; +import playCircleSvg from "./icons/play-circle.svg"; +import { PerformanceCanvasImage } from "@helpers/react/PerformanceCanvasImage"; +import { useXRButton } from "@helpers/react/useXRButton"; +import { ReadonlySignal } from "@preact/signals-react"; +import { useFrame } from "@react-three/fiber"; +import { Handle, HandleTarget, HandleState } from "@react-three/handle"; +import { Container, Text, Image } from "@react-three/uikit"; +import { Button } from "@react-three/uikit-default"; +import React, { useRef, useState, useEffect } from "react"; +import { + Color, + Group, + Mesh, + MeshStandardMaterial, + Object3D, + Vector3, +} from "three"; +import { damp } from "three/src/math/MathUtils.js"; // Face-camera rotation constants const FACE_CAMERA_DAMPING = 10; // Higher = faster rotation toward camera @@ -80,6 +87,18 @@ interface CloudXRUIProps { panelHiddenAtStart?: boolean; /** Immersive XR active; used to apply panelHiddenAtStart on session enter. */ isXRMode?: boolean; + /** Show recording controls (Record/Replay/Save) — enabled via Troubleshooting settings. */ + showRecordingControls?: boolean; + recorderMode?: "idle" | "recording" | "replaying"; + hasSavedRecording?: boolean; + recordedFrameCount?: number; + onStartRecord?: () => void; + onStopRecord?: () => void; + onStartReplay?: () => void; + onStopReplay?: () => void; + onSaveRecording?: () => void; + /** Show/hide rolling path trace for hands/controllers. Driven by the 2D settings panel. */ + showTrace?: boolean; } // Reusable objects for face-camera rotation (avoid allocations in render loop) @@ -87,8 +106,8 @@ const cameraPositionHelper = new Vector3(); const uiPositionHelper = new Vector3(); // Handle hover colors (module-level to avoid per-render allocations) -const HANDLE_COLOR_DEFAULT = new Color('#666666'); -const HANDLE_COLOR_HOVER = new Color('#aaaaaa'); +const HANDLE_COLOR_DEFAULT = new Color("#666666"); +const HANDLE_COLOR_HOVER = new Color("#aaaaaa"); // Workaround for @pmndrs/handle defaultApply behavior: defaultApply copies // state.current.quaternion to the target on every drag frame AND on drag release. @@ -98,7 +117,10 @@ const HANDLE_COLOR_HOVER = new Color('#aaaaaa'); // By providing a custom apply that skips quaternion, face-camera owns rotation fully. // Scale is intentionally omitted too: scale={false} keeps it constant, so copying // it would be a no-op. If scale is ever enabled on this Handle, add it back here. -function applyPositionSkipRotation(state: HandleState, target: Object3D): void { +function applyPositionSkipRotation( + state: HandleState, + target: Object3D, +): void { target.position.copy(state.current.position); } @@ -106,9 +128,9 @@ export default function CloudXR3DUI({ onStartTeleop, onDisconnect, onResetTeleop, - serverAddress = '127.0.0.1', - sessionStatus = 'Disconnected', - playLabel = 'Play', + serverAddress = "127.0.0.1", + sessionStatus = "Disconnected", + playLabel = "Play", playInProgress = false, countdownSeconds, onCountdownIncrease, @@ -121,8 +143,18 @@ export default function CloudXR3DUI({ poseToRenderText, panelHiddenAtStart = false, isXRMode = false, + showRecordingControls = false, + recorderMode = "idle", + hasSavedRecording = false, + recordedFrameCount = 0, + onStartRecord, + onStopRecord, + onStartReplay, + onStopReplay, + onSaveRecording, + showTrace = false, }: CloudXRUIProps) { - const MINIMIZE_ON_PLAY_KEY = 'cxr.isaac.minimizeOnPlay'; + const MINIMIZE_ON_PLAY_KEY = "cxr.isaac.minimizeOnPlay"; const groupRef = useRef(null); const handleRef = useRef(null); @@ -131,7 +163,7 @@ export default function CloudXR3DUI({ const [minimizeOnPlay, setMinimizeOnPlay] = useState(() => { try { const saved = localStorage.getItem(MINIMIZE_ON_PLAY_KEY); - return saved === 'true'; + return saved === "true"; } catch { return false; } @@ -190,7 +222,12 @@ export default function CloudXR3DUI({ diff = diff - Math.round(diff / (2 * Math.PI)) * (2 * Math.PI); targetY = currentY + diff; - groupRef.current.rotation.y = damp(currentY, targetY, FACE_CAMERA_DAMPING, dt); + groupRef.current.rotation.y = damp( + currentY, + targetY, + FACE_CAMERA_DAMPING, + dt, + ); }); return ( @@ -199,7 +236,7 @@ export default function CloudXR3DUI({ ref={groupRef} position={position} rotation={rotation} - pointerEventsType={{ deny: 'grab' }} + pointerEventsType={{ deny: "grab" }} > {/* Drag Handle Bar - grab to reposition the panel */} { - const mat = handleRef.current?.material as MeshStandardMaterial | undefined; + const mat = handleRef.current?.material as + | MeshStandardMaterial + | undefined; if (mat) { mat.color.copy(HANDLE_COLOR_HOVER); mat.opacity = panelHidden ? 0.55 : 0.9; } }} onPointerLeave={() => { - const mat = handleRef.current?.material as MeshStandardMaterial | undefined; + const mat = handleRef.current?.material as + | MeshStandardMaterial + | undefined; if (mat) { mat.color.copy(HANDLE_COLOR_DEFAULT); mat.opacity = panelHidden ? 0.35 : 0.6; } }} > - + {panelHidden ? ( @@ -280,21 +327,23 @@ export default function CloudXR3DUI({ padding={24} > @@ -411,7 +466,7 @@ export default function CloudXR3DUI({ gap={14} marginTop={20} cursor="pointer" - {...xrButton('minimize', () => setMinimizeOnPlay(v => !v))} + {...xrButton("minimize", () => setMinimizeOnPlay((v) => !v))} > + {/* Recording controls — shown only when enabled in Troubleshooting settings */} + {showRecordingControls && ( + + + Recording + + {recorderMode !== "idle" && ( + + {recorderMode === "recording" + ? `● REC ${recordedFrameCount} frames` + : `▶ Replaying`} + + )} + + {/* Record / Stop Record */} + {recorderMode !== "replaying" && ( + + )} + {/* Replay / Stop Replay */} + {recorderMode !== "recording" && ( + + )} + {/* Save / Load — only when idle */} + {recorderMode === "idle" && hasSavedRecording && ( + + )} + + + )} {/* Right Column - Controls */} @@ -449,7 +616,12 @@ export default function CloudXR3DUI({ justifyContent="center" > {/* Title */} - + Controls @@ -461,10 +633,18 @@ export default function CloudXR3DUI({ marginTop={4} marginBottom={4} > - + Server: {serverAddress} - + Status: {sessionStatus} @@ -481,7 +661,7 @@ export default function CloudXR3DUI({ Countdown diff --git a/deps/cloudxr/webxr_client/src/RecorderComponent.tsx b/deps/cloudxr/webxr_client/src/RecorderComponent.tsx new file mode 100644 index 000000000..9985018a6 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/RecorderComponent.tsx @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * RecorderComponent.tsx - Null-rendering R3F component that drives the recorder each frame. + * + * Runs at priority -1001 so it executes before CloudXRComponent (-1000), ensuring + * beginFrame() advances the replay pointer before the CloudXR SDK reads the + * monkey-patched XRFrame methods. + */ + +import { useFrame } from "@react-three/fiber"; +import { useRef } from "react"; + +import type { XRInputRecorder } from "./xrInputRecorder"; + +interface RecorderComponentProps { + recorder: XRInputRecorder; + /** Called approximately every 30 frames during recording with current frame count. */ + onFrameRecord?: (count: number) => void; + /** True when the CloudXR session is in the Connected state — gates replay frame advancement. */ + isConnected: boolean; +} + +export function RecorderComponent({ + recorder, + onFrameRecord, + isConnected, +}: RecorderComponentProps) { + const tickRef = useRef(0); + + useFrame((state) => { + const xrFrame = state.gl.xr.getFrame() as XRFrame | null; + if (!xrFrame) return; + + // Only advance replay/record when the session is visible (avoids poisoning recordings + // with null poses on Quest sleep) AND CloudXR is Connected (avoids consuming replay + // frames during Connecting state before the server is ready). + const session = state.gl.xr.getSession() as XRSession | null; + const isVisible = session?.visibilityState === "visible"; + + recorder.setSceneRefSpace(state.gl.xr.getReferenceSpace()); + recorder.beginFrame(xrFrame, isConnected && isVisible); + + if (recorder.mode === "recording" && onFrameRecord) { + tickRef.current++; + if (tickRef.current % 30 === 0) { + onFrameRecord(recorder.recordedFrameCount); + } + } else { + tickRef.current = 0; + } + }, -1001); + + return null; +} diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.tsx new file mode 100644 index 000000000..f2d9cc9f1 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/RecorderContext.tsx @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * RecorderContext.tsx - React context for XRInputRecorder state. + * + * Owns the recorder instance and the React-visible state (mode, saved + * recording, showTrace). Heavy per-frame work (beginFrame) runs in + * RecorderComponent via useFrame; this context provides actions and state + * to descendant components. + */ + +import React, { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from "react"; + +import { XRInputRecorder, type Recording } from "./xrInputRecorder"; + +export interface RecorderContextValue { + recorder: XRInputRecorder; + mode: "idle" | "recording" | "replaying"; + savedRecording: Recording | null; + recordedFrameCount: number; + startRecord: () => void; + stopRecord: () => void; + startReplay: () => void; + stopReplay: () => void; + onSaveRecording: () => void; + onLoadRecording: () => void; + setSavedRecording: (r: Recording) => void; + onFrameRecord: (count: number) => void; +} + +const RecorderContext = createContext(null); + +export function RecorderProvider({ children }: { children: React.ReactNode }) { + const recorder = useMemo(() => new XRInputRecorder(), []); + const [mode, setMode] = useState<"idle" | "recording" | "replaying">("idle"); + const [savedRecording, setSavedRecordingState] = useState( + null, + ); + const [recordedFrameCount, setRecordedFrameCount] = useState(0); + const fileInputRef = useRef(null); + + const startRecord = useCallback(() => { + if (recorder.mode !== "idle") return; + recorder.startRecording(); + setMode("recording"); + setRecordedFrameCount(0); + }, [recorder]); + + const stopRecord = useCallback(() => { + if (recorder.mode !== "recording") return; + recorder.stopRecording(); + const rec = recorder.getRecording(); + setSavedRecordingState(rec); + setRecordedFrameCount(rec.frames.length); + setMode("idle"); + }, [recorder]); + + const startReplay = useCallback(() => { + if (recorder.mode !== "idle" || !savedRecording) return; + recorder.startReplay(savedRecording, true); + setMode("replaying"); + }, [recorder, savedRecording]); + + const stopReplay = useCallback(() => { + if (recorder.mode !== "replaying") return; + recorder.stopReplay(); + setMode("idle"); + }, [recorder]); + + const onSaveRecording = useCallback(() => { + if (!savedRecording) return; + const blob = new Blob([JSON.stringify(savedRecording)], { + type: "application/json", + }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "isaacteleop-input-recording.json"; + a.click(); + URL.revokeObjectURL(url); + }, [savedRecording]); + + const setSavedRecording = useCallback((r: Recording) => { + setSavedRecordingState(r); + setRecordedFrameCount(r.frames.length); + }, []); + + useEffect(() => { + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".json,application/json"; + input.style.display = "none"; + input.addEventListener("change", (e: Event) => { + const file = (e.target as HTMLInputElement).files?.[0]; + if (!file) return; + file + .text() + .then((text) => { + try { + const recording = XRInputRecorder.importJSON(text); + setSavedRecording(recording); + } catch (err) { + console.error("[Recorder] Failed to load recording:", err); + } + }) + .catch((err) => { + console.error("[Recorder] Failed to read file:", err); + }); + (e.target as HTMLInputElement).value = ""; + }); + document.body.appendChild(input); + fileInputRef.current = input; + return () => { + document.body.removeChild(input); + fileInputRef.current = null; + }; + }, [setSavedRecording]); + + const onLoadRecording = useCallback(() => { + fileInputRef.current?.click(); + }, []); + + const onFrameRecord = useCallback((count: number) => { + setRecordedFrameCount(count); + }, []); + + const value: RecorderContextValue = { + recorder, + mode, + savedRecording, + recordedFrameCount, + startRecord, + stopRecord, + startReplay, + stopReplay, + onSaveRecording, + onLoadRecording, + setSavedRecording, + onFrameRecord, + }; + + return ( + + {children} + + ); +} + +export function useRecorder(): RecorderContextValue { + const ctx = useContext(RecorderContext); + if (!ctx) throw new Error("useRecorder must be used inside RecorderProvider"); + return ctx; +} diff --git a/deps/cloudxr/webxr_client/src/TraceVisualization.tsx b/deps/cloudxr/webxr_client/src/TraceVisualization.tsx new file mode 100644 index 000000000..e5acbbfc0 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/TraceVisualization.tsx @@ -0,0 +1,292 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * TraceVisualization.tsx - R3F component that renders hand/controller path traces. + * + * Renders rolling-window point-cloud traces for left and right grip/wrist positions. + * No meshes — the existing IsaacTeleop hand/controller models already show current pose. + * + * In idle mode (not recording/replaying) poses are queried directly from the live + * XR session in the scene's reference space so traces build up at all times. + * During recording/replay, poses come from the recorder's current frame. + * + * All geometry is pre-allocated and mutated directly; positions/visibility are + * never driven by React state so this runs at full XR frame rate. + */ + +import { useFrame } from "@react-three/fiber"; +import { useRef } from "react"; +import { + BufferAttribute, + BufferGeometry, + Color, + Points, + PointsMaterial, +} from "three"; + +import type { + XRInputRecorder, + RecordedFrame, + SerializedJoint, +} from "./xrInputRecorder"; + +// ---- constants ------------------------------------------------------------- + + +const TRACE_LEN = 500; +// Fixed screen-space pixel size — reliable in WebXR stereo +const TRACE_DOT_SIZE = 6; + +// ---- TraceBuffer ----------------------------------------------------------- + +class TraceBuffer { + private readonly buf: Float32Array; + private writeIdx = 0; + private count = 0; + + constructor(private readonly capacity: number) { + this.buf = new Float32Array(capacity * 3); + } + + push(x: number, y: number, z: number): void { + const i = this.writeIdx * 3; + this.buf[i] = x; + this.buf[i + 1] = y; + this.buf[i + 2] = z; + this.writeIdx = (this.writeIdx + 1) % this.capacity; + if (this.count < this.capacity) this.count++; + } + + /** + * Fill positions/colors oldest→newest with brightness ramping 0.1→1.0. + * Returns the number of valid points written. + */ + fill( + positions: Float32Array, + colors: Float32Array, + br: number, + bg: number, + bb: number, + ): number { + if (this.count === 0) return 0; + const start = this.count < this.capacity ? 0 : this.writeIdx; + const last = this.count - 1; + for (let i = 0; i < this.count; i++) { + const src = ((start + i) % this.capacity) * 3; + const dst = i * 3; + positions[dst] = this.buf[src]; + positions[dst + 1] = this.buf[src + 1]; + positions[dst + 2] = this.buf[src + 2]; + const t = last > 0 ? i / last : 1; + const brightness = 0.1 + 0.9 * t; + colors[dst] = br * brightness; + colors[dst + 1] = bg * brightness; + colors[dst + 2] = bb * brightness; + } + return this.count; + } + + clear(): void { + this.writeIdx = 0; + this.count = 0; + } +} + +// ---- helpers --------------------------------------------------------------- + +interface TraceChannel { + buf: TraceBuffer; + points: Points; + positions: Float32Array; + colors: Float32Array; + br: number; + bg: number; + bb: number; +} + +function makeTraceChannel(color: string): TraceChannel { + const positions = new Float32Array(TRACE_LEN * 3); + const colors = new Float32Array(TRACE_LEN * 3); + const geo = new BufferGeometry(); + geo.setAttribute("position", new BufferAttribute(positions, 3)); + geo.setAttribute("color", new BufferAttribute(colors, 3)); + geo.setDrawRange(0, 0); + const mat = new PointsMaterial({ + vertexColors: true, + size: TRACE_DOT_SIZE, + sizeAttenuation: false, + }); + const points = new Points(geo, mat); + points.visible = false; + const c = new Color(color); + return { + buf: new TraceBuffer(TRACE_LEN), + points, + positions, + colors, + br: c.r, + bg: c.g, + bb: c.b, + }; +} + +type PoseLike = { px: number; py: number; pz: number } | null | undefined; +type JointLike = (PoseLike & { radius?: number }) | null | undefined; + +// ---- component ------------------------------------------------------------- + +interface TraceVisualizationProps { + recorder: XRInputRecorder; + showTrace: boolean; +} + +export function TraceVisualization({ + recorder, + showTrace, +}: TraceVisualizationProps) { + // Stable trace channels initialised on first render + const traceRef = useRef<{ + leftGrip: TraceChannel; + rightGrip: TraceChannel; + leftWrist: TraceChannel; + rightWrist: TraceChannel; + } | null>(null); + + if (traceRef.current === null) { + traceRef.current = { + leftGrip: makeTraceChannel("#4488ff"), + rightGrip: makeTraceChannel("#44ff88"), + leftWrist: makeTraceChannel("#ff4422"), + rightWrist: makeTraceChannel("#ff44cc"), + }; + } + const traceChannels = Object.values(traceRef.current); + + useFrame((state) => { + const t = traceRef.current!; + + // Use recorded/replayed frame when active, otherwise query live XR poses + let frame: RecordedFrame | null = recorder.currentFrame; + + if (!frame) { + const xrFrame = state.gl.xr.getFrame() as XRFrame | null; + const refSpace = + state.gl.xr.getReferenceSpace() as XRReferenceSpace | null; + if (xrFrame && refSpace) { + const livePoses: RecordedFrame["poses"] = { + leftGrip: null, + leftAim: null, + rightGrip: null, + rightAim: null, + }; + const liveJoints: { + left: (SerializedJoint | null)[]; + right: (SerializedJoint | null)[]; + } = { left: [], right: [] }; + let hasJoints = false; + + for (const src of xrFrame.session.inputSources) { + const h = src.handedness; + if (h !== "left" && h !== "right") continue; + if (src.gripSpace) { + const p = xrFrame.getPose(src.gripSpace, refSpace); + if (p) { + const { position: pos, orientation: ori } = p.transform; + const sp = { + px: pos.x, + py: pos.y, + pz: pos.z, + ox: ori.x, + oy: ori.y, + oz: ori.z, + ow: ori.w, + }; + if (h === "left") livePoses.leftGrip = sp; + else livePoses.rightGrip = sp; + } + } + if (src.hand) { + hasJoints = true; + let i = 0; + for (const [, joint] of src.hand.entries()) { + const jp = xrFrame.getJointPose?.(joint, refSpace); + if (jp) { + const { position: pos, orientation: ori } = jp.transform; + liveJoints[h][i] = { + px: pos.x, + py: pos.y, + pz: pos.z, + ox: ori.x, + oy: ori.y, + oz: ori.z, + ow: ori.w, + radius: jp.radius ?? 0.005, + }; + } else { + liveJoints[h][i] = null; + } + i++; + } + } + } + + frame = { + poses: livePoses, + gamepads: { left: null, right: null }, + handJoints: hasJoints ? liveJoints : undefined, + }; + } + } + + const updateTrace = (slot: TraceChannel, pose: PoseLike) => { + if (!showTrace) { + slot.points.visible = false; + return; + } + if (frame && pose) { + slot.buf.push(pose.px, pose.py, pose.pz); + const count = slot.buf.fill( + slot.positions, + slot.colors, + slot.br, + slot.bg, + slot.bb, + ); + const geo = slot.points.geometry as BufferGeometry; + geo.setDrawRange(0, count); + (geo.attributes.position as BufferAttribute).needsUpdate = true; + (geo.attributes.color as BufferAttribute).needsUpdate = true; + slot.points.visible = true; + } + // showTrace && tracking loss: leave visible so accumulated history stays rendered + }; + + updateTrace(t.leftGrip, frame?.poses.leftGrip); + updateTrace(t.rightGrip, frame?.poses.rightGrip); + updateTrace(t.leftWrist, frame?.handJoints?.left?.[0] as JointLike); + updateTrace(t.rightWrist, frame?.handJoints?.right?.[0] as JointLike); + }); + + return ( + <> + {traceChannels.map((ch, i) => ( + + ))} + + ); +} diff --git a/deps/cloudxr/webxr_client/src/config/params.ts b/deps/cloudxr/webxr_client/src/config/params.ts index e9cd4c6d3..5df6c2d2d 100644 --- a/deps/cloudxr/webxr_client/src/config/params.ts +++ b/deps/cloudxr/webxr_client/src/config/params.ts @@ -34,7 +34,7 @@ export interface UrlParam { /** id of the bound form control in index.html. Present only for form-backed params. */ elementId?: string; /** How a form-backed value is applied to the control. Defaults to 'value'. */ - kind?: 'value' | 'checked'; + kind?: "value" | "checked"; /** Optional check for the raw URL string; invalid values are ignored. */ isValid?: (raw: string) => boolean; /** @@ -49,50 +49,204 @@ const oneOf = (...allowed: string[]) => (raw: string): boolean => allowed.includes(raw); -const isBool = oneOf('true', 'false'); -const isNumber = (raw: string): boolean => raw.trim() !== '' && Number.isFinite(Number(raw)); +const isBool = oneOf("true", "false"); +const isNumber = (raw: string): boolean => + raw.trim() !== "" && Number.isFinite(Number(raw)); export const URL_PARAMS: UrlParam[] = [ // --- Form-backed settings (seeded into a control, then read through the form) --- - { key: 'serverIP', elementId: 'serverIpInput', description: 'CloudXR server IP/hostname (default: page URL hostname).' }, - { key: 'port', elementId: 'portInput', isValid: isNumber, description: 'CloudXR server port.' }, - { key: 'serverType', elementId: 'serverType', isValid: oneOf('manual', 'nvcf'), description: 'Server backend: manual or nvcf.' }, - { key: 'codec', elementId: 'codec', isValid: oneOf('h264', 'h265', 'av1'), description: 'Preferred video codec: h264, h265, or av1.' }, - { key: 'immersiveMode', elementId: 'immersive', isValid: oneOf('ar', 'vr'), description: 'WebXR session mode: ar or vr.' }, + { + key: "serverIP", + elementId: "serverIpInput", + description: "CloudXR server IP/hostname (default: page URL hostname).", + }, + { + key: "port", + elementId: "portInput", + isValid: isNumber, + description: "CloudXR server port.", + }, + { + key: "serverType", + elementId: "serverType", + isValid: oneOf("manual", "nvcf"), + description: "Server backend: manual or nvcf.", + }, + { + key: "codec", + elementId: "codec", + isValid: oneOf("h264", "h265", "av1"), + description: "Preferred video codec: h264, h265, or av1.", + }, + { + key: "immersiveMode", + elementId: "immersive", + isValid: oneOf("ar", "vr"), + description: "WebXR session mode: ar or vr.", + }, // deviceProfile is intentionally omitted: it is a preset that bulk-fills the fields below via a // change handler, which programmatic seeding does not trigger. Set the individual fields instead. - { key: 'deviceFrameRate', elementId: 'deviceFrameRate', isValid: isNumber, description: 'Target device frame rate in FPS (e.g. 72, 90, 120).' }, - { key: 'maxStreamingBitrateMbps', elementId: 'maxStreamingBitrateMbps', isValid: isNumber, description: 'Maximum streaming bitrate in Mbps.' }, - { key: 'perEyeWidth', elementId: 'perEyeWidth', isValid: isNumber, description: 'Per-eye render width in pixels (multiple of 16, min 128).' }, - { key: 'perEyeHeight', elementId: 'perEyeHeight', isValid: isNumber, description: 'Per-eye render height in pixels (multiple of 64, min 128).' }, - { key: 'reprojectionGridCols', elementId: 'reprojectionGridCols', isValid: isNumber, description: 'Depth reprojection mesh columns (>= 2, with rows; blank = factor mode).' }, - { key: 'reprojectionGridRows', elementId: 'reprojectionGridRows', isValid: isNumber, description: 'Depth reprojection mesh rows (>= 2, with columns; blank = factor mode).' }, - { key: 'enablePoseSmoothing', elementId: 'enablePoseSmoothing', isValid: isBool, description: 'Smooth predicted positions to reduce jitter (true/false).' }, - { key: 'posePredictionFactor', elementId: 'posePredictionFactor', isValid: isNumber, description: 'Pose prediction horizon scale, 0.0 (off) to 1.0 (full).' }, - { key: 'enableTexSubImage2D', elementId: 'enableTexSubImage2D', isValid: isBool, description: 'Use texSubImage2D texture updates; faster on Quest (true/false).' }, - { key: 'useQuestColorWorkaround', elementId: 'useQuestColorWorkaround', isValid: isBool, description: 'Display P3 color workaround for Quest 3 Browser (true/false).' }, - { key: 'referenceSpace', elementId: 'referenceSpace', isValid: oneOf('auto', 'local-floor', 'local', 'viewer', 'unbounded'), description: 'XR reference space: auto, local-floor, local, viewer, or unbounded.' }, - { key: 'xrOffsetX', elementId: 'xrOffsetX', isValid: isNumber, description: 'Reference-space X offset (horizontal) in centimeters.' }, - { key: 'xrOffsetY', elementId: 'xrOffsetY', isValid: isNumber, description: 'Reference-space Y offset (vertical) in centimeters.' }, - { key: 'xrOffsetZ', elementId: 'xrOffsetZ', isValid: isNumber, description: 'Reference-space Z offset (depth) in centimeters.' }, - { key: 'controlPanelPosition', elementId: 'controlPanelPosition', isValid: oneOf('left', 'center', 'right'), description: 'In-XR control panel start position: left, center, or right.' }, - { key: 'controllerModelVisibility', elementId: 'controllerModelVisibility', isValid: oneOf('show', 'hide'), description: 'Show or hide controller model meshes in XR.' }, - { key: 'panelHiddenAtStart', elementId: 'panelHiddenAtStart', isValid: isBool, description: 'Start with the in-XR control panel hidden (true/false).' }, - { key: 'headless', elementId: 'cloudxrHeadless', kind: 'checked', isValid: isBool, description: 'Headless: skip all client render code, keep tracking (true/false).' }, - { key: 'autoRefreshMode', elementId: 'cloudxrAutoRefreshMode', isValid: oneOf('never', 'clean', 'any'), description: 'Reload page after session ends: never, clean, or any.' }, - { key: 'proxyUrl', elementId: 'proxyUrl', description: 'Proxy URL for routing (HTTPS); leave empty for direct WSS.' }, - { key: 'mediaAddress', elementId: 'mediaAddress', description: 'WebRTC media server address for NAT traversal (optional).' }, - { key: 'mediaPort', elementId: 'mediaPort', isValid: isNumber, description: 'WebRTC media server port (0 = auto).' }, + { + key: "deviceFrameRate", + elementId: "deviceFrameRate", + isValid: isNumber, + description: "Target device frame rate in FPS (e.g. 72, 90, 120).", + }, + { + key: "maxStreamingBitrateMbps", + elementId: "maxStreamingBitrateMbps", + isValid: isNumber, + description: "Maximum streaming bitrate in Mbps.", + }, + { + key: "perEyeWidth", + elementId: "perEyeWidth", + isValid: isNumber, + description: "Per-eye render width in pixels (multiple of 16, min 128).", + }, + { + key: "perEyeHeight", + elementId: "perEyeHeight", + isValid: isNumber, + description: "Per-eye render height in pixels (multiple of 64, min 128).", + }, + { + key: "reprojectionGridCols", + elementId: "reprojectionGridCols", + isValid: isNumber, + description: + "Depth reprojection mesh columns (>= 2, with rows; blank = factor mode).", + }, + { + key: "reprojectionGridRows", + elementId: "reprojectionGridRows", + isValid: isNumber, + description: + "Depth reprojection mesh rows (>= 2, with columns; blank = factor mode).", + }, + { + key: "enablePoseSmoothing", + elementId: "enablePoseSmoothing", + isValid: isBool, + description: "Smooth predicted positions to reduce jitter (true/false).", + }, + { + key: "posePredictionFactor", + elementId: "posePredictionFactor", + isValid: isNumber, + description: "Pose prediction horizon scale, 0.0 (off) to 1.0 (full).", + }, + { + key: "enableTexSubImage2D", + elementId: "enableTexSubImage2D", + isValid: isBool, + description: + "Use texSubImage2D texture updates; faster on Quest (true/false).", + }, + { + key: "useQuestColorWorkaround", + elementId: "useQuestColorWorkaround", + isValid: isBool, + description: + "Display P3 color workaround for Quest 3 Browser (true/false).", + }, + { + key: "referenceSpace", + elementId: "referenceSpace", + isValid: oneOf("auto", "local-floor", "local", "viewer", "unbounded"), + description: + "XR reference space: auto, local-floor, local, viewer, or unbounded.", + }, + { + key: "xrOffsetX", + elementId: "xrOffsetX", + isValid: isNumber, + description: "Reference-space X offset (horizontal) in centimeters.", + }, + { + key: "xrOffsetY", + elementId: "xrOffsetY", + isValid: isNumber, + description: "Reference-space Y offset (vertical) in centimeters.", + }, + { + key: "xrOffsetZ", + elementId: "xrOffsetZ", + isValid: isNumber, + description: "Reference-space Z offset (depth) in centimeters.", + }, + { + key: "controlPanelPosition", + elementId: "controlPanelPosition", + isValid: oneOf("left", "center", "right"), + description: "In-XR control panel start position: left, center, or right.", + }, + { + key: "controllerModelVisibility", + elementId: "controllerModelVisibility", + isValid: oneOf("show", "hide"), + description: "Show or hide controller model meshes in XR.", + }, + { + key: "showTraceInXR", + elementId: "showTraceInXR", + isValid: isBool, + description: + "Show rolling path-trace dots for hands/controllers in XR (true/false).", + }, + { + key: "showRecordingControls", + elementId: "showRecordingControls", + isValid: isBool, + description: + "Show Record / Replay / Save controls in the in-XR panel (true/false).", + }, + { + key: "panelHiddenAtStart", + elementId: "panelHiddenAtStart", + isValid: isBool, + description: "Start with the in-XR control panel hidden (true/false).", + }, + { + key: "headless", + elementId: "cloudxrHeadless", + kind: "checked", + isValid: isBool, + description: + "Headless: skip all client render code, keep tracking (true/false).", + }, + { + key: "autoRefreshMode", + elementId: "cloudxrAutoRefreshMode", + isValid: oneOf("never", "clean", "any"), + description: "Reload page after session ends: never, clean, or any.", + }, + { + key: "proxyUrl", + elementId: "proxyUrl", + description: "Proxy URL for routing (HTTPS); leave empty for direct WSS.", + }, + { + key: "mediaAddress", + elementId: "mediaAddress", + description: "WebRTC media server address for NAT traversal (optional).", + }, + { + key: "mediaPort", + elementId: "mediaPort", + isValid: isNumber, + description: "WebRTC media server port (0 = auto).", + }, // --- Direct params: read straight from the URL by app logic (no control, never stored) --- // TURN/ICE for NAT traversal and OOB hub, typically set in USB-local mode by oob_teleop_env.py. // No `isValid` here: the consumers apply their own interpretation (exact matching, regex, etc.). // No `description`: these are set by tooling, not hand-edited, and some are secrets — keep them // out of the user-facing help panel. - { key: 'turnServer' }, - { key: 'turnUsername' }, - { key: 'turnCredential' }, - { key: 'iceRelayOnly' }, - { key: 'oobEnable' }, - { key: 'controlToken' }, + { key: "turnServer" }, + { key: "turnUsername" }, + { key: "turnCredential" }, + { key: "iceRelayOnly" }, + { key: "oobEnable" }, + { key: "controlToken" }, ]; diff --git a/deps/cloudxr/webxr_client/src/config/resolve.test.ts b/deps/cloudxr/webxr_client/src/config/resolve.test.ts index 076bea219..171fe5196 100644 --- a/deps/cloudxr/webxr_client/src/config/resolve.test.ts +++ b/deps/cloudxr/webxr_client/src/config/resolve.test.ts @@ -134,6 +134,8 @@ function sampleValid(key: string): string { referenceSpace: 'auto', controlPanelPosition: 'center', controllerModelVisibility: 'show', + showTraceInXR: 'false', + showRecordingControls: 'false', autoRefreshMode: 'clean', enablePoseSmoothing: 'true', enableTexSubImage2D: 'true', diff --git a/deps/cloudxr/webxr_client/src/index.html b/deps/cloudxr/webxr_client/src/index.html index 52e3059fa..249372268 100644 --- a/deps/cloudxr/webxr_client/src/index.html +++ b/deps/cloudxr/webxr_client/src/index.html @@ -1,4 +1,4 @@ - + - - - - - - + + + + + NVIDIA Isaac Teleop Web Client - + - + - +
-
-
-

NVIDIA Isaac Teleop Web Client - -

-
- -
- Not sure which to pick? -
-

Pick the most specific entry that accurately describes your setup. - If your specific robot configuration isn't listed, just choose the - backend; if your backend isn't listed either, choose Simulation or - Real Robot.

-

Used for telemetry and app-specific client settings.

-
-
+
+
+

+ NVIDIA Isaac Teleop Web Client + +

+
+ +
+ Not sure which to pick? +
+

+ Pick the most specific entry that accurately describes your + setup. If your specific robot configuration isn't listed, just + choose the backend; if your backend isn't listed either, choose + Simulation or Real Robot. +

+

Used for telemetry and app-specific client settings.

-
+ +
+
+ +
+ + +
+
+

Setup & Advanced

+ +
+

+ Pick your immersive mode and codec, then connect. Everything else is + grouped and folded below — open a section only when you need it. Any + setting can also be preset with a URL query parameter (see "URL + parameters" at the bottom). +

+ + +
+ + +
+ +
+ + +
+ Select the preferred streaming codec. Availability depends on + browser and server support. +
+
-
- - -
-
-

Setup & Advanced

- + + +
+ + +
+
+ Grid rules: leave either field blank to use factor mode, or + set both to integers >= 2 for explicit mesh resolution.
-

- Pick your immersive mode and codec, then connect. Everything else is grouped and folded - below — open a section only when you need it. Any setting can also be preset with a URL - query parameter (see "URL parameters" at the bottom). -

- - -
- - +
+
+ + +
+ Networking +
+
+ + + +
+ Maximum streaming bitrate (Megabits per second). Lower it on + constrained networks.
- -
- - -
- Select the preferred streaming codec. Availability depends on browser and server support. -
+
+ +
+ + + +
+
- -

Advanced settings

- - -
- - - -
- Load recommended defaults for a target device. You can still override settings below. -
-
+
+ +
+ + + + + +
+ Configure WebRTC media server address and port for NAT + traversal. Only needed when the streaming server is behind + NAT. Leave empty to use server-provided ICE information.
+
+
+
+ +
+ In-XR comfort +
+
+ + +
+ Show or hide the rolling path trace for hands and controllers + in XR. Dots fade from dim (oldest) to bright (newest). +
+
+ +
+ + +
+ Show or hide WebXR controller model meshes in XR. Input + remains active either way. +
+
+ +
+ + +
+ Where the in-XR control panel appears when you enter the + session: left, center, or right. +
+
+ +
+ + +
+ If you hide the control panel, only a small control stays + visible so you can show it again. +
+
+
+
-
- Image quality -
-
- - - -
- Select the target device frame rate for the XR session -
-
- -
- - - -
- - -
-
- Configure the per-eye resolution. Width must be a multiple of 16 (min 128); height must be a multiple of 64 (min 128). -
- - -
- - -
-
- Grid rules: leave either field blank to use factor mode, or set both to integers >= 2 for explicit mesh resolution. -
-
-
-
- -
- Networking -
-
- - - -
- Maximum streaming bitrate (Megabits per second). Lower it on constrained networks. -
-
- -
- - - -
- -
-
- -
- - - - - -
- Configure WebRTC media server address and port for NAT traversal. Only needed when the streaming server is behind NAT. Leave empty to use server-provided ICE information. -
-
-
-
- -
- In-XR comfort -
-
- - -
- Show or hide WebXR controller model meshes in XR. Input remains active either way. -
-
- -
- - -
- Where the in-XR control panel appears when you enter the session: left, center, or right. -
-
- -
- - -
- If you hide the control panel, only a small control stays visible so you can show it again. -
-
-
-
- -
- Teleoperator adjustment -
- -
- - -
- Select the preferred reference space for XR tracking. "Auto" uses fallback logic (local-floor → local → viewer). Other options will attempt to use the specified space only. -
- - -
- - -
- -
- - -
- -
- - -
-
- Offsets shift the XR origin (in centimeters) to position the operator. On headsets whose keyboard has no minus key, type the number and tap ± to make it negative. -
-
-
-
- -
- Troubleshooting -
-
- - -
- Enable or disable secondary smoothing on predicted positions to reduce jitter. This only affects position, not orientation. -
-
- -
- - -
- Scale the pose prediction horizon (0.0 = no prediction, 1.0 = full prediction). This multiplier affects both position and orientation prediction strength. -
-
- -
- - -
- Enable texSubImage2D for texture updates. Improves performance on Meta Quest devices but may cause issues on headsets with older Chromium forks. -
-
- -
- - -
- Enable Display P3 color space workaround. Recommended for Quest 3 Browser. -
-
- -
- -
- - Per frame: skip all render code in WebXR client. -
-
- -
- - -
- Reload the page after the XR session ends to return to a fresh state. Choose to refresh - only on a clean end (Disconnect, headset exit, server stop), on any end including a - streaming error, or never. -
-
-
-
- - -
- URL parameters -

- Append any of these to the page URL to preset a setting, e.g. - ?codec=h264&perEyeWidth=1920. URL values apply for that load only - and are not saved. -

-
    -
    -
    -
    +
    + URL parameters +

    + Append any of these to the page URL to preset a setting, e.g. + ?codec=h264&perEyeWidth=1920. URL values apply + for that load only and are not saved. +

    +
      +
      +
      +
      - - + diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts new file mode 100644 index 000000000..26fae605a --- /dev/null +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -0,0 +1,398 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for XRInputRecorder. + * + * WebXR globals (XRFrame, XRSession) don't exist in node — minimal stubs are + * installed as globals before any test that calls startRecording/startReplay. + */ + +import { XRInputRecorder, type Recording } from './xrInputRecorder'; + +// ---- minimal WebXR stubs --------------------------------------------------- + +class FakeXRSession { + inputSources: XRInputSource[] = []; +} + +class FakeXRFrame { + readonly session: FakeXRSession; + readonly predictedDisplayTime: number; + constructor(session: FakeXRSession, predictedDisplayTime = 0) { + this.session = session; + this.predictedDisplayTime = predictedDisplayTime; + } + getPose(_space: unknown, _ref: unknown): XRPose | null { + return null; + } + getJointPose(_joint: unknown, _ref: unknown): XRJointPose | null { + return null; + } +} + +function makeFrame( + sources: Partial[] = [], + time = 0, +): XRFrame { + const session = new FakeXRSession(); + session.inputSources = sources as XRInputSource[]; + return new FakeXRFrame(session, time) as unknown as XRFrame; +} + +/** Record N frames and return the resulting Recording. */ +function makeRecording(frameCount: number): Recording { + const r = new XRInputRecorder(); + r.startRecording(); + for (let i = 0; i < frameCount; i++) r.beginFrame(makeFrame()); + r.stopRecording(); + return r.getRecording(); +} + +beforeAll(() => { + (global as any).XRFrame = FakeXRFrame; + (global as any).XRSession = FakeXRSession; +}); + +// ---- mode state machine ---------------------------------------------------- + +describe('mode transitions', () => { + test('starts idle', () => { + const r = new XRInputRecorder(); + expect(r.mode).toBe('idle'); + }); + + test('idle → recording → idle', () => { + const r = new XRInputRecorder(); + r.startRecording(); + expect(r.mode).toBe('recording'); + r.stopRecording(); + expect(r.mode).toBe('idle'); + }); + + test('idle → replaying → idle', () => { + const r = new XRInputRecorder(); + r.startReplay(makeRecording(1)); + expect(r.mode).toBe('replaying'); + r.stopReplay(); + expect(r.mode).toBe('idle'); + }); + + test('startRecording throws when already active', () => { + const r = new XRInputRecorder(); + r.startRecording(); + expect(() => r.startRecording()).toThrow(); + r.stopRecording(); + }); + + test('startReplay throws when already active', () => { + const r = new XRInputRecorder(); + r.startReplay(makeRecording(1)); + expect(() => r.startReplay(makeRecording(1))).toThrow(); + r.stopReplay(); + }); + + test('stopRecording is safe when idle', () => { + const r = new XRInputRecorder(); + expect(() => r.stopRecording()).not.toThrow(); + }); + + test('stopReplay is safe when idle', () => { + const r = new XRInputRecorder(); + expect(() => r.stopReplay()).not.toThrow(); + }); +}); + +// ---- frame accumulation ----------------------------------------------------- + +describe('recordedFrameCount', () => { + test('zero before first recording', () => { + const r = new XRInputRecorder(); + expect(r.recordedFrameCount).toBe(0); + }); + + test('increments with each beginFrame', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame()); + expect(r.recordedFrameCount).toBe(1); + r.beginFrame(makeFrame()); + expect(r.recordedFrameCount).toBe(2); + r.stopRecording(); + }); + + test('resets to 0 on a second startRecording', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame()); + r.stopRecording(); + r.startRecording(); + expect(r.recordedFrameCount).toBe(0); + r.stopRecording(); + }); + + test('does not record when connected=false', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame(), false); + r.beginFrame(makeFrame(), false); + expect(r.recordedFrameCount).toBe(0); + r.stopRecording(); + }); + + test('getRecording frames length matches recordedFrameCount', () => { + const r = new XRInputRecorder(); + r.startRecording(); + for (let i = 0; i < 7; i++) r.beginFrame(makeFrame()); + const count = r.recordedFrameCount; + r.stopRecording(); + expect(r.getRecording().frames).toHaveLength(count); + }); +}); + +// ---- currentFrame ----------------------------------------------------------- + +describe('currentFrame', () => { + test('null in idle mode', () => { + expect(new XRInputRecorder().currentFrame).toBeNull(); + }); + + test('null before first beginFrame during recording', () => { + const r = new XRInputRecorder(); + r.startRecording(); + expect(r.currentFrame).toBeNull(); + r.stopRecording(); + }); + + test('non-null after beginFrame during recording', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame()); + expect(r.currentFrame).not.toBeNull(); + r.stopRecording(); + }); + + test('null after stopRecording', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame()); + r.stopRecording(); + expect(r.currentFrame).toBeNull(); + }); + + test('non-null during replay after beginFrame', () => { + const r = new XRInputRecorder(); + r.startReplay(makeRecording(2)); + r.beginFrame(makeFrame()); + expect(r.currentFrame).not.toBeNull(); + r.stopReplay(); + }); + + test('null after stopReplay', () => { + const r = new XRInputRecorder(); + r.startReplay(makeRecording(1)); + r.beginFrame(makeFrame()); + r.stopReplay(); + expect(r.currentFrame).toBeNull(); + }); +}); + +// ---- replay frame advancement ----------------------------------------------- + +describe('replay frame advancement', () => { + test('advances index each beginFrame', () => { + const r = new XRInputRecorder(); + r.startReplay(makeRecording(3)); + expect(r.replayFrameIndex).toBe(0); + r.beginFrame(makeFrame()); + expect(r.replayFrameIndex).toBe(1); + r.beginFrame(makeFrame()); + expect(r.replayFrameIndex).toBe(2); + r.stopReplay(); + }); + + test('loops when loop=true (default)', () => { + const r = new XRInputRecorder(); + r.startReplay(makeRecording(2), true); + r.beginFrame(makeFrame()); // consumes frame 0 → index 1 + r.beginFrame(makeFrame()); // consumes frame 1 → wraps to 0 + expect(r.replayFrameIndex).toBe(0); + r.stopReplay(); + }); + + test('clamps to last frame when loop=false', () => { + const r = new XRInputRecorder(); + r.startReplay(makeRecording(2), false); + r.beginFrame(makeFrame()); // index → 1 + r.beginFrame(makeFrame()); // index → clamped to 1 (last) + r.beginFrame(makeFrame()); // stays at 1 + expect(r.replayFrameIndex).toBe(1); + r.stopReplay(); + }); + + test('does not advance when connected=false', () => { + const r = new XRInputRecorder(); + r.startReplay(makeRecording(3)); + r.beginFrame(makeFrame(), false); + r.beginFrame(makeFrame(), false); + expect(r.replayFrameIndex).toBe(0); + r.stopReplay(); + }); + + test('beginFrame during recording does not advance replayFrameIndex', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame()); + r.beginFrame(makeFrame()); + r.stopRecording(); + expect(r.replayFrameIndex).toBe(0); + }); +}); + +// ---- serialization ---------------------------------------------------------- + +describe('exportJSON / importJSON', () => { + test('round-trip preserves version and frame count', () => { + const r = new XRInputRecorder(); + r.startRecording(); + for (let i = 0; i < 4; i++) r.beginFrame(makeFrame()); + r.stopRecording(); + const loaded = XRInputRecorder.importJSON(r.exportJSON()); + expect(loaded.version).toBe(1); + expect(loaded.frames).toHaveLength(4); + }); + + test('importJSON throws on unsupported version', () => { + expect(() => + XRInputRecorder.importJSON(JSON.stringify({ version: 99, frames: [] })), + ).toThrow(/version/i); + }); + + test('importJSON throws when frames is not an array', () => { + expect(() => + XRInputRecorder.importJSON(JSON.stringify({ version: 1, frames: null })), + ).toThrow(/malformed/i); + }); +}); + +describe('getRecording', () => { + test('returns a snapshot — further recording does not mutate it', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame()); + r.stopRecording(); + const snap = r.getRecording(); + const snapLen = snap.frames.length; + + r.startRecording(); + r.beginFrame(makeFrame()); + r.stopRecording(); + + expect(snap.frames).toHaveLength(snapLen); + }); +}); + +// ---- timestamp & head capture (recorded, not replayed) ---------------------- + +describe('timestamp & head capture', () => { + test('captures predictedDisplayTime per frame as t', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame([], 123)); + r.beginFrame(makeFrame([], 456)); + r.stopRecording(); + const frames = r.getRecording().frames; + expect(frames[0].t).toBe(123); + expect(frames[1].t).toBe(456); + }); + + test('records recordedAt epoch on the Recording', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame()); + r.stopRecording(); + expect(typeof r.getRecording().recordedAt).toBe('number'); + }); + + test('captures head pose via getViewerPose when scene ref space is set', () => { + const r = new XRInputRecorder(); + const frame = makeFrame(); + (frame as any).getViewerPose = () => ({ + transform: { + position: { x: 1, y: 2, z: 3 }, + orientation: { x: 0, y: 0, z: 0, w: 1 }, + }, + }); + r.setSceneRefSpace({} as unknown as XRSpace); + r.startRecording(); + r.beginFrame(frame); + r.stopRecording(); + expect(r.getRecording().frames[0].head).toEqual({ + px: 1, + py: 2, + pz: 3, + ox: 0, + oy: 0, + oz: 0, + ow: 1, + }); + }); + + test('head is null when no scene ref space is set', () => { + const r = new XRInputRecorder(); + r.startRecording(); + r.beginFrame(makeFrame()); + r.stopRecording(); + expect(r.getRecording().frames[0].head).toBeNull(); + }); +}); + +// ---- prototype patching ----------------------------------------------------- + +describe('prototype patching', () => { + test('startRecording replaces XRFrame.prototype.getPose', () => { + const original = FakeXRFrame.prototype.getPose; + const r = new XRInputRecorder(); + r.startRecording(); + expect(FakeXRFrame.prototype.getPose).not.toBe(original); + r.stopRecording(); + }); + + test('stopRecording restores XRFrame.prototype.getPose', () => { + const original = FakeXRFrame.prototype.getPose; + const r = new XRInputRecorder(); + r.startRecording(); + r.stopRecording(); + expect(FakeXRFrame.prototype.getPose).toBe(original); + }); + + test('startReplay replaces XRFrame.prototype.getPose', () => { + const original = FakeXRFrame.prototype.getPose; + const r = new XRInputRecorder(); + r.startReplay(makeRecording(1)); + expect(FakeXRFrame.prototype.getPose).not.toBe(original); + r.stopReplay(); + }); + + test('stopReplay restores XRFrame.prototype.getPose', () => { + const original = FakeXRFrame.prototype.getPose; + const r = new XRInputRecorder(); + r.startReplay(makeRecording(1)); + r.stopReplay(); + expect(FakeXRFrame.prototype.getPose).toBe(original); + }); +}); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts new file mode 100644 index 000000000..4256ad72f --- /dev/null +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -0,0 +1,556 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * XR Input Recorder / Replayer + * + * Monkey-patches XRFrame.prototype.getPose and XRFrame.prototype.getJointPose + * to intercept controller and hand-tracking poses during replay. + * Gamepad button/axis state is captured per-frame and replayed via a proxy. + * + * Frame boundaries are driven explicitly via beginFrame() called from the + * main requestAnimationFrame loop — this avoids relying on XRFrame object + * identity, which Chrome on Quest may reuse across callbacks (object pooling). + */ + +// ---- serialization types --------------------------------------------------- + +export type SerializedPose = { + px: number; + py: number; + pz: number; + ox: number; + oy: number; + oz: number; + ow: number; +} | null; + +export type SerializedJoint = { + px: number; + py: number; + pz: number; + ox: number; + oy: number; + oz: number; + ow: number; + radius: number; +} | null; + +type SerializedGamepad = { + buttons: Array<{ value: number; pressed: boolean; touched: boolean }>; + axes: number[]; +}; + +export type RecordedFrame = { + // Frame timestamp (ms) from XRFrame.predictedDisplayTime. Recorded, not read by replay. + t: number; + poses: { + leftGrip: SerializedPose; + leftAim: SerializedPose; + rightGrip: SerializedPose; + rightAim: SerializedPose; + }; + // Head/viewer pose in scene reference space; null when no scene ref space is set. + // Recorded, not read by replay. + head: SerializedPose; + gamepads: { + left: SerializedGamepad | null; + right: SerializedGamepad | null; + }; + // Absent when the source device has no hand tracking. + handJoints?: { + left: (SerializedJoint | null)[]; + right: (SerializedJoint | null)[]; + }; +}; + +export type Recording = { + version: 1; + // Epoch ms at record start. + recordedAt?: number; + frames: RecordedFrame[]; +}; + +// ---- helpers --------------------------------------------------------------- + +function serializePose(pose: XRPose | null | undefined): SerializedPose { + if (!pose) return null; + const { position: p, orientation: o } = pose.transform; + return { px: p.x, py: p.y, pz: p.z, ox: o.x, oy: o.y, oz: o.z, ow: o.w }; +} + +function serializeGamepad( + gp: Gamepad | null | undefined, +): SerializedGamepad | null { + if (!gp) return null; + return { + buttons: Array.from(gp.buttons).map((b) => ({ + value: b.value, + pressed: b.pressed, + touched: b.touched, + })), + axes: Array.from(gp.axes), + }; +} + +function makeFakePose(sp: SerializedPose): XRPose | undefined { + if (!sp) return undefined; + const transform = new XRRigidTransform( + { x: sp.px, y: sp.py, z: sp.pz, w: 1 }, + { x: sp.ox, y: sp.oy, z: sp.oz, w: sp.ow }, + ); + return { + transform, + emulatedPosition: true, + linearVelocity: null as unknown as DOMPointReadOnly, + angularVelocity: null as unknown as DOMPointReadOnly, + } as unknown as XRPose; +} + +function makeFakeJointPose(sj: SerializedJoint): XRJointPose | undefined { + if (!sj) return undefined; + const transform = new XRRigidTransform( + { x: sj.px, y: sj.py, z: sj.pz, w: 1 }, + { x: sj.ox, y: sj.oy, z: sj.oz, w: sj.ow }, + ); + return { + transform, + emulatedPosition: true, + linearVelocity: null as unknown as DOMPointReadOnly, + angularVelocity: null as unknown as DOMPointReadOnly, + radius: sj.radius, + } as unknown as XRJointPose; +} + +function makeFakeGamepad(sg: SerializedGamepad): Gamepad { + return { + buttons: sg.buttons.map((b) => ({ + value: b.value, + pressed: b.pressed, + touched: b.touched, + })), + axes: sg.axes, + id: "recorded", + index: -1, + connected: true, + timestamp: 0, + mapping: "xr-standard" as GamepadMappingType, + hapticActuators: [], + vibrationActuator: null, + } as unknown as Gamepad; +} + +function proxyInputSource( + src: XRInputSource, + fakeGamepad: Gamepad, +): XRInputSource { + return new Proxy(src, { + get(target, prop) { + if (prop === "gamepad") return fakeGamepad; + const val = Reflect.get(target, prop, target); + return typeof val === "function" ? val.bind(target) : val; + }, + }); +} + +function emptyFrame(): RecordedFrame { + return { + t: 0, + poses: { leftGrip: null, leftAim: null, rightGrip: null, rightAim: null }, + head: null, + gamepads: { left: null, right: null }, + }; +} + +// ---- XRInputRecorder ------------------------------------------------------- + +export class XRInputRecorder { + private _mode: "idle" | "recording" | "replaying" = "idle"; + private _frames: RecordedFrame[] = []; + private _replayFrames: RecordedFrame[] = []; + private _replayIndex = 0; + private _loopReplay = true; + + private _currentEntry: RecordedFrame | null = null; + private _currentReplay: RecordedFrame = emptyFrame(); + private _recordedAt: number | undefined = undefined; + + private _jointSpaceMap = new WeakMap< + XRJointSpace, + { hand: "left" | "right"; index: number } + >(); + + private _origGetPose: typeof XRFrame.prototype.getPose | null = null; + private _origGetJointPose: typeof XRFrame.prototype.getJointPose | null = + null; + private _origInputSourcesDesc: PropertyDescriptor | null = null; + /** Scene reference space used to proactively capture grip/joint poses for the trace. */ + private _sceneRefSpace: XRSpace | null = null; + + setSceneRefSpace(rs: XRSpace | null): void { + this._sceneRefSpace = rs; + } + + get mode() { + return this._mode; + } + get recordedFrameCount() { + return this._frames.length; + } + get replayFrameIndex() { + return this._replayIndex; + } + + // ---- lifecycle ----------------------------------------------------------- + + startRecording(): void { + if (this._mode !== "idle") + throw new Error("XRInputRecorder: already active"); + this._frames = []; + this._currentEntry = null; + this._recordedAt = Date.now(); + this._installPatches(); + this._mode = "recording"; + } + + stopRecording(): void { + if (this._mode !== "recording") return; + this._removePatches(); + this._currentEntry = null; + this._mode = "idle"; + } + + startReplay(recording: Recording, loop = true): void { + if (this._mode !== "idle") + throw new Error("XRInputRecorder: already active"); + this._replayFrames = recording.frames; + this._replayIndex = 0; + this._loopReplay = loop; + this._currentReplay = emptyFrame(); + this._installPatches(); + this._mode = "replaying"; + } + + stopReplay(): void { + if (this._mode !== "replaying") return; + this._removePatches(); + this._mode = "idle"; + } + + /** + * Must be called once at the start of every requestAnimationFrame callback. + * connected=true gates replay frame advancement so frames are not consumed + * while the CloudXR session is still initialising. + */ + beginFrame(frame: XRFrame, connected = true): void { + if (this._mode === "recording" && connected) { + const entry = emptyFrame(); + entry.t = frame.predictedDisplayTime; + for (const src of frame.session.inputSources) { + const gp = serializeGamepad(src.gamepad); + if (src.handedness === "left") entry.gamepads.left = gp; + else if (src.handedness === "right") entry.gamepads.right = gp; + } + + // Proactively capture grip/aim poses and hand joints in scene space so + // the trace works even if CloudXR never calls getPose(gripSpace, ...). + // These run through the original (unpatched) getPose, then _onGetPose + // will not overwrite them (null-check guard) if CloudXR uses a different + // reference space. + const rs = this._sceneRefSpace; + const origGet = this._origGetPose; + const origGetJoint = this._origGetJointPose; + if (rs && origGet) { + // getViewerPose is unpatched, so this reads the real head pose. + if (typeof frame.getViewerPose === "function") { + entry.head = serializePose( + frame.getViewerPose(rs as XRReferenceSpace), + ); + } + for (const src of frame.session.inputSources) { + const h = src.handedness; + if (h !== "left" && h !== "right") continue; + if (src.gripSpace) { + const p = origGet.call(frame, src.gripSpace, rs); + if (h === "left") entry.poses.leftGrip = serializePose(p); + else entry.poses.rightGrip = serializePose(p); + } + if (src.targetRaySpace) { + const p = origGet.call(frame, src.targetRaySpace, rs); + if (h === "left") entry.poses.leftAim = serializePose(p); + else entry.poses.rightAim = serializePose(p); + } + if (src.hand && origGetJoint) { + this._ensureJointMap(frame.session); + if (!entry.handJoints) entry.handJoints = { left: [], right: [] }; + let i = 0; + for (const [, joint] of src.hand.entries()) { + const jp = origGetJoint.call(frame, joint, rs); + entry.handJoints[h][i] = jp + ? { + px: jp.transform.position.x, + py: jp.transform.position.y, + pz: jp.transform.position.z, + ox: jp.transform.orientation.x, + oy: jp.transform.orientation.y, + oz: jp.transform.orientation.z, + ow: jp.transform.orientation.w, + radius: jp.radius ?? 0.005, + } + : null; + i++; + } + } + } + } + + this._currentEntry = entry; + this._frames.push(entry); + } else if (this._mode === "replaying" && connected) { + this._currentReplay = + this._replayFrames[this._replayIndex] ?? emptyFrame(); + this._replayIndex++; + if (this._replayIndex >= this._replayFrames.length) { + this._replayIndex = this._loopReplay + ? 0 + : this._replayFrames.length - 1; + } + } + } + + /** The frame data for the current tick: recording entry when recording, replay frame when replaying. */ + get currentFrame(): RecordedFrame | null { + if (this._mode === "recording") return this._currentEntry; + if (this._mode === "replaying") return this._currentReplay; + return null; + } + + // ---- serialization ------------------------------------------------------- + + exportJSON(): string { + return JSON.stringify({ + version: 1, + recordedAt: this._recordedAt, + frames: this._frames, + } satisfies Recording); + } + + static importJSON(json: string): Recording { + const r = JSON.parse(json) as Recording; + if (r.version !== 1) + throw new Error(`Unsupported recording version: ${r.version}`); + if (!Array.isArray(r.frames)) + throw new Error("Malformed recording: frames is not an array"); + return r; + } + + getRecording(): Recording { + return { + version: 1, + recordedAt: this._recordedAt, + frames: [...this._frames], + }; + } + + // ---- patch install / remove ---------------------------------------------- + + private _installPatches(): void { + const self = this; + + this._origGetPose = XRFrame.prototype.getPose; + XRFrame.prototype.getPose = function ( + this: XRFrame, + space: XRSpace, + baseSpace: XRSpace, + ): XRPose | undefined { + return self._onGetPose(this, space, baseSpace); + }; + + this._origGetJointPose = XRFrame.prototype.getJointPose; + XRFrame.prototype.getJointPose = function ( + this: XRFrame, + joint: XRJointSpace, + baseSpace: XRSpace, + ): XRJointPose | undefined { + return self._onGetJointPose(this, joint, baseSpace); + }; + + // Intercept XRSession.inputSources so gamepad state is proxied during replay + // even when the CloudXR SDK reads it via xrFrame.session.inputSources directly. + const inputSourcesDesc = Object.getOwnPropertyDescriptor( + XRSession.prototype, + "inputSources", + ); + if (inputSourcesDesc?.get) { + this._origInputSourcesDesc = inputSourcesDesc; + const origGet = inputSourcesDesc.get; + Object.defineProperty(XRSession.prototype, "inputSources", { + get(this: XRSession) { + const real = origGet.call(this); + if (self._mode !== "replaying") return real; + const rframe = self._currentReplay; + return Array.from(real as XRInputSourceArray).map( + (src: XRInputSource) => { + const sg = + src.handedness === "left" + ? rframe.gamepads.left + : src.handedness === "right" + ? rframe.gamepads.right + : null; + return sg ? proxyInputSource(src, makeFakeGamepad(sg)) : src; + }, + ); + }, + configurable: true, + }); + } + } + + private _removePatches(): void { + if (this._origGetPose) { + XRFrame.prototype.getPose = this._origGetPose; + this._origGetPose = null; + } + if (this._origGetJointPose) { + XRFrame.prototype.getJointPose = this._origGetJointPose; + this._origGetJointPose = null; + } + if (this._origInputSourcesDesc) { + Object.defineProperty( + XRSession.prototype, + "inputSources", + this._origInputSourcesDesc, + ); + this._origInputSourcesDesc = null; + } + } + + // ---- getPose handler ----------------------------------------------------- + + private _onGetPose( + frame: XRFrame, + space: XRSpace, + baseSpace: XRSpace, + ): XRPose | undefined { + if (this._mode === "recording") { + const real = this._origGetPose!.call(frame, space, baseSpace); + const entry = this._currentEntry; + if (entry) { + for (const src of frame.session.inputSources) { + if (space === src.gripSpace) { + // Always overwrite proactive scene-space capture so the pose is stored + // in the caller's actual reference space (localFloorSpace), decoupling + // the recording from CloudXR's coordinate transform. + if (src.handedness === "left") + entry.poses.leftGrip = serializePose(real); + else if (src.handedness === "right") + entry.poses.rightGrip = serializePose(real); + break; + } + if (space === src.targetRaySpace) { + if (src.handedness === "left") + entry.poses.leftAim = serializePose(real); + else if (src.handedness === "right") + entry.poses.rightAim = serializePose(real); + break; + } + } + } + return real; + } + + if (this._mode === "replaying") { + const rframe = this._currentReplay; + for (const src of frame.session.inputSources) { + if (space === src.gripSpace) { + if (src.handedness === "left") + return makeFakePose(rframe.poses.leftGrip); + if (src.handedness === "right") + return makeFakePose(rframe.poses.rightGrip); + } + // targetRaySpace is intentionally not intercepted: the real pointer + // pose is preserved so the in-XR UI stays interactable during replay. + } + return this._origGetPose!.call(frame, space, baseSpace); + } + + return this._origGetPose!.call(frame, space, baseSpace); + } + + // ---- getJointPose handler ------------------------------------------------ + + private _onGetJointPose( + frame: XRFrame, + joint: XRJointSpace, + baseSpace: XRSpace, + ): XRJointPose | undefined { + if (this._mode === "recording") { + this._ensureJointMap(frame.session); + const real = this._origGetJointPose!.call(frame, joint, baseSpace); + const info = this._jointSpaceMap.get(joint); + const entry = this._currentEntry; + if (info && entry) { + if (!entry.handJoints) entry.handJoints = { left: [], right: [] }; + entry.handJoints[info.hand][info.index] = real + ? { + px: real.transform.position.x, + py: real.transform.position.y, + pz: real.transform.position.z, + ox: real.transform.orientation.x, + oy: real.transform.orientation.y, + oz: real.transform.orientation.z, + ow: real.transform.orientation.w, + radius: real.radius ?? 0.005, + } + : null; + } + return real; + } + + if (this._mode === "replaying") { + const rframe = this._currentReplay; + if (!rframe.handJoints) + return this._origGetJointPose!.call(frame, joint, baseSpace); + this._ensureJointMap(frame.session); + const info = this._jointSpaceMap.get(joint); + if (info) + return makeFakeJointPose( + rframe.handJoints[info.hand][info.index] ?? null, + ); + return this._origGetJointPose!.call(frame, joint, baseSpace); + } + + return this._origGetJointPose!.call(frame, joint, baseSpace); + } + + // ---- joint map builder --------------------------------------------------- + + private _ensureJointMap(session: XRSession): void { + for (const src of session.inputSources) { + if (!src.hand) continue; + const hand = src.handedness; + if (hand !== "left" && hand !== "right") continue; + let index = 0; + for (const [, jointSpace] of src.hand.entries()) { + if (!this._jointSpaceMap.has(jointSpace)) { + this._jointSpaceMap.set(jointSpace, { hand, index }); + } + index++; + } + } + } +}