From 8967e64af63ddff021839f3f7407f5b8d2ac502c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 4 May 2026 13:04:05 -0400 Subject: [PATCH 1/4] Add stream encoder and quality controls to the UI --- README.md | 9 +- cli/DFPrivateSimulatorDisplayBridge.m | 12 +- cli/XCWH264Encoder.m | 223 +++--- cli/XCWPrivateSimulatorSession.m | 15 +- client/src/app/AppShell.tsx | 40 +- client/src/features/input/usePointerInput.ts | 39 +- .../src/features/simulators/SimulatorMenu.tsx | 74 ++ client/src/features/stream/stats.ts | 3 + client/src/features/stream/streamTypes.ts | 14 + .../src/features/stream/streamWorkerClient.ts | 351 ++++++++- client/src/features/stream/useLiveStream.ts | 84 ++- client/src/features/toolbar/DebugPanel.tsx | 4 +- client/src/features/toolbar/Toolbar.tsx | 18 + client/src/features/viewport/DeviceChrome.tsx | 2 +- .../features/viewport/SimulatorViewport.tsx | 2 +- client/src/styles/components.css | 10 +- docs/api/health.md | 2 +- docs/api/rest.md | 22 + docs/cli/commands.md | 8 +- docs/cli/flags.md | 4 +- docs/guide/daemon.md | 4 +- docs/guide/video.md | 5 +- package.json | 3 + scripts/check-stream-reliability.mjs | 141 ++++ scripts/e2e-webrtc-reliability.mjs | 580 +++++++++++++++ server/src/api/routes.rs | 79 +- server/src/main.rs | 81 +- server/src/metrics/counters.rs | 16 + server/src/simulators/session.rs | 66 +- server/src/transport/webrtc.rs | 695 +++++++++++------- skills/simdeck/SKILL.md | 10 +- 31 files changed, 2073 insertions(+), 543 deletions(-) create mode 100644 scripts/check-stream-reliability.mjs create mode 100644 scripts/e2e-webrtc-reliability.mjs diff --git a/README.md b/README.md index af358c44..7b843465 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ the outbound bridge alive until you press Ctrl-C. It uses software H.264 by default with realtime stream settings for remote viewing, and prints the active codec/profile when it starts. Studio defaults to the `smooth` stream quality profile (`1170` longest edge, dynamic up to `60` fps). Use -`--stream-quality quality|balanced|smooth|economy|ci-software` to override it, +`--stream-quality quality|balanced|fast|smooth|economy|ci-software` to override it, or pass `--video-codec hardware` when a dedicated hardware encoder is preferable. The remote viewer renders live video with the browser's native video element; the canvas is only used for input geometry. @@ -116,11 +116,12 @@ more important than full-resolution smoothness: simdeck daemon start --video-codec software --low-latency ``` -Local browser streams default to 60 fps. On high-refresh local displays, opt in -to a paced hardware stream up to 120 fps: +Local browser streams default to realtime WebRTC delivery with the `quality` +profile on VideoToolbox H.264: full resolution, 120 fps, and a high bitrate floor. On +high-refresh local displays, raise the local stream target explicitly: ```sh -simdeck daemon restart --local-stream-fps 120 +simdeck daemon restart --local-stream-fps 240 ``` Restart the CoreSimulator service layer when `simctl` reports a stale service diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 5e733593..dfa930aa 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -3372,13 +3372,15 @@ - (void)disconnect { self->_latestPixelBuffer = nil; } - [self->_headlessHostWindow orderOut:nil]; - [self->_headlessHostWindow close]; - self->_headlessHostWindow = nil; - self->_headlessHostView = nil; + DFRunOnMainSync(^{ + [self->_headlessHostWindow orderOut:nil]; + [self->_headlessHostWindow close]; + self->_headlessHostWindow = nil; + self->_headlessHostView = nil; + [self.displayView removeFromSuperview]; + }); [self updateStatus:@"Disconnected"]; - [self.displayView removeFromSuperview]; }; if (dispatch_get_specific(DFPrivateSimulatorCallbackQueueKey) != NULL) { diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 3f0a2bae..5c7259e2 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -7,15 +7,15 @@ #import #include -static const int32_t XCWMaximumEncodedDimension = 1920; +static const int32_t XCWMaximumEncodedDimension = 4096; static const int32_t XCWMaximumRealtimeHardwareEncodedDimension = 1440; static const int32_t XCWMaximumSoftwareEncodedDimension = 1600; static const int32_t XCWMaximumLowLatencySoftwareEncodedDimension = 1170; -static const int32_t XCWTargetRealTimeFrameRate = 60; +static const int32_t XCWTargetRealTimeFrameRate = 120; static const int32_t XCWTargetRealtimeHardwareFrameRate = 30; -static const int32_t XCWTargetSoftwareFrameRate = 60; +static const int32_t XCWTargetSoftwareFrameRate = 120; static const int32_t XCWMinimumLocalStreamFrameRate = 15; -static const int32_t XCWMaximumLocalStreamFrameRate = 120; +static const int32_t XCWMaximumLocalStreamFrameRate = 240; static const int32_t XCWTargetLowLatencySoftwareFrameRate = 15; static const NSUInteger XCWMaximumInFlightFrames = 2; static const int32_t XCWMinimumAverageBitRate = 18000000; @@ -127,7 +127,7 @@ static int32_t XCWRealtimeTargetFrameRate(void) { return XCWIntFromEnvironment(@"SIMDECK_REALTIME_FPS", XCWTargetRealtimeHardwareFrameRate, 15, - 60); + XCWMaximumLocalStreamFrameRate); } static uint64_t XCWRealtimeFrameIntervalUs(void) { @@ -474,6 +474,11 @@ @interface XCWH264Encoder () - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer targetWidth:(int32_t)targetWidth targetHeight:(int32_t)targetHeight; +- (nullable CVPixelBufferRef)copyPixelBufferFromScalingPoolWithWidth:(int32_t)targetWidth + height:(int32_t)targetHeight + pixelFormat:(OSType)pixelFormat; +- (void)handleCompressionOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + submittedAtUs:(uint64_t)submittedAtUs; @end @@ -489,8 +494,11 @@ @implementation XCWH264Encoder { int32_t _height; uint64_t _timestampOriginUs; VTPixelTransferSessionRef _pixelTransferSession; - CVPixelBufferRef _scaledPixelBuffer; + CVPixelBufferPoolRef _scaledPixelBufferPool; + int32_t _scaledPixelBufferWidth; + int32_t _scaledPixelBufferHeight; OSType _scaledPixelFormat; + BOOL _scalingActive; XCWVideoEncoderMode _encoderMode; BOOL _lowLatencyMode; BOOL _realtimeStreamMode; @@ -582,6 +590,9 @@ - (void)requestKeyFrame { - (void)reconfigureForStreamQualityChange { dispatch_async(_queue, ^{ [self invalidateCompressionSessionLocked]; + self->_encoderMode = XCWVideoEncoderModeFromEnvironment(); + self->_lowLatencyMode = (self->_encoderMode == XCWVideoEncoderModeH264Software) && XCWLowLatencyModeFromEnvironment(); + self->_codecType = XCWVideoCodecTypeForMode(self->_encoderMode); self->_needsKeyFrame = YES; self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; self->_softwarePacedFrameCount = 0; @@ -654,7 +665,10 @@ - (void)invalidate { } - (NSUInteger)maximumInFlightFrameCountLocked { - return (_realtimeStreamMode || (_encoderMode == XCWVideoEncoderModeH264Software && _lowLatencyMode)) ? 1 : XCWMaximumInFlightFrames; + if (_realtimeStreamMode || _lowLatencyMode) { + return 1; + } + return XCWMaximumInFlightFrames; } - (uint64_t)minimumSoftwareFrameIntervalUsLocked { @@ -748,7 +762,7 @@ - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { } - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { - if (_encoderMode != XCWVideoEncoderModeH264Software || latencyUs == 0) { + if (_encoderMode != XCWVideoEncoderModeH264Software || !_lowLatencyMode || latencyUs == 0) { return; } if (_softwareFrameIntervalUs == 0) { @@ -787,7 +801,7 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { } - (void)adaptHardwarePacingForLatencyUs:(uint64_t)latencyUs { - if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || latencyUs == 0) { + if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_lowLatencyMode || latencyUs == 0) { return; } if (_hardwareFrameIntervalUs == 0) { @@ -859,6 +873,7 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { if (targetWidth <= 0 || targetHeight <= 0) { return NO; } + _scalingActive = sourceWidth != targetWidth || sourceHeight != targetHeight; uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || [self shouldPaceHardwareFrameAtTimeUs:nowUs]) { @@ -929,7 +944,7 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height if (encoderID.length > 0) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EncoderID] = encoderID; } - if (_encoderMode != XCWVideoEncoderModeH264Software && _realtimeStreamMode) { + if (_encoderMode != XCWVideoEncoderModeH264Software && _lowLatencyMode) { if (@available(macOS 11.3, *)) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = @YES; } @@ -938,6 +953,8 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder] = @NO; } else if (_encoderMode == XCWVideoEncoderModeH264Hardware) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder] = @YES; + } else if (_encoderMode == XCWVideoEncoderModeAuto && _realtimeStreamMode) { + encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder] = @YES; } VTCompressionSessionRef session = NULL; @@ -969,7 +986,9 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height if (@available(macOS 10.14, *)) { VTSessionSetProperty(session, kVTCompressionPropertyKey_MaximizePowerEfficiency, kCFBooleanFalse); } - XCWApplyCompressionPresetIfAvailable(session); + if (_lowLatencyMode) { + XCWApplyCompressionPresetIfAvailable(session); + } VTSessionSetProperty(session, kVTCompressionPropertyKey_AllowTemporalCompression, kCFBooleanTrue); VTSessionSetProperty(session, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse); if (@available(macOS 10.14, *)) { @@ -984,7 +1003,7 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height } VTSessionSetProperty(session, kVTCompressionPropertyKey_H264EntropyMode, kVTH264EntropyMode_CAVLC); VTSessionSetProperty(session, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)@(expectedFrameRate)); - BOOL shortKeyframeInterval = _lowLatencyMode || _realtimeStreamMode; + BOOL shortKeyframeInterval = _lowLatencyMode; VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(shortKeyframeInterval ? MAX(1, expectedFrameRate / 2) : expectedFrameRate * 2)); VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(shortKeyframeInterval ? 0.5 : 2.0)); VTSessionSetProperty(session, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(averageBitRate)); @@ -998,7 +1017,7 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height if (@available(macOS 11.0, *)) { VTSessionSetProperty(session, kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, - kCFBooleanTrue); + _lowLatencyMode ? kCFBooleanTrue : kCFBooleanFalse); } if (@available(macOS 15.0, *)) { VTSessionSetProperty(session, @@ -1038,6 +1057,7 @@ - (void)invalidateCompressionSessionLocked { _lastHardwareSubmissionUs = 0; _hardwareAccelerated = NO; _selectedEncoderID = nil; + _scalingActive = NO; [self invalidateScalingResourcesLocked]; } @@ -1051,15 +1071,14 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix return pixelBuffer; } - if (_encoderMode == XCWVideoEncoderModeH264Software) { - return [self copySoftwareScaledPixelBuffer:pixelBuffer - targetWidth:targetWidth - targetHeight:targetHeight]; - } - if (_pixelTransferSession == NULL) { OSStatus sessionStatus = VTPixelTransferSessionCreate(kCFAllocatorDefault, &_pixelTransferSession); if (sessionStatus != noErr || _pixelTransferSession == NULL) { + if (_encoderMode == XCWVideoEncoderModeH264Software) { + return [self copySoftwareScaledPixelBuffer:pixelBuffer + targetWidth:targetWidth + targetHeight:targetHeight]; + } return NULL; } #ifdef kVTPixelTransferPropertyKey_RealTime @@ -1075,43 +1094,28 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix } OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); - BOOL needsNewBuffer = (_scaledPixelBuffer == NULL) - || ((int32_t)CVPixelBufferGetWidth(_scaledPixelBuffer) != targetWidth) - || ((int32_t)CVPixelBufferGetHeight(_scaledPixelBuffer) != targetHeight) - || (_scaledPixelFormat != sourcePixelFormat); - if (needsNewBuffer) { - if (_scaledPixelBuffer != NULL) { - CVPixelBufferRelease(_scaledPixelBuffer); - _scaledPixelBuffer = NULL; - } - - NSDictionary *attributes = @{ - (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, - }; - CVPixelBufferRef scaledPixelBuffer = NULL; - OSStatus bufferStatus = CVPixelBufferCreate(kCFAllocatorDefault, - targetWidth, - targetHeight, - sourcePixelFormat, - (__bridge CFDictionaryRef)attributes, - &scaledPixelBuffer); - if (bufferStatus != noErr || scaledPixelBuffer == NULL) { - return NULL; - } - _scaledPixelBuffer = scaledPixelBuffer; - _scaledPixelFormat = sourcePixelFormat; + CVPixelBufferRef scaledPixelBuffer = [self copyPixelBufferFromScalingPoolWithWidth:targetWidth + height:targetHeight + pixelFormat:sourcePixelFormat]; + if (scaledPixelBuffer == NULL) { + return NULL; } OSStatus transferStatus = VTPixelTransferSessionTransferImage(_pixelTransferSession, pixelBuffer, - _scaledPixelBuffer); + scaledPixelBuffer); _lastScaleStatus = transferStatus; if (transferStatus != noErr) { + CVPixelBufferRelease(scaledPixelBuffer); + if (_encoderMode == XCWVideoEncoderModeH264Software) { + return [self copySoftwareScaledPixelBuffer:pixelBuffer + targetWidth:targetWidth + targetHeight:targetHeight]; + } return NULL; } - CVPixelBufferRetain(_scaledPixelBuffer); - return _scaledPixelBuffer; + return scaledPixelBuffer; } - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer @@ -1123,43 +1127,24 @@ - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pix return NULL; } - BOOL needsNewBuffer = (_scaledPixelBuffer == NULL) - || ((int32_t)CVPixelBufferGetWidth(_scaledPixelBuffer) != targetWidth) - || ((int32_t)CVPixelBufferGetHeight(_scaledPixelBuffer) != targetHeight) - || (_scaledPixelFormat != sourcePixelFormat); - if (needsNewBuffer) { - if (_scaledPixelBuffer != NULL) { - CVPixelBufferRelease(_scaledPixelBuffer); - _scaledPixelBuffer = NULL; - } - - NSDictionary *attributes = @{ - (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, - }; - CVPixelBufferRef scaledPixelBuffer = NULL; - OSStatus bufferStatus = CVPixelBufferCreate(kCFAllocatorDefault, - targetWidth, - targetHeight, - sourcePixelFormat, - (__bridge CFDictionaryRef)attributes, - &scaledPixelBuffer); - if (bufferStatus != noErr || scaledPixelBuffer == NULL) { - _lastScaleStatus = bufferStatus; - return NULL; - } - _scaledPixelBuffer = scaledPixelBuffer; - _scaledPixelFormat = sourcePixelFormat; + CVPixelBufferRef scaledPixelBuffer = [self copyPixelBufferFromScalingPoolWithWidth:targetWidth + height:targetHeight + pixelFormat:sourcePixelFormat]; + if (scaledPixelBuffer == NULL) { + return NULL; } CVReturn sourceLockStatus = CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); if (sourceLockStatus != kCVReturnSuccess) { + CVPixelBufferRelease(scaledPixelBuffer); _lastScaleStatus = sourceLockStatus; return NULL; } - CVReturn targetLockStatus = CVPixelBufferLockBaseAddress(_scaledPixelBuffer, 0); + CVReturn targetLockStatus = CVPixelBufferLockBaseAddress(scaledPixelBuffer, 0); if (targetLockStatus != kCVReturnSuccess) { CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + CVPixelBufferRelease(scaledPixelBuffer); _lastScaleStatus = targetLockStatus; return NULL; } @@ -1171,32 +1156,79 @@ - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pix .rowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer), }; vImage_Buffer targetBuffer = { - .data = CVPixelBufferGetBaseAddress(_scaledPixelBuffer), - .height = (vImagePixelCount)CVPixelBufferGetHeight(_scaledPixelBuffer), - .width = (vImagePixelCount)CVPixelBufferGetWidth(_scaledPixelBuffer), - .rowBytes = CVPixelBufferGetBytesPerRow(_scaledPixelBuffer), + .data = CVPixelBufferGetBaseAddress(scaledPixelBuffer), + .height = (vImagePixelCount)CVPixelBufferGetHeight(scaledPixelBuffer), + .width = (vImagePixelCount)CVPixelBufferGetWidth(scaledPixelBuffer), + .rowBytes = CVPixelBufferGetBytesPerRow(scaledPixelBuffer), }; vImage_Flags scaleFlags = _realtimeStreamMode ? kvImageNoFlags : kvImageHighQualityResampling; vImage_Error scaleStatus = vImageScale_ARGB8888(&sourceBuffer, &targetBuffer, NULL, scaleFlags); - CVPixelBufferUnlockBaseAddress(_scaledPixelBuffer, 0); + CVPixelBufferUnlockBaseAddress(scaledPixelBuffer, 0); CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); _lastScaleStatus = scaleStatus; if (scaleStatus != kvImageNoError) { + CVPixelBufferRelease(scaledPixelBuffer); return NULL; } - CVPixelBufferRetain(_scaledPixelBuffer); - return _scaledPixelBuffer; + return scaledPixelBuffer; +} + +- (nullable CVPixelBufferRef)copyPixelBufferFromScalingPoolWithWidth:(int32_t)targetWidth + height:(int32_t)targetHeight + pixelFormat:(OSType)pixelFormat { + BOOL needsNewPool = (_scaledPixelBufferPool == NULL) + || (_scaledPixelBufferWidth != targetWidth) + || (_scaledPixelBufferHeight != targetHeight) + || (_scaledPixelFormat != pixelFormat); + if (needsNewPool) { + if (_scaledPixelBufferPool != NULL) { + CVPixelBufferPoolRelease(_scaledPixelBufferPool); + _scaledPixelBufferPool = NULL; + } + + NSDictionary *attributes = @{ + (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(pixelFormat), + (__bridge NSString *)kCVPixelBufferWidthKey: @(targetWidth), + (__bridge NSString *)kCVPixelBufferHeightKey: @(targetHeight), + (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + CVPixelBufferPoolRef pool = NULL; + CVReturn poolStatus = CVPixelBufferPoolCreate(kCFAllocatorDefault, + NULL, + (__bridge CFDictionaryRef)attributes, + &pool); + if (poolStatus != kCVReturnSuccess || pool == NULL) { + _lastScaleStatus = poolStatus; + return NULL; + } + _scaledPixelBufferPool = pool; + _scaledPixelBufferWidth = targetWidth; + _scaledPixelBufferHeight = targetHeight; + _scaledPixelFormat = pixelFormat; + } + + CVPixelBufferRef scaledPixelBuffer = NULL; + CVReturn bufferStatus = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, + _scaledPixelBufferPool, + &scaledPixelBuffer); + if (bufferStatus != kCVReturnSuccess || scaledPixelBuffer == NULL) { + _lastScaleStatus = bufferStatus; + return NULL; + } + return scaledPixelBuffer; } - (void)invalidateScalingResourcesLocked { - if (_scaledPixelBuffer != NULL) { - CVPixelBufferRelease(_scaledPixelBuffer); - _scaledPixelBuffer = NULL; + if (_scaledPixelBufferPool != NULL) { + CVPixelBufferPoolRelease(_scaledPixelBufferPool); + _scaledPixelBufferPool = NULL; } + _scaledPixelBufferWidth = 0; + _scaledPixelBufferHeight = 0; _scaledPixelFormat = 0; if (_pixelTransferSession != NULL) { VTPixelTransferSessionInvalidate(_pixelTransferSession); @@ -1300,6 +1332,24 @@ - (void)completeFailedFrame { }); } +- (void)handleCompressionOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + submittedAtUs:(uint64_t)submittedAtUs { + if (sampleBuffer == NULL) { + [self completeFailedFrame]; + return; + } + + CFRetain(sampleBuffer); + dispatch_async(_queue, ^{ + [self handleEncodedSampleBuffer:sampleBuffer submittedAtUs:submittedAtUs]; + CFRelease(sampleBuffer); + if (self->_inFlightFrameCount > 0) { + self->_inFlightFrameCount -= 1; + } + [self drainPendingFramesLocked]; + }); +} + @end static void XCWH264EncoderOutputCallback(void *outputCallbackRefCon, @@ -1314,7 +1364,6 @@ static void XCWH264EncoderOutputCallback(void *outputCallbackRefCon, } XCWH264Encoder *encoder = (__bridge XCWH264Encoder *)outputCallbackRefCon; - [encoder handleEncodedSampleBuffer:sampleBuffer - submittedAtUs:(uint64_t)(uintptr_t)sourceFrameRefCon]; - [encoder completeInFlightFrame]; + [encoder handleCompressionOutputSampleBuffer:sampleBuffer + submittedAtUs:(uint64_t)(uintptr_t)sourceFrameRefCon]; } diff --git a/cli/XCWPrivateSimulatorSession.m b/cli/XCWPrivateSimulatorSession.m index ac945438..bed1b153 100644 --- a/cli/XCWPrivateSimulatorSession.m +++ b/cli/XCWPrivateSimulatorSession.m @@ -163,8 +163,8 @@ - (void)refreshCurrentFrame { } - (void)requestKeyFrameRefresh { - [self refreshCurrentFrame]; [_videoEncoder requestKeyFrame]; + [self refreshCurrentFrame]; } - (void)requestFrameRefresh { @@ -189,18 +189,9 @@ - (id)addEncodedFrameListener:(XCWPrivateSimulatorEncodedFrameHandler)handler { NSUUID *token = [NSUUID UUID]; dispatch_sync(_stateQueue, ^{ self->_encodedFrameListeners[token] = [handler copy]; - if (self->_latestKeyFrameData.length > 0) { - handler(self->_latestKeyFrameData, - self->_latestKeyFrameSequenceValue, - self->_latestKeyFrameTimestampUs, - YES, - self->_latestKeyFrameCodec, - self->_latestKeyFrameDecoderConfig, - self->_latestKeyFrameDimensions); - } }); - [self refreshCurrentFrame]; [_videoEncoder requestKeyFrame]; + [self refreshCurrentFrame]; return token; } @@ -209,7 +200,7 @@ - (void)removeEncodedFrameListener:(id)token { return; } - dispatch_async(_stateQueue, ^{ + dispatch_sync(_stateQueue, ^{ [self->_encodedFrameListeners removeObjectForKey:(NSUUID *)token]; }); } diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index d3d2cfa9..cb4b7047 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -40,6 +40,12 @@ import { usePointerInput } from "../features/input/usePointerInput"; import { simulatorRuntimeLabel } from "../features/simulators/simulatorDisplay"; import { useSimulatorList } from "../features/simulators/useSimulatorList"; import { sendWebRtcControlMessage } from "../features/stream/streamWorkerClient"; +import type { + StreamConfig, + StreamEncoder, + StreamFps, + StreamQualityPreset, +} from "../features/stream/streamTypes"; import { useLiveStream } from "../features/stream/useLiveStream"; import { DebugPanel } from "../features/toolbar/DebugPanel"; import { Toolbar } from "../features/toolbar/Toolbar"; @@ -86,6 +92,16 @@ const REACT_NATIVE_ACCESSIBILITY_REFRESH_MS = 500; const DEFAULT_ACCESSIBILITY_MAX_DEPTH = 10; const LOGICAL_INSPECTOR_MAX_DEPTH = 80; const AUTH_REQUIRED_MESSAGE = "SimDeck API access token is required."; +const LOCAL_STREAM_DEFAULTS: StreamConfig = { + encoder: "hardware", + fps: 120, + quality: "quality", +}; +const REMOTE_STREAM_DEFAULTS: StreamConfig = { + encoder: "software", + fps: 30, + quality: "balanced", +}; clearLegacyVolatileUiState(); function buildChromeUrl(udid: string, stamp: number): string { @@ -277,6 +293,9 @@ export function AppShell({ const [touchOverlayVisible, setTouchOverlayVisible] = useState(() => readStoredFlag(TOUCH_OVERLAY_VISIBLE_STORAGE_KEY, true), ); + const [streamConfig, setStreamConfig] = useState(() => + remoteStream ? REMOTE_STREAM_DEFAULTS : LOCAL_STREAM_DEFAULTS, + ); const [touchIndicators, setTouchIndicators] = useState([]); const menuRef = useRef(null); @@ -390,8 +409,21 @@ export function AppShell({ canvasElement: streamCanvasElement, remote: remoteStream, simulator: selectedSimulator, + streamConfig, }); + const updateStreamEncoder = useCallback((encoder: StreamEncoder) => { + setStreamConfig((current) => ({ ...current, encoder })); + }, []); + + const updateStreamFps = useCallback((fps: StreamFps) => { + setStreamConfig((current) => ({ ...current, fps })); + }, []); + + const updateStreamQuality = useCallback((quality: StreamQualityPreset) => { + setStreamConfig((current) => ({ ...current, quality })); + }, []); + useEffect(() => { if ( !selectedSimulator || @@ -880,9 +912,7 @@ export function AppShell({ ? streamStatus.detail ? `${streamStatus.error} ${streamStatus.detail}` : streamStatus.error - : streamStatus.state === "connecting" && !hasFrame - ? (streamStatus.detail ?? "") - : ""; + : ""; const viewportStatusOverlayLabel = simulatorStatusOverlayLabel || streamStatusMessage || @@ -1392,6 +1422,9 @@ export function AppShell({ setStreamStamp(Date.now()); }, false); }} + onStreamEncoderChange={updateStreamEncoder} + onStreamFpsChange={updateStreamFps} + onStreamQualityChange={updateStreamQuality} onShutdown={() => { if (!selectedSimulator) { return; @@ -1438,6 +1471,7 @@ export function AppShell({ !selectedSimulator.isBooted && !selectedSimulatorTransitionKind, )} + streamConfig={streamConfig} showStopButton={Boolean( selectedSimulator?.isBooted && !selectedSimulatorTransitionKind, )} diff --git a/client/src/features/input/usePointerInput.ts b/client/src/features/input/usePointerInput.ts index cd085123..8e45e57a 100644 --- a/client/src/features/input/usePointerInput.ts +++ b/client/src/features/input/usePointerInput.ts @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from "react"; +import { useRef, useState } from "react"; import type { ChromeProfile, TouchPhase } from "../../api/types"; import { normalizedPointerCoordinatesForOrientation } from "./gestureMath"; @@ -33,48 +33,14 @@ export function usePointerInput({ onTouchPreview, }: UsePointerInputOptions) { const activePointerRef = useRef(null); - const moveFrameRef = useRef(0); const panningRef = useRef<{ startX: number; startY: number; startPanX: number; startPanY: number; } | null>(null); - const queuedMoveRef = useRef(null); const [isPanning, setIsPanning] = useState(false); - useEffect(() => { - return () => { - if (moveFrameRef.current) { - cancelAnimationFrame(moveFrameRef.current); - } - }; - }, []); - - function queueMove(coords: Point) { - queuedMoveRef.current = coords; - if (moveFrameRef.current) { - return; - } - - moveFrameRef.current = requestAnimationFrame(() => { - moveFrameRef.current = 0; - const nextCoords = queuedMoveRef.current; - queuedMoveRef.current = null; - if (nextCoords) { - onTouch("moved", nextCoords); - } - }); - } - - function clearQueuedMove() { - if (moveFrameRef.current) { - cancelAnimationFrame(moveFrameRef.current); - moveFrameRef.current = 0; - } - queuedMoveRef.current = null; - } - function startPanning(event: React.PointerEvent) { if (event.pointerType !== "mouse") { return; @@ -155,7 +121,7 @@ export function usePointerInput({ ); if (coords) { onTouchPreview?.("moved", coords); - queueMove(coords); + onTouch("moved", coords); } } @@ -168,7 +134,6 @@ export function usePointerInput({ return; } activePointerRef.current = null; - clearQueuedMove(); const coords = normalizedPointerCoordinatesForOrientation( event, rotationQuarterTurns, diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index 858ad68d..6e244cfb 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -1,6 +1,12 @@ import type { RefObject } from "react"; import type { SimulatorMetadata } from "../../api/types"; +import type { + StreamConfig, + StreamEncoder, + StreamFps, + StreamQualityPreset, +} from "../stream/streamTypes"; import { SimulatorRow } from "./SimulatorRow"; interface SimulatorMenuProps { @@ -16,6 +22,9 @@ interface SimulatorMenuProps { onOpenBundlePrompt: () => void; onOpenUrlPrompt: () => void; onRotateLeft: () => void; + onStreamEncoderChange: (encoder: StreamEncoder) => void; + onStreamFpsChange: (fps: StreamFps) => void; + onStreamQualityChange: (quality: StreamQualityPreset) => void; onToggleAppearance: () => void; onToggleDebug: () => void; onToggleMenu: () => void; @@ -23,6 +32,7 @@ interface SimulatorMenuProps { search: string; selectedSimulator: SimulatorMetadata | null; setSelectedUDID: (udid: string) => void; + streamConfig: StreamConfig; touchOverlayVisible: boolean; } @@ -39,6 +49,9 @@ export function SimulatorMenu({ onOpenBundlePrompt, onOpenUrlPrompt, onRotateLeft, + onStreamEncoderChange, + onStreamFpsChange, + onStreamQualityChange, onToggleAppearance, onToggleDebug, onToggleMenu, @@ -46,6 +59,7 @@ export function SimulatorMenu({ search, selectedSimulator, setSelectedUDID, + streamConfig, touchOverlayVisible, }: SimulatorMenuProps) { return ( @@ -94,6 +108,46 @@ export function SimulatorMenu({ ) : null} {selectedSimulator ? ( <> +
+
+ Stream +
+ {STREAM_ENCODERS.map((option) => ( + + ))} +
+
+ {STREAM_FPS_OPTIONS.map((option) => ( + + ))} +
+
+ {STREAM_QUALITY_OPTIONS.map((option) => ( + + ))} +
+
) : null} {isBooted && diff --git a/client/src/features/viewport/SimulatorViewport.tsx b/client/src/features/viewport/SimulatorViewport.tsx index 72736401..19f74eaf 100644 --- a/client/src/features/viewport/SimulatorViewport.tsx +++ b/client/src/features/viewport/SimulatorViewport.tsx @@ -207,7 +207,7 @@ export function SimulatorViewport({ className="canvas-loading" role="status" > - Loading simulator chrome... +
) : null} {debugPanel ?
{debugPanel}
: null} diff --git a/client/src/styles/components.css b/client/src/styles/components.css index 89a22bb6..8e81f11c 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -335,6 +335,10 @@ gap: 4px; } +.menu-segment.two { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + .menu-option { min-width: 0; height: 30px; @@ -1179,6 +1183,7 @@ cursor: crosshair; touch-action: none; position: relative; + contain: paint; z-index: 1; } @@ -1194,7 +1199,7 @@ display: block; width: 100%; height: 100%; - background: #000000; + background: transparent; pointer-events: none; } @@ -1208,6 +1213,9 @@ background: #000000; object-fit: fill; pointer-events: none; + backface-visibility: hidden; + transform: translate3d(0, 0, 0); + will-change: transform; } .accessibility-picker-layer { diff --git a/docs/api/health.md b/docs/api/health.md index 91e3e344..357da069 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -28,7 +28,7 @@ Returns the static bootstrap information the browser client needs, plus a freshn | `videoCodec` | Requested encoder mode. One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | | `lowLatency` | `true` when software H.264 low-latency mode was enabled at daemon startup. | | `realtimeStream` | `true` when the WebRTC stream is configured to favor freshness and realtime pacing. | -| `localStreamFps` | Local quality stream frame cap, from 15 to 120 fps. Defaults to 60. | +| `localStreamFps` | Local quality stream frame target, from 15 to 240 fps. Defaults to 60. | | `streamQuality` | Active realtime quality profile and encoder limits such as `maxEdge`, `fps`, and bitrate. | | `webRtc.iceServers` | ICE servers the browser should use when creating the WebRTC peer connection. | | `webRtc.iceTransportPolicy` | Browser ICE transport policy. One of `all` or `relay`. | diff --git a/docs/api/rest.md b/docs/api/rest.md index ffc1c7e1..a848e99f 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -68,6 +68,28 @@ Content-Type: application/json Required fields: `clientId` and `kind`. Every other field is optional but typed in `ClientStreamStats`. +### `GET /api/stream-quality` + +Returns the active stream encoder settings and available quality profiles. + +### `POST /api/stream-quality` + +Updates the active stream encoder settings for newly encoded frames. The browser +UI uses this before WebRTC negotiation when the user selects encoder, FPS, or +quality. + +```json +{ + "videoCodec": "hardware", + "fps": 120, + "profile": "quality" +} +``` + +`videoCodec` accepts `hardware` or `software` from the UI, and the API also +accepts `auto`. `fps` is clamped to the local stream range. `profile` accepts +`quality`, `balanced`, `fast`, `smooth`, `economy`, or `ci-software`. + ## Simulator inventory ### `GET /api/simulators` diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 52ad183c..a1719477 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -33,7 +33,7 @@ Start or reuse the project daemon and serve the browser UI. simdeck ui [--port 4310] [--bind 127.0.0.1] [--advertise-host ] [--client-root ] [--video-codec auto|hardware|software] [--low-latency] [--stream-quality ] - [--local-stream-fps <15-120>] [--open] + [--local-stream-fps <15-240>] [--open] ``` `--open` opens the authenticated local URL after the daemon is ready. @@ -62,7 +62,7 @@ Start or reuse the project daemon without opening the browser: simdeck daemon start [--port 4310] [--bind 127.0.0.1] [--advertise-host ] [--client-root ] [--video-codec auto|hardware|software] [--low-latency] - [--stream-quality ] [--local-stream-fps <15-120>] + [--stream-quality ] [--local-stream-fps <15-240>] ``` Output: @@ -98,7 +98,7 @@ options as `daemon start`: simdeck daemon restart [--port 4310] [--bind 127.0.0.1] [--advertise-host ] [--client-root ] [--video-codec auto|hardware|software] [--low-latency] - [--stream-quality ] [--local-stream-fps <15-120>] + [--stream-quality ] [--local-stream-fps <15-240>] ``` ### `daemon stop` @@ -127,7 +127,7 @@ that starts after login and stays available. simdeck service on [--port 4310] [--bind 127.0.0.1] [--advertise-host ] [--client-root ] [--video-codec auto|hardware|software] [--low-latency] - [--stream-quality ] [--local-stream-fps <15-120>] + [--stream-quality ] [--local-stream-fps <15-240>] [--access-token ] simdeck service restart [same options as service on] simdeck service off diff --git a/docs/cli/flags.md b/docs/cli/flags.md index dabccc99..be801b64 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -36,8 +36,8 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas | `--client-root` | bundled `client/dist` | Override the static browser client directory. | | `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | -| `--stream-quality` | auto/default | Optional realtime stream quality profile: `quality`, `balanced`, `smooth`, `economy`, or `ci-software`. | -| `--local-stream-fps` | `60` | Local quality stream frame cap, from 15 to 120 fps. | +| `--stream-quality` | `smooth` | Realtime stream quality profile: `quality`, `balanced`, `fast`, `smooth`, `economy`, or `ci-software`. | +| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | `studio expose` defaults to software H.264. Pass `--video-codec hardware` to diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index b0334bed..4303dce0 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -62,8 +62,8 @@ This starts or reuses the project daemon, serves the bundled browser client, and | `--client-root` | bundled `client/dist` | Override the static browser client directory. | | `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. | -| `--stream-quality` | auto/default | Optional realtime stream quality profile, including `ci-software` for CI providers. | -| `--local-stream-fps` | `60` | Local quality stream frame cap, from 15 to 120 fps. | +| `--stream-quality` | `smooth` | Realtime stream quality profile, including `ci-software` for CI providers. | +| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | Example: diff --git a/docs/guide/video.md b/docs/guide/video.md index ed142a66..050c3997 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -32,6 +32,7 @@ It is CLI-only because it is meant for less capable machines where freshness matters more than maximum smoothness. The requested encoder mode is reported to clients in the JSON `videoCodec` field on `GET /api/health`. +The browser UI exposes stream controls for encoder, FPS, and quality. Local browser sessions default to hardware H.264, 120 fps, and `quality`/full resolution; remote browser sessions default to software H.264, 30 fps, and `balanced`. ## Remote WebRTC ICE @@ -77,8 +78,8 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques A few practical guidelines: -- **Start on the default for local preview.** `auto` lets VideoToolbox choose without requiring the shared hardware encoder. -- **Use `--local-stream-fps 120` only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher caps pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames. +- **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with the `quality` profile: full resolution, 120 fps, and a high bitrate floor. Pass `--video-codec software` only when the shared hardware encoder is unavailable or performs worse on that host. +- **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames. - **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. - **Studio providers default to software H.264 plus `--stream-quality smooth`.** This profile uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder. - **The remote browser renders the live stream as a native `