diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index c20b464f..55ce8f79 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -135,6 +135,16 @@ function simulatorDisplaySize( }; } +function simulatorDisplayReady(simulator: SimulatorMetadata): boolean { + const display = simulator.privateDisplay; + return Boolean( + simulator.isBooted && + display?.displayReady && + display.displayWidth > 0 && + display.displayHeight > 0, + ); +} + function mergeAccessibilitySources( ...sources: unknown[] ): AccessibilitySource[] { @@ -190,6 +200,9 @@ export function AppShell() { ); const [menuOpen, setMenuOpen] = useState(false); const [localError, setLocalError] = useState(""); + const [failedStreamUDIDs, setFailedStreamUDIDs] = useState>( + () => new Set(), + ); const [pairingCode, setPairingCode] = useState(""); const [pairingError, setPairingError] = useState(""); const [pairingBusy, setPairingBusy] = useState(false); @@ -292,6 +305,8 @@ export function AppShell() { simulators.find((simulator) => simulatorMatchesIdentifier(simulator, selectedUDID), ) ?? + filteredSimulators.find((simulator) => simulatorDisplayReady(simulator)) ?? + filteredSimulators.find((simulator) => simulator.isBooted) ?? filteredSimulators[0] ?? null; const selectedSimulatorDetail = @@ -333,6 +348,36 @@ export function AppShell() { canvasElement: streamCanvasElement, simulator: selectedSimulator, }); + + useEffect(() => { + if ( + !selectedSimulator || + !streamError || + readDeviceQueryParam() || + !isStreamAttachFailure(streamError) + ) { + return; + } + const failedUDID = selectedSimulator.udid; + setFailedStreamUDIDs((current) => { + if (current.has(failedUDID)) { + return current; + } + return new Set(current).add(failedUDID); + }); + const nextSimulator = simulators.find( + (simulator) => + simulator.isBooted && + simulator.udid !== failedUDID && + !failedStreamUDIDs.has(simulator.udid), + ); + if (nextSimulator) { + setSelectedUDID(nextSimulator.udid); + setLocalError( + `${selectedSimulator.name} did not expose a live simulator screen. Switched to ${nextSimulator.name}.`, + ); + } + }, [failedStreamUDIDs, selectedSimulator, simulators, streamError]); const shouldRenderChrome = selectedSimulator != null && shouldRenderNativeChrome(selectedSimulator); const viewportChromeProfile = shouldRenderChrome ? chromeProfile : null; @@ -1389,3 +1434,13 @@ function readDeviceQueryParam(): string | undefined { const trimmed = value?.trim(); return trimmed ? trimmed : undefined; } + +function isStreamAttachFailure(message: string): boolean { + const normalized = message.toLowerCase(); + return ( + normalized.includes("headless screen") || + normalized.includes("screen adapter") || + normalized.includes("coresimulator did not provide") || + normalized.includes("did not expose any live screens") + ); +} diff --git a/docs/api/health.md b/docs/api/health.md index 4545b8f1..3cbfb5e0 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -11,7 +11,7 @@ Returns the static bootstrap information the browser client needs, plus a freshn "ok": true, "httpPort": 4310, "timestamp": 1714094761.234, - "videoCodec": "h264-software", + "videoCodec": "h264", "lowLatency": false, "webRtc": { "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], diff --git a/docs/api/rest.md b/docs/api/rest.md index eb80eb20..351914f7 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -22,7 +22,7 @@ Returns server health and the active video encoder mode. "ok": true, "httpPort": 4310, "timestamp": 1714094761.234, - "videoCodec": "h264-software", + "videoCodec": "h264", "lowLatency": false, "webRtc": { "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], diff --git a/docs/cli/flags.md b/docs/cli/flags.md index fb242216..9fcd0859 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -34,7 +34,7 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas | `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | | `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | | `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `h264-software` | One of `h264` or `h264-software`. See [Video Pipeline](/guide/video). | +| `--video-codec` | `h264` | One of `h264` or `h264-software`. See [Video Pipeline](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | | `--stream-quality` | auto/default | Optional realtime stream quality profile: `quality`, `balanced`, `smooth`, `economy`, or `ci-software`. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index 98b7a45b..c95e57cf 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -60,7 +60,7 @@ This starts or reuses the project daemon, serves the bundled browser client, and | `--bind ` | `127.0.0.1` | Bind address. Use `0.0.0.0` for [LAN access](/guide/lan-access). | | `--advertise-host` | matches local host | Hostname or IP advertised to browser clients. | | `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `h264-software` | One of `h264` or `h264-software`. See [Video](/guide/video). | +| `--video-codec` | `h264` | One of `h264` or `h264-software`. See [Video](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. | | `--stream-quality` | auto/default | Optional realtime stream quality profile, including `ci-software` for CI providers. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | diff --git a/docs/guide/lan-access.md b/docs/guide/lan-access.md index 980f07fd..ca162299 100644 --- a/docs/guide/lan-access.md +++ b/docs/guide/lan-access.md @@ -48,7 +48,7 @@ Whatever you advertise must be resolvable from the remote client. { "ok": true, "httpPort": 4310, - "videoCodec": "h264-software", + "videoCodec": "h264", "lowLatency": false, "webRtc": { "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], diff --git a/docs/guide/video.md b/docs/guide/video.md index 3503710d..80f96f7b 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -6,10 +6,10 @@ SimDeck streams the iOS Simulator over WebRTC using browser-native H.264 video p The server can encode the simulator display in two modes, picked at startup with `--video-codec`: -| Value | Encoder | When to use it | -| --------------------------- | ------------------------------- | -------------------------------------------------------------- | -| `h264` | Hardware H.264 via VideoToolbox | Best local performance when the hardware encoder is available. | -| `h264-software` _(default)_ | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable. | +| Value | Encoder | When to use it | +| ------------------ | ------------------------------- | -------------------------------------------------------------- | +| `h264` _(default)_ | Hardware H.264 via VideoToolbox | Best local performance when the hardware encoder is available. | +| `h264-software` | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable. | Restart the daemon to change encoder mode: @@ -76,10 +76,9 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques A few practical guidelines: -- **Start on the default for compatibility.** `h264-software` works without requiring the hardware encoder, but full-resolution latency can be high. -- **Switch to `h264` on local Apple Silicon when hardware encode is available.** Hardware H.264 gives the smoothest local preview with the least CPU. +- **Start on the default for local preview.** `h264` gives the smoothest preview when VideoToolbox can provide a hardware encoder. - **Switch to `h264-software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. -- **Use `--stream-quality ci-software` for Studio providers on virtualized CI Macs when hardware encode is unavailable.** This profile uses software H.264 at an 844-pixel longest edge, targets 20 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness. +- **Use `--stream-quality ci-software` for Studio providers on virtualized CI Macs when hardware encode is unavailable.** This profile uses software H.264 at a 960-pixel longest edge, targets 24 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness. - **Use `h264-software --low-latency` only when you need the older extra-conservative software profile.** It caps at 15 fps, uses a single pending frame, reduces the longest edge to 1170 pixels, and backs off before software encode latency turns into seconds of stream delay. ## Tuning with metrics diff --git a/scripts/integration/cli.mjs b/scripts/integration/cli.mjs index efc7d500..9d11c2a2 100644 --- a/scripts/integration/cli.mjs +++ b/scripts/integration/cli.mjs @@ -213,10 +213,14 @@ async function main() { async () => { fs.writeFileSync( stdoutPng, - runBuffer(simdeck, ["screenshot", simulatorUDID, "--stdout"], { - timeoutMs: 300_000, - maxBuffer: 64 * 1024 * 1024, - }), + runBuffer( + simdeck, + ["--server-url", serverUrl, "screenshot", simulatorUDID, "--stdout"], + { + timeoutMs: 300_000, + maxBuffer: 64 * 1024 * 1024, + }, + ), ); assertPng(stdoutPng); }, diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 0e77e7b3..a0344cf2 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -106,10 +106,10 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ StreamQualityProfile { id: "ci-software", label: "CI Software", - max_edge: 844, - fps: 20, - min_bitrate: 800_000, - bits_per_pixel: 1, + max_edge: 960, + fps: 24, + min_bitrate: 1_200_000, + bits_per_pixel: 2, }, StreamQualityProfile { id: "quality", @@ -591,17 +591,17 @@ async fn set_stream_quality( .max_edge .or_else(|| profile.map(|profile| profile.max_edge)) .unwrap_or(1440) - .clamp(720, 1920); + .clamp(320, 1920); let fps = payload .fps .or_else(|| profile.map(|profile| profile.fps)) .unwrap_or(30) - .clamp(15, 60); + .clamp(10, 60); let min_bitrate = payload .min_bitrate .or_else(|| profile.map(|profile| profile.min_bitrate)) .unwrap_or(3_000_000) - .clamp(750_000, 20_000_000); + .clamp(200_000, 20_000_000); let bits_per_pixel = payload .bits_per_pixel .or_else(|| profile.map(|profile| profile.bits_per_pixel)) @@ -645,12 +645,12 @@ fn stream_quality_state() -> Value { min_bitrate: 3_000_000, bits_per_pixel: 4, }); - let max_edge = env_u32("SIMDECK_REALTIME_MAX_EDGE", fallback.max_edge, 720, 1920); - let fps = env_u32("SIMDECK_REALTIME_FPS", fallback.fps, 15, 60); + let max_edge = env_u32("SIMDECK_REALTIME_MAX_EDGE", fallback.max_edge, 320, 1920); + let fps = env_u32("SIMDECK_REALTIME_FPS", fallback.fps, 10, 60); let min_bitrate = env_u32( "SIMDECK_REALTIME_MIN_BITRATE", fallback.min_bitrate, - 750_000, + 200_000, 20_000_000, ); let bits_per_pixel = env_u32( diff --git a/server/src/main.rs b/server/src/main.rs index be29139c..bc1ec4f4 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -47,8 +47,8 @@ const SERVER_FD_RESTART_THRESHOLD: usize = 4096; const SERVER_HEALTH_WATCHDOG_INITIAL_DELAY: Duration = Duration::from_secs(15); const SERVER_HEALTH_WATCHDOG_INTERVAL: Duration = Duration::from_secs(5); const SERVER_HEALTH_WATCHDOG_PROBE_TIMEOUT: Duration = Duration::from_secs(3); -const SERVER_HEALTH_WATCHDOG_STALE_HEARTBEAT: Duration = Duration::from_secs(10); -const SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD: usize = 3; +const SERVER_HEALTH_WATCHDOG_STALE_HEARTBEAT: Duration = Duration::from_secs(60); +const SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD: usize = 12; #[derive(Parser)] #[command(name = "simdeck")] @@ -79,7 +79,7 @@ enum Command { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -106,7 +106,7 @@ enum Command { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -380,7 +380,7 @@ enum DaemonCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -396,7 +396,7 @@ enum DaemonCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -420,7 +420,7 @@ enum DaemonCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -463,7 +463,7 @@ enum ServiceCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -481,7 +481,7 @@ enum ServiceCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -652,10 +652,10 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result Ok(StreamQualityEnvironment { profile: "ci-software", - max_edge: 844, - fps: 20, - min_bitrate: 800_000, - bits_per_pixel: 1, + max_edge: 960, + fps: 24, + min_bitrate: 1_200_000, + bits_per_pixel: 2, }), _ => anyhow::bail!("Unknown stream quality profile `{profile}`."), } @@ -710,7 +710,7 @@ impl Default for DaemonLaunchOptions { bind: IpAddr::V4(Ipv4Addr::LOCALHOST), advertise_host: None, client_root: None, - video_codec: VideoCodecMode::H264Software, + video_codec: VideoCodecMode::H264, low_latency: false, realtime_stream: false, stream_quality_profile: None, @@ -1071,17 +1071,24 @@ fn project_root() -> anyhow::Result { } fn choose_daemon_port(preferred: u16) -> anyhow::Result { + choose_daemon_port_for_bind(preferred, IpAddr::V4(Ipv4Addr::LOCALHOST)) +} + +fn choose_daemon_port_for_bind(preferred: u16, bind: IpAddr) -> anyhow::Result { let start = preferred.max(1024); for port in start..start.saturating_add(200) { - if port_available(port) { + if port_available(bind, port) { return Ok(port); } } anyhow::bail!("No available SimDeck daemon port near {preferred}") } -fn port_available(port: u16) -> bool { - TcpListener::bind((Ipv4Addr::LOCALHOST, port)).is_ok() +fn port_available(bind: IpAddr, port: u16) -> bool { + if bind.is_unspecified() && TcpListener::bind((Ipv4Addr::LOCALHOST, port)).is_err() { + return false; + } + TcpListener::bind((bind, port)).is_ok() } fn open_browser(url: &str) -> anyhow::Result<()> { @@ -1184,9 +1191,9 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { } let project_root = project_root()?; - let port = choose_daemon_port(4310)?; let bind = IpAddr::V4(Ipv4Addr::UNSPECIFIED); - let video_codec = VideoCodecMode::H264Software; + let port = choose_daemon_port_for_bind(4310, bind)?; + let video_codec = VideoCodecMode::H264; let low_latency = false; let advertise_host = detect_lan_ip() .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST))