Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
7f939c8
Experiment with lean CI realtime stream
DjDeveloperr May 1, 2026
64cfd8e
Honor lean CI realtime profile
DjDeveloperr May 1, 2026
f0e67fe
Optimize CI streaming experiments
DjDeveloperr May 1, 2026
3da6519
Restore usable CI software stream quality
DjDeveloperr May 1, 2026
51988cf
Retry CoreSimulator headless screen attach
DjDeveloperr May 1, 2026
4e4f0b7
Handle direct SimulatorKit screen adapters
DjDeveloperr May 1, 2026
fa9f37b
Restore local hardware H264 defaults
DjDeveloperr May 1, 2026
66b8827
Probe foreground bind address for port selection
DjDeveloperr May 1, 2026
a721e98
Avoid loading loops on failed simulator streams
DjDeveloperr May 1, 2026
229bfb5
Restore smooth local WebRTC defaults
DjDeveloperr May 1, 2026
1824195
Reduce local WebRTC stream pressure
DjDeveloperr May 1, 2026
41552f9
Revert "Reduce local WebRTC stream pressure"
DjDeveloperr May 1, 2026
2f1d4a7
Revert "Restore smooth local WebRTC defaults"
DjDeveloperr May 1, 2026
4c57870
Uncap local hardware WebRTC streaming
DjDeveloperr May 1, 2026
522bf2b
Restore main WebRTC streaming implementation
DjDeveloperr May 1, 2026
0f36930
Remove local hardware stream FPS cap
DjDeveloperr May 1, 2026
95f8abf
Avoid reconnecting established WebRTC streams on frame stalls
DjDeveloperr May 1, 2026
6903fd8
Tolerate transient WebRTC disconnected state
DjDeveloperr May 1, 2026
cc6e0ea
Use realtime cleanup without capping local hardware
DjDeveloperr May 1, 2026
65ec204
Restore smooth local WebRTC defaults
DjDeveloperr May 1, 2026
5d20181
Restore main local preview hot path
DjDeveloperr May 1, 2026
c3794ee
Clean up CI stream profile branch
DjDeveloperr May 1, 2026
f4ae8d4
Fix PR CI formatting and clippy
DjDeveloperr May 1, 2026
1c9afda
Use integration server for stdout screenshot test
DjDeveloperr May 1, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions client/src/app/AppShell.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,16 @@ function simulatorDisplaySize(
};
}

function simulatorDisplayReady(simulator: SimulatorMetadata): boolean {
const display = simulator.privateDisplay;
return Boolean(
simulator.isBooted &&
display?.displayReady &&
display.displayWidth > 0 &&
display.displayHeight > 0,
);
}

function mergeAccessibilitySources(
...sources: unknown[]
): AccessibilitySource[] {
Expand Down Expand Up @@ -190,6 +200,9 @@ export function AppShell() {
);
const [menuOpen, setMenuOpen] = useState(false);
const [localError, setLocalError] = useState("");
const [failedStreamUDIDs, setFailedStreamUDIDs] = useState<Set<string>>(
() => new Set(),
);
const [pairingCode, setPairingCode] = useState("");
const [pairingError, setPairingError] = useState("");
const [pairingBusy, setPairingBusy] = useState(false);
Expand Down Expand Up @@ -292,6 +305,8 @@ export function AppShell() {
simulators.find((simulator) =>
simulatorMatchesIdentifier(simulator, selectedUDID),
) ??
filteredSimulators.find((simulator) => simulatorDisplayReady(simulator)) ??
filteredSimulators.find((simulator) => simulator.isBooted) ??
filteredSimulators[0] ??
null;
const selectedSimulatorDetail =
Expand Down Expand Up @@ -333,6 +348,36 @@ export function AppShell() {
canvasElement: streamCanvasElement,
simulator: selectedSimulator,
});

useEffect(() => {
if (
!selectedSimulator ||
!streamError ||
readDeviceQueryParam() ||
!isStreamAttachFailure(streamError)
) {
return;
}
const failedUDID = selectedSimulator.udid;
setFailedStreamUDIDs((current) => {
if (current.has(failedUDID)) {
return current;
}
return new Set(current).add(failedUDID);
});
const nextSimulator = simulators.find(
(simulator) =>
simulator.isBooted &&
simulator.udid !== failedUDID &&
!failedStreamUDIDs.has(simulator.udid),
);
if (nextSimulator) {
setSelectedUDID(nextSimulator.udid);
setLocalError(
`${selectedSimulator.name} did not expose a live simulator screen. Switched to ${nextSimulator.name}.`,
);
}
}, [failedStreamUDIDs, selectedSimulator, simulators, streamError]);
const shouldRenderChrome =
selectedSimulator != null && shouldRenderNativeChrome(selectedSimulator);
const viewportChromeProfile = shouldRenderChrome ? chromeProfile : null;
Expand Down Expand Up @@ -1389,3 +1434,13 @@ function readDeviceQueryParam(): string | undefined {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;
}

function isStreamAttachFailure(message: string): boolean {
const normalized = message.toLowerCase();
return (
normalized.includes("headless screen") ||
normalized.includes("screen adapter") ||
normalized.includes("coresimulator did not provide") ||
normalized.includes("did not expose any live screens")
);
}
2 changes: 1 addition & 1 deletion docs/api/health.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Returns the static bootstrap information the browser client needs, plus a freshn
"ok": true,
"httpPort": 4310,
"timestamp": 1714094761.234,
"videoCodec": "h264-software",
"videoCodec": "h264",
"lowLatency": false,
"webRtc": {
"iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }],
Expand Down
2 changes: 1 addition & 1 deletion docs/api/rest.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Returns server health and the active video encoder mode.
"ok": true,
"httpPort": 4310,
"timestamp": 1714094761.234,
"videoCodec": "h264-software",
"videoCodec": "h264",
"lowLatency": false,
"webRtc": {
"iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }],
Expand Down
2 changes: 1 addition & 1 deletion docs/cli/flags.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas
| `--bind <ip>` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). |
| `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. |
| `--client-root` | bundled `client/dist` | Override the static browser client directory. |
| `--video-codec` | `h264-software` | One of `h264` or `h264-software`. See [Video Pipeline](/guide/video). |
| `--video-codec` | `h264` | One of `h264` or `h264-software`. See [Video Pipeline](/guide/video). |
| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. |
| `--stream-quality` | auto/default | Optional realtime stream quality profile: `quality`, `balanced`, `smooth`, `economy`, or `ci-software`. |
| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. |
Expand Down
2 changes: 1 addition & 1 deletion docs/guide/daemon.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ This starts or reuses the project daemon, serves the bundled browser client, and
| `--bind <ip>` | `127.0.0.1` | Bind address. Use `0.0.0.0` for [LAN access](/guide/lan-access). |
| `--advertise-host` | matches local host | Hostname or IP advertised to browser clients. |
| `--client-root` | bundled `client/dist` | Override the static browser client directory. |
| `--video-codec` | `h264-software` | One of `h264` or `h264-software`. See [Video](/guide/video). |
| `--video-codec` | `h264` | One of `h264` or `h264-software`. See [Video](/guide/video). |
| `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. |
| `--stream-quality` | auto/default | Optional realtime stream quality profile, including `ci-software` for CI providers. |
| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. |
Expand Down
2 changes: 1 addition & 1 deletion docs/guide/lan-access.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ Whatever you advertise must be resolvable from the remote client.
{
"ok": true,
"httpPort": 4310,
"videoCodec": "h264-software",
"videoCodec": "h264",
"lowLatency": false,
"webRtc": {
"iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }],
Expand Down
13 changes: 6 additions & 7 deletions docs/guide/video.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ SimDeck streams the iOS Simulator over WebRTC using browser-native H.264 video p

The server can encode the simulator display in two modes, picked at startup with `--video-codec`:

| Value | Encoder | When to use it |
| --------------------------- | ------------------------------- | -------------------------------------------------------------- |
| `h264` | Hardware H.264 via VideoToolbox | Best local performance when the hardware encoder is available. |
| `h264-software` _(default)_ | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable. |
| Value | Encoder | When to use it |
| ------------------ | ------------------------------- | -------------------------------------------------------------- |
| `h264` _(default)_ | Hardware H.264 via VideoToolbox | Best local performance when the hardware encoder is available. |
| `h264-software` | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable. |

Restart the daemon to change encoder mode:

Expand Down Expand Up @@ -76,10 +76,9 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques

A few practical guidelines:

- **Start on the default for compatibility.** `h264-software` works without requiring the hardware encoder, but full-resolution latency can be high.
- **Switch to `h264` on local Apple Silicon when hardware encode is available.** Hardware H.264 gives the smoothest local preview with the least CPU.
- **Start on the default for local preview.** `h264` gives the smoothest preview when VideoToolbox can provide a hardware encoder.
- **Switch to `h264-software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency.
- **Use `--stream-quality ci-software` for Studio providers on virtualized CI Macs when hardware encode is unavailable.** This profile uses software H.264 at an 844-pixel longest edge, targets 20 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness.
- **Use `--stream-quality ci-software` for Studio providers on virtualized CI Macs when hardware encode is unavailable.** This profile uses software H.264 at a 960-pixel longest edge, targets 24 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness.
- **Use `h264-software --low-latency` only when you need the older extra-conservative software profile.** It caps at 15 fps, uses a single pending frame, reduces the longest edge to 1170 pixels, and backs off before software encode latency turns into seconds of stream delay.

## Tuning with metrics
Expand Down
12 changes: 8 additions & 4 deletions scripts/integration/cli.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -213,10 +213,14 @@ async function main() {
async () => {
fs.writeFileSync(
stdoutPng,
runBuffer(simdeck, ["screenshot", simulatorUDID, "--stdout"], {
timeoutMs: 300_000,
maxBuffer: 64 * 1024 * 1024,
}),
runBuffer(
simdeck,
["--server-url", serverUrl, "screenshot", simulatorUDID, "--stdout"],
{
timeoutMs: 300_000,
maxBuffer: 64 * 1024 * 1024,
},
),
);
assertPng(stdoutPng);
},
Expand Down
20 changes: 10 additions & 10 deletions server/src/api/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,10 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[
StreamQualityProfile {
id: "ci-software",
label: "CI Software",
max_edge: 844,
fps: 20,
min_bitrate: 800_000,
bits_per_pixel: 1,
max_edge: 960,
fps: 24,
min_bitrate: 1_200_000,
bits_per_pixel: 2,
},
StreamQualityProfile {
id: "quality",
Expand Down Expand Up @@ -591,17 +591,17 @@ async fn set_stream_quality(
.max_edge
.or_else(|| profile.map(|profile| profile.max_edge))
.unwrap_or(1440)
.clamp(720, 1920);
.clamp(320, 1920);
let fps = payload
.fps
.or_else(|| profile.map(|profile| profile.fps))
.unwrap_or(30)
.clamp(15, 60);
.clamp(10, 60);
let min_bitrate = payload
.min_bitrate
.or_else(|| profile.map(|profile| profile.min_bitrate))
.unwrap_or(3_000_000)
.clamp(750_000, 20_000_000);
.clamp(200_000, 20_000_000);
let bits_per_pixel = payload
.bits_per_pixel
.or_else(|| profile.map(|profile| profile.bits_per_pixel))
Expand Down Expand Up @@ -645,12 +645,12 @@ fn stream_quality_state() -> Value {
min_bitrate: 3_000_000,
bits_per_pixel: 4,
});
let max_edge = env_u32("SIMDECK_REALTIME_MAX_EDGE", fallback.max_edge, 720, 1920);
let fps = env_u32("SIMDECK_REALTIME_FPS", fallback.fps, 15, 60);
let max_edge = env_u32("SIMDECK_REALTIME_MAX_EDGE", fallback.max_edge, 320, 1920);
let fps = env_u32("SIMDECK_REALTIME_FPS", fallback.fps, 10, 60);
let min_bitrate = env_u32(
"SIMDECK_REALTIME_MIN_BITRATE",
fallback.min_bitrate,
750_000,
200_000,
20_000_000,
);
let bits_per_pixel = env_u32(
Expand Down
45 changes: 26 additions & 19 deletions server/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ const SERVER_FD_RESTART_THRESHOLD: usize = 4096;
const SERVER_HEALTH_WATCHDOG_INITIAL_DELAY: Duration = Duration::from_secs(15);
const SERVER_HEALTH_WATCHDOG_INTERVAL: Duration = Duration::from_secs(5);
const SERVER_HEALTH_WATCHDOG_PROBE_TIMEOUT: Duration = Duration::from_secs(3);
const SERVER_HEALTH_WATCHDOG_STALE_HEARTBEAT: Duration = Duration::from_secs(10);
const SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD: usize = 3;
const SERVER_HEALTH_WATCHDOG_STALE_HEARTBEAT: Duration = Duration::from_secs(60);
const SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD: usize = 12;

#[derive(Parser)]
#[command(name = "simdeck")]
Expand Down Expand Up @@ -79,7 +79,7 @@ enum Command {
advertise_host: Option<String>,
#[arg(long)]
client_root: Option<PathBuf>,
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)]
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264)]
video_codec: VideoCodecMode,
#[arg(long)]
low_latency: bool,
Expand All @@ -106,7 +106,7 @@ enum Command {
advertise_host: Option<String>,
#[arg(long)]
client_root: Option<PathBuf>,
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)]
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264)]
video_codec: VideoCodecMode,
#[arg(long)]
low_latency: bool,
Expand Down Expand Up @@ -380,7 +380,7 @@ enum DaemonCommand {
advertise_host: Option<String>,
#[arg(long)]
client_root: Option<PathBuf>,
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)]
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264)]
video_codec: VideoCodecMode,
#[arg(long)]
low_latency: bool,
Expand All @@ -396,7 +396,7 @@ enum DaemonCommand {
advertise_host: Option<String>,
#[arg(long)]
client_root: Option<PathBuf>,
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)]
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264)]
video_codec: VideoCodecMode,
#[arg(long)]
low_latency: bool,
Expand All @@ -420,7 +420,7 @@ enum DaemonCommand {
advertise_host: Option<String>,
#[arg(long)]
client_root: Option<PathBuf>,
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)]
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264)]
video_codec: VideoCodecMode,
#[arg(long)]
low_latency: bool,
Expand Down Expand Up @@ -463,7 +463,7 @@ enum ServiceCommand {
advertise_host: Option<String>,
#[arg(long)]
client_root: Option<PathBuf>,
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)]
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264)]
video_codec: VideoCodecMode,
#[arg(long)]
low_latency: bool,
Expand All @@ -481,7 +481,7 @@ enum ServiceCommand {
advertise_host: Option<String>,
#[arg(long)]
client_root: Option<PathBuf>,
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)]
#[arg(long, value_enum, default_value_t = VideoCodecMode::H264)]
video_codec: VideoCodecMode,
#[arg(long)]
low_latency: bool,
Expand Down Expand Up @@ -652,10 +652,10 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result<StreamQuality
}),
"ci-software" => Ok(StreamQualityEnvironment {
profile: "ci-software",
max_edge: 844,
fps: 20,
min_bitrate: 800_000,
bits_per_pixel: 1,
max_edge: 960,
fps: 24,
min_bitrate: 1_200_000,
bits_per_pixel: 2,
}),
_ => anyhow::bail!("Unknown stream quality profile `{profile}`."),
}
Expand Down Expand Up @@ -710,7 +710,7 @@ impl Default for DaemonLaunchOptions {
bind: IpAddr::V4(Ipv4Addr::LOCALHOST),
advertise_host: None,
client_root: None,
video_codec: VideoCodecMode::H264Software,
video_codec: VideoCodecMode::H264,
low_latency: false,
realtime_stream: false,
stream_quality_profile: None,
Expand Down Expand Up @@ -1071,17 +1071,24 @@ fn project_root() -> anyhow::Result<PathBuf> {
}

fn choose_daemon_port(preferred: u16) -> anyhow::Result<u16> {
choose_daemon_port_for_bind(preferred, IpAddr::V4(Ipv4Addr::LOCALHOST))
}

fn choose_daemon_port_for_bind(preferred: u16, bind: IpAddr) -> anyhow::Result<u16> {
let start = preferred.max(1024);
for port in start..start.saturating_add(200) {
if port_available(port) {
if port_available(bind, port) {
return Ok(port);
}
}
anyhow::bail!("No available SimDeck daemon port near {preferred}")
}

fn port_available(port: u16) -> bool {
TcpListener::bind((Ipv4Addr::LOCALHOST, port)).is_ok()
fn port_available(bind: IpAddr, port: u16) -> bool {
if bind.is_unspecified() && TcpListener::bind((Ipv4Addr::LOCALHOST, port)).is_err() {
return false;
}
TcpListener::bind((bind, port)).is_ok()
}

fn open_browser(url: &str) -> anyhow::Result<()> {
Expand Down Expand Up @@ -1184,9 +1191,9 @@ fn run_foreground_ui(selector: Option<String>) -> anyhow::Result<()> {
}

let project_root = project_root()?;
let port = choose_daemon_port(4310)?;
let bind = IpAddr::V4(Ipv4Addr::UNSPECIFIED);
let video_codec = VideoCodecMode::H264Software;
let port = choose_daemon_port_for_bind(4310, bind)?;
let video_codec = VideoCodecMode::H264;
let low_latency = false;
let advertise_host = detect_lan_ip()
.unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST))
Expand Down
Loading