From 842b4bb3bfc28785be2b60c9bc7edbf3d9d886a0 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 5 Jul 2026 17:42:28 +0300 Subject: [PATCH 1/7] fix(recording): keep system audio on the recording timeline when delivery is intermittent WASAPI loopback only produces packets while something renders audio, so a system-audio track could start at the first sound (not the recording start), lose long silent stretches to the 1s gap-insertion cap, and end short of the stop point due to a frame-of-reference bug in tail padding. A late first packet also became the latest start_time, anchoring playback past the head of every other track. - Anchor system-audio pipelines at the recording epoch (AudioAnchor::PipelineEpoch): head silence is synthesized from the epoch to the first packet, start_time is reported as ~0, and a track that never receives a packet still spans the recording. Mic/camera keep first-frame (device-ready) anchoring. - Insert the full wall-clock-validated gap instead of truncating to 1s; long fills are chunked into <=1s frames. Gap health events are only emitted for device-backed sources where a gap is a real anomaly. - Fix tail padding to compare the epoch-relative target against the track's own timeline (subtract the start offset) and fill the true trailing gap instead of capping at 300ms. - Keep a silent render stream open on the captured endpoint (Windows) so loopback delivers packets through system silence, and bump cpal to zero capture buffers flagged AUDCLNT_BUFFERFLAGS_SILENT instead of passing unspecified contents through. - Sync test coverage: sync_matrix system-audio cases (late first sound, dead zone, silent throughout, bursty notifications, concurrent mic + system audio) verifying start anchoring, silence materialization and wall-clock content positions; playback selftest fixture now records a continuous mic plus loopback-style burst-delivered system audio with recorder-faithful clip offsets; meta tests lock the anchor semantics. --- Cargo.lock | 25 +- Cargo.toml | 6 +- apps/cli/src/selftest/playback.rs | 214 +++++--- crates/project/src/meta.rs | 71 +++ crates/recording/src/output_pipeline/core.rs | 470 ++++++++++++++++-- .../src/sources/screen_capture/windows.rs | 8 + crates/recording/src/studio_recording.rs | 11 +- crates/recording/tests/sync_matrix.rs | 400 ++++++++++++++- crates/scap-cpal/Cargo.toml | 1 + crates/scap-cpal/src/lib.rs | 65 ++- 10 files changed, 1144 insertions(+), 127 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index daf25e32521..6609b5d3132 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1175,7 +1175,7 @@ dependencies = [ "cidre", "clap", "clap_complete", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "dirs 6.0.0", "ffmpeg-next", "flume", @@ -1214,7 +1214,7 @@ name = "cap-audio" version = "0.1.0" dependencies = [ "cidre", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "serde", "serde_json", @@ -1354,7 +1354,7 @@ dependencies = [ name = "cap-cpal-ffmpeg" version = "0.1.0" dependencies = [ - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "workspace-hack", ] @@ -1427,7 +1427,7 @@ dependencies = [ "cocoa", "core-foundation 0.10.1", "core-graphics 0.24.0", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "device_query", "dirs 6.0.0", "dotenvy_macro", @@ -1525,7 +1525,7 @@ dependencies = [ "cap-media-info", "cap-project", "cap-rendering", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "flume", "futures", @@ -1698,7 +1698,7 @@ dependencies = [ name = "cap-media-info" version = "0.1.0" dependencies = [ - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "thiserror 1.0.69", "workspace-hack", @@ -1803,7 +1803,7 @@ dependencies = [ "core-foundation 0.10.1", "core-graphics 0.24.0", "coreaudio-rs", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "device_query", "dhat", "either", @@ -1926,7 +1926,7 @@ dependencies = [ "cidre", "clap", "colored", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "flume", "indicatif", @@ -1953,7 +1953,7 @@ name = "cap-timestamp" version = "0.1.0" dependencies = [ "cidre", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "windows 0.60.0", "workspace-hack", ] @@ -2630,7 +2630,7 @@ dependencies = [ [[package]] name = "cpal" version = "0.15.3" -source = "git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca#3cc779a7b4ca51770211f1b7dc19f107978af707" +source = "git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3#6013cb5f8bd34ddfbf7b215b9540fd211ce829fc" dependencies = [ "alsa", "core-foundation-sys", @@ -8828,8 +8828,9 @@ dependencies = [ name = "scap-cpal" version = "0.1.0" dependencies = [ - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "thiserror 1.0.69", + "tracing", "workspace-hack", ] @@ -8852,7 +8853,7 @@ name = "scap-ffmpeg" version = "0.1.0" dependencies = [ "cidre", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "futures", "scap-cpal", diff --git a/Cargo.toml b/Cargo.toml index 78a1cfe3591..5c9c2cab521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,10 @@ members = [ [workspace.dependencies] anyhow = { version = "1.0.86" } # This includes a currently-unreleased fix that ensures the audio stream is actually -# stopped and released on drop on macOS -cpal = { git = "https://github.com/CapSoftware/cpal", rev = "3cc779a7b4ca" } +# stopped and released on drop on macOS, plus zeroing of WASAPI capture +# packets flagged AUDCLNT_BUFFERFLAGS_SILENT (loopback of an idle endpoint +# otherwise surfaces unspecified buffer contents as audio). +cpal = { git = "https://github.com/CapSoftware/cpal", rev = "6013cb5f8bd3" } ffmpeg = { package = "ffmpeg-next", git = "https://github.com/CapSoftware/rust-ffmpeg", rev = "49db1fede112" } tokio = { version = "1.39.3", features = [ "macros", diff --git a/apps/cli/src/selftest/playback.rs b/apps/cli/src/selftest/playback.rs index 6dd63c71ed5..182eddc4952 100644 --- a/apps/cli/src/selftest/playback.rs +++ b/apps/cli/src/selftest/playback.rs @@ -585,7 +585,14 @@ mod fixture { const AUDIO_RATE: u32 = 48_000; const AUDIO_CHUNK_SECS: f64 = 0.02; const BEEP_FREQ: f32 = 1_000.0; - const BEEP_AMPLITUDE: f32 = 0.5; + /// Two tracks beep in unison (mic + system audio); keep their sum + /// comfortably below full scale. + const BEEP_AMPLITUDE: f32 = 0.35; + /// The mic device becomes ready shortly after the recording starts, like + /// real capture hardware. Its start_time (the latest across tracks, since + /// system audio anchors at the epoch) becomes the playback anchor, so the + /// fixture exercises the cross-track start_time offset math. + const FIXTURE_MIC_START_SECS: f64 = 0.3; struct Pattern { events: Vec, @@ -625,6 +632,7 @@ mod fixture { std::fs::create_dir_all(&segment_dir) .map_err(|e| format!("failed to create fixture directories: {e}"))?; let display_path = segment_dir.join("display.mp4"); + let mic_path = segment_dir.join("audio-input.ogg"); let audio_path = segment_dir.join("system_audio.ogg"); let timestamps = Timestamps::now(); @@ -669,55 +677,98 @@ mod fixture { }) }; - // Audio leg: silence with 1 kHz beep bursts aligned to the flashes. + // Audio legs: 1 kHz beep bursts aligned to the flashes on BOTH audio + // tracks so any relative shift between them (or against video) splits + // the beep clusters and fails the sync gates. + // + // The mic delivers continuously from device-ready to stop, like real + // capture hardware. System audio delivers loopback-style: chunks + // exist ONLY while a beep plays (WASAPI loopback produces no packets + // while the system is silent), so the recorder must synthesize the + // head/gap/tail silence to keep the track on the recording timeline. let audio_info = AudioInfo::new(Sample::F32(Type::Packed), AUDIO_RATE, 2) .map_err(|e| format!("audio info: {e:?}"))?; - let (audio_tx, audio_rx) = futures::channel::mpsc::channel::(32); - let audio_emit = { + + let beep_chunk = move |chunk_t: f64, events: &[f64]| { + let chunk_frames = (f64::from(AUDIO_RATE) * AUDIO_CHUNK_SECS) as usize; + let mut frame = ffmpeg::frame::Audio::new( + ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), + chunk_frames, + audio_info.channel_layout(), + ); + frame.set_rate(AUDIO_RATE); + let data = frame.data_mut(0); + let samples = unsafe { + std::slice::from_raw_parts_mut(data.as_mut_ptr().cast::(), data.len() / 4) + }; + for (i, sample) in samples.iter_mut().enumerate() { + let t = chunk_t + (i / 2) as f64 / f64::from(AUDIO_RATE); + *sample = if in_flash(events, t) { + (t as f32 * BEEP_FREQ * 2.0 * std::f32::consts::PI).sin() * BEEP_AMPLITUDE + } else { + 0.0 + }; + } + frame + }; + + // Both emitters return their sender when done: real capture sources + // keep the channel open until the recording stops, and the muxer's + // stop-time tail fill only runs on stop-cancellation, not on + // channel-closure. + let (mic_tx, mic_rx) = futures::channel::mpsc::channel::(32); + let mic_emit = { let events = pattern.events.clone(); let total_secs = pattern.total_secs; let base = timestamps.instant(); - let mut tx = audio_tx; - let info = audio_info; + let mut tx = mic_tx; + let beep_chunk = beep_chunk.clone(); tokio::spawn(async move { use futures::SinkExt; - let chunk_frames = (f64::from(AUDIO_RATE) * AUDIO_CHUNK_SECS) as usize; + let first_chunk = (FIXTURE_MIC_START_SECS / AUDIO_CHUNK_SECS).ceil() as usize; let total_chunks = (total_secs / AUDIO_CHUNK_SECS).ceil() as usize; - for k in 0..total_chunks { + for k in first_chunk..total_chunks { let chunk_t = k as f64 * AUDIO_CHUNK_SECS; tokio::time::sleep_until((base + Duration::from_secs_f64(chunk_t)).into()) .await; - let mut frame = ffmpeg::frame::Audio::new( - ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), - chunk_frames, - info.channel_layout(), - ); - frame.set_rate(AUDIO_RATE); - let data = frame.data_mut(0); - let samples = unsafe { - std::slice::from_raw_parts_mut( - data.as_mut_ptr().cast::(), - data.len() / 4, - ) - }; - for (i, sample) in samples.iter_mut().enumerate() { - let n = (k * chunk_frames + i / 2) as f64; - let t = n / f64::from(AUDIO_RATE); - *sample = if in_flash(&events, t) { - (t as f32 * BEEP_FREQ * 2.0 * std::f32::consts::PI).sin() - * BEEP_AMPLITUDE - } else { - 0.0 - }; - } let frame = AudioFrame::new( - frame, + beep_chunk(chunk_t, &events), Timestamp::Instant(base + Duration::from_secs_f64(chunk_t)), ); if tx.send(frame).await.is_err() { break; } } + tx + }) + }; + + let (sys_tx, sys_rx) = futures::channel::mpsc::channel::(32); + let sys_emit = { + let events = pattern.events.clone(); + let base = timestamps.instant(); + let mut tx = sys_tx; + let beep_chunk = beep_chunk.clone(); + tokio::spawn(async move { + use futures::SinkExt; + for &event in &events { + let first_chunk = (event / AUDIO_CHUNK_SECS).floor() as usize; + let last_chunk = + ((event + FIXTURE_FLASH_SECS) / AUDIO_CHUNK_SECS).ceil() as usize; + for k in first_chunk..last_chunk { + let chunk_t = k as f64 * AUDIO_CHUNK_SECS; + tokio::time::sleep_until((base + Duration::from_secs_f64(chunk_t)).into()) + .await; + let frame = AudioFrame::new( + beep_chunk(chunk_t, &events), + Timestamp::Instant(base + Duration::from_secs_f64(chunk_t)), + ); + if tx.send(frame).await.is_err() { + return tx; + } + } + } + tx }) }; @@ -729,21 +780,33 @@ mod fixture { .build::(()) .await .map_err(|e| format!("video pipeline: {e}"))?; - let audio_pipeline = OutputPipeline::builder(audio_path.clone()) + let mic_pipeline = OutputPipeline::builder(mic_path.clone()) .with_audio_source::(ChannelAudioSourceConfig::new( - audio_info, audio_rx, + audio_info, mic_rx, )) .with_timestamps(timestamps) .build::(()) .await - .map_err(|e| format!("audio pipeline: {e}"))?; + .map_err(|e| format!("mic pipeline: {e}"))?; + let sys_pipeline = OutputPipeline::builder(audio_path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new( + audio_info, sys_rx, + )) + .with_timestamps(timestamps) + // System audio anchors at the recording epoch, exactly like the + // studio recorder configures it. + .with_audio_anchor(cap_recording::AudioAnchor::PipelineEpoch) + .build::(()) + .await + .map_err(|e| format!("system audio pipeline: {e}"))?; video_emit .await .map_err(|e| format!("video emit join: {e}"))?; - audio_emit + let mic_held_tx = mic_emit.await.map_err(|e| format!("mic emit join: {e}"))?; + let sys_held_tx = sys_emit .await - .map_err(|e| format!("audio emit join: {e}"))?; + .map_err(|e| format!("system audio emit join: {e}"))?; // Let the stream tails flush through the encoders. tokio::time::sleep(Duration::from_millis(500)).await; @@ -751,10 +814,15 @@ mod fixture { .stop() .await .map_err(|e| format!("video pipeline stop: {e}"))?; - let finished_audio = audio_pipeline + let finished_mic = mic_pipeline .stop() .await - .map_err(|e| format!("audio pipeline stop: {e}"))?; + .map_err(|e| format!("mic pipeline stop: {e}"))?; + let finished_sys = sys_pipeline + .stop() + .await + .map_err(|e| format!("system audio pipeline stop: {e}"))?; + drop((mic_held_tx, sys_held_tx)); // Persist metadata the way the studio recorder does: start times are // each track's first timestamp on the shared clock, and the timeline @@ -762,7 +830,10 @@ mod fixture { let display_start = finished_video .first_timestamp .signed_duration_since_secs(timestamps); - let audio_start = finished_audio + let mic_start = finished_mic + .first_timestamp + .signed_duration_since_secs(timestamps); + let sys_start = finished_sys .first_timestamp .signed_duration_since_secs(timestamps); let display_duration = finished_video @@ -770,33 +841,43 @@ mod fixture { .map(|(first, last)| (last - first).as_secs_f64() + 1.0 / f64::from(FIXTURE_FPS)) .ok_or("fixture video reported no timestamp span")?; + let to_project_gap_summary = + |s: cap_recording::AudioGapSummary| cap_project::AudioGapSummary { + total_overlap_trimmed_ms: s.total_overlap_trimmed_ms, + startup_overlap_trimmed_ms: s.startup_overlap_trimmed_ms, + overlap_dropped_frames: s.overlap_dropped_frames, + startup_overlap_drops: s.startup_overlap_drops, + }; + + let segment = MultipleSegment { + display: VideoMeta { + path: RelativePathBuf::from("content/segments/segment-0/display.mp4"), + fps: FIXTURE_FPS, + start_time: Some(display_start), + device_id: None, + }, + camera: None, + mic: Some(AudioMeta { + path: RelativePathBuf::from("content/segments/segment-0/audio-input.ogg"), + start_time: Some(mic_start), + device_id: None, + gap_summary: finished_mic.audio_gap_summary.map(to_project_gap_summary), + }), + system_audio: Some(AudioMeta { + path: RelativePathBuf::from("content/segments/segment-0/system_audio.ogg"), + start_time: Some(sys_start), + device_id: None, + gap_summary: finished_sys.audio_gap_summary.map(to_project_gap_summary), + }), + cursor: None, + keyboard: None, + }; + // Clip offsets exactly as the studio recorder persists them. + let offsets = segment.calculate_audio_offsets(); + let meta = StudioRecordingMeta::MultipleSegments { inner: MultipleSegments { - segments: vec![MultipleSegment { - display: VideoMeta { - path: RelativePathBuf::from("content/segments/segment-0/display.mp4"), - fps: FIXTURE_FPS, - start_time: Some(display_start), - device_id: None, - }, - camera: None, - mic: None, - system_audio: Some(AudioMeta { - path: RelativePathBuf::from("content/segments/segment-0/system_audio.ogg"), - start_time: Some(audio_start), - device_id: None, - gap_summary: finished_audio.audio_gap_summary.map(|s| { - cap_project::AudioGapSummary { - total_overlap_trimmed_ms: s.total_overlap_trimmed_ms, - startup_overlap_trimmed_ms: s.startup_overlap_trimmed_ms, - overlap_dropped_frames: s.overlap_dropped_frames, - startup_overlap_drops: s.startup_overlap_drops, - } - }), - }), - cursor: None, - keyboard: None, - }], + segments: vec![segment], cursors: Default::default(), status: Some(StudioRecordingStatus::Complete), }, @@ -833,7 +914,8 @@ mod fixture { }), clips: vec![ClipConfiguration { index: 0, - offsets: Default::default(), + offsets, + ..Default::default() }], ..Default::default() }; diff --git a/crates/project/src/meta.rs b/crates/project/src/meta.rs index dcf5f9341fc..857003efe26 100644 --- a/crates/project/src/meta.rs +++ b/crates/project/src/meta.rs @@ -733,4 +733,75 @@ mod test { }"#, ); } + + mod audio_offsets { + use crate::{AudioMeta, MultipleSegment, VideoMeta}; + use relative_path::RelativePathBuf; + + fn video(start_time: Option) -> VideoMeta { + VideoMeta { + path: RelativePathBuf::from("display.mp4"), + fps: 30, + start_time, + device_id: None, + } + } + + fn audio(start_time: Option) -> AudioMeta { + AudioMeta { + path: RelativePathBuf::from("audio.ogg"), + start_time, + device_id: None, + gap_summary: None, + } + } + + fn segment( + display_start: f64, + mic_start: Option, + system_start: Option, + ) -> MultipleSegment { + MultipleSegment { + display: video(Some(display_start)), + camera: None, + mic: mic_start.map(|s| audio(Some(s))), + system_audio: system_start.map(|s| audio(Some(s))), + cursor: None, + keyboard: None, + } + } + + // The recorder anchors system audio at the recording epoch + // (start_time ~ 0.0), which keeps it from ever being the latest + // start_time: the mic/display anchor — and therefore where playback + // starts and how the mic aligns to video — must be identical with + // and without a system audio track. + #[test] + fn epoch_anchored_system_audio_does_not_move_the_anchor() { + let without = segment(0.58, Some(0.55), None); + let with = segment(0.58, Some(0.55), Some(0.0)); + + assert_eq!(without.latest_start_time(), Some(0.58)); + assert_eq!(with.latest_start_time(), Some(0.58)); + + let offsets_without = without.calculate_audio_offsets(); + let offsets_with = with.calculate_audio_offsets(); + assert_eq!(offsets_without.mic, offsets_with.mic); + assert!((offsets_with.mic - 0.03).abs() < 1e-6); + // System audio positions itself by its own start. + assert!((offsets_with.system_audio - 0.58).abs() < 1e-6); + } + + // Legacy recordings (pre-epoch-anchor) stamped system audio with its + // first packet time; those files keep their historical alignment: + // a later system start is still the anchor for them. + #[test] + fn legacy_first_packet_system_audio_keeps_historical_anchor() { + let legacy = segment(0.5824678, Some(0.5559852), Some(0.6586015)); + assert_eq!(legacy.latest_start_time(), Some(0.6586015)); + let offsets = legacy.calculate_audio_offsets(); + assert!((offsets.mic - (0.6586015 - 0.5559852) as f32).abs() < 1e-6); + assert_eq!(offsets.system_audio, 0.0); + } + } } diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index 3d8393c6bc4..a24f019cdb6 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -719,13 +719,21 @@ const WIRED_GAP_THRESHOLD: Duration = Duration::from_millis(70); const WIRELESS_GAP_THRESHOLD: Duration = Duration::from_millis(160); const AUDIO_WALL_CLOCK_TOLERANCE: Duration = Duration::from_millis(100); const AUDIO_OVERLAP_TOLERANCE: Duration = Duration::from_millis(5); -const MAX_SILENCE_INSERTION: Duration = Duration::from_secs(1); -const MAX_AUDIO_TAIL_PADDING: Duration = Duration::from_millis(300); - -fn audio_tail_padding_duration(audio_elapsed: Duration, target_elapsed: Duration) -> Duration { - target_elapsed - .saturating_sub(audio_elapsed) - .min(MAX_AUDIO_TAIL_PADDING) +const LONG_SILENCE_LOG_THRESHOLD: Duration = Duration::from_secs(1); +/// Cap on individual synthesized-silence frames; long fills are emitted as a +/// sequence of frames so a multi-second dead zone doesn't allocate one giant +/// buffer. +const SILENCE_FRAME_MAX: Duration = Duration::from_secs(1); + +/// How much trailing silence the track needs to reach the stop point. +/// `track_target_elapsed` must be in the track's own timeline (epoch-relative +/// target minus the track's start offset); both the previous overshoot +/// (mic tracks padded past the stop point) and the previous shortfall +/// (a system-audio track whose last sound came long before stop stayed +/// short) came from comparing an epoch-relative target against the +/// track-local timeline. +fn audio_tail_padding_duration(audio_elapsed: Duration, track_target_elapsed: Duration) -> Duration { + track_target_elapsed.saturating_sub(audio_elapsed) } const STARTUP_OVERLAP_DROP_FRAME_COUNT: u64 = 3; @@ -810,6 +818,18 @@ impl AudioGapTracker { } } + fn started(&self) -> bool { + self.first_frame_ts.is_some() + } + + /// Offset of this track's timeline zero from the pipeline epoch: the + /// capture time of the first muxed frame, or zero when the track is + /// anchored at the epoch itself. + fn track_start_offset(&self) -> Option { + let secs = self.first_frame_ts?.signed_duration_since_secs(self.reference); + Some(Duration::from_secs_f64(secs.max(0.0))) + } + fn capture_elapsed( &self, current_frame_ts: Timestamp, @@ -849,7 +869,14 @@ impl AudioGapTracker { let gap = capture_elapsed.saturating_sub(sample_based_elapsed); if gap > self.gap_threshold { - Some(gap.min(MAX_SILENCE_INSERTION)) + // The full gap is inserted: capture_elapsed is already clamped to + // wall-clock elapsed (+tolerance), so a large value here is a real + // silent stretch (e.g. WASAPI loopback delivers nothing while the + // system plays no sound), not a bogus timestamp. Truncating it + // (the old 1s cap) placed the audio that follows a long dead zone + // up to the truncated amount too early until repeated insertions + // converged, smearing the first seconds after the gap. + Some(gap) } else { None } @@ -894,6 +921,34 @@ impl AudioGapTracker { } } +/// Send `total_samples` of synthesized silence starting at the track-local +/// sample position `start_samples`, split into frames of at most +/// [`SILENCE_FRAME_MAX`]. Each frame's mux timestamp is derived from the +/// running sample count so long fills stay sample-accurate. +async fn send_silence_frames( + muxer: &Arc>, + audio_info: &AudioInfo, + frame_ts: Timestamp, + start_samples: u64, + total_samples: u64, +) -> anyhow::Result<()> { + let sample_rate = audio_info.sample_rate; + let chunk_max = ((sample_rate.max(1) as u64) * SILENCE_FRAME_MAX.as_millis() as u64) / 1000; + let chunk_max = chunk_max.max(1); + let mut sent = 0u64; + while sent < total_samples { + let n = (total_samples - sent).min(chunk_max); + let elapsed = Duration::from_nanos(samples_to_nanos(start_samples + sent, sample_rate)); + let silence = create_silence_frame(audio_info, n as usize); + muxer + .lock() + .await + .send_audio_frame(AudioFrame::new(silence, frame_ts), elapsed)?; + sent += n; + } + Ok(()) +} + fn create_silence_frame(audio_info: &AudioInfo, sample_count: usize) -> ffmpeg::frame::Audio { let mut frame = ffmpeg::frame::Audio::new( audio_info.sample_format, @@ -1433,10 +1488,30 @@ impl OutputPipeline { audio_sources: vec![], timestamps, master_clock: None, + audio_anchor: AudioAnchor::FirstFrame, } } } +/// Where the audio track's timeline zero (and therefore its persisted +/// `start_time`) is anchored. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum AudioAnchor { + /// Timeline zero is the capture timestamp of the first frame the muxer + /// sees. Right for device-backed sources (microphone, camera audio): + /// the device produces samples continuously once live, so the first + /// frame marks "device ready" and downstream start-time alignment cuts + /// all tracks to the latest-starting device. + FirstFrame, + /// Timeline zero is the pipeline epoch; silence is synthesized from the + /// epoch up to the first captured frame. Right for intermittent sources + /// (WASAPI loopback system audio) where the first packet marks "first + /// sound played", not "source ready" — anchoring such a track at its + /// first frame would let a late first sound become the cross-track + /// alignment anchor and cut the head off every other track. + PipelineEpoch, +} + pub struct SetupCtx { tasks: TaskPool, health_tx: HealthSender, @@ -1496,6 +1571,7 @@ pub struct OutputPipelineBuilder { audio_sources: Vec, timestamps: Timestamps, master_clock: Option>, + audio_anchor: AudioAnchor, } pub struct NoVideo; @@ -1534,6 +1610,13 @@ impl OutputPipelineBuilder { pub fn set_master_clock(&mut self, master_clock: Arc) { self.master_clock = Some(master_clock); } + + /// Anchor the audio track at the pipeline epoch instead of the first + /// captured frame. See [`AudioAnchor::PipelineEpoch`]. + pub fn with_audio_anchor(mut self, anchor: AudioAnchor) -> Self { + self.audio_anchor = anchor; + self + } } impl OutputPipelineBuilder { @@ -1547,6 +1630,7 @@ impl OutputPipelineBuilder { audio_sources: self.audio_sources, timestamps: self.timestamps, master_clock: self.master_clock, + audio_anchor: self.audio_anchor, } } } @@ -1612,6 +1696,7 @@ impl OutputPipelineBuilder> { timestamps, path, master_clock, + audio_anchor, .. } = self; @@ -1690,6 +1775,7 @@ impl OutputPipelineBuilder> { video_start_gate, build_ctx.stop_signal, audio_gap_summary.clone(), + audio_anchor, ) .await?; @@ -1719,6 +1805,7 @@ impl OutputPipelineBuilder { timestamps, path, master_clock, + audio_anchor, .. } = self; @@ -1774,6 +1861,7 @@ impl OutputPipelineBuilder { None, build_ctx.stop_signal, audio_gap_summary.clone(), + audio_anchor, ) .await?; @@ -1845,6 +1933,7 @@ async fn finish_build( video_start_gate: Option, stop_signal: PipelineStopSignal, gap_summary_slot: Arc>, + audio_anchor: AudioAnchor, ) -> anyhow::Result<()> { if let Some(audio) = audio { audio.configure( @@ -1857,6 +1946,7 @@ async fn finish_build( has_video, video_start_gate, gap_summary_slot, + audio_anchor, ); } @@ -2302,12 +2392,20 @@ impl PreparedAudioSources { has_video: bool, video_start_gate: Option, gap_summary_slot: Arc>, + audio_anchor: AudioAnchor, ) { let audio_info = self.audio_info; let has_wireless_source = self.has_wireless_source; let health_tx = setup_ctx.health_tx().clone(); let master_clock = setup_ctx.master_clock().clone(); + if audio_anchor == AudioAnchor::PipelineEpoch && video_start_gate.is_some() { + warn!( + "PipelineEpoch audio anchor is ignored when a video start gate \ + aligns audio to the video track" + ); + } + setup_ctx.tasks().spawn("mux-audio", { let stop_token = stop_token.child_token(); let muxer = muxer.clone(); @@ -2339,6 +2437,8 @@ impl PreparedAudioSources { has_video, origin: FrameProcessOrigin::Live, observed_at: Instant::now(), + timestamps, + anchor: audio_anchor, }, AudioFrameProcessState { timestamp_generator: &mut timestamp_generator, @@ -2399,6 +2499,8 @@ impl PreparedAudioSources { has_video, origin: FrameProcessOrigin::Drain, observed_at: Instant::now(), + timestamps, + anchor: audio_anchor, }, AudioFrameProcessState { timestamp_generator: &mut timestamp_generator, @@ -2450,21 +2552,44 @@ impl PreparedAudioSources { if let Some(target_elapsed) = cancellation_target_elapsed && !audio_degraded { + // An epoch-anchored track that never received a frame + // (e.g. WASAPI loopback with no sound played the whole + // recording) still spans the recording: anchor it now so + // the fill below covers the full duration and the track + // reports a valid start. + if audio_anchor == AudioAnchor::PipelineEpoch && !gap_tracker.started() { + let epoch_ts = Timestamp::Instant(timestamps.instant()); + gap_tracker.mark_started(epoch_ts, timestamps.instant()); + if let Some(first_tx) = first_tx.take() { + let _ = first_tx.send(epoch_ts); + } + info!("No audio frames arrived; anchoring silent track at epoch"); + } + let audio_elapsed = timestamp_generator.next_timestamp(0); - let tail_padding = audio_tail_padding_duration(audio_elapsed, target_elapsed); + // target_elapsed is epoch-relative; the audio timeline is + // track-local (zero = first muxed frame, or the epoch when + // head-anchored), so remove the track's start offset + // before comparing. + let track_target = gap_tracker + .track_start_offset() + .map(|offset| target_elapsed.saturating_sub(offset)) + .unwrap_or(target_elapsed); + let tail_padding = audio_tail_padding_duration(audio_elapsed, track_target); + let start_samples = timestamp_generator.total_samples; let tail_samples = timestamp_generator.advance_by_duration(tail_padding); if tail_samples > 0 { - let silence = create_silence_frame(&audio_info, tail_samples as usize); - let silence_frame = AudioFrame::new( - silence, - Timestamp::Instant(timestamps.instant() + audio_elapsed), - ); - - if let Err(e) = muxer - .lock() - .await - .send_audio_frame(silence_frame, audio_elapsed) + let frame_ts = Timestamp::Instant(timestamps.instant() + audio_elapsed); + + if let Err(e) = send_silence_frames( + &muxer, + &audio_info, + frame_ts, + start_samples, + tail_samples, + ) + .await { if has_video { warn!( @@ -2492,6 +2617,7 @@ impl PreparedAudioSources { padding_ms = tail_padding.as_millis() as u64, samples = tail_samples, audio_end_ms = audio_elapsed.as_millis() as u64, + track_target_ms = track_target.as_millis() as u64, target_ms = target_elapsed.as_millis() as u64, "Padded audio tail with silence" ); @@ -2570,6 +2696,8 @@ struct AudioFrameProcessContext<'a, TMutex: AudioMuxer> { has_video: bool, origin: FrameProcessOrigin, observed_at: Instant, + timestamps: Timestamps, + anchor: AudioAnchor, } struct AudioFrameProcessState<'a> { @@ -2622,11 +2750,78 @@ async fn process_audio_frame( } } + let observed_at = ctx.observed_at; + + // Epoch-anchored tracks (intermittent sources like WASAPI loopback): + // timeline zero is the pipeline epoch, so the stretch between the epoch + // and the first captured frame is real recorded silence — synthesize it + // and report the epoch as the track start. Pause time never reaches the + // sample timeline, so it is excised from the head like everywhere else. + if ctx.anchor == AudioAnchor::PipelineEpoch + && ctx.video_start_gate.is_none() + && !state.gap_tracker.started() + { + let epoch_ts = Timestamp::Instant(ctx.timestamps.instant()); + state.gap_tracker.mark_started(epoch_ts, ctx.timestamps.instant()); + + let head_secs = frame.timestamp.signed_duration_since_secs(ctx.timestamps); + let head = Duration::from_secs_f64(head_secs.max(0.0)) + .saturating_sub(total_pause_duration) + // A capture timestamp can't credibly predate more wall time than + // has actually elapsed since the epoch. + .min(observed_at.saturating_duration_since(ctx.timestamps.instant())); + + if !head.is_zero() { + let start_samples = state.timestamp_generator.total_samples; + let head_samples = state.timestamp_generator.advance_by_duration(head); + + if head_samples > 0 { + info!( + head_ms = head.as_millis() as u64, + samples = head_samples, + "Anchoring audio track at pipeline epoch; \ + filling head with silence up to first captured frame" + ); + + if let Err(e) = send_silence_frames( + ctx.muxer, + ctx.audio_info, + epoch_ts, + start_samples, + head_samples, + ) + .await + { + if ctx.has_video { + warn!( + "Audio muxer rejected head silence, \ + degrading to video-only: {e}" + ); + emit_health( + ctx.health_tx, + PipelineHealthEvent::AudioDegradedToVideoOnly { + reason: format!("Head silence rejected: {e}"), + }, + ); + return Ok(AudioFrameOutcome::AudioDegraded); + } + return Err(anyhow!("Audio muxer stopped accepting head silence: {e}")); + } + } + } + } + if let Some(first_tx) = state.first_tx.take() { - let _ = first_tx.send(frame.timestamp); + let anchor_ts = if ctx.anchor == AudioAnchor::PipelineEpoch + && ctx.video_start_gate.is_none() + { + Timestamp::Instant(ctx.timestamps.instant()) + } else { + frame.timestamp + }; + let _ = first_tx.send(anchor_ts); } - let observed_at = ctx.observed_at; state.gap_tracker.mark_started(frame.timestamp, observed_at); let sample_based_before = state.timestamp_generator.next_timestamp(0); @@ -2678,35 +2873,44 @@ async fn process_audio_frame( total_pause_duration, observed_at, ) { + let start_samples = state.timestamp_generator.total_samples; let silence_samples = state.timestamp_generator.advance_by_duration(gap_duration); if silence_samples > 0 { - let silence = create_silence_frame(ctx.audio_info, silence_samples as usize); - - let silence_frame = AudioFrame::new(silence, frame.timestamp); - - if gap_duration >= MAX_SILENCE_INSERTION { - error!( + if gap_duration >= LONG_SILENCE_LOG_THRESHOLD { + // Long gaps are expected for intermittent sources (loopback + // system audio while nothing plays); the wall-clock clamp in + // capture_elapsed already vouched that this much real time + // passed. + info!( gap_ms = gap_duration.as_millis(), - "Audio gap exceeded 1s cap, \ - something may be seriously wrong" + "Long audio gap; filling with silence" ); } state.gap_tracker.record_insertion(gap_duration); - emit_health( - ctx.health_tx, - PipelineHealthEvent::AudioGapDetected { - gap_ms: gap_duration.as_millis() as u64, - }, - ); + // For device-backed sources a delivery gap is a real anomaly + // worth surfacing; for epoch-anchored intermittent sources + // (loopback system audio) silent stretches are the normal shape + // of the stream, not a health problem. + if ctx.anchor == AudioAnchor::FirstFrame { + emit_health( + ctx.health_tx, + PipelineHealthEvent::AudioGapDetected { + gap_ms: gap_duration.as_millis() as u64, + }, + ); + } - if let Err(e) = ctx - .muxer - .lock() - .await - .send_audio_frame(silence_frame, sample_based_before) + if let Err(e) = send_silence_frames( + ctx.muxer, + ctx.audio_info, + frame.timestamp, + start_samples, + silence_samples, + ) + .await { if ctx.has_video { warn!( @@ -3361,7 +3565,7 @@ mod tests { } #[test] - fn allows_wall_clock_confirmed_stall_up_to_cap() { + fn wall_clock_confirmed_stall_inserts_full_gap() { let timestamps = Timestamps::now(); let first_ts = Timestamp::Instant(timestamps.instant()); let first_wall_clock = Instant::now(); @@ -3378,7 +3582,33 @@ mod tests { ) .expect("wall-clock-confirmed stall should insert silence"); - assert_eq!(gap, MAX_SILENCE_INSERTION); + // The full wall-clock-validated gap is inserted; truncating it + // would place post-gap audio too early. + assert_eq!(gap, Duration::from_millis(1460)); + } + + #[test] + fn long_dead_zone_inserts_full_gap_in_one_detection() { + // WASAPI loopback delivers nothing while the system is silent; a + // frame arriving after a long dead zone must account for the + // whole stretch at once so its content lands at capture time. + let timestamps = Timestamps::now(); + let first_ts = Timestamp::Instant(timestamps.instant()); + let first_wall_clock = Instant::now(); + let mut tracker = AudioGapTracker::new(false, timestamps); + + tracker.mark_started(first_ts, first_wall_clock); + + let gap = tracker + .detect_gap( + Timestamp::Instant(timestamps.instant() + Duration::from_secs(30)), + Duration::from_secs(2), + Duration::ZERO, + first_wall_clock + Duration::from_secs(30), + ) + .expect("dead zone should insert silence"); + + assert_eq!(gap, Duration::from_secs(28)); } #[test] @@ -3431,10 +3661,13 @@ mod tests { } #[test] - fn caps_tail_padding() { + fn fills_long_tail_gap_completely() { + // A track whose source stopped delivering long before the stop + // point is padded to the full track-relative target so it spans + // the recording (the old 300ms cap left such tracks short). assert_eq!( audio_tail_padding_duration(Duration::from_millis(100), Duration::from_secs(2)), - MAX_AUDIO_TAIL_PADDING + Duration::from_millis(1900) ); } } @@ -5184,6 +5417,7 @@ mod tests { first_tx: Option>, frame_count: u64, dropped_during_pause: u64, + anchor: AudioAnchor, } impl AudioTimelineHarness { @@ -5210,6 +5444,14 @@ mod tests { first_tx: None, frame_count: 0, dropped_during_pause: 0, + anchor: AudioAnchor::FirstFrame, + } + } + + fn new_epoch_anchored() -> Self { + Self { + anchor: AudioAnchor::PipelineEpoch, + ..Self::new() } } @@ -5248,6 +5490,8 @@ mod tests { has_video: true, origin: FrameProcessOrigin::Live, observed_at, + timestamps: self.timestamps, + anchor: self.anchor, }, AudioFrameProcessState { timestamp_generator: &mut self.timestamp_generator, @@ -5423,6 +5667,144 @@ mod tests { ); } + // System audio (WASAPI loopback) may deliver its first packet long after + // the recording starts — the first packet marks "first sound played", + // not "source ready". An epoch-anchored track reports the pipeline + // epoch as its start and synthesizes head silence, so a late first + // sound can never become the cross-track alignment anchor and cut the + // head off the display/mic tracks. + #[tokio::test(flavor = "current_thread")] + async fn epoch_anchor_fills_head_and_reports_epoch_start() { + let mut harness = AudioTimelineHarness::new_epoch_anchored(); + let (tx, mut rx) = oneshot::channel(); + harness.first_tx = Some(tx); + + let first_frame_at = Duration::from_millis(2_500); + assert!(matches!( + harness.process_at(first_frame_at, first_frame_at, 960).await, + AudioFrameOutcome::Sent + )); + + let start = rx + .try_recv() + .unwrap() + .expect("first timestamp must be reported"); + assert!( + start.signed_duration_since_secs(harness.timestamps).abs() < 1e-9, + "track start must be the pipeline epoch, not the first frame" + ); + + let committed = harness.committed_audio(); + let expected = + first_frame_at + Duration::from_secs_f64(960.0 / TEST_SAMPLE_RATE as f64); + assert!( + abs_skew(committed, expected) <= Duration::from_millis(1), + "timeline must cover head silence + frame, got {committed:?}" + ); + + let sent = harness.sent(); + let head_samples: usize = sent[..sent.len() - 1].iter().map(|f| f.samples).sum(); + assert_eq!( + head_samples, + (TEST_SAMPLE_RATE as f64 * 2.5) as usize, + "head silence must cover exactly epoch..first frame" + ); + assert!( + sent[..sent.len() - 1] + .iter() + .all(|f| f.samples <= TEST_SAMPLE_RATE as usize), + "head silence must be chunked to at most 1s frames" + ); + let real = sent.last().unwrap().clone(); + assert_eq!(real.samples, 960); + assert!( + abs_skew(real.timestamp, first_frame_at) <= Duration::from_millis(1), + "first real frame must land at its capture offset from the epoch" + ); + for pair in sent.windows(2) { + assert!(pair[1].timestamp >= pair[0].timestamp); + } + assert_eq!( + harness.total_silence(), + Duration::ZERO, + "head anchoring must not count as gap-repair silence" + ); + } + + // While the system plays nothing, loopback delivers nothing; when + // sound resumes after a long dead zone the resumed content must land + // at its capture time in one detection, not smeared early by a + // truncated insertion. + #[tokio::test(flavor = "current_thread")] + async fn epoch_anchor_dead_zone_resumes_at_capture_time() { + let mut harness = AudioTimelineHarness::new_epoch_anchored(); + + let frame_dur = Duration::from_secs_f64(960.0 / TEST_SAMPLE_RATE as f64); + assert!(matches!( + harness + .process_at(Duration::from_millis(50), Duration::from_millis(50), 960) + .await, + AudioFrameOutcome::Sent + )); + + let resume_at = Duration::from_secs(30); + assert!(matches!( + harness.process_at(resume_at, resume_at, 960).await, + AudioFrameOutcome::Sent + )); + + let committed = harness.committed_audio(); + assert!( + abs_skew(committed, resume_at + frame_dur) <= Duration::from_millis(5), + "post-dead-zone audio must land at capture time, got {committed:?}" + ); + + let sent = harness.sent(); + let last = sent.last().unwrap(); + assert_eq!(last.samples, 960); + assert!( + abs_skew(last.timestamp, resume_at) <= Duration::from_millis(5), + "resumed frame must be muxed at its capture offset, got {:?}", + last.timestamp + ); + assert!( + sent.iter().all(|f| f.samples <= TEST_SAMPLE_RATE as usize), + "gap silence must be chunked to at most 1s frames" + ); + for pair in sent.windows(2) { + assert!(pair[1].timestamp >= pair[0].timestamp); + } + } + + // Device-backed tracks (microphone) keep the first-frame anchor: the + // track starts when the device produces its first samples. + #[tokio::test(flavor = "current_thread")] + async fn first_frame_anchor_reports_first_frame_start() { + let mut harness = AudioTimelineHarness::new(); + let (tx, mut rx) = oneshot::channel(); + harness.first_tx = Some(tx); + + let first_frame_at = Duration::from_millis(2_500); + assert!(matches!( + harness.process_at(first_frame_at, first_frame_at, 960).await, + AudioFrameOutcome::Sent + )); + + let start = rx + .try_recv() + .unwrap() + .expect("first timestamp must be reported"); + assert!( + (start.signed_duration_since_secs(harness.timestamps) - 2.5).abs() < 1e-6, + "mic-style tracks must still report the first frame as start" + ); + + let sent = harness.sent(); + assert_eq!(sent.len(), 1, "no head silence for first-frame anchoring"); + assert_eq!(sent[0].samples, 960); + assert_eq!(sent[0].timestamp, Duration::ZERO); + } + // 5m30s simulated recording at 48kHz with a 0.1% slow mic clock and eight stalls // (5-293ms) that later fill. The sample-count audio timeline must keep tracking the // device capture clock within a bounded window (gap-corrected, never runaway), and diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 7441b6002e1..e8286d414ef 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -1344,6 +1344,14 @@ impl output_pipeline::AudioSource for SystemAudioSource { async move { let capturer = setup_result.map_err(|e| anyhow!("{e}"))?; + if capturer.has_silence_keepalive() { + info!("System audio loopback silence keepalive active"); + } else { + warn!( + "System audio loopback has no silence keepalive; \ + capture will only produce packets while other apps play audio" + ); + } if let Ok(mut guard) = state.lock() { guard.capturer = Some(capturer); } diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index b79278a4148..463a41a0096 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -15,7 +15,8 @@ use crate::{ feeds::{camera::CameraFeedLock, microphone::MicrophoneFeedLock}, ffmpeg::{FragmentedAudioMuxer, FragmentedAudioMuxerConfig, OggMuxer}, output_pipeline::{ - AudioGapSummary, DoneFut, FinishedOutputPipeline, OutputPipeline, PipelineDoneError, + AudioAnchor, AudioGapSummary, DoneFut, FinishedOutputPipeline, OutputPipeline, + PipelineDoneError, }, screen_capture::ScreenCaptureConfig, sources::{self, screen_capture}, @@ -1132,6 +1133,7 @@ async fn stop_recording( .map(|(i, segment)| ClipConfiguration { index: i as u32, offsets: segment.calculate_audio_offsets(), + ..Default::default() }) .collect::>() }); @@ -1660,11 +1662,17 @@ async fn create_segment_pipeline( }; let system_audio = if let Some(system_audio_source) = system_audio { + // System audio is intermittent (WASAPI loopback only delivers while + // sound plays), so its first packet is not a "source ready" marker: + // anchor the track at the recording epoch. This keeps a late first + // sound from becoming the latest start_time and cutting the head off + // the display/mic/camera tracks at playback. let pipeline = if segment_fragmented { let output_path = dir.join("system_audio.m4a"); OutputPipeline::builder(output_path) .with_audio_source::(system_audio_source) .with_timestamps(start_time) + .with_audio_anchor(AudioAnchor::PipelineEpoch) .build::(FragmentedAudioMuxerConfig { shared_pause_state: shared_pause_state.clone(), }) @@ -1674,6 +1682,7 @@ async fn create_segment_pipeline( OutputPipeline::builder(dir.join("system_audio.ogg")) .with_audio_source::(system_audio_source) .with_timestamps(start_time) + .with_audio_anchor(AudioAnchor::PipelineEpoch) .build::(()) .instrument(error_span!("system-audio-out")) .await diff --git a/crates/recording/tests/sync_matrix.rs b/crates/recording/tests/sync_matrix.rs index 431b511de26..fb1d394f663 100644 --- a/crates/recording/tests/sync_matrix.rs +++ b/crates/recording/tests/sync_matrix.rs @@ -534,7 +534,6 @@ async fn run_audio_case(case: AudioCase) -> Result { let emit = { let base = timestamps.instant(); let mut tx = tx; - let info = info; tokio::spawn(async move { use futures::SinkExt; for k in 0..total_chunks { @@ -603,6 +602,359 @@ fn bytemuck_cast_f32(data: &mut [u8]) -> &mut [f32] { unsafe { std::slice::from_raw_parts_mut(data.as_mut_ptr().cast::(), len) } } +/// Tolerance for where audible content sits inside a decoded track. Bounded +/// by the emission chunk size, the muxer gap threshold (70ms), codec frame +/// granularity and CI scheduler noise. +const CONTENT_POSITION_TOLERANCE_SECS: f64 = 0.25; + +/// System-audio (WASAPI-loopback-style) delivery: packets only exist while +/// something plays. The recorder sees a late first packet, dead zones with no +/// delivery at all, and possibly nothing after the last sound. The muxed +/// track must still span the whole recording with every burst at its true +/// wall-clock position and `first_timestamp` at the recording epoch. +struct SystemAudioCase { + /// (start_secs, end_secs) of each burst of real packet delivery. + bursts: Vec<(f64, f64)>, + /// Wall-clock stop point of the recording. + total_secs: f64, +} + +/// Emits loopback-style bursts on `tx` in real time, using 80ms chunks (the +/// buffer size our Windows loopback capturer targets). +/// +/// Returns the sender when done: real capture sources hold their channel +/// open until the recording stops (the mic feed and the system-audio watcher +/// both keep sender clones), so the test must too — dropping it early would +/// end the mux loop through channel-closure instead of stop-cancellation. +fn spawn_burst_emitter( + mut tx: futures::channel::mpsc::Sender, + info: AudioInfo, + timestamps: Timestamps, + bursts: Vec<(f64, f64)>, +) -> tokio::task::JoinHandle> { + const CHUNK_SECS: f64 = 0.08; + let rate = info.sample_rate; + let channels = info.channels; + let base = timestamps.instant(); + + tokio::spawn(async move { + use futures::SinkExt; + for (start, end) in bursts { + let chunk_frames = (f64::from(rate) * CHUNK_SECS) as usize; + let chunks = ((end - start) / CHUNK_SECS).round() as usize; + for k in 0..chunks { + let t = start + k as f64 * CHUNK_SECS; + tokio::time::sleep_until((base + Duration::from_secs_f64(t)).into()).await; + let mut frame = ffmpeg::frame::Audio::new( + ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), + chunk_frames, + info.channel_layout(), + ); + frame.set_rate(rate); + for (i, sample) in bytemuck_cast_f32(frame.data_mut(0)).iter_mut().enumerate() { + let n = (i / channels) as f32 + (t * f64::from(rate)) as f32; + *sample = (n * 440.0 * 2.0 * std::f32::consts::PI / rate as f32).sin() * 0.4; + } + let frame = + AudioFrame::new(frame, Timestamp::Instant(base + Duration::from_secs_f64(t))); + if tx.send(frame).await.is_err() { + return tx; + } + } + } + tx + }) +} + +/// Decodes the first channel and returns (windowed rms envelope, window secs). +fn read_audio_envelope(path: &Path, win_secs: f64) -> Result<(Vec, f64), String> { + let mut ictx = ffmpeg::format::input(&path).map_err(|e| format!("open {e}"))?; + let stream = ictx + .streams() + .best(ffmpeg::media::Type::Audio) + .ok_or("no audio stream")?; + let index = stream.index(); + let ctx = ffmpeg::codec::context::Context::from_parameters(stream.parameters()) + .map_err(|e| format!("params: {e}"))?; + let mut decoder = ctx.decoder().audio().map_err(|e| format!("decoder: {e}"))?; + + let mut samples: Vec = Vec::new(); + let mut rate = 0u32; + let mut frame = ffmpeg::frame::Audio::empty(); + let drain = |decoder: &mut ffmpeg::decoder::Audio, + samples: &mut Vec, + rate: &mut u32, + frame: &mut ffmpeg::frame::Audio| { + while decoder.receive_frame(frame).is_ok() { + *rate = frame.rate(); + if let ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Planar) = + frame.format() + { + samples.extend(frame.plane::(0)[..frame.samples()].iter().map(|&v| f64::from(v))); + } + } + }; + for (s, packet) in ictx.packets() { + if s.index() != index { + continue; + } + if decoder.send_packet(&packet).is_ok() { + drain(&mut decoder, &mut samples, &mut rate, &mut frame); + } + } + let _ = decoder.send_eof(); + drain(&mut decoder, &mut samples, &mut rate, &mut frame); + + if rate == 0 || samples.is_empty() { + return Err("no audio decoded".to_string()); + } + let win = ((f64::from(rate) * win_secs) as usize).max(1); + let env: Vec = samples + .chunks(win) + .map(|c| (c.iter().map(|v| v * v).sum::() / c.len() as f64).sqrt()) + .collect(); + Ok((env, win as f64 / f64::from(rate))) +} + +/// Regions of the envelope above `thresh`, merged across single-window dips, +/// dropping blips shorter than 0.15s. +fn active_spans(env: &[f64], win_secs: f64, thresh: f64) -> Vec<(f64, f64)> { + let mut spans: Vec<(f64, f64)> = Vec::new(); + let mut current: Option<(usize, usize)> = None; + for (i, &v) in env.iter().enumerate() { + if v > thresh { + current = match current { + Some((s, _)) => Some((s, i)), + None => Some((i, i)), + }; + } else if let Some((s, e)) = current + && i > e + 1 + { + spans.push((s as f64 * win_secs, (e + 1) as f64 * win_secs)); + current = None; + } + } + if let Some((s, e)) = current { + spans.push((s as f64 * win_secs, (e + 1) as f64 * win_secs)); + } + spans.retain(|(s, e)| e - s >= 0.15); + spans +} + +fn check_spans( + decoded: &[(f64, f64)], + expected: &[(f64, f64)], + track_offset_secs: f64, +) -> Result<(), String> { + if decoded.len() != expected.len() { + return Err(format!( + "expected {} audible bursts at {:?}, decoded {} at {:?} \ + (track offset {track_offset_secs:.3}s)", + expected.len(), + expected, + decoded.len(), + decoded, + )); + } + for ((ds, de), (es, ee)) in decoded.iter().zip(expected) { + // Positions are compared on the recording (epoch) timeline: the + // decoded in-track position plus the track's start offset. + let (ds, de) = (ds + track_offset_secs, de + track_offset_secs); + if (ds - es).abs() > CONTENT_POSITION_TOLERANCE_SECS + || (de - ee).abs() > CONTENT_POSITION_TOLERANCE_SECS + { + return Err(format!( + "burst decoded at {ds:.3}..{de:.3}s on the recording timeline, \ + emitted at {es:.3}..{ee:.3}s" + )); + } + } + Ok(()) +} + +async fn run_system_audio_case(case: SystemAudioCase) -> Result { + use cap_recording::AudioAnchor; + + let SystemAudioCase { bursts, total_secs } = case; + let temp = tempfile::tempdir().map_err(|e| format!("tempdir: {e}"))?; + let out_path = temp.path().join("system_audio.ogg"); + + let info = AudioInfo::new(Sample::F32(Type::Packed), 48_000, 2) + .map_err(|e| format!("audio info: {e:?}"))?; + let (tx, rx) = futures::channel::mpsc::channel::(32); + let timestamps = Timestamps::now(); + + let emit = spawn_burst_emitter(tx, info, timestamps, bursts.clone()); + + let pipeline = OutputPipeline::builder(out_path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new(info, rx)) + .with_timestamps(timestamps) + .with_audio_anchor(AudioAnchor::PipelineEpoch) + .build::(()) + .await + .map_err(|e| format!("pipeline build: {e}"))?; + + let held_tx = emit.await.map_err(|e| format!("emit join: {e}"))?; + tokio::time::sleep_until((timestamps.instant() + Duration::from_secs_f64(total_secs)).into()) + .await; + let stop_lag = timestamps.instant().elapsed().as_secs_f64() - total_secs; + let finished = pipeline.stop().await.map_err(|e| format!("stop: {e}"))?; + drop(held_tx); + if stop_lag > 1.5 { + return Ok(format!( + "skipped: runner fell {stop_lag:.1}s behind real-time emission" + )); + } + + // An intermittent track is anchored at the recording epoch, never at its + // first packet: a late first sound must not be able to become the + // cross-track start_time anchor. + let start_offset = finished + .first_timestamp + .signed_duration_since_secs(timestamps); + if start_offset.abs() > 0.05 { + return Err(format!( + "track start {start_offset:.3}s from the epoch; \ + an intermittent source must anchor at the recording start" + )); + } + + // Head silence, dead-zone silence and tail silence must all be + // materialized: the track spans the whole recording. + let (duration, _, _) = read_audio_stats(&out_path)?; + if (duration - total_secs).abs() > AUDIO_DURATION_TOLERANCE_SECS + stop_lag { + return Err(format!( + "decoded duration {duration:.3}s vs recording span {total_secs:.3}s; \ + silence between/around bursts was not materialized" + )); + } + + // Every burst must sit at its true wall-clock position. + let (env, win) = read_audio_envelope(&out_path, 0.05)?; + let spans = active_spans(&env, win, 0.08); + check_spans(&spans, &bursts, start_offset)?; + + Ok(format!( + "duration {duration:.3}s, start {start_offset:+.3}s, {} bursts in place", + spans.len() + )) +} + +/// The reported regression shape: a continuous mic and an intermittent +/// system-audio source recorded simultaneously (separate pipelines sharing +/// the recording epoch, exactly like studio mode). Every piece of content in +/// BOTH tracks must resolve to its true wall-clock position through each +/// track's own start_time — the presence of system audio must not move the +/// mic, and vice versa. +async fn run_mic_with_system_audio_case() -> Result { + use cap_recording::AudioAnchor; + + const MIC_START_SECS: f64 = 0.15; + const TOTAL_SECS: f64 = 4.5; + let system_bursts: Vec<(f64, f64)> = vec![(1.2, 2.0), (3.4, 4.1)]; + + let temp = tempfile::tempdir().map_err(|e| format!("tempdir: {e}"))?; + let mic_path = temp.path().join("audio-input.ogg"); + let sys_path = temp.path().join("system_audio.ogg"); + + let info = AudioInfo::new(Sample::F32(Type::Packed), 48_000, 2) + .map_err(|e| format!("audio info: {e:?}"))?; + let timestamps = Timestamps::now(); + + let (mic_tx, mic_rx) = futures::channel::mpsc::channel::(32); + let (sys_tx, sys_rx) = futures::channel::mpsc::channel::(32); + + // The mic delivers continuously from device-ready at 0.15s to stop. + let mic_emit = spawn_burst_emitter( + mic_tx, + info, + timestamps, + vec![(MIC_START_SECS, TOTAL_SECS)], + ); + let sys_emit = spawn_burst_emitter(sys_tx, info, timestamps, system_bursts.clone()); + + let mic_pipeline = OutputPipeline::builder(mic_path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new(info, mic_rx)) + .with_timestamps(timestamps) + .build::(()) + .await + .map_err(|e| format!("mic pipeline build: {e}"))?; + let sys_pipeline = OutputPipeline::builder(sys_path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new(info, sys_rx)) + .with_timestamps(timestamps) + .with_audio_anchor(AudioAnchor::PipelineEpoch) + .build::(()) + .await + .map_err(|e| format!("system pipeline build: {e}"))?; + + let (mic_join, sys_join) = tokio::join!(mic_emit, sys_emit); + let mic_held_tx = mic_join.map_err(|e| format!("mic emit join: {e}"))?; + let sys_held_tx = sys_join.map_err(|e| format!("sys emit join: {e}"))?; + tokio::time::sleep_until((timestamps.instant() + Duration::from_secs_f64(TOTAL_SECS)).into()) + .await; + let stop_lag = timestamps.instant().elapsed().as_secs_f64() - TOTAL_SECS; + let (mic_finished, sys_finished) = tokio::join!(mic_pipeline.stop(), sys_pipeline.stop()); + let mic_finished = mic_finished.map_err(|e| format!("mic stop: {e}"))?; + let sys_finished = sys_finished.map_err(|e| format!("sys stop: {e}"))?; + drop((mic_held_tx, sys_held_tx)); + if stop_lag > 1.5 { + return Ok(format!( + "skipped: runner fell {stop_lag:.1}s behind real-time emission" + )); + } + + // Mic keeps first-frame anchoring: its start_time is device-ready. + let mic_start = mic_finished + .first_timestamp + .signed_duration_since_secs(timestamps); + if (mic_start - MIC_START_SECS).abs() > 0.05 { + return Err(format!( + "mic start_time {mic_start:.3}s, expected {MIC_START_SECS:.3}s: \ + recording system audio simultaneously must not move the mic anchor" + )); + } + + let sys_start = sys_finished + .first_timestamp + .signed_duration_since_secs(timestamps); + if sys_start.abs() > 0.05 { + return Err(format!( + "system audio start_time {sys_start:.3}s, expected the epoch" + )); + } + + // Content must land at its true wall-clock position through each track's + // own start offset — this is exactly the invariant the editor's + // start_time-based alignment depends on. + let (mic_env, mic_win) = read_audio_envelope(&mic_path, 0.05)?; + let mic_spans = active_spans(&mic_env, mic_win, 0.08); + check_spans(&mic_spans, &[(MIC_START_SECS, TOTAL_SECS)], mic_start)?; + + let (sys_env, sys_win) = read_audio_envelope(&sys_path, 0.05)?; + let sys_spans = active_spans(&sys_env, sys_win, 0.08); + check_spans(&sys_spans, &system_bursts, sys_start)?; + + // Both tracks span to the stop point (tail silence materialized). + let (mic_duration, _, _) = read_audio_stats(&mic_path)?; + let mic_expected = TOTAL_SECS - MIC_START_SECS; + if (mic_duration - mic_expected).abs() > AUDIO_DURATION_TOLERANCE_SECS + stop_lag { + return Err(format!( + "mic duration {mic_duration:.3}s vs expected {mic_expected:.3}s" + )); + } + let (sys_duration, _, _) = read_audio_stats(&sys_path)?; + if (sys_duration - TOTAL_SECS).abs() > AUDIO_DURATION_TOLERANCE_SECS + stop_lag { + return Err(format!( + "system audio duration {sys_duration:.3}s vs recording span {TOTAL_SECS:.3}s" + )); + } + + Ok(format!( + "mic start {mic_start:.3}s dur {mic_duration:.3}s; \ + system start {sys_start:+.3}s dur {sys_duration:.3}s; all content in place" + )) +} + /// Concatenates a fragmented-mp4 segment directory (init.mp4 + *.m4s) into a /// single playable file. fn concat_fmp4(dir: &Path, scratch: &Path) -> Result { @@ -856,6 +1208,52 @@ async fn synthetic_device_matrix_preserves_sync() { record(&mut results, name, outcome); } + // System-audio delivery shapes: WASAPI-loopback-style intermittent + // sources that only produce packets while sound plays. These guard the + // epoch anchoring + silence materialization that keep an intermittent + // track from desyncing (or re-anchoring) the whole recording. + let system_audio_cases: Vec<(&str, SystemAudioCase)> = vec![ + ( + "system-audio/late-first-sound", + SystemAudioCase { + bursts: vec![(1.4, 2.6)], + total_secs: 4.0, + }, + ), + ( + "system-audio/dead-zone", + SystemAudioCase { + bursts: vec![(0.2, 1.2), (3.0, 3.8)], + total_secs: 4.5, + }, + ), + ( + "system-audio/silent-throughout", + SystemAudioCase { + bursts: vec![], + total_secs: 3.0, + }, + ), + ( + "system-audio/bursty-notifications", + SystemAudioCase { + bursts: vec![(0.4, 0.8), (1.6, 2.0), (2.8, 3.2)], + total_secs: 4.0, + }, + ), + ]; + + for (name, case) in system_audio_cases { + let outcome = run_system_audio_case(case).await; + record(&mut results, name.to_string(), outcome); + } + + record( + &mut results, + "system-audio/with-mic".to_string(), + run_mic_with_system_audio_case().await, + ); + // Non-predetermined coverage: random device shapes and delivery // pathologies, combined audio+video like a real studio recording. let mut rng = Rng(seed); diff --git a/crates/scap-cpal/Cargo.toml b/crates/scap-cpal/Cargo.toml index 955935f94c9..e214335e379 100644 --- a/crates/scap-cpal/Cargo.toml +++ b/crates/scap-cpal/Cargo.toml @@ -7,6 +7,7 @@ license = "MIT" [dependencies] cpal.workspace = true thiserror.workspace = true +tracing.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } [lints] diff --git a/crates/scap-cpal/src/lib.rs b/crates/scap-cpal/src/lib.rs index 0a81c1bb11b..b19c7dee1be 100644 --- a/crates/scap-cpal/src/lib.rs +++ b/crates/scap-cpal/src/lib.rs @@ -37,6 +37,34 @@ fn safe_buffer_size(supported: &SupportedBufferSize, sample_rate: u32) -> Buffer } } +/// WASAPI loopback capture only receives packets while some client is +/// rendering to the endpoint: with nothing playing, the capture event never +/// fires, the track's first packet arrives at the first sound (not at +/// recording start) and long silent stretches produce no frames at all. +/// Keep a silent render stream open on the captured device for the lifetime +/// of the capturer so packets flow continuously (the same workaround OBS +/// uses for desktop audio). +#[cfg(windows)] +fn build_silence_keepalive(device: &cpal::Device) -> Option { + use cpal::traits::DeviceTrait; + + let supported_config = device.default_output_config().ok()?; + let mut config: StreamConfig = supported_config.clone().into(); + config.buffer_size = BufferSize::Default; + + device + .build_output_stream_raw( + &config, + supported_config.sample_format(), + |data, _| { + data.bytes_mut().fill(0); + }, + |_| {}, + None, + ) + .ok() +} + pub fn create_capturer( mut data_callback: impl FnMut(&cpal::Data, &InputCallbackInfo, &StreamConfig) + Send + 'static, error_callback: impl FnMut(StreamError) + Send + 'static, @@ -72,8 +100,15 @@ pub fn create_capturer( ) .map_err(|e| CapturerError::BuildStream(e.to_string()))?; + // A failed keepalive is non-fatal: capture still works whenever other + // clients render audio (the pre-keepalive behavior). + #[cfg(windows)] + let keepalive = build_silence_keepalive(&output_device); + Ok(Capturer { stream, + #[cfg(windows)] + keepalive, config, _output_device: output_device, _host: host, @@ -85,6 +120,8 @@ unsafe impl Send for Capturer {} pub struct Capturer { stream: Stream, + #[cfg(windows)] + keepalive: Option, config: StreamConfig, _output_device: cpal::Device, _host: cpal::Host, @@ -93,14 +130,40 @@ pub struct Capturer { impl Capturer { pub fn play(&self) -> Result<(), PlayStreamError> { + #[cfg(windows)] + if let Some(keepalive) = &self.keepalive + && let Err(e) = keepalive.play() + { + // Non-fatal: capture continues whenever other clients render. + tracing::warn!("loopback silence keepalive failed to start: {e}"); + } self.stream.play() } pub fn pause(&self) -> Result<(), PauseStreamError> { - self.stream.pause() + let result = self.stream.pause(); + #[cfg(windows)] + if let Some(keepalive) = &self.keepalive { + let _ = keepalive.pause(); + } + result } pub fn config(&self) -> &StreamConfig { &self.config } + + /// Whether the silent keepalive render stream is active (Windows only). + /// Without it, loopback capture only produces packets while other + /// applications play audio. + pub fn has_silence_keepalive(&self) -> bool { + #[cfg(windows)] + { + self.keepalive.is_some() + } + #[cfg(not(windows))] + { + false + } + } } From f1dac2cbc3298105b30c2bb91a326ca7aee3c1a8 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 5 Jul 2026 17:45:00 +0300 Subject: [PATCH 2/7] chore: cargo fmt --- .../examples/editor-startup-benchmark.rs | 3 +- crates/recording/src/output_pipeline/core.rs | 34 ++++++++++++------- crates/recording/tests/sync_matrix.rs | 20 +++++------ crates/rendering/src/zoom_spring.rs | 3 +- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/crates/editor/examples/editor-startup-benchmark.rs b/crates/editor/examples/editor-startup-benchmark.rs index 395678abc90..2004b0e82de 100644 --- a/crates/editor/examples/editor-startup-benchmark.rs +++ b/crates/editor/examples/editor-startup-benchmark.rs @@ -6,8 +6,7 @@ use std::{ use cap_editor::{ EditorFrameOutput, EditorInstance, FrameLayout, Renderer, create_segments, - finish_renderer_layers_creation, - start_renderer_layers_creation, + finish_renderer_layers_creation, start_renderer_layers_creation, }; use cap_project::{ProjectConfiguration, RecordingMeta, RecordingMetaInner}; use cap_rendering::{ProjectRecordingsMeta, RenderVideoConstants}; diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index a24f019cdb6..2666155598c 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -732,7 +732,10 @@ const SILENCE_FRAME_MAX: Duration = Duration::from_secs(1); /// (a system-audio track whose last sound came long before stop stayed /// short) came from comparing an epoch-relative target against the /// track-local timeline. -fn audio_tail_padding_duration(audio_elapsed: Duration, track_target_elapsed: Duration) -> Duration { +fn audio_tail_padding_duration( + audio_elapsed: Duration, + track_target_elapsed: Duration, +) -> Duration { track_target_elapsed.saturating_sub(audio_elapsed) } @@ -826,7 +829,9 @@ impl AudioGapTracker { /// capture time of the first muxed frame, or zero when the track is /// anchored at the epoch itself. fn track_start_offset(&self) -> Option { - let secs = self.first_frame_ts?.signed_duration_since_secs(self.reference); + let secs = self + .first_frame_ts? + .signed_duration_since_secs(self.reference); Some(Duration::from_secs_f64(secs.max(0.0))) } @@ -2762,7 +2767,9 @@ async fn process_audio_frame( && !state.gap_tracker.started() { let epoch_ts = Timestamp::Instant(ctx.timestamps.instant()); - state.gap_tracker.mark_started(epoch_ts, ctx.timestamps.instant()); + state + .gap_tracker + .mark_started(epoch_ts, ctx.timestamps.instant()); let head_secs = frame.timestamp.signed_duration_since_secs(ctx.timestamps); let head = Duration::from_secs_f64(head_secs.max(0.0)) @@ -2812,13 +2819,12 @@ async fn process_audio_frame( } if let Some(first_tx) = state.first_tx.take() { - let anchor_ts = if ctx.anchor == AudioAnchor::PipelineEpoch - && ctx.video_start_gate.is_none() - { - Timestamp::Instant(ctx.timestamps.instant()) - } else { - frame.timestamp - }; + let anchor_ts = + if ctx.anchor == AudioAnchor::PipelineEpoch && ctx.video_start_gate.is_none() { + Timestamp::Instant(ctx.timestamps.instant()) + } else { + frame.timestamp + }; let _ = first_tx.send(anchor_ts); } @@ -5681,7 +5687,9 @@ mod tests { let first_frame_at = Duration::from_millis(2_500); assert!(matches!( - harness.process_at(first_frame_at, first_frame_at, 960).await, + harness + .process_at(first_frame_at, first_frame_at, 960) + .await, AudioFrameOutcome::Sent )); @@ -5786,7 +5794,9 @@ mod tests { let first_frame_at = Duration::from_millis(2_500); assert!(matches!( - harness.process_at(first_frame_at, first_frame_at, 960).await, + harness + .process_at(first_frame_at, first_frame_at, 960) + .await, AudioFrameOutcome::Sent )); diff --git a/crates/recording/tests/sync_matrix.rs b/crates/recording/tests/sync_matrix.rs index fb1d394f663..734ce7f9948 100644 --- a/crates/recording/tests/sync_matrix.rs +++ b/crates/recording/tests/sync_matrix.rs @@ -682,15 +682,19 @@ fn read_audio_envelope(path: &Path, win_secs: f64) -> Result<(Vec, f64), St let mut rate = 0u32; let mut frame = ffmpeg::frame::Audio::empty(); let drain = |decoder: &mut ffmpeg::decoder::Audio, - samples: &mut Vec, - rate: &mut u32, - frame: &mut ffmpeg::frame::Audio| { + samples: &mut Vec, + rate: &mut u32, + frame: &mut ffmpeg::frame::Audio| { while decoder.receive_frame(frame).is_ok() { *rate = frame.rate(); if let ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Planar) = frame.format() { - samples.extend(frame.plane::(0)[..frame.samples()].iter().map(|&v| f64::from(v))); + samples.extend( + frame.plane::(0)[..frame.samples()] + .iter() + .map(|&v| f64::from(v)), + ); } } }; @@ -865,12 +869,8 @@ async fn run_mic_with_system_audio_case() -> Result { let (sys_tx, sys_rx) = futures::channel::mpsc::channel::(32); // The mic delivers continuously from device-ready at 0.15s to stop. - let mic_emit = spawn_burst_emitter( - mic_tx, - info, - timestamps, - vec![(MIC_START_SECS, TOTAL_SECS)], - ); + let mic_emit = + spawn_burst_emitter(mic_tx, info, timestamps, vec![(MIC_START_SECS, TOTAL_SECS)]); let sys_emit = spawn_burst_emitter(sys_tx, info, timestamps, system_bursts.clone()); let mic_pipeline = OutputPipeline::builder(mic_path.clone()) diff --git a/crates/rendering/src/zoom_spring.rs b/crates/rendering/src/zoom_spring.rs index 119cceaab20..34f9d2af2dc 100644 --- a/crates/rendering/src/zoom_spring.rs +++ b/crates/rendering/src/zoom_spring.rs @@ -237,8 +237,7 @@ fn map_timeline_to_recording_secs(map: &[TimeMapSegment], timeline_secs: f64) -> /// One precomputed step. Sample times are implicit: `samples[i]` is the state /// at `i * STEP_MS`, so lookup is pure index math. -#[derive(Clone, Copy)] -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] struct TimelineSample { amount: f32, center: XY, From ff418c302d97567c10e9c1a9fb9517f2b5ba9fc3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:19:48 +0300 Subject: [PATCH 3/7] fix: clippy needless_update and clone_on_copy findings --- apps/cli/src/selftest/playback.rs | 8 +------- crates/recording/src/studio_recording.rs | 1 - 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/apps/cli/src/selftest/playback.rs b/apps/cli/src/selftest/playback.rs index 182eddc4952..eede85ec43d 100644 --- a/apps/cli/src/selftest/playback.rs +++ b/apps/cli/src/selftest/playback.rs @@ -722,7 +722,6 @@ mod fixture { let total_secs = pattern.total_secs; let base = timestamps.instant(); let mut tx = mic_tx; - let beep_chunk = beep_chunk.clone(); tokio::spawn(async move { use futures::SinkExt; let first_chunk = (FIXTURE_MIC_START_SECS / AUDIO_CHUNK_SECS).ceil() as usize; @@ -748,7 +747,6 @@ mod fixture { let events = pattern.events.clone(); let base = timestamps.instant(); let mut tx = sys_tx; - let beep_chunk = beep_chunk.clone(); tokio::spawn(async move { use futures::SinkExt; for &event in &events { @@ -912,11 +910,7 @@ mod fixture { keyboard_segments: Vec::new(), audio_segments: Vec::new(), }), - clips: vec![ClipConfiguration { - index: 0, - offsets, - ..Default::default() - }], + clips: vec![ClipConfiguration { index: 0, offsets }], ..Default::default() }; project_config diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 463a41a0096..a735afdf0d2 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -1133,7 +1133,6 @@ async fn stop_recording( .map(|(i, segment)| ClipConfiguration { index: i as u32, offsets: segment.calculate_audio_offsets(), - ..Default::default() }) .collect::>() }); From e40d244eba5f2b093be9a6f9297b26e6839e18f3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:49:19 +0300 Subject: [PATCH 4/7] test(recording): validate WASAPI loopback against a real endpoint in CI Hosted Windows runners have no audio device, so the loopback keepalive, AUDCLNT_BUFFERFLAGS_SILENT zeroing and QPC capture timestamps could never run in CI. Install the Scream virtual sound card in the sync-tests Windows job and add an endpoint integration test that asserts packets flow through engine silence (keepalive), silence decodes as digital zeros (SILENT flag handling), capture timestamps track real time, and a tone played on the endpoint is captured. Skips loudly on machines with no endpoint; CAP_REQUIRE_AUDIO_ENDPOINT=1 (set in CI) makes a missing endpoint a hard failure so a broken driver install cannot silently skip. --- .github/workflows/sync-tests.yml | 31 +++ crates/recording/tests/windows_loopback.rs | 228 +++++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 crates/recording/tests/windows_loopback.rs diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index 2197eb57514..ebc8aee4677 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -81,6 +81,26 @@ jobs: sudo apt-get update sudo apt-get install -y mesa-vulkan-drivers libvulkan1 + # Hosted Windows runners have no audio endpoint, so WASAPI loopback + # capture (system audio) cannot run at all without a virtual sound + # card. Scream provides a signed null render device; with it installed, + # the loopback keepalive, AUDCLNT_BUFFERFLAGS_SILENT zeroing and QPC + # capture timestamps are exercised against a real audio engine. + - name: Install virtual audio device (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + Invoke-WebRequest -Uri https://github.com/duncanthrax/scream/releases/download/4.0/Scream4.0.zip -OutFile Scream4.0.zip + Expand-Archive -Path Scream4.0.zip -DestinationPath Scream + $cert = (Get-AuthenticodeSignature Scream\Install\driver\x64\Scream.sys).SignerCertificate + $store = [System.Security.Cryptography.X509Certificates.X509Store]::new("TrustedPublisher", "LocalMachine") + $store.Open("ReadWrite") + $store.Add($cert) + $store.Close() + Scream\Install\helpers\devcon-x64.exe install Scream\Install\driver\x64\Scream.inf *Scream + if ($LASTEXITCODE -gt 1) { throw "devcon install failed with exit code $LASTEXITCODE" } + Start-Service -Name Audiosrv -ErrorAction SilentlyContinue + - name: Timestamp pipeline unit + property tests shell: bash run: | @@ -88,6 +108,17 @@ jobs: cargo test --locked -p cap-recording --lib cargo test --locked -p cap-rendering + # Runs against the Scream endpoint installed above; the env var turns + # a missing endpoint (driver install regression) into a hard failure + # instead of a silent skip. + - name: WASAPI loopback endpoint tests (Windows) + if: runner.os == 'Windows' + shell: bash + env: + CAP_REQUIRE_AUDIO_ENDPOINT: "1" + run: | + cargo test --locked -p cap-recording --test windows_loopback -- --nocapture + - name: Synthetic device matrix id: matrix continue-on-error: true diff --git a/crates/recording/tests/windows_loopback.rs b/crates/recording/tests/windows_loopback.rs new file mode 100644 index 00000000000..078fc45bac8 --- /dev/null +++ b/crates/recording/tests/windows_loopback.rs @@ -0,0 +1,228 @@ +//! Real-endpoint validation of the WASAPI loopback capture path. +//! +//! The synthetic sync matrix proves the pipeline math, but the Windows +//! loopback behaviors this exercises only exist against a real audio +//! endpoint: the silent-render keepalive (packets must flow while the +//! system plays nothing), AUDCLNT_BUFFERFLAGS_SILENT zeroing (engine +//! silence must decode as digital zeros, not stale buffer contents), and +//! GetBuffer QPC capture timestamps. +//! +//! On developer machines without a usable render endpoint the test skips +//! loudly. CI installs a virtual audio device (Scream) and sets +//! `CAP_REQUIRE_AUDIO_ENDPOINT=1`, which turns both the missing-endpoint +//! skip and the content assertions into hard failures — silence on CI is +//! digital, so any nonzero sample there is a real defect. +#![cfg(windows)] + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use cap_timestamp::{Timestamp, Timestamps}; + +#[derive(Debug, Clone, Copy)] +struct PacketEvent { + arrived_at: Instant, + /// Capture timestamp (GetBuffer qpc_position) relative to the test epoch. + capture_secs: f64, + rms: f64, + /// None when the device mix format is something we don't inspect. + content_known: bool, +} + +fn rms_of(data: &cpal::Data) -> Option { + match data.sample_format() { + cpal::SampleFormat::F32 => { + let s = data.as_slice::()?; + if s.is_empty() { + return Some(0.0); + } + Some((s.iter().map(|&v| f64::from(v) * f64::from(v)).sum::() / s.len() as f64).sqrt()) + } + cpal::SampleFormat::I16 => { + let s = data.as_slice::()?; + if s.is_empty() { + return Some(0.0); + } + Some( + (s.iter() + .map(|&v| { + let f = f64::from(v) / f64::from(i16::MAX); + f * f + }) + .sum::() + / s.len() as f64) + .sqrt(), + ) + } + _ => None, + } +} + +/// Plays a 440 Hz tone on the default render device until dropped. +fn play_tone() -> Option { + use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; + + let host = cpal::default_host(); + let device = host.default_output_device()?; + let supported = device.default_output_config().ok()?; + let channels = supported.channels() as usize; + let rate = supported.sample_rate().0 as f32; + let config: cpal::StreamConfig = supported.clone().into(); + + let mut n: u64 = 0; + let stream = match supported.sample_format() { + cpal::SampleFormat::F32 => device + .build_output_stream( + &config, + move |data: &mut [f32], _| { + for frame in data.chunks_mut(channels) { + let v = (n as f32 * 440.0 * 2.0 * std::f32::consts::PI / rate).sin() * 0.5; + frame.fill(v); + n += 1; + } + }, + |_| {}, + None, + ) + .ok()?, + cpal::SampleFormat::I16 => device + .build_output_stream( + &config, + move |data: &mut [i16], _| { + for frame in data.chunks_mut(channels) { + let v = (n as f32 * 440.0 * 2.0 * std::f32::consts::PI / rate).sin() * 0.5; + frame.fill((v * f32::from(i16::MAX)) as i16); + n += 1; + } + }, + |_| {}, + None, + ) + .ok()?, + _ => return None, + }; + stream.play().ok()?; + Some(stream) +} + +#[test] +fn loopback_delivers_through_silence_and_captures_tone() { + let require_endpoint = std::env::var("CAP_REQUIRE_AUDIO_ENDPOINT").is_ok(); + + let reference = Timestamps::now(); + let events: Arc>> = Arc::new(Mutex::new(Vec::new())); + + let capturer = { + let events = events.clone(); + scap_cpal::create_capturer( + move |data, info, _config| { + let capture_secs = Timestamp::from_cpal(info.timestamp().capture) + .signed_duration_since_secs(reference); + let rms = rms_of(data); + events.lock().unwrap().push(PacketEvent { + arrived_at: Instant::now(), + capture_secs, + rms: rms.unwrap_or(0.0), + content_known: rms.is_some(), + }); + }, + |e| eprintln!("loopback stream error: {e}"), + ) + }; + + let capturer = match capturer { + Ok(c) => c, + Err(e) => { + assert!( + !require_endpoint, + "CAP_REQUIRE_AUDIO_ENDPOINT is set but the loopback capturer \ + could not be created: {e}. On CI this means the virtual audio \ + driver did not install correctly." + ); + eprintln!("skipping: no usable render endpoint ({e})"); + return; + } + }; + + assert!( + capturer.has_silence_keepalive(), + "the silent-render keepalive must be active; without it loopback \ + only produces packets while other applications play audio" + ); + + capturer.play().expect("loopback stream failed to start"); + + // Phase 1: nothing plays. The keepalive alone must keep packets flowing, + // and with an idle engine every sample must be digital zero (the SILENT + // buffer-flag path). + let phase1 = Duration::from_secs(3); + std::thread::sleep(phase1); + let silence_events: Vec = events.lock().unwrap().drain(..).collect(); + + let packets = silence_events.len(); + assert!( + packets >= 10, + "only {packets} loopback packets arrived during {phase1:?} of system \ + silence; the silent-render keepalive is not keeping the endpoint hot" + ); + + // Capture timestamps must track real time (QPC-derived, not garbage). + let first = silence_events.first().unwrap(); + let last = silence_events.last().unwrap(); + let capture_span = last.capture_secs - first.capture_secs; + let wall_span = last.arrived_at.duration_since(first.arrived_at).as_secs_f64(); + assert!( + (capture_span - wall_span).abs() < 0.5, + "loopback capture timestamps advanced {capture_span:.3}s while wall \ + time advanced {wall_span:.3}s" + ); + assert!( + first.capture_secs > -1.0 && first.capture_secs < 5.0, + "first capture timestamp {:.3}s is not near the test epoch", + first.capture_secs + ); + + if require_endpoint { + // CI: the runner plays nothing, so silence is digital. Any nonzero + // sample means unspecified SILENT-flagged buffer contents leaked + // through as audio. + let loudest = silence_events + .iter() + .filter(|e| e.content_known) + .map(|e| e.rms) + .fold(0.0f64, f64::max); + assert!( + loudest == 0.0, + "captured rms {loudest} during engine silence; SILENT-flagged \ + packets are not being zeroed" + ); + } + + // Phase 2: play a tone into the endpoint; the loopback must capture it. + let Some(tone) = play_tone() else { + assert!( + !require_endpoint, + "CAP_REQUIRE_AUDIO_ENDPOINT is set but no tone could be played \ + on the default render device" + ); + eprintln!("skipping tone phase: could not open an output stream"); + return; + }; + std::thread::sleep(Duration::from_millis(1500)); + drop(tone); + std::thread::sleep(Duration::from_millis(200)); + let tone_events: Vec = events.lock().unwrap().drain(..).collect(); + + let heard = tone_events + .iter() + .filter(|e| e.content_known) + .map(|e| e.rms) + .fold(0.0f64, f64::max); + assert!( + heard > 0.05, + "loopback captured no audible content while a 440 Hz tone played \ + (max rms {heard}); real render output is not reaching the capture path" + ); + + capturer.pause().ok(); +} From ace458d373d160e933dcf65682ec3b29ae8259c3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:51:44 +0300 Subject: [PATCH 5/7] chore: cargo fmt --- crates/recording/tests/windows_loopback.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/recording/tests/windows_loopback.rs b/crates/recording/tests/windows_loopback.rs index 078fc45bac8..091078f8675 100644 --- a/crates/recording/tests/windows_loopback.rs +++ b/crates/recording/tests/windows_loopback.rs @@ -36,7 +36,10 @@ fn rms_of(data: &cpal::Data) -> Option { if s.is_empty() { return Some(0.0); } - Some((s.iter().map(|&v| f64::from(v) * f64::from(v)).sum::() / s.len() as f64).sqrt()) + Some( + (s.iter().map(|&v| f64::from(v) * f64::from(v)).sum::() / s.len() as f64) + .sqrt(), + ) } cpal::SampleFormat::I16 => { let s = data.as_slice::()?; @@ -170,7 +173,10 @@ fn loopback_delivers_through_silence_and_captures_tone() { let first = silence_events.first().unwrap(); let last = silence_events.last().unwrap(); let capture_span = last.capture_secs - first.capture_secs; - let wall_span = last.arrived_at.duration_since(first.arrived_at).as_secs_f64(); + let wall_span = last + .arrived_at + .duration_since(first.arrived_at) + .as_secs_f64(); assert!( (capture_span - wall_span).abs() < 0.5, "loopback capture timestamps advanced {capture_span:.3}s while wall \ From f7822eb6f1dfc019172a6950d505e4e961fe97fd Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:56:03 +0300 Subject: [PATCH 6/7] ci: verify Scream driver download by SHA-256 before install --- .github/workflows/sync-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index ebc8aee4677..90589607ca4 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -91,6 +91,9 @@ jobs: shell: pwsh run: | Invoke-WebRequest -Uri https://github.com/duncanthrax/scream/releases/download/4.0/Scream4.0.zip -OutFile Scream4.0.zip + $expected = "fa33e25f9a46c61e4e0cd83362c51c3d2a45c6fe4091aad7507e240e40f1a520" + $actual = (Get-FileHash -Algorithm SHA256 Scream4.0.zip).Hash.ToLowerInvariant() + if ($actual -ne $expected) { throw "Scream4.0.zip SHA-256 mismatch: got $actual, expected $expected" } Expand-Archive -Path Scream4.0.zip -DestinationPath Scream $cert = (Get-AuthenticodeSignature Scream\Install\driver\x64\Scream.sys).SignerCertificate $store = [System.Security.Cryptography.X509Certificates.X509Store]::new("TrustedPublisher", "LocalMachine") From 69dc60f7a157a2fdeea76a90fee1afd01eff7d83 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 5 Jul 2026 19:53:44 +0300 Subject: [PATCH 7/7] ci: pin Scream 3.6 for the virtual audio device and bound the install step 3.8/4.0 driver signatures postdate the 2021 kernel cross-signing deprecation; PnP blocks their install on an interactive consent prompt, which hangs a headless runner. 3.6's signature chain installs silently. A step timeout turns any future hang into a fast failure. --- .github/workflows/sync-tests.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index 90589607ca4..639f4368f1b 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -86,21 +86,27 @@ jobs: # card. Scream provides a signed null render device; with it installed, # the loopback keepalive, AUDCLNT_BUFFERFLAGS_SILENT zeroing and QPC # capture timestamps are exercised against a real audio engine. + # + # Pinned to 3.6: later releases (3.8/4.0) were signed after the 2021 + # kernel cross-signing deprecation and PnP blocks their install on an + # interactive consent prompt, which hangs headless runners. The step + # timeout turns any such hang into a fast failure. - name: Install virtual audio device (Windows) if: runner.os == 'Windows' + timeout-minutes: 10 shell: pwsh run: | - Invoke-WebRequest -Uri https://github.com/duncanthrax/scream/releases/download/4.0/Scream4.0.zip -OutFile Scream4.0.zip - $expected = "fa33e25f9a46c61e4e0cd83362c51c3d2a45c6fe4091aad7507e240e40f1a520" - $actual = (Get-FileHash -Algorithm SHA256 Scream4.0.zip).Hash.ToLowerInvariant() - if ($actual -ne $expected) { throw "Scream4.0.zip SHA-256 mismatch: got $actual, expected $expected" } - Expand-Archive -Path Scream4.0.zip -DestinationPath Scream - $cert = (Get-AuthenticodeSignature Scream\Install\driver\x64\Scream.sys).SignerCertificate + Invoke-WebRequest -Uri https://github.com/duncanthrax/scream/releases/download/3.6/Scream3.6.zip -OutFile Scream3.6.zip + $expected = "25ea5e778b4e6995a98d448b9b5f6d321f681663f1aeeec69d8e63183d008b19" + $actual = (Get-FileHash -Algorithm SHA256 Scream3.6.zip).Hash.ToLowerInvariant() + if ($actual -ne $expected) { throw "Scream3.6.zip SHA-256 mismatch: got $actual, expected $expected" } + Expand-Archive -Path Scream3.6.zip -DestinationPath Scream + $cert = (Get-AuthenticodeSignature Scream\Install\driver\Scream.sys).SignerCertificate $store = [System.Security.Cryptography.X509Certificates.X509Store]::new("TrustedPublisher", "LocalMachine") $store.Open("ReadWrite") $store.Add($cert) $store.Close() - Scream\Install\helpers\devcon-x64.exe install Scream\Install\driver\x64\Scream.inf *Scream + Scream\Install\helpers\devcon.exe install Scream\Install\driver\Scream.inf *Scream if ($LASTEXITCODE -gt 1) { throw "devcon install failed with exit code $LASTEXITCODE" } Start-Service -Name Audiosrv -ErrorAction SilentlyContinue