diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index 2197eb57514..639f4368f1b 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -81,6 +81,35 @@ jobs: sudo apt-get update sudo apt-get install -y mesa-vulkan-drivers libvulkan1 + # Hosted Windows runners have no audio endpoint, so WASAPI loopback + # capture (system audio) cannot run at all without a virtual sound + # card. Scream provides a signed null render device; with it installed, + # the loopback keepalive, AUDCLNT_BUFFERFLAGS_SILENT zeroing and QPC + # capture timestamps are exercised against a real audio engine. + # + # Pinned to 3.6: later releases (3.8/4.0) were signed after the 2021 + # kernel cross-signing deprecation and PnP blocks their install on an + # interactive consent prompt, which hangs headless runners. The step + # timeout turns any such hang into a fast failure. + - name: Install virtual audio device (Windows) + if: runner.os == 'Windows' + timeout-minutes: 10 + shell: pwsh + run: | + Invoke-WebRequest -Uri https://github.com/duncanthrax/scream/releases/download/3.6/Scream3.6.zip -OutFile Scream3.6.zip + $expected = "25ea5e778b4e6995a98d448b9b5f6d321f681663f1aeeec69d8e63183d008b19" + $actual = (Get-FileHash -Algorithm SHA256 Scream3.6.zip).Hash.ToLowerInvariant() + if ($actual -ne $expected) { throw "Scream3.6.zip SHA-256 mismatch: got $actual, expected $expected" } + Expand-Archive -Path Scream3.6.zip -DestinationPath Scream + $cert = (Get-AuthenticodeSignature Scream\Install\driver\Scream.sys).SignerCertificate + $store = [System.Security.Cryptography.X509Certificates.X509Store]::new("TrustedPublisher", "LocalMachine") + $store.Open("ReadWrite") + $store.Add($cert) + $store.Close() + Scream\Install\helpers\devcon.exe install Scream\Install\driver\Scream.inf *Scream + if ($LASTEXITCODE -gt 1) { throw "devcon install failed with exit code $LASTEXITCODE" } + Start-Service -Name Audiosrv -ErrorAction SilentlyContinue + - name: Timestamp pipeline unit + property tests shell: bash run: | @@ -88,6 +117,17 @@ jobs: cargo test --locked -p cap-recording --lib cargo test --locked -p cap-rendering + # Runs against the Scream endpoint installed above; the env var turns + # a missing endpoint (driver install regression) into a hard failure + # instead of a silent skip. + - name: WASAPI loopback endpoint tests (Windows) + if: runner.os == 'Windows' + shell: bash + env: + CAP_REQUIRE_AUDIO_ENDPOINT: "1" + run: | + cargo test --locked -p cap-recording --test windows_loopback -- --nocapture + - name: Synthetic device matrix id: matrix continue-on-error: true diff --git a/Cargo.lock b/Cargo.lock index daf25e32521..6609b5d3132 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1175,7 +1175,7 @@ dependencies = [ "cidre", "clap", "clap_complete", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "dirs 6.0.0", "ffmpeg-next", "flume", @@ -1214,7 +1214,7 @@ name = "cap-audio" version = "0.1.0" dependencies = [ "cidre", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "serde", "serde_json", @@ -1354,7 +1354,7 @@ dependencies = [ name = "cap-cpal-ffmpeg" version = "0.1.0" dependencies = [ - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "workspace-hack", ] @@ -1427,7 +1427,7 @@ dependencies = [ "cocoa", "core-foundation 0.10.1", "core-graphics 0.24.0", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "device_query", "dirs 6.0.0", "dotenvy_macro", @@ -1525,7 +1525,7 @@ dependencies = [ "cap-media-info", "cap-project", "cap-rendering", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "flume", "futures", @@ -1698,7 +1698,7 @@ dependencies = [ name = "cap-media-info" version = "0.1.0" dependencies = [ - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "thiserror 1.0.69", "workspace-hack", @@ -1803,7 +1803,7 @@ dependencies = [ "core-foundation 0.10.1", "core-graphics 0.24.0", "coreaudio-rs", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "device_query", "dhat", "either", @@ -1926,7 +1926,7 @@ dependencies = [ "cidre", "clap", "colored", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "flume", "indicatif", @@ -1953,7 +1953,7 @@ name = "cap-timestamp" version = "0.1.0" dependencies = [ "cidre", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "windows 0.60.0", "workspace-hack", ] @@ -2630,7 +2630,7 @@ dependencies = [ [[package]] name = "cpal" version = "0.15.3" -source = "git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca#3cc779a7b4ca51770211f1b7dc19f107978af707" +source = "git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3#6013cb5f8bd34ddfbf7b215b9540fd211ce829fc" dependencies = [ "alsa", "core-foundation-sys", @@ -8828,8 +8828,9 @@ dependencies = [ name = "scap-cpal" version = "0.1.0" dependencies = [ - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "thiserror 1.0.69", + "tracing", "workspace-hack", ] @@ -8852,7 +8853,7 @@ name = "scap-ffmpeg" version = "0.1.0" dependencies = [ "cidre", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "futures", "scap-cpal", diff --git a/Cargo.toml b/Cargo.toml index 78a1cfe3591..5c9c2cab521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,10 @@ members = [ [workspace.dependencies] anyhow = { version = "1.0.86" } # This includes a currently-unreleased fix that ensures the audio stream is actually -# stopped and released on drop on macOS -cpal = { git = "https://github.com/CapSoftware/cpal", rev = "3cc779a7b4ca" } +# stopped and released on drop on macOS, plus zeroing of WASAPI capture +# packets flagged AUDCLNT_BUFFERFLAGS_SILENT (loopback of an idle endpoint +# otherwise surfaces unspecified buffer contents as audio). +cpal = { git = "https://github.com/CapSoftware/cpal", rev = "6013cb5f8bd3" } ffmpeg = { package = "ffmpeg-next", git = "https://github.com/CapSoftware/rust-ffmpeg", rev = "49db1fede112" } tokio = { version = "1.39.3", features = [ "macros", diff --git a/apps/cli/src/selftest/playback.rs b/apps/cli/src/selftest/playback.rs index 6dd63c71ed5..eede85ec43d 100644 --- a/apps/cli/src/selftest/playback.rs +++ b/apps/cli/src/selftest/playback.rs @@ -585,7 +585,14 @@ mod fixture { const AUDIO_RATE: u32 = 48_000; const AUDIO_CHUNK_SECS: f64 = 0.02; const BEEP_FREQ: f32 = 1_000.0; - const BEEP_AMPLITUDE: f32 = 0.5; + /// Two tracks beep in unison (mic + system audio); keep their sum + /// comfortably below full scale. + const BEEP_AMPLITUDE: f32 = 0.35; + /// The mic device becomes ready shortly after the recording starts, like + /// real capture hardware. Its start_time (the latest across tracks, since + /// system audio anchors at the epoch) becomes the playback anchor, so the + /// fixture exercises the cross-track start_time offset math. + const FIXTURE_MIC_START_SECS: f64 = 0.3; struct Pattern { events: Vec, @@ -625,6 +632,7 @@ mod fixture { std::fs::create_dir_all(&segment_dir) .map_err(|e| format!("failed to create fixture directories: {e}"))?; let display_path = segment_dir.join("display.mp4"); + let mic_path = segment_dir.join("audio-input.ogg"); let audio_path = segment_dir.join("system_audio.ogg"); let timestamps = Timestamps::now(); @@ -669,55 +677,96 @@ mod fixture { }) }; - // Audio leg: silence with 1 kHz beep bursts aligned to the flashes. + // Audio legs: 1 kHz beep bursts aligned to the flashes on BOTH audio + // tracks so any relative shift between them (or against video) splits + // the beep clusters and fails the sync gates. + // + // The mic delivers continuously from device-ready to stop, like real + // capture hardware. System audio delivers loopback-style: chunks + // exist ONLY while a beep plays (WASAPI loopback produces no packets + // while the system is silent), so the recorder must synthesize the + // head/gap/tail silence to keep the track on the recording timeline. let audio_info = AudioInfo::new(Sample::F32(Type::Packed), AUDIO_RATE, 2) .map_err(|e| format!("audio info: {e:?}"))?; - let (audio_tx, audio_rx) = futures::channel::mpsc::channel::(32); - let audio_emit = { + + let beep_chunk = move |chunk_t: f64, events: &[f64]| { + let chunk_frames = (f64::from(AUDIO_RATE) * AUDIO_CHUNK_SECS) as usize; + let mut frame = ffmpeg::frame::Audio::new( + ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), + chunk_frames, + audio_info.channel_layout(), + ); + frame.set_rate(AUDIO_RATE); + let data = frame.data_mut(0); + let samples = unsafe { + std::slice::from_raw_parts_mut(data.as_mut_ptr().cast::(), data.len() / 4) + }; + for (i, sample) in samples.iter_mut().enumerate() { + let t = chunk_t + (i / 2) as f64 / f64::from(AUDIO_RATE); + *sample = if in_flash(events, t) { + (t as f32 * BEEP_FREQ * 2.0 * std::f32::consts::PI).sin() * BEEP_AMPLITUDE + } else { + 0.0 + }; + } + frame + }; + + // Both emitters return their sender when done: real capture sources + // keep the channel open until the recording stops, and the muxer's + // stop-time tail fill only runs on stop-cancellation, not on + // channel-closure. + let (mic_tx, mic_rx) = futures::channel::mpsc::channel::(32); + let mic_emit = { let events = pattern.events.clone(); let total_secs = pattern.total_secs; let base = timestamps.instant(); - let mut tx = audio_tx; - let info = audio_info; + let mut tx = mic_tx; tokio::spawn(async move { use futures::SinkExt; - let chunk_frames = (f64::from(AUDIO_RATE) * AUDIO_CHUNK_SECS) as usize; + let first_chunk = (FIXTURE_MIC_START_SECS / AUDIO_CHUNK_SECS).ceil() as usize; let total_chunks = (total_secs / AUDIO_CHUNK_SECS).ceil() as usize; - for k in 0..total_chunks { + for k in first_chunk..total_chunks { let chunk_t = k as f64 * AUDIO_CHUNK_SECS; tokio::time::sleep_until((base + Duration::from_secs_f64(chunk_t)).into()) .await; - let mut frame = ffmpeg::frame::Audio::new( - ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), - chunk_frames, - info.channel_layout(), - ); - frame.set_rate(AUDIO_RATE); - let data = frame.data_mut(0); - let samples = unsafe { - std::slice::from_raw_parts_mut( - data.as_mut_ptr().cast::(), - data.len() / 4, - ) - }; - for (i, sample) in samples.iter_mut().enumerate() { - let n = (k * chunk_frames + i / 2) as f64; - let t = n / f64::from(AUDIO_RATE); - *sample = if in_flash(&events, t) { - (t as f32 * BEEP_FREQ * 2.0 * std::f32::consts::PI).sin() - * BEEP_AMPLITUDE - } else { - 0.0 - }; - } let frame = AudioFrame::new( - frame, + beep_chunk(chunk_t, &events), Timestamp::Instant(base + Duration::from_secs_f64(chunk_t)), ); if tx.send(frame).await.is_err() { break; } } + tx + }) + }; + + let (sys_tx, sys_rx) = futures::channel::mpsc::channel::(32); + let sys_emit = { + let events = pattern.events.clone(); + let base = timestamps.instant(); + let mut tx = sys_tx; + tokio::spawn(async move { + use futures::SinkExt; + for &event in &events { + let first_chunk = (event / AUDIO_CHUNK_SECS).floor() as usize; + let last_chunk = + ((event + FIXTURE_FLASH_SECS) / AUDIO_CHUNK_SECS).ceil() as usize; + for k in first_chunk..last_chunk { + let chunk_t = k as f64 * AUDIO_CHUNK_SECS; + tokio::time::sleep_until((base + Duration::from_secs_f64(chunk_t)).into()) + .await; + let frame = AudioFrame::new( + beep_chunk(chunk_t, &events), + Timestamp::Instant(base + Duration::from_secs_f64(chunk_t)), + ); + if tx.send(frame).await.is_err() { + return tx; + } + } + } + tx }) }; @@ -729,21 +778,33 @@ mod fixture { .build::(()) .await .map_err(|e| format!("video pipeline: {e}"))?; - let audio_pipeline = OutputPipeline::builder(audio_path.clone()) + let mic_pipeline = OutputPipeline::builder(mic_path.clone()) .with_audio_source::(ChannelAudioSourceConfig::new( - audio_info, audio_rx, + audio_info, mic_rx, )) .with_timestamps(timestamps) .build::(()) .await - .map_err(|e| format!("audio pipeline: {e}"))?; + .map_err(|e| format!("mic pipeline: {e}"))?; + let sys_pipeline = OutputPipeline::builder(audio_path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new( + audio_info, sys_rx, + )) + .with_timestamps(timestamps) + // System audio anchors at the recording epoch, exactly like the + // studio recorder configures it. + .with_audio_anchor(cap_recording::AudioAnchor::PipelineEpoch) + .build::(()) + .await + .map_err(|e| format!("system audio pipeline: {e}"))?; video_emit .await .map_err(|e| format!("video emit join: {e}"))?; - audio_emit + let mic_held_tx = mic_emit.await.map_err(|e| format!("mic emit join: {e}"))?; + let sys_held_tx = sys_emit .await - .map_err(|e| format!("audio emit join: {e}"))?; + .map_err(|e| format!("system audio emit join: {e}"))?; // Let the stream tails flush through the encoders. tokio::time::sleep(Duration::from_millis(500)).await; @@ -751,10 +812,15 @@ mod fixture { .stop() .await .map_err(|e| format!("video pipeline stop: {e}"))?; - let finished_audio = audio_pipeline + let finished_mic = mic_pipeline .stop() .await - .map_err(|e| format!("audio pipeline stop: {e}"))?; + .map_err(|e| format!("mic pipeline stop: {e}"))?; + let finished_sys = sys_pipeline + .stop() + .await + .map_err(|e| format!("system audio pipeline stop: {e}"))?; + drop((mic_held_tx, sys_held_tx)); // Persist metadata the way the studio recorder does: start times are // each track's first timestamp on the shared clock, and the timeline @@ -762,7 +828,10 @@ mod fixture { let display_start = finished_video .first_timestamp .signed_duration_since_secs(timestamps); - let audio_start = finished_audio + let mic_start = finished_mic + .first_timestamp + .signed_duration_since_secs(timestamps); + let sys_start = finished_sys .first_timestamp .signed_duration_since_secs(timestamps); let display_duration = finished_video @@ -770,33 +839,43 @@ mod fixture { .map(|(first, last)| (last - first).as_secs_f64() + 1.0 / f64::from(FIXTURE_FPS)) .ok_or("fixture video reported no timestamp span")?; + let to_project_gap_summary = + |s: cap_recording::AudioGapSummary| cap_project::AudioGapSummary { + total_overlap_trimmed_ms: s.total_overlap_trimmed_ms, + startup_overlap_trimmed_ms: s.startup_overlap_trimmed_ms, + overlap_dropped_frames: s.overlap_dropped_frames, + startup_overlap_drops: s.startup_overlap_drops, + }; + + let segment = MultipleSegment { + display: VideoMeta { + path: RelativePathBuf::from("content/segments/segment-0/display.mp4"), + fps: FIXTURE_FPS, + start_time: Some(display_start), + device_id: None, + }, + camera: None, + mic: Some(AudioMeta { + path: RelativePathBuf::from("content/segments/segment-0/audio-input.ogg"), + start_time: Some(mic_start), + device_id: None, + gap_summary: finished_mic.audio_gap_summary.map(to_project_gap_summary), + }), + system_audio: Some(AudioMeta { + path: RelativePathBuf::from("content/segments/segment-0/system_audio.ogg"), + start_time: Some(sys_start), + device_id: None, + gap_summary: finished_sys.audio_gap_summary.map(to_project_gap_summary), + }), + cursor: None, + keyboard: None, + }; + // Clip offsets exactly as the studio recorder persists them. + let offsets = segment.calculate_audio_offsets(); + let meta = StudioRecordingMeta::MultipleSegments { inner: MultipleSegments { - segments: vec![MultipleSegment { - display: VideoMeta { - path: RelativePathBuf::from("content/segments/segment-0/display.mp4"), - fps: FIXTURE_FPS, - start_time: Some(display_start), - device_id: None, - }, - camera: None, - mic: None, - system_audio: Some(AudioMeta { - path: RelativePathBuf::from("content/segments/segment-0/system_audio.ogg"), - start_time: Some(audio_start), - device_id: None, - gap_summary: finished_audio.audio_gap_summary.map(|s| { - cap_project::AudioGapSummary { - total_overlap_trimmed_ms: s.total_overlap_trimmed_ms, - startup_overlap_trimmed_ms: s.startup_overlap_trimmed_ms, - overlap_dropped_frames: s.overlap_dropped_frames, - startup_overlap_drops: s.startup_overlap_drops, - } - }), - }), - cursor: None, - keyboard: None, - }], + segments: vec![segment], cursors: Default::default(), status: Some(StudioRecordingStatus::Complete), }, @@ -831,10 +910,7 @@ mod fixture { keyboard_segments: Vec::new(), audio_segments: Vec::new(), }), - clips: vec![ClipConfiguration { - index: 0, - offsets: Default::default(), - }], + clips: vec![ClipConfiguration { index: 0, offsets }], ..Default::default() }; project_config diff --git a/crates/editor/examples/editor-startup-benchmark.rs b/crates/editor/examples/editor-startup-benchmark.rs index 395678abc90..2004b0e82de 100644 --- a/crates/editor/examples/editor-startup-benchmark.rs +++ b/crates/editor/examples/editor-startup-benchmark.rs @@ -6,8 +6,7 @@ use std::{ use cap_editor::{ EditorFrameOutput, EditorInstance, FrameLayout, Renderer, create_segments, - finish_renderer_layers_creation, - start_renderer_layers_creation, + finish_renderer_layers_creation, start_renderer_layers_creation, }; use cap_project::{ProjectConfiguration, RecordingMeta, RecordingMetaInner}; use cap_rendering::{ProjectRecordingsMeta, RenderVideoConstants}; diff --git a/crates/project/src/meta.rs b/crates/project/src/meta.rs index dcf5f9341fc..857003efe26 100644 --- a/crates/project/src/meta.rs +++ b/crates/project/src/meta.rs @@ -733,4 +733,75 @@ mod test { }"#, ); } + + mod audio_offsets { + use crate::{AudioMeta, MultipleSegment, VideoMeta}; + use relative_path::RelativePathBuf; + + fn video(start_time: Option) -> VideoMeta { + VideoMeta { + path: RelativePathBuf::from("display.mp4"), + fps: 30, + start_time, + device_id: None, + } + } + + fn audio(start_time: Option) -> AudioMeta { + AudioMeta { + path: RelativePathBuf::from("audio.ogg"), + start_time, + device_id: None, + gap_summary: None, + } + } + + fn segment( + display_start: f64, + mic_start: Option, + system_start: Option, + ) -> MultipleSegment { + MultipleSegment { + display: video(Some(display_start)), + camera: None, + mic: mic_start.map(|s| audio(Some(s))), + system_audio: system_start.map(|s| audio(Some(s))), + cursor: None, + keyboard: None, + } + } + + // The recorder anchors system audio at the recording epoch + // (start_time ~ 0.0), which keeps it from ever being the latest + // start_time: the mic/display anchor — and therefore where playback + // starts and how the mic aligns to video — must be identical with + // and without a system audio track. + #[test] + fn epoch_anchored_system_audio_does_not_move_the_anchor() { + let without = segment(0.58, Some(0.55), None); + let with = segment(0.58, Some(0.55), Some(0.0)); + + assert_eq!(without.latest_start_time(), Some(0.58)); + assert_eq!(with.latest_start_time(), Some(0.58)); + + let offsets_without = without.calculate_audio_offsets(); + let offsets_with = with.calculate_audio_offsets(); + assert_eq!(offsets_without.mic, offsets_with.mic); + assert!((offsets_with.mic - 0.03).abs() < 1e-6); + // System audio positions itself by its own start. + assert!((offsets_with.system_audio - 0.58).abs() < 1e-6); + } + + // Legacy recordings (pre-epoch-anchor) stamped system audio with its + // first packet time; those files keep their historical alignment: + // a later system start is still the anchor for them. + #[test] + fn legacy_first_packet_system_audio_keeps_historical_anchor() { + let legacy = segment(0.5824678, Some(0.5559852), Some(0.6586015)); + assert_eq!(legacy.latest_start_time(), Some(0.6586015)); + let offsets = legacy.calculate_audio_offsets(); + assert!((offsets.mic - (0.6586015 - 0.5559852) as f32).abs() < 1e-6); + assert_eq!(offsets.system_audio, 0.0); + } + } } diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index 3d8393c6bc4..2666155598c 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -719,13 +719,24 @@ const WIRED_GAP_THRESHOLD: Duration = Duration::from_millis(70); const WIRELESS_GAP_THRESHOLD: Duration = Duration::from_millis(160); const AUDIO_WALL_CLOCK_TOLERANCE: Duration = Duration::from_millis(100); const AUDIO_OVERLAP_TOLERANCE: Duration = Duration::from_millis(5); -const MAX_SILENCE_INSERTION: Duration = Duration::from_secs(1); -const MAX_AUDIO_TAIL_PADDING: Duration = Duration::from_millis(300); - -fn audio_tail_padding_duration(audio_elapsed: Duration, target_elapsed: Duration) -> Duration { - target_elapsed - .saturating_sub(audio_elapsed) - .min(MAX_AUDIO_TAIL_PADDING) +const LONG_SILENCE_LOG_THRESHOLD: Duration = Duration::from_secs(1); +/// Cap on individual synthesized-silence frames; long fills are emitted as a +/// sequence of frames so a multi-second dead zone doesn't allocate one giant +/// buffer. +const SILENCE_FRAME_MAX: Duration = Duration::from_secs(1); + +/// How much trailing silence the track needs to reach the stop point. +/// `track_target_elapsed` must be in the track's own timeline (epoch-relative +/// target minus the track's start offset); both the previous overshoot +/// (mic tracks padded past the stop point) and the previous shortfall +/// (a system-audio track whose last sound came long before stop stayed +/// short) came from comparing an epoch-relative target against the +/// track-local timeline. +fn audio_tail_padding_duration( + audio_elapsed: Duration, + track_target_elapsed: Duration, +) -> Duration { + track_target_elapsed.saturating_sub(audio_elapsed) } const STARTUP_OVERLAP_DROP_FRAME_COUNT: u64 = 3; @@ -810,6 +821,20 @@ impl AudioGapTracker { } } + fn started(&self) -> bool { + self.first_frame_ts.is_some() + } + + /// Offset of this track's timeline zero from the pipeline epoch: the + /// capture time of the first muxed frame, or zero when the track is + /// anchored at the epoch itself. + fn track_start_offset(&self) -> Option { + let secs = self + .first_frame_ts? + .signed_duration_since_secs(self.reference); + Some(Duration::from_secs_f64(secs.max(0.0))) + } + fn capture_elapsed( &self, current_frame_ts: Timestamp, @@ -849,7 +874,14 @@ impl AudioGapTracker { let gap = capture_elapsed.saturating_sub(sample_based_elapsed); if gap > self.gap_threshold { - Some(gap.min(MAX_SILENCE_INSERTION)) + // The full gap is inserted: capture_elapsed is already clamped to + // wall-clock elapsed (+tolerance), so a large value here is a real + // silent stretch (e.g. WASAPI loopback delivers nothing while the + // system plays no sound), not a bogus timestamp. Truncating it + // (the old 1s cap) placed the audio that follows a long dead zone + // up to the truncated amount too early until repeated insertions + // converged, smearing the first seconds after the gap. + Some(gap) } else { None } @@ -894,6 +926,34 @@ impl AudioGapTracker { } } +/// Send `total_samples` of synthesized silence starting at the track-local +/// sample position `start_samples`, split into frames of at most +/// [`SILENCE_FRAME_MAX`]. Each frame's mux timestamp is derived from the +/// running sample count so long fills stay sample-accurate. +async fn send_silence_frames( + muxer: &Arc>, + audio_info: &AudioInfo, + frame_ts: Timestamp, + start_samples: u64, + total_samples: u64, +) -> anyhow::Result<()> { + let sample_rate = audio_info.sample_rate; + let chunk_max = ((sample_rate.max(1) as u64) * SILENCE_FRAME_MAX.as_millis() as u64) / 1000; + let chunk_max = chunk_max.max(1); + let mut sent = 0u64; + while sent < total_samples { + let n = (total_samples - sent).min(chunk_max); + let elapsed = Duration::from_nanos(samples_to_nanos(start_samples + sent, sample_rate)); + let silence = create_silence_frame(audio_info, n as usize); + muxer + .lock() + .await + .send_audio_frame(AudioFrame::new(silence, frame_ts), elapsed)?; + sent += n; + } + Ok(()) +} + fn create_silence_frame(audio_info: &AudioInfo, sample_count: usize) -> ffmpeg::frame::Audio { let mut frame = ffmpeg::frame::Audio::new( audio_info.sample_format, @@ -1433,10 +1493,30 @@ impl OutputPipeline { audio_sources: vec![], timestamps, master_clock: None, + audio_anchor: AudioAnchor::FirstFrame, } } } +/// Where the audio track's timeline zero (and therefore its persisted +/// `start_time`) is anchored. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum AudioAnchor { + /// Timeline zero is the capture timestamp of the first frame the muxer + /// sees. Right for device-backed sources (microphone, camera audio): + /// the device produces samples continuously once live, so the first + /// frame marks "device ready" and downstream start-time alignment cuts + /// all tracks to the latest-starting device. + FirstFrame, + /// Timeline zero is the pipeline epoch; silence is synthesized from the + /// epoch up to the first captured frame. Right for intermittent sources + /// (WASAPI loopback system audio) where the first packet marks "first + /// sound played", not "source ready" — anchoring such a track at its + /// first frame would let a late first sound become the cross-track + /// alignment anchor and cut the head off every other track. + PipelineEpoch, +} + pub struct SetupCtx { tasks: TaskPool, health_tx: HealthSender, @@ -1496,6 +1576,7 @@ pub struct OutputPipelineBuilder { audio_sources: Vec, timestamps: Timestamps, master_clock: Option>, + audio_anchor: AudioAnchor, } pub struct NoVideo; @@ -1534,6 +1615,13 @@ impl OutputPipelineBuilder { pub fn set_master_clock(&mut self, master_clock: Arc) { self.master_clock = Some(master_clock); } + + /// Anchor the audio track at the pipeline epoch instead of the first + /// captured frame. See [`AudioAnchor::PipelineEpoch`]. + pub fn with_audio_anchor(mut self, anchor: AudioAnchor) -> Self { + self.audio_anchor = anchor; + self + } } impl OutputPipelineBuilder { @@ -1547,6 +1635,7 @@ impl OutputPipelineBuilder { audio_sources: self.audio_sources, timestamps: self.timestamps, master_clock: self.master_clock, + audio_anchor: self.audio_anchor, } } } @@ -1612,6 +1701,7 @@ impl OutputPipelineBuilder> { timestamps, path, master_clock, + audio_anchor, .. } = self; @@ -1690,6 +1780,7 @@ impl OutputPipelineBuilder> { video_start_gate, build_ctx.stop_signal, audio_gap_summary.clone(), + audio_anchor, ) .await?; @@ -1719,6 +1810,7 @@ impl OutputPipelineBuilder { timestamps, path, master_clock, + audio_anchor, .. } = self; @@ -1774,6 +1866,7 @@ impl OutputPipelineBuilder { None, build_ctx.stop_signal, audio_gap_summary.clone(), + audio_anchor, ) .await?; @@ -1845,6 +1938,7 @@ async fn finish_build( video_start_gate: Option, stop_signal: PipelineStopSignal, gap_summary_slot: Arc>, + audio_anchor: AudioAnchor, ) -> anyhow::Result<()> { if let Some(audio) = audio { audio.configure( @@ -1857,6 +1951,7 @@ async fn finish_build( has_video, video_start_gate, gap_summary_slot, + audio_anchor, ); } @@ -2302,12 +2397,20 @@ impl PreparedAudioSources { has_video: bool, video_start_gate: Option, gap_summary_slot: Arc>, + audio_anchor: AudioAnchor, ) { let audio_info = self.audio_info; let has_wireless_source = self.has_wireless_source; let health_tx = setup_ctx.health_tx().clone(); let master_clock = setup_ctx.master_clock().clone(); + if audio_anchor == AudioAnchor::PipelineEpoch && video_start_gate.is_some() { + warn!( + "PipelineEpoch audio anchor is ignored when a video start gate \ + aligns audio to the video track" + ); + } + setup_ctx.tasks().spawn("mux-audio", { let stop_token = stop_token.child_token(); let muxer = muxer.clone(); @@ -2339,6 +2442,8 @@ impl PreparedAudioSources { has_video, origin: FrameProcessOrigin::Live, observed_at: Instant::now(), + timestamps, + anchor: audio_anchor, }, AudioFrameProcessState { timestamp_generator: &mut timestamp_generator, @@ -2399,6 +2504,8 @@ impl PreparedAudioSources { has_video, origin: FrameProcessOrigin::Drain, observed_at: Instant::now(), + timestamps, + anchor: audio_anchor, }, AudioFrameProcessState { timestamp_generator: &mut timestamp_generator, @@ -2450,21 +2557,44 @@ impl PreparedAudioSources { if let Some(target_elapsed) = cancellation_target_elapsed && !audio_degraded { + // An epoch-anchored track that never received a frame + // (e.g. WASAPI loopback with no sound played the whole + // recording) still spans the recording: anchor it now so + // the fill below covers the full duration and the track + // reports a valid start. + if audio_anchor == AudioAnchor::PipelineEpoch && !gap_tracker.started() { + let epoch_ts = Timestamp::Instant(timestamps.instant()); + gap_tracker.mark_started(epoch_ts, timestamps.instant()); + if let Some(first_tx) = first_tx.take() { + let _ = first_tx.send(epoch_ts); + } + info!("No audio frames arrived; anchoring silent track at epoch"); + } + let audio_elapsed = timestamp_generator.next_timestamp(0); - let tail_padding = audio_tail_padding_duration(audio_elapsed, target_elapsed); + // target_elapsed is epoch-relative; the audio timeline is + // track-local (zero = first muxed frame, or the epoch when + // head-anchored), so remove the track's start offset + // before comparing. + let track_target = gap_tracker + .track_start_offset() + .map(|offset| target_elapsed.saturating_sub(offset)) + .unwrap_or(target_elapsed); + let tail_padding = audio_tail_padding_duration(audio_elapsed, track_target); + let start_samples = timestamp_generator.total_samples; let tail_samples = timestamp_generator.advance_by_duration(tail_padding); if tail_samples > 0 { - let silence = create_silence_frame(&audio_info, tail_samples as usize); - let silence_frame = AudioFrame::new( - silence, - Timestamp::Instant(timestamps.instant() + audio_elapsed), - ); - - if let Err(e) = muxer - .lock() - .await - .send_audio_frame(silence_frame, audio_elapsed) + let frame_ts = Timestamp::Instant(timestamps.instant() + audio_elapsed); + + if let Err(e) = send_silence_frames( + &muxer, + &audio_info, + frame_ts, + start_samples, + tail_samples, + ) + .await { if has_video { warn!( @@ -2492,6 +2622,7 @@ impl PreparedAudioSources { padding_ms = tail_padding.as_millis() as u64, samples = tail_samples, audio_end_ms = audio_elapsed.as_millis() as u64, + track_target_ms = track_target.as_millis() as u64, target_ms = target_elapsed.as_millis() as u64, "Padded audio tail with silence" ); @@ -2570,6 +2701,8 @@ struct AudioFrameProcessContext<'a, TMutex: AudioMuxer> { has_video: bool, origin: FrameProcessOrigin, observed_at: Instant, + timestamps: Timestamps, + anchor: AudioAnchor, } struct AudioFrameProcessState<'a> { @@ -2622,11 +2755,79 @@ async fn process_audio_frame( } } + let observed_at = ctx.observed_at; + + // Epoch-anchored tracks (intermittent sources like WASAPI loopback): + // timeline zero is the pipeline epoch, so the stretch between the epoch + // and the first captured frame is real recorded silence — synthesize it + // and report the epoch as the track start. Pause time never reaches the + // sample timeline, so it is excised from the head like everywhere else. + if ctx.anchor == AudioAnchor::PipelineEpoch + && ctx.video_start_gate.is_none() + && !state.gap_tracker.started() + { + let epoch_ts = Timestamp::Instant(ctx.timestamps.instant()); + state + .gap_tracker + .mark_started(epoch_ts, ctx.timestamps.instant()); + + let head_secs = frame.timestamp.signed_duration_since_secs(ctx.timestamps); + let head = Duration::from_secs_f64(head_secs.max(0.0)) + .saturating_sub(total_pause_duration) + // A capture timestamp can't credibly predate more wall time than + // has actually elapsed since the epoch. + .min(observed_at.saturating_duration_since(ctx.timestamps.instant())); + + if !head.is_zero() { + let start_samples = state.timestamp_generator.total_samples; + let head_samples = state.timestamp_generator.advance_by_duration(head); + + if head_samples > 0 { + info!( + head_ms = head.as_millis() as u64, + samples = head_samples, + "Anchoring audio track at pipeline epoch; \ + filling head with silence up to first captured frame" + ); + + if let Err(e) = send_silence_frames( + ctx.muxer, + ctx.audio_info, + epoch_ts, + start_samples, + head_samples, + ) + .await + { + if ctx.has_video { + warn!( + "Audio muxer rejected head silence, \ + degrading to video-only: {e}" + ); + emit_health( + ctx.health_tx, + PipelineHealthEvent::AudioDegradedToVideoOnly { + reason: format!("Head silence rejected: {e}"), + }, + ); + return Ok(AudioFrameOutcome::AudioDegraded); + } + return Err(anyhow!("Audio muxer stopped accepting head silence: {e}")); + } + } + } + } + if let Some(first_tx) = state.first_tx.take() { - let _ = first_tx.send(frame.timestamp); + let anchor_ts = + if ctx.anchor == AudioAnchor::PipelineEpoch && ctx.video_start_gate.is_none() { + Timestamp::Instant(ctx.timestamps.instant()) + } else { + frame.timestamp + }; + let _ = first_tx.send(anchor_ts); } - let observed_at = ctx.observed_at; state.gap_tracker.mark_started(frame.timestamp, observed_at); let sample_based_before = state.timestamp_generator.next_timestamp(0); @@ -2678,35 +2879,44 @@ async fn process_audio_frame( total_pause_duration, observed_at, ) { + let start_samples = state.timestamp_generator.total_samples; let silence_samples = state.timestamp_generator.advance_by_duration(gap_duration); if silence_samples > 0 { - let silence = create_silence_frame(ctx.audio_info, silence_samples as usize); - - let silence_frame = AudioFrame::new(silence, frame.timestamp); - - if gap_duration >= MAX_SILENCE_INSERTION { - error!( + if gap_duration >= LONG_SILENCE_LOG_THRESHOLD { + // Long gaps are expected for intermittent sources (loopback + // system audio while nothing plays); the wall-clock clamp in + // capture_elapsed already vouched that this much real time + // passed. + info!( gap_ms = gap_duration.as_millis(), - "Audio gap exceeded 1s cap, \ - something may be seriously wrong" + "Long audio gap; filling with silence" ); } state.gap_tracker.record_insertion(gap_duration); - emit_health( - ctx.health_tx, - PipelineHealthEvent::AudioGapDetected { - gap_ms: gap_duration.as_millis() as u64, - }, - ); + // For device-backed sources a delivery gap is a real anomaly + // worth surfacing; for epoch-anchored intermittent sources + // (loopback system audio) silent stretches are the normal shape + // of the stream, not a health problem. + if ctx.anchor == AudioAnchor::FirstFrame { + emit_health( + ctx.health_tx, + PipelineHealthEvent::AudioGapDetected { + gap_ms: gap_duration.as_millis() as u64, + }, + ); + } - if let Err(e) = ctx - .muxer - .lock() - .await - .send_audio_frame(silence_frame, sample_based_before) + if let Err(e) = send_silence_frames( + ctx.muxer, + ctx.audio_info, + frame.timestamp, + start_samples, + silence_samples, + ) + .await { if ctx.has_video { warn!( @@ -3361,7 +3571,7 @@ mod tests { } #[test] - fn allows_wall_clock_confirmed_stall_up_to_cap() { + fn wall_clock_confirmed_stall_inserts_full_gap() { let timestamps = Timestamps::now(); let first_ts = Timestamp::Instant(timestamps.instant()); let first_wall_clock = Instant::now(); @@ -3378,7 +3588,33 @@ mod tests { ) .expect("wall-clock-confirmed stall should insert silence"); - assert_eq!(gap, MAX_SILENCE_INSERTION); + // The full wall-clock-validated gap is inserted; truncating it + // would place post-gap audio too early. + assert_eq!(gap, Duration::from_millis(1460)); + } + + #[test] + fn long_dead_zone_inserts_full_gap_in_one_detection() { + // WASAPI loopback delivers nothing while the system is silent; a + // frame arriving after a long dead zone must account for the + // whole stretch at once so its content lands at capture time. + let timestamps = Timestamps::now(); + let first_ts = Timestamp::Instant(timestamps.instant()); + let first_wall_clock = Instant::now(); + let mut tracker = AudioGapTracker::new(false, timestamps); + + tracker.mark_started(first_ts, first_wall_clock); + + let gap = tracker + .detect_gap( + Timestamp::Instant(timestamps.instant() + Duration::from_secs(30)), + Duration::from_secs(2), + Duration::ZERO, + first_wall_clock + Duration::from_secs(30), + ) + .expect("dead zone should insert silence"); + + assert_eq!(gap, Duration::from_secs(28)); } #[test] @@ -3431,10 +3667,13 @@ mod tests { } #[test] - fn caps_tail_padding() { + fn fills_long_tail_gap_completely() { + // A track whose source stopped delivering long before the stop + // point is padded to the full track-relative target so it spans + // the recording (the old 300ms cap left such tracks short). assert_eq!( audio_tail_padding_duration(Duration::from_millis(100), Duration::from_secs(2)), - MAX_AUDIO_TAIL_PADDING + Duration::from_millis(1900) ); } } @@ -5184,6 +5423,7 @@ mod tests { first_tx: Option>, frame_count: u64, dropped_during_pause: u64, + anchor: AudioAnchor, } impl AudioTimelineHarness { @@ -5210,6 +5450,14 @@ mod tests { first_tx: None, frame_count: 0, dropped_during_pause: 0, + anchor: AudioAnchor::FirstFrame, + } + } + + fn new_epoch_anchored() -> Self { + Self { + anchor: AudioAnchor::PipelineEpoch, + ..Self::new() } } @@ -5248,6 +5496,8 @@ mod tests { has_video: true, origin: FrameProcessOrigin::Live, observed_at, + timestamps: self.timestamps, + anchor: self.anchor, }, AudioFrameProcessState { timestamp_generator: &mut self.timestamp_generator, @@ -5423,6 +5673,148 @@ mod tests { ); } + // System audio (WASAPI loopback) may deliver its first packet long after + // the recording starts — the first packet marks "first sound played", + // not "source ready". An epoch-anchored track reports the pipeline + // epoch as its start and synthesizes head silence, so a late first + // sound can never become the cross-track alignment anchor and cut the + // head off the display/mic tracks. + #[tokio::test(flavor = "current_thread")] + async fn epoch_anchor_fills_head_and_reports_epoch_start() { + let mut harness = AudioTimelineHarness::new_epoch_anchored(); + let (tx, mut rx) = oneshot::channel(); + harness.first_tx = Some(tx); + + let first_frame_at = Duration::from_millis(2_500); + assert!(matches!( + harness + .process_at(first_frame_at, first_frame_at, 960) + .await, + AudioFrameOutcome::Sent + )); + + let start = rx + .try_recv() + .unwrap() + .expect("first timestamp must be reported"); + assert!( + start.signed_duration_since_secs(harness.timestamps).abs() < 1e-9, + "track start must be the pipeline epoch, not the first frame" + ); + + let committed = harness.committed_audio(); + let expected = + first_frame_at + Duration::from_secs_f64(960.0 / TEST_SAMPLE_RATE as f64); + assert!( + abs_skew(committed, expected) <= Duration::from_millis(1), + "timeline must cover head silence + frame, got {committed:?}" + ); + + let sent = harness.sent(); + let head_samples: usize = sent[..sent.len() - 1].iter().map(|f| f.samples).sum(); + assert_eq!( + head_samples, + (TEST_SAMPLE_RATE as f64 * 2.5) as usize, + "head silence must cover exactly epoch..first frame" + ); + assert!( + sent[..sent.len() - 1] + .iter() + .all(|f| f.samples <= TEST_SAMPLE_RATE as usize), + "head silence must be chunked to at most 1s frames" + ); + let real = sent.last().unwrap().clone(); + assert_eq!(real.samples, 960); + assert!( + abs_skew(real.timestamp, first_frame_at) <= Duration::from_millis(1), + "first real frame must land at its capture offset from the epoch" + ); + for pair in sent.windows(2) { + assert!(pair[1].timestamp >= pair[0].timestamp); + } + assert_eq!( + harness.total_silence(), + Duration::ZERO, + "head anchoring must not count as gap-repair silence" + ); + } + + // While the system plays nothing, loopback delivers nothing; when + // sound resumes after a long dead zone the resumed content must land + // at its capture time in one detection, not smeared early by a + // truncated insertion. + #[tokio::test(flavor = "current_thread")] + async fn epoch_anchor_dead_zone_resumes_at_capture_time() { + let mut harness = AudioTimelineHarness::new_epoch_anchored(); + + let frame_dur = Duration::from_secs_f64(960.0 / TEST_SAMPLE_RATE as f64); + assert!(matches!( + harness + .process_at(Duration::from_millis(50), Duration::from_millis(50), 960) + .await, + AudioFrameOutcome::Sent + )); + + let resume_at = Duration::from_secs(30); + assert!(matches!( + harness.process_at(resume_at, resume_at, 960).await, + AudioFrameOutcome::Sent + )); + + let committed = harness.committed_audio(); + assert!( + abs_skew(committed, resume_at + frame_dur) <= Duration::from_millis(5), + "post-dead-zone audio must land at capture time, got {committed:?}" + ); + + let sent = harness.sent(); + let last = sent.last().unwrap(); + assert_eq!(last.samples, 960); + assert!( + abs_skew(last.timestamp, resume_at) <= Duration::from_millis(5), + "resumed frame must be muxed at its capture offset, got {:?}", + last.timestamp + ); + assert!( + sent.iter().all(|f| f.samples <= TEST_SAMPLE_RATE as usize), + "gap silence must be chunked to at most 1s frames" + ); + for pair in sent.windows(2) { + assert!(pair[1].timestamp >= pair[0].timestamp); + } + } + + // Device-backed tracks (microphone) keep the first-frame anchor: the + // track starts when the device produces its first samples. + #[tokio::test(flavor = "current_thread")] + async fn first_frame_anchor_reports_first_frame_start() { + let mut harness = AudioTimelineHarness::new(); + let (tx, mut rx) = oneshot::channel(); + harness.first_tx = Some(tx); + + let first_frame_at = Duration::from_millis(2_500); + assert!(matches!( + harness + .process_at(first_frame_at, first_frame_at, 960) + .await, + AudioFrameOutcome::Sent + )); + + let start = rx + .try_recv() + .unwrap() + .expect("first timestamp must be reported"); + assert!( + (start.signed_duration_since_secs(harness.timestamps) - 2.5).abs() < 1e-6, + "mic-style tracks must still report the first frame as start" + ); + + let sent = harness.sent(); + assert_eq!(sent.len(), 1, "no head silence for first-frame anchoring"); + assert_eq!(sent[0].samples, 960); + assert_eq!(sent[0].timestamp, Duration::ZERO); + } + // 5m30s simulated recording at 48kHz with a 0.1% slow mic clock and eight stalls // (5-293ms) that later fill. The sample-count audio timeline must keep tracking the // device capture clock within a bounded window (gap-corrected, never runaway), and diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 7441b6002e1..e8286d414ef 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -1344,6 +1344,14 @@ impl output_pipeline::AudioSource for SystemAudioSource { async move { let capturer = setup_result.map_err(|e| anyhow!("{e}"))?; + if capturer.has_silence_keepalive() { + info!("System audio loopback silence keepalive active"); + } else { + warn!( + "System audio loopback has no silence keepalive; \ + capture will only produce packets while other apps play audio" + ); + } if let Ok(mut guard) = state.lock() { guard.capturer = Some(capturer); } diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index b79278a4148..a735afdf0d2 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -15,7 +15,8 @@ use crate::{ feeds::{camera::CameraFeedLock, microphone::MicrophoneFeedLock}, ffmpeg::{FragmentedAudioMuxer, FragmentedAudioMuxerConfig, OggMuxer}, output_pipeline::{ - AudioGapSummary, DoneFut, FinishedOutputPipeline, OutputPipeline, PipelineDoneError, + AudioAnchor, AudioGapSummary, DoneFut, FinishedOutputPipeline, OutputPipeline, + PipelineDoneError, }, screen_capture::ScreenCaptureConfig, sources::{self, screen_capture}, @@ -1660,11 +1661,17 @@ async fn create_segment_pipeline( }; let system_audio = if let Some(system_audio_source) = system_audio { + // System audio is intermittent (WASAPI loopback only delivers while + // sound plays), so its first packet is not a "source ready" marker: + // anchor the track at the recording epoch. This keeps a late first + // sound from becoming the latest start_time and cutting the head off + // the display/mic/camera tracks at playback. let pipeline = if segment_fragmented { let output_path = dir.join("system_audio.m4a"); OutputPipeline::builder(output_path) .with_audio_source::(system_audio_source) .with_timestamps(start_time) + .with_audio_anchor(AudioAnchor::PipelineEpoch) .build::(FragmentedAudioMuxerConfig { shared_pause_state: shared_pause_state.clone(), }) @@ -1674,6 +1681,7 @@ async fn create_segment_pipeline( OutputPipeline::builder(dir.join("system_audio.ogg")) .with_audio_source::(system_audio_source) .with_timestamps(start_time) + .with_audio_anchor(AudioAnchor::PipelineEpoch) .build::(()) .instrument(error_span!("system-audio-out")) .await diff --git a/crates/recording/tests/sync_matrix.rs b/crates/recording/tests/sync_matrix.rs index 431b511de26..734ce7f9948 100644 --- a/crates/recording/tests/sync_matrix.rs +++ b/crates/recording/tests/sync_matrix.rs @@ -534,7 +534,6 @@ async fn run_audio_case(case: AudioCase) -> Result { let emit = { let base = timestamps.instant(); let mut tx = tx; - let info = info; tokio::spawn(async move { use futures::SinkExt; for k in 0..total_chunks { @@ -603,6 +602,359 @@ fn bytemuck_cast_f32(data: &mut [u8]) -> &mut [f32] { unsafe { std::slice::from_raw_parts_mut(data.as_mut_ptr().cast::(), len) } } +/// Tolerance for where audible content sits inside a decoded track. Bounded +/// by the emission chunk size, the muxer gap threshold (70ms), codec frame +/// granularity and CI scheduler noise. +const CONTENT_POSITION_TOLERANCE_SECS: f64 = 0.25; + +/// System-audio (WASAPI-loopback-style) delivery: packets only exist while +/// something plays. The recorder sees a late first packet, dead zones with no +/// delivery at all, and possibly nothing after the last sound. The muxed +/// track must still span the whole recording with every burst at its true +/// wall-clock position and `first_timestamp` at the recording epoch. +struct SystemAudioCase { + /// (start_secs, end_secs) of each burst of real packet delivery. + bursts: Vec<(f64, f64)>, + /// Wall-clock stop point of the recording. + total_secs: f64, +} + +/// Emits loopback-style bursts on `tx` in real time, using 80ms chunks (the +/// buffer size our Windows loopback capturer targets). +/// +/// Returns the sender when done: real capture sources hold their channel +/// open until the recording stops (the mic feed and the system-audio watcher +/// both keep sender clones), so the test must too — dropping it early would +/// end the mux loop through channel-closure instead of stop-cancellation. +fn spawn_burst_emitter( + mut tx: futures::channel::mpsc::Sender, + info: AudioInfo, + timestamps: Timestamps, + bursts: Vec<(f64, f64)>, +) -> tokio::task::JoinHandle> { + const CHUNK_SECS: f64 = 0.08; + let rate = info.sample_rate; + let channels = info.channels; + let base = timestamps.instant(); + + tokio::spawn(async move { + use futures::SinkExt; + for (start, end) in bursts { + let chunk_frames = (f64::from(rate) * CHUNK_SECS) as usize; + let chunks = ((end - start) / CHUNK_SECS).round() as usize; + for k in 0..chunks { + let t = start + k as f64 * CHUNK_SECS; + tokio::time::sleep_until((base + Duration::from_secs_f64(t)).into()).await; + let mut frame = ffmpeg::frame::Audio::new( + ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), + chunk_frames, + info.channel_layout(), + ); + frame.set_rate(rate); + for (i, sample) in bytemuck_cast_f32(frame.data_mut(0)).iter_mut().enumerate() { + let n = (i / channels) as f32 + (t * f64::from(rate)) as f32; + *sample = (n * 440.0 * 2.0 * std::f32::consts::PI / rate as f32).sin() * 0.4; + } + let frame = + AudioFrame::new(frame, Timestamp::Instant(base + Duration::from_secs_f64(t))); + if tx.send(frame).await.is_err() { + return tx; + } + } + } + tx + }) +} + +/// Decodes the first channel and returns (windowed rms envelope, window secs). +fn read_audio_envelope(path: &Path, win_secs: f64) -> Result<(Vec, f64), String> { + let mut ictx = ffmpeg::format::input(&path).map_err(|e| format!("open {e}"))?; + let stream = ictx + .streams() + .best(ffmpeg::media::Type::Audio) + .ok_or("no audio stream")?; + let index = stream.index(); + let ctx = ffmpeg::codec::context::Context::from_parameters(stream.parameters()) + .map_err(|e| format!("params: {e}"))?; + let mut decoder = ctx.decoder().audio().map_err(|e| format!("decoder: {e}"))?; + + let mut samples: Vec = Vec::new(); + let mut rate = 0u32; + let mut frame = ffmpeg::frame::Audio::empty(); + let drain = |decoder: &mut ffmpeg::decoder::Audio, + samples: &mut Vec, + rate: &mut u32, + frame: &mut ffmpeg::frame::Audio| { + while decoder.receive_frame(frame).is_ok() { + *rate = frame.rate(); + if let ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Planar) = + frame.format() + { + samples.extend( + frame.plane::(0)[..frame.samples()] + .iter() + .map(|&v| f64::from(v)), + ); + } + } + }; + for (s, packet) in ictx.packets() { + if s.index() != index { + continue; + } + if decoder.send_packet(&packet).is_ok() { + drain(&mut decoder, &mut samples, &mut rate, &mut frame); + } + } + let _ = decoder.send_eof(); + drain(&mut decoder, &mut samples, &mut rate, &mut frame); + + if rate == 0 || samples.is_empty() { + return Err("no audio decoded".to_string()); + } + let win = ((f64::from(rate) * win_secs) as usize).max(1); + let env: Vec = samples + .chunks(win) + .map(|c| (c.iter().map(|v| v * v).sum::() / c.len() as f64).sqrt()) + .collect(); + Ok((env, win as f64 / f64::from(rate))) +} + +/// Regions of the envelope above `thresh`, merged across single-window dips, +/// dropping blips shorter than 0.15s. +fn active_spans(env: &[f64], win_secs: f64, thresh: f64) -> Vec<(f64, f64)> { + let mut spans: Vec<(f64, f64)> = Vec::new(); + let mut current: Option<(usize, usize)> = None; + for (i, &v) in env.iter().enumerate() { + if v > thresh { + current = match current { + Some((s, _)) => Some((s, i)), + None => Some((i, i)), + }; + } else if let Some((s, e)) = current + && i > e + 1 + { + spans.push((s as f64 * win_secs, (e + 1) as f64 * win_secs)); + current = None; + } + } + if let Some((s, e)) = current { + spans.push((s as f64 * win_secs, (e + 1) as f64 * win_secs)); + } + spans.retain(|(s, e)| e - s >= 0.15); + spans +} + +fn check_spans( + decoded: &[(f64, f64)], + expected: &[(f64, f64)], + track_offset_secs: f64, +) -> Result<(), String> { + if decoded.len() != expected.len() { + return Err(format!( + "expected {} audible bursts at {:?}, decoded {} at {:?} \ + (track offset {track_offset_secs:.3}s)", + expected.len(), + expected, + decoded.len(), + decoded, + )); + } + for ((ds, de), (es, ee)) in decoded.iter().zip(expected) { + // Positions are compared on the recording (epoch) timeline: the + // decoded in-track position plus the track's start offset. + let (ds, de) = (ds + track_offset_secs, de + track_offset_secs); + if (ds - es).abs() > CONTENT_POSITION_TOLERANCE_SECS + || (de - ee).abs() > CONTENT_POSITION_TOLERANCE_SECS + { + return Err(format!( + "burst decoded at {ds:.3}..{de:.3}s on the recording timeline, \ + emitted at {es:.3}..{ee:.3}s" + )); + } + } + Ok(()) +} + +async fn run_system_audio_case(case: SystemAudioCase) -> Result { + use cap_recording::AudioAnchor; + + let SystemAudioCase { bursts, total_secs } = case; + let temp = tempfile::tempdir().map_err(|e| format!("tempdir: {e}"))?; + let out_path = temp.path().join("system_audio.ogg"); + + let info = AudioInfo::new(Sample::F32(Type::Packed), 48_000, 2) + .map_err(|e| format!("audio info: {e:?}"))?; + let (tx, rx) = futures::channel::mpsc::channel::(32); + let timestamps = Timestamps::now(); + + let emit = spawn_burst_emitter(tx, info, timestamps, bursts.clone()); + + let pipeline = OutputPipeline::builder(out_path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new(info, rx)) + .with_timestamps(timestamps) + .with_audio_anchor(AudioAnchor::PipelineEpoch) + .build::(()) + .await + .map_err(|e| format!("pipeline build: {e}"))?; + + let held_tx = emit.await.map_err(|e| format!("emit join: {e}"))?; + tokio::time::sleep_until((timestamps.instant() + Duration::from_secs_f64(total_secs)).into()) + .await; + let stop_lag = timestamps.instant().elapsed().as_secs_f64() - total_secs; + let finished = pipeline.stop().await.map_err(|e| format!("stop: {e}"))?; + drop(held_tx); + if stop_lag > 1.5 { + return Ok(format!( + "skipped: runner fell {stop_lag:.1}s behind real-time emission" + )); + } + + // An intermittent track is anchored at the recording epoch, never at its + // first packet: a late first sound must not be able to become the + // cross-track start_time anchor. + let start_offset = finished + .first_timestamp + .signed_duration_since_secs(timestamps); + if start_offset.abs() > 0.05 { + return Err(format!( + "track start {start_offset:.3}s from the epoch; \ + an intermittent source must anchor at the recording start" + )); + } + + // Head silence, dead-zone silence and tail silence must all be + // materialized: the track spans the whole recording. + let (duration, _, _) = read_audio_stats(&out_path)?; + if (duration - total_secs).abs() > AUDIO_DURATION_TOLERANCE_SECS + stop_lag { + return Err(format!( + "decoded duration {duration:.3}s vs recording span {total_secs:.3}s; \ + silence between/around bursts was not materialized" + )); + } + + // Every burst must sit at its true wall-clock position. + let (env, win) = read_audio_envelope(&out_path, 0.05)?; + let spans = active_spans(&env, win, 0.08); + check_spans(&spans, &bursts, start_offset)?; + + Ok(format!( + "duration {duration:.3}s, start {start_offset:+.3}s, {} bursts in place", + spans.len() + )) +} + +/// The reported regression shape: a continuous mic and an intermittent +/// system-audio source recorded simultaneously (separate pipelines sharing +/// the recording epoch, exactly like studio mode). Every piece of content in +/// BOTH tracks must resolve to its true wall-clock position through each +/// track's own start_time — the presence of system audio must not move the +/// mic, and vice versa. +async fn run_mic_with_system_audio_case() -> Result { + use cap_recording::AudioAnchor; + + const MIC_START_SECS: f64 = 0.15; + const TOTAL_SECS: f64 = 4.5; + let system_bursts: Vec<(f64, f64)> = vec![(1.2, 2.0), (3.4, 4.1)]; + + let temp = tempfile::tempdir().map_err(|e| format!("tempdir: {e}"))?; + let mic_path = temp.path().join("audio-input.ogg"); + let sys_path = temp.path().join("system_audio.ogg"); + + let info = AudioInfo::new(Sample::F32(Type::Packed), 48_000, 2) + .map_err(|e| format!("audio info: {e:?}"))?; + let timestamps = Timestamps::now(); + + let (mic_tx, mic_rx) = futures::channel::mpsc::channel::(32); + let (sys_tx, sys_rx) = futures::channel::mpsc::channel::(32); + + // The mic delivers continuously from device-ready at 0.15s to stop. + let mic_emit = + spawn_burst_emitter(mic_tx, info, timestamps, vec![(MIC_START_SECS, TOTAL_SECS)]); + let sys_emit = spawn_burst_emitter(sys_tx, info, timestamps, system_bursts.clone()); + + let mic_pipeline = OutputPipeline::builder(mic_path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new(info, mic_rx)) + .with_timestamps(timestamps) + .build::(()) + .await + .map_err(|e| format!("mic pipeline build: {e}"))?; + let sys_pipeline = OutputPipeline::builder(sys_path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new(info, sys_rx)) + .with_timestamps(timestamps) + .with_audio_anchor(AudioAnchor::PipelineEpoch) + .build::(()) + .await + .map_err(|e| format!("system pipeline build: {e}"))?; + + let (mic_join, sys_join) = tokio::join!(mic_emit, sys_emit); + let mic_held_tx = mic_join.map_err(|e| format!("mic emit join: {e}"))?; + let sys_held_tx = sys_join.map_err(|e| format!("sys emit join: {e}"))?; + tokio::time::sleep_until((timestamps.instant() + Duration::from_secs_f64(TOTAL_SECS)).into()) + .await; + let stop_lag = timestamps.instant().elapsed().as_secs_f64() - TOTAL_SECS; + let (mic_finished, sys_finished) = tokio::join!(mic_pipeline.stop(), sys_pipeline.stop()); + let mic_finished = mic_finished.map_err(|e| format!("mic stop: {e}"))?; + let sys_finished = sys_finished.map_err(|e| format!("sys stop: {e}"))?; + drop((mic_held_tx, sys_held_tx)); + if stop_lag > 1.5 { + return Ok(format!( + "skipped: runner fell {stop_lag:.1}s behind real-time emission" + )); + } + + // Mic keeps first-frame anchoring: its start_time is device-ready. + let mic_start = mic_finished + .first_timestamp + .signed_duration_since_secs(timestamps); + if (mic_start - MIC_START_SECS).abs() > 0.05 { + return Err(format!( + "mic start_time {mic_start:.3}s, expected {MIC_START_SECS:.3}s: \ + recording system audio simultaneously must not move the mic anchor" + )); + } + + let sys_start = sys_finished + .first_timestamp + .signed_duration_since_secs(timestamps); + if sys_start.abs() > 0.05 { + return Err(format!( + "system audio start_time {sys_start:.3}s, expected the epoch" + )); + } + + // Content must land at its true wall-clock position through each track's + // own start offset — this is exactly the invariant the editor's + // start_time-based alignment depends on. + let (mic_env, mic_win) = read_audio_envelope(&mic_path, 0.05)?; + let mic_spans = active_spans(&mic_env, mic_win, 0.08); + check_spans(&mic_spans, &[(MIC_START_SECS, TOTAL_SECS)], mic_start)?; + + let (sys_env, sys_win) = read_audio_envelope(&sys_path, 0.05)?; + let sys_spans = active_spans(&sys_env, sys_win, 0.08); + check_spans(&sys_spans, &system_bursts, sys_start)?; + + // Both tracks span to the stop point (tail silence materialized). + let (mic_duration, _, _) = read_audio_stats(&mic_path)?; + let mic_expected = TOTAL_SECS - MIC_START_SECS; + if (mic_duration - mic_expected).abs() > AUDIO_DURATION_TOLERANCE_SECS + stop_lag { + return Err(format!( + "mic duration {mic_duration:.3}s vs expected {mic_expected:.3}s" + )); + } + let (sys_duration, _, _) = read_audio_stats(&sys_path)?; + if (sys_duration - TOTAL_SECS).abs() > AUDIO_DURATION_TOLERANCE_SECS + stop_lag { + return Err(format!( + "system audio duration {sys_duration:.3}s vs recording span {TOTAL_SECS:.3}s" + )); + } + + Ok(format!( + "mic start {mic_start:.3}s dur {mic_duration:.3}s; \ + system start {sys_start:+.3}s dur {sys_duration:.3}s; all content in place" + )) +} + /// Concatenates a fragmented-mp4 segment directory (init.mp4 + *.m4s) into a /// single playable file. fn concat_fmp4(dir: &Path, scratch: &Path) -> Result { @@ -856,6 +1208,52 @@ async fn synthetic_device_matrix_preserves_sync() { record(&mut results, name, outcome); } + // System-audio delivery shapes: WASAPI-loopback-style intermittent + // sources that only produce packets while sound plays. These guard the + // epoch anchoring + silence materialization that keep an intermittent + // track from desyncing (or re-anchoring) the whole recording. + let system_audio_cases: Vec<(&str, SystemAudioCase)> = vec![ + ( + "system-audio/late-first-sound", + SystemAudioCase { + bursts: vec![(1.4, 2.6)], + total_secs: 4.0, + }, + ), + ( + "system-audio/dead-zone", + SystemAudioCase { + bursts: vec![(0.2, 1.2), (3.0, 3.8)], + total_secs: 4.5, + }, + ), + ( + "system-audio/silent-throughout", + SystemAudioCase { + bursts: vec![], + total_secs: 3.0, + }, + ), + ( + "system-audio/bursty-notifications", + SystemAudioCase { + bursts: vec![(0.4, 0.8), (1.6, 2.0), (2.8, 3.2)], + total_secs: 4.0, + }, + ), + ]; + + for (name, case) in system_audio_cases { + let outcome = run_system_audio_case(case).await; + record(&mut results, name.to_string(), outcome); + } + + record( + &mut results, + "system-audio/with-mic".to_string(), + run_mic_with_system_audio_case().await, + ); + // Non-predetermined coverage: random device shapes and delivery // pathologies, combined audio+video like a real studio recording. let mut rng = Rng(seed); diff --git a/crates/recording/tests/windows_loopback.rs b/crates/recording/tests/windows_loopback.rs new file mode 100644 index 00000000000..091078f8675 --- /dev/null +++ b/crates/recording/tests/windows_loopback.rs @@ -0,0 +1,234 @@ +//! Real-endpoint validation of the WASAPI loopback capture path. +//! +//! The synthetic sync matrix proves the pipeline math, but the Windows +//! loopback behaviors this exercises only exist against a real audio +//! endpoint: the silent-render keepalive (packets must flow while the +//! system plays nothing), AUDCLNT_BUFFERFLAGS_SILENT zeroing (engine +//! silence must decode as digital zeros, not stale buffer contents), and +//! GetBuffer QPC capture timestamps. +//! +//! On developer machines without a usable render endpoint the test skips +//! loudly. CI installs a virtual audio device (Scream) and sets +//! `CAP_REQUIRE_AUDIO_ENDPOINT=1`, which turns both the missing-endpoint +//! skip and the content assertions into hard failures — silence on CI is +//! digital, so any nonzero sample there is a real defect. +#![cfg(windows)] + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use cap_timestamp::{Timestamp, Timestamps}; + +#[derive(Debug, Clone, Copy)] +struct PacketEvent { + arrived_at: Instant, + /// Capture timestamp (GetBuffer qpc_position) relative to the test epoch. + capture_secs: f64, + rms: f64, + /// None when the device mix format is something we don't inspect. + content_known: bool, +} + +fn rms_of(data: &cpal::Data) -> Option { + match data.sample_format() { + cpal::SampleFormat::F32 => { + let s = data.as_slice::()?; + if s.is_empty() { + return Some(0.0); + } + Some( + (s.iter().map(|&v| f64::from(v) * f64::from(v)).sum::() / s.len() as f64) + .sqrt(), + ) + } + cpal::SampleFormat::I16 => { + let s = data.as_slice::()?; + if s.is_empty() { + return Some(0.0); + } + Some( + (s.iter() + .map(|&v| { + let f = f64::from(v) / f64::from(i16::MAX); + f * f + }) + .sum::() + / s.len() as f64) + .sqrt(), + ) + } + _ => None, + } +} + +/// Plays a 440 Hz tone on the default render device until dropped. +fn play_tone() -> Option { + use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; + + let host = cpal::default_host(); + let device = host.default_output_device()?; + let supported = device.default_output_config().ok()?; + let channels = supported.channels() as usize; + let rate = supported.sample_rate().0 as f32; + let config: cpal::StreamConfig = supported.clone().into(); + + let mut n: u64 = 0; + let stream = match supported.sample_format() { + cpal::SampleFormat::F32 => device + .build_output_stream( + &config, + move |data: &mut [f32], _| { + for frame in data.chunks_mut(channels) { + let v = (n as f32 * 440.0 * 2.0 * std::f32::consts::PI / rate).sin() * 0.5; + frame.fill(v); + n += 1; + } + }, + |_| {}, + None, + ) + .ok()?, + cpal::SampleFormat::I16 => device + .build_output_stream( + &config, + move |data: &mut [i16], _| { + for frame in data.chunks_mut(channels) { + let v = (n as f32 * 440.0 * 2.0 * std::f32::consts::PI / rate).sin() * 0.5; + frame.fill((v * f32::from(i16::MAX)) as i16); + n += 1; + } + }, + |_| {}, + None, + ) + .ok()?, + _ => return None, + }; + stream.play().ok()?; + Some(stream) +} + +#[test] +fn loopback_delivers_through_silence_and_captures_tone() { + let require_endpoint = std::env::var("CAP_REQUIRE_AUDIO_ENDPOINT").is_ok(); + + let reference = Timestamps::now(); + let events: Arc>> = Arc::new(Mutex::new(Vec::new())); + + let capturer = { + let events = events.clone(); + scap_cpal::create_capturer( + move |data, info, _config| { + let capture_secs = Timestamp::from_cpal(info.timestamp().capture) + .signed_duration_since_secs(reference); + let rms = rms_of(data); + events.lock().unwrap().push(PacketEvent { + arrived_at: Instant::now(), + capture_secs, + rms: rms.unwrap_or(0.0), + content_known: rms.is_some(), + }); + }, + |e| eprintln!("loopback stream error: {e}"), + ) + }; + + let capturer = match capturer { + Ok(c) => c, + Err(e) => { + assert!( + !require_endpoint, + "CAP_REQUIRE_AUDIO_ENDPOINT is set but the loopback capturer \ + could not be created: {e}. On CI this means the virtual audio \ + driver did not install correctly." + ); + eprintln!("skipping: no usable render endpoint ({e})"); + return; + } + }; + + assert!( + capturer.has_silence_keepalive(), + "the silent-render keepalive must be active; without it loopback \ + only produces packets while other applications play audio" + ); + + capturer.play().expect("loopback stream failed to start"); + + // Phase 1: nothing plays. The keepalive alone must keep packets flowing, + // and with an idle engine every sample must be digital zero (the SILENT + // buffer-flag path). + let phase1 = Duration::from_secs(3); + std::thread::sleep(phase1); + let silence_events: Vec = events.lock().unwrap().drain(..).collect(); + + let packets = silence_events.len(); + assert!( + packets >= 10, + "only {packets} loopback packets arrived during {phase1:?} of system \ + silence; the silent-render keepalive is not keeping the endpoint hot" + ); + + // Capture timestamps must track real time (QPC-derived, not garbage). + let first = silence_events.first().unwrap(); + let last = silence_events.last().unwrap(); + let capture_span = last.capture_secs - first.capture_secs; + let wall_span = last + .arrived_at + .duration_since(first.arrived_at) + .as_secs_f64(); + assert!( + (capture_span - wall_span).abs() < 0.5, + "loopback capture timestamps advanced {capture_span:.3}s while wall \ + time advanced {wall_span:.3}s" + ); + assert!( + first.capture_secs > -1.0 && first.capture_secs < 5.0, + "first capture timestamp {:.3}s is not near the test epoch", + first.capture_secs + ); + + if require_endpoint { + // CI: the runner plays nothing, so silence is digital. Any nonzero + // sample means unspecified SILENT-flagged buffer contents leaked + // through as audio. + let loudest = silence_events + .iter() + .filter(|e| e.content_known) + .map(|e| e.rms) + .fold(0.0f64, f64::max); + assert!( + loudest == 0.0, + "captured rms {loudest} during engine silence; SILENT-flagged \ + packets are not being zeroed" + ); + } + + // Phase 2: play a tone into the endpoint; the loopback must capture it. + let Some(tone) = play_tone() else { + assert!( + !require_endpoint, + "CAP_REQUIRE_AUDIO_ENDPOINT is set but no tone could be played \ + on the default render device" + ); + eprintln!("skipping tone phase: could not open an output stream"); + return; + }; + std::thread::sleep(Duration::from_millis(1500)); + drop(tone); + std::thread::sleep(Duration::from_millis(200)); + let tone_events: Vec = events.lock().unwrap().drain(..).collect(); + + let heard = tone_events + .iter() + .filter(|e| e.content_known) + .map(|e| e.rms) + .fold(0.0f64, f64::max); + assert!( + heard > 0.05, + "loopback captured no audible content while a 440 Hz tone played \ + (max rms {heard}); real render output is not reaching the capture path" + ); + + capturer.pause().ok(); +} diff --git a/crates/rendering/src/zoom_spring.rs b/crates/rendering/src/zoom_spring.rs index 119cceaab20..34f9d2af2dc 100644 --- a/crates/rendering/src/zoom_spring.rs +++ b/crates/rendering/src/zoom_spring.rs @@ -237,8 +237,7 @@ fn map_timeline_to_recording_secs(map: &[TimeMapSegment], timeline_secs: f64) -> /// One precomputed step. Sample times are implicit: `samples[i]` is the state /// at `i * STEP_MS`, so lookup is pure index math. -#[derive(Clone, Copy)] -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] struct TimelineSample { amount: f32, center: XY, diff --git a/crates/scap-cpal/Cargo.toml b/crates/scap-cpal/Cargo.toml index 955935f94c9..e214335e379 100644 --- a/crates/scap-cpal/Cargo.toml +++ b/crates/scap-cpal/Cargo.toml @@ -7,6 +7,7 @@ license = "MIT" [dependencies] cpal.workspace = true thiserror.workspace = true +tracing.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } [lints] diff --git a/crates/scap-cpal/src/lib.rs b/crates/scap-cpal/src/lib.rs index 0a81c1bb11b..b19c7dee1be 100644 --- a/crates/scap-cpal/src/lib.rs +++ b/crates/scap-cpal/src/lib.rs @@ -37,6 +37,34 @@ fn safe_buffer_size(supported: &SupportedBufferSize, sample_rate: u32) -> Buffer } } +/// WASAPI loopback capture only receives packets while some client is +/// rendering to the endpoint: with nothing playing, the capture event never +/// fires, the track's first packet arrives at the first sound (not at +/// recording start) and long silent stretches produce no frames at all. +/// Keep a silent render stream open on the captured device for the lifetime +/// of the capturer so packets flow continuously (the same workaround OBS +/// uses for desktop audio). +#[cfg(windows)] +fn build_silence_keepalive(device: &cpal::Device) -> Option { + use cpal::traits::DeviceTrait; + + let supported_config = device.default_output_config().ok()?; + let mut config: StreamConfig = supported_config.clone().into(); + config.buffer_size = BufferSize::Default; + + device + .build_output_stream_raw( + &config, + supported_config.sample_format(), + |data, _| { + data.bytes_mut().fill(0); + }, + |_| {}, + None, + ) + .ok() +} + pub fn create_capturer( mut data_callback: impl FnMut(&cpal::Data, &InputCallbackInfo, &StreamConfig) + Send + 'static, error_callback: impl FnMut(StreamError) + Send + 'static, @@ -72,8 +100,15 @@ pub fn create_capturer( ) .map_err(|e| CapturerError::BuildStream(e.to_string()))?; + // A failed keepalive is non-fatal: capture still works whenever other + // clients render audio (the pre-keepalive behavior). + #[cfg(windows)] + let keepalive = build_silence_keepalive(&output_device); + Ok(Capturer { stream, + #[cfg(windows)] + keepalive, config, _output_device: output_device, _host: host, @@ -85,6 +120,8 @@ unsafe impl Send for Capturer {} pub struct Capturer { stream: Stream, + #[cfg(windows)] + keepalive: Option, config: StreamConfig, _output_device: cpal::Device, _host: cpal::Host, @@ -93,14 +130,40 @@ pub struct Capturer { impl Capturer { pub fn play(&self) -> Result<(), PlayStreamError> { + #[cfg(windows)] + if let Some(keepalive) = &self.keepalive + && let Err(e) = keepalive.play() + { + // Non-fatal: capture continues whenever other clients render. + tracing::warn!("loopback silence keepalive failed to start: {e}"); + } self.stream.play() } pub fn pause(&self) -> Result<(), PauseStreamError> { - self.stream.pause() + let result = self.stream.pause(); + #[cfg(windows)] + if let Some(keepalive) = &self.keepalive { + let _ = keepalive.pause(); + } + result } pub fn config(&self) -> &StreamConfig { &self.config } + + /// Whether the silent keepalive render stream is active (Windows only). + /// Without it, loopback capture only produces packets while other + /// applications play audio. + pub fn has_silence_keepalive(&self) -> bool { + #[cfg(windows)] + { + self.keepalive.is_some() + } + #[cfg(not(windows))] + { + false + } + } }