From c91a1b5c510440e5cebe3bb1188a670fd031657e Mon Sep 17 00:00:00 2001 From: Louis Beaumont Date: Fri, 19 Jun 2026 12:59:29 -0700 Subject: [PATCH] feat(coreaudio): render an AEC far-end reference into VoiceProcessingIO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VPIO cancels echo by subtracting whatever the app renders through the unit's output element. screenpipe doesn't play the far-end (other apps do), so its VPIO was configured input-only — output element disabled, nothing rendered — and Apple's AEC had no downlink to subtract, removing 0 dB (mediar/screenpipe#3938). Add an optional MacosVoiceProcessingInputConfig.aec_reference (a shared mono-f32 ring). When set: enable the output element, force it to mono f32, set kAUVoiceIOProperty_MuteOutput=1 (reference is for AEC only, never replayed), and install a render callback that drains the ring into the output element each cycle. The caller (engine) feeds it the captured system-audio lane. None keeps the historical input-only behavior. Render callback is realtime-safe: try_lock + zero-fill on contention/underflow. Co-Authored-By: Claude Opus 4.8 --- src/host/coreaudio/macos/mod.rs | 70 ++++++++++++++++++++++++++++----- src/lib.rs | 28 ++++++++++++- 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/src/host/coreaudio/macos/mod.rs b/src/host/coreaudio/macos/mod.rs index e8f11f3..c305b80 100644 --- a/src/host/coreaudio/macos/mod.rs +++ b/src/host/coreaudio/macos/mod.rs @@ -699,12 +699,37 @@ impl Device { // Do not force nominal sample rate before VoiceProcessingIO; let the unit // keep the hardware rate and expose its actual input format instead. - let mut audio_unit = build_voice_processing_audio_unit( - self, - config, - sample_format, - &use_voice_processing_input_config.expect("voice processing config is required"), - )?; + let vp_config = + use_voice_processing_input_config.expect("voice processing config is required"); + let aec_reference = vp_config.aec_reference.clone(); + let mut audio_unit = build_voice_processing_audio_unit(self, config, sample_format, &vp_config)?; + + // VoiceProcessingIO AEC reference: render the captured system-audio lane + // into the (enabled, muted) output element so Apple's canceller has a + // downlink to subtract from the mic. `build_voice_processing_audio_unit` + // forced that element to mono f32, so we write straight through. + // Realtime-safe: `try_lock` + zero-fill, never blocks or allocates. + if let Some(reference) = aec_reference { + type RefArgs = render_callback::Args; + audio_unit.set_render_callback(move |args: RefArgs| unsafe { + let AudioBuffer { + mDataByteSize: data_byte_size, + mData: data, + .. + } = (*args.data.data).mBuffers[0]; + let frames = data_byte_size as usize / mem::size_of::(); + let out = slice::from_raw_parts_mut(data as *mut f32, frames); + match reference.try_lock() { + Ok(mut ring) => { + for sample in out.iter_mut() { + *sample = ring.pop_front().unwrap_or(0.0); + } + } + Err(_) => out.iter_mut().for_each(|s| *s = 0.0), + } + Ok(()) + })?; + } // Set the stream in interleaved mode. let asbd = asbd_from_config(config, sample_format); @@ -902,7 +927,10 @@ impl Device { } #[cfg(target_os = "macos")] -fn configure_voice_processing_io(audio_unit: &mut AudioUnit) -> Result<(), coreaudio::Error> { +fn configure_voice_processing_io( + audio_unit: &mut AudioUnit, + enable_output: bool, +) -> Result<(), coreaudio::Error> { let enabled: u32 = 1; let disabled: u32 = 0; audio_unit.set_property( @@ -911,11 +939,15 @@ fn configure_voice_processing_io(audio_unit: &mut AudioUnit) -> Result<(), corea Element::Input, Some(&enabled), )?; + // The output (downlink) element stays disabled unless we have a far-end + // reference to render into it. Enabling it without rendering anything is + // the historical 0 dB no-op (the render is silence, #3938); enabling it AND + // feeding the system-audio reference is what lets VPIO's AEC subtract echo. audio_unit.set_property( kAudioOutputUnitProperty_EnableIO, Scope::Output, Element::Output, - Some(&disabled), + Some(if enable_output { &enabled } else { &disabled }), )?; Ok(()) } @@ -938,7 +970,9 @@ fn apply_voice_processing_flags( .voice_processing_enable_agc .unwrap_or(false), ); - let mute_output = 0u32; + // Mute the output element when we render a far-end reference into it, so the + // reference is used for AEC only and never replayed to the speakers. + let mute_output = u32::from(voice_processing.aec_reference.is_some()); apply_voice_processing_property(audio_unit, K_AU_VOICE_IO_BYPASS_VOICE_PROCESSING, &bypass); apply_voice_processing_property(audio_unit, K_AU_VOICE_IO_VOICE_PROCESSING_ENABLE_AGC, &agc); apply_voice_processing_property(audio_unit, K_AU_VOICE_IO_MUTE_OUTPUT, &mute_output); @@ -967,7 +1001,8 @@ fn build_voice_processing_audio_unit( voice_processing: &crate::MacosVoiceProcessingInputConfig, ) -> Result { let mut audio_unit = AudioUnit::new(coreaudio::audio_unit::IOType::VoiceProcessingIO)?; - configure_voice_processing_io(&mut audio_unit)?; + let has_reference = voice_processing.aec_reference.is_some(); + configure_voice_processing_io(&mut audio_unit, has_reference)?; audio_unit.set_property( kAudioOutputUnitProperty_CurrentDevice, @@ -979,6 +1014,21 @@ fn build_voice_processing_audio_unit( let asbd = asbd_from_config(config, sample_format); configure_voice_processing_stream_format(&mut audio_unit, &asbd)?; + // When feeding a reference, the app supplies the downlink on the *input + // scope of the output element*. Force it to mono f32 so the render callback + // can write the reference straight through without per-channel conversion. + if has_reference { + let mut mono = config.clone(); + mono.channels = 1; + let ref_asbd = asbd_from_config(&mono, SampleFormat::F32); + audio_unit.set_property( + kAudioUnitProperty_StreamFormat, + Scope::Input, + Element::Output, + Some(&ref_asbd), + )?; + } + apply_voice_processing_flags(&mut audio_unit, voice_processing); Ok(audio_unit) diff --git a/src/lib.rs b/src/lib.rs index d579d86..3de3351 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -302,8 +302,21 @@ impl DuckingLevel { } } +/// Shared far-end (downlink) reference for VoiceProcessingIO's echo canceller. +/// +/// Apple's VPIO cancels echo by subtracting whatever the app renders through +/// the unit's own output element. screenpipe doesn't play the far-end (other +/// apps do), so we capture the system-audio lane and feed it here: the engine +/// pushes mono f32 samples at the unit's input sample rate, and VPIO's realtime +/// render callback drains them into the (muted) output element as the AEC +/// reference. The ring is intentionally simple; the render callback uses +/// `try_lock` and zero-fills on contention/underflow, so a missed lock costs a +/// little un-cancelled echo, never an audio glitch. +#[cfg(target_os = "macos")] +pub type AecReference = std::sync::Arc>>; + #[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug, Default)] +#[derive(Clone, Debug, Default)] pub struct MacosVoiceProcessingInputConfig { /// Enable or disable Apple AEC processing in VoiceProcessingIO. pub enable_voice_processing: bool, @@ -315,6 +328,11 @@ pub struct MacosVoiceProcessingInputConfig { pub enable_advanced_ducking: bool, /// Ducking level for non-voice audio. pub ducking_level: DuckingLevel, + /// When set, VPIO's output element is enabled and muted, and this ring is + /// rendered into it as the AEC far-end reference — the missing downlink + /// that makes VPIO's echo cancellation actually do anything (see + /// [`AecReference`]). `None` keeps the historical input-only behavior. + pub aec_reference: Option, } #[cfg(target_os = "macos")] @@ -327,8 +345,16 @@ impl MacosVoiceProcessingInputConfig { voice_processing_bypass: Some(false), enable_advanced_ducking: false, ducking_level: DuckingLevel::Min, + aec_reference: None, } } + + /// Attach the far-end reference ring so VPIO's echo canceller has a downlink + /// to subtract (see [`AecReference`]). Without it VPIO removes 0 dB. + pub fn with_aec_reference(mut self, reference: AecReference) -> Self { + self.aec_reference = Some(reference); + self + } } /// Describes the minimum and maximum supported buffer size for the device