From 1af4862568b934a39567dd991a9f16209335211c Mon Sep 17 00:00:00 2001 From: divanshu-go Date: Wed, 20 May 2026 00:51:52 +0530 Subject: [PATCH 1/4] Fix macOS VoiceProcessingIO input setup for screenpipe AEC. Disable the VPIO output bus, apply AEC properties on both IO elements, stop forcing sample rate before unit creation, and add screenpipe_aec() defaults for callers. --- src/host/coreaudio/macos/mod.rs | 128 ++++++++++++++++++-------------- src/lib.rs | 14 ++++ 2 files changed, 86 insertions(+), 56 deletions(-) diff --git a/src/host/coreaudio/macos/mod.rs b/src/host/coreaudio/macos/mod.rs index 4b8acfa..efd3ada 100644 --- a/src/host/coreaudio/macos/mod.rs +++ b/src/host/coreaudio/macos/mod.rs @@ -53,6 +53,8 @@ const K_AU_VOICE_IO_BYPASS_VOICE_PROCESSING: u32 = 2100; #[cfg(target_os = "macos")] const K_AU_VOICE_IO_VOICE_PROCESSING_ENABLE_AGC: u32 = 2101; #[cfg(target_os = "macos")] +const K_AU_VOICE_IO_MUTE_OUTPUT: u32 = 2104; +#[cfg(target_os = "macos")] const K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_CONFIGURATION: u32 = 2108; #[cfg(target_os = "macos")] @@ -695,9 +697,8 @@ impl Device { let scope = Scope::Output; let element = Element::Input; - // Potentially change the device sample rate to match the config. - set_sample_rate(self.audio_device_id, config.sample_rate)?; - + // Do not force nominal sample rate before VoiceProcessingIO; let the unit + // keep the hardware rate and expose its actual input format instead. let mut audio_unit = build_voice_processing_audio_unit( self, config, @@ -901,29 +902,80 @@ impl Device { } #[cfg(target_os = "macos")] -fn build_voice_processing_audio_unit( - device: &Device, - config: &StreamConfig, - sample_format: SampleFormat, - voice_processing: &crate::MacosVoiceProcessingInputConfig, -) -> Result { - let mut audio_unit = AudioUnit::new(coreaudio::audio_unit::IOType::VoiceProcessingIO)?; - - let enable_input = 1u32; +fn configure_voice_processing_io(audio_unit: &mut AudioUnit) -> Result<(), coreaudio::Error> { + let enabled: u32 = 1; + let disabled: u32 = 0; audio_unit.set_property( kAudioOutputUnitProperty_EnableIO, Scope::Input, Element::Input, - Some(&enable_input), + Some(&enabled), )?; - - let enable_output = 1u32; audio_unit.set_property( kAudioOutputUnitProperty_EnableIO, Scope::Output, Element::Output, - Some(&enable_output), + Some(&disabled), )?; + Ok(()) +} + +#[cfg(target_os = "macos")] +fn apply_voice_processing_property( + audio_unit: &mut AudioUnit, + property: u32, + value: &u32, +) { + for element in [Element::Input, Element::Output] { + let _ = audio_unit.set_property(property, Scope::Global, element, Some(value)); + } +} + +#[cfg(target_os = "macos")] +fn apply_voice_processing_flags( + audio_unit: &mut AudioUnit, + voice_processing: &crate::MacosVoiceProcessingInputConfig, +) { + let bypass = u32::from(voice_processing.voice_processing_bypass.unwrap_or(false)); + let agc = u32::from(voice_processing.voice_processing_enable_agc.unwrap_or(false)); + let mute_output = 0u32; + apply_voice_processing_property( + audio_unit, + K_AU_VOICE_IO_BYPASS_VOICE_PROCESSING, + &bypass, + ); + apply_voice_processing_property( + audio_unit, + K_AU_VOICE_IO_VOICE_PROCESSING_ENABLE_AGC, + &agc, + ); + apply_voice_processing_property(audio_unit, K_AU_VOICE_IO_MUTE_OUTPUT, &mute_output); + + if voice_processing.enable_advanced_ducking || voice_processing.ducking_level.as_u32() != 0 { + let ducking = AUVoiceIOOtherAudioDuckingConfiguration { + m_enable_advanced_ducking: u8::from(voice_processing.enable_advanced_ducking), + m_ducking_level: voice_processing.ducking_level.as_u32(), + }; + for element in [Element::Input, Element::Output] { + let _ = audio_unit.set_property( + K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_CONFIGURATION, + Scope::Global, + element, + Some(&ducking), + ); + } + } +} + +#[cfg(target_os = "macos")] +fn build_voice_processing_audio_unit( + device: &Device, + config: &StreamConfig, + sample_format: SampleFormat, + voice_processing: &crate::MacosVoiceProcessingInputConfig, +) -> Result { + let mut audio_unit = AudioUnit::new(coreaudio::audio_unit::IOType::VoiceProcessingIO)?; + configure_voice_processing_io(&mut audio_unit)?; audio_unit.set_property( kAudioOutputUnitProperty_CurrentDevice, @@ -933,51 +985,15 @@ fn build_voice_processing_audio_unit( )?; let asbd = asbd_from_config(config, sample_format); - audio_unit.set_property( + // Some VoiceProcessingIO paths expose stream format as read-only; best-effort. + let _ = audio_unit.set_property( kAudioUnitProperty_StreamFormat, Scope::Output, Element::Input, Some(&asbd), - )?; - audio_unit.set_property( - kAudioUnitProperty_StreamFormat, - Scope::Input, - Element::Output, - Some(&asbd), - )?; - - if let Some(agc_enabled) = voice_processing.voice_processing_enable_agc { - let agc_value: u32 = u32::from(agc_enabled); - audio_unit.set_property( - K_AU_VOICE_IO_VOICE_PROCESSING_ENABLE_AGC, - Scope::Global, - Element::Output, - Some(&agc_value), - )?; - } + ); - if let Some(bypass_enabled) = voice_processing.voice_processing_bypass { - let bypass: u32 = u32::from(bypass_enabled); - audio_unit.set_property( - K_AU_VOICE_IO_BYPASS_VOICE_PROCESSING, - Scope::Global, - Element::Output, - Some(&bypass), - )?; - } - - if voice_processing.enable_advanced_ducking || voice_processing.ducking_level.as_u32() != 0 { - let ducking = AUVoiceIOOtherAudioDuckingConfiguration { - m_enable_advanced_ducking: u8::from(voice_processing.enable_advanced_ducking), - m_ducking_level: voice_processing.ducking_level.as_u32(), - }; - audio_unit.set_property( - K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_CONFIGURATION, - Scope::Global, - Element::Output, - Some(&ducking), - )?; - } + apply_voice_processing_flags(&mut audio_unit, voice_processing); Ok(audio_unit) } diff --git a/src/lib.rs b/src/lib.rs index 1494f1e..d579d86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -317,6 +317,20 @@ pub struct MacosVoiceProcessingInputConfig { pub ducking_level: DuckingLevel, } +#[cfg(target_os = "macos")] +impl MacosVoiceProcessingInputConfig { + /// Screenpipe defaults: processing on, AGC off, bypass off, output muted, light ducking. + pub fn screenpipe_aec() -> Self { + Self { + enable_voice_processing: true, + voice_processing_enable_agc: Some(false), + voice_processing_bypass: Some(false), + enable_advanced_ducking: false, + ducking_level: DuckingLevel::Min, + } + } +} + /// Describes the minimum and maximum supported buffer size for the device #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SupportedBufferSize { From 9fb6ce2d3d3689253ccd284229b393baad5e78e9 Mon Sep 17 00:00:00 2001 From: divanshu-go Date: Wed, 20 May 2026 01:01:54 +0530 Subject: [PATCH 2/4] cargo fmt --- src/host/coreaudio/macos/mod.rs | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/host/coreaudio/macos/mod.rs b/src/host/coreaudio/macos/mod.rs index efd3ada..9ed1d8b 100644 --- a/src/host/coreaudio/macos/mod.rs +++ b/src/host/coreaudio/macos/mod.rs @@ -921,11 +921,7 @@ fn configure_voice_processing_io(audio_unit: &mut AudioUnit) -> Result<(), corea } #[cfg(target_os = "macos")] -fn apply_voice_processing_property( - audio_unit: &mut AudioUnit, - property: u32, - value: &u32, -) { +fn apply_voice_processing_property(audio_unit: &mut AudioUnit, property: u32, value: &u32) { for element in [Element::Input, Element::Output] { let _ = audio_unit.set_property(property, Scope::Global, element, Some(value)); } @@ -937,18 +933,14 @@ fn apply_voice_processing_flags( voice_processing: &crate::MacosVoiceProcessingInputConfig, ) { let bypass = u32::from(voice_processing.voice_processing_bypass.unwrap_or(false)); - let agc = u32::from(voice_processing.voice_processing_enable_agc.unwrap_or(false)); - let mute_output = 0u32; - apply_voice_processing_property( - audio_unit, - K_AU_VOICE_IO_BYPASS_VOICE_PROCESSING, - &bypass, - ); - apply_voice_processing_property( - audio_unit, - K_AU_VOICE_IO_VOICE_PROCESSING_ENABLE_AGC, - &agc, + let agc = u32::from( + voice_processing + .voice_processing_enable_agc + .unwrap_or(false), ); + let mute_output = 0u32; + apply_voice_processing_property(audio_unit, K_AU_VOICE_IO_BYPASS_VOICE_PROCESSING, &bypass); + apply_voice_processing_property(audio_unit, K_AU_VOICE_IO_VOICE_PROCESSING_ENABLE_AGC, &agc); apply_voice_processing_property(audio_unit, K_AU_VOICE_IO_MUTE_OUTPUT, &mute_output); if voice_processing.enable_advanced_ducking || voice_processing.ducking_level.as_u32() != 0 { From 9e2b188c7bbc15d2915fbdd031135994cb8347e1 Mon Sep 17 00:00:00 2001 From: divanshu-go Date: Wed, 20 May 2026 02:20:03 +0530 Subject: [PATCH 3/4] fix --- examples/feedback.rs | 4 ++ examples/record_wav.rs | 68 ++++++++++++++++++---------- examples/record_wav_aec.rs | 90 ++++++++++++++++++++++++-------------- 3 files changed, 106 insertions(+), 56 deletions(-) diff --git a/examples/feedback.rs b/examples/feedback.rs index 557f202..6772bf7 100644 --- a/examples/feedback.rs +++ b/examples/feedback.rs @@ -151,6 +151,10 @@ fn main() -> anyhow::Result<()> { "Attempting to build both streams with f32 samples and `{:?}`.", config ); + #[cfg(target_os = "macos")] + let input_stream = + input_device.build_input_stream(&config, input_data_fn, err_fn, None, None)?; + #[cfg(not(target_os = "macos"))] let input_stream = input_device.build_input_stream(&config, input_data_fn, err_fn, None)?; let output_stream = output_device.build_output_stream(&config, output_data_fn, err_fn, None)?; println!("Successfully built streams."); diff --git a/examples/record_wav.rs b/examples/record_wav.rs index 4841bfa..dd87626 100644 --- a/examples/record_wav.rs +++ b/examples/record_wav.rs @@ -100,31 +100,20 @@ fn main() -> Result<(), anyhow::Error> { eprintln!("an error occurred on stream: {}", err); }; + let stream_config = config.config(); let stream = match config.sample_format() { - cpal::SampleFormat::I8 => device.build_input_stream( - &config.into(), - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )?, - cpal::SampleFormat::I16 => device.build_input_stream( - &config.into(), - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )?, - cpal::SampleFormat::I32 => device.build_input_stream( - &config.into(), - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )?, - cpal::SampleFormat::F32 => device.build_input_stream( - &config.into(), - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )?, + cpal::SampleFormat::I8 => { + build_recording_stream::(&device, &stream_config, writer_2, err_fn)? + } + cpal::SampleFormat::I16 => { + build_recording_stream::(&device, &stream_config, writer_2, err_fn)? + } + cpal::SampleFormat::I32 => { + build_recording_stream::(&device, &stream_config, writer_2, err_fn)? + } + cpal::SampleFormat::F32 => { + build_recording_stream::(&device, &stream_config, writer_2, err_fn)? + } sample_format => { return Err(anyhow::Error::msg(format!( "Unsupported sample format '{sample_format}'" @@ -161,6 +150,37 @@ fn wav_spec_from_config(config: &cpal::SupportedStreamConfig) -> hound::WavSpec type WavWriterHandle = Arc>>>>; +fn build_recording_stream( + device: &cpal::Device, + config: &cpal::StreamConfig, + writer: WavWriterHandle, + err_fn: impl FnMut(cpal::StreamError) + Send + 'static, +) -> Result +where + T: Sample + cpal::SizedSample, + U: Sample + hound::Sample + FromSample, +{ + #[cfg(target_os = "macos")] + { + device.build_input_stream( + config, + move |data: &[T], _: &cpal::InputCallbackInfo| write_input_data::(data, &writer), + err_fn, + None, + None, + ) + } + #[cfg(not(target_os = "macos"))] + { + device.build_input_stream( + config, + move |data: &[T], _: &cpal::InputCallbackInfo| write_input_data::(data, &writer), + err_fn, + None, + ) + } +} + fn write_input_data(input: &[T], writer: &WavWriterHandle) where T: Sample, diff --git a/examples/record_wav_aec.rs b/examples/record_wav_aec.rs index 923206f..691c336 100644 --- a/examples/record_wav_aec.rs +++ b/examples/record_wav_aec.rs @@ -1,7 +1,7 @@ -//! Records a WAV file using the default input device with Screenpipe's Windows WASAPI AEC enabled. +//! Records a WAV file using the default input device with echo cancellation requested. //! -//! AEC is requested only on Windows WASAPI input streams. Unsupported devices continue recording -//! normally without echo cancellation. +//! On Windows, sets `StreamConfig.windows_input_aec`. On macOS, passes +//! `MacosVoiceProcessingInputConfig::screenpipe_aec()`. Other platforms record normally. use clap::Parser; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; @@ -13,7 +13,10 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; #[derive(Parser, Debug)] -#[command(version, about = "Record a WAV file with Windows WASAPI AEC requested")] +#[command( + version, + about = "Record a WAV file with platform AEC requested when available" +)] struct Opt { /// The audio input device to use. #[arg(short, long, default_value_t = String::from("default"))] @@ -31,8 +34,8 @@ struct Opt { fn main() -> Result<(), anyhow::Error> { let opt = Opt::parse(); - #[cfg(not(target_os = "windows"))] - eprintln!("AEC is only requested by the Windows WASAPI backend; recording normally here."); + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + eprintln!("AEC is only requested on Windows WASAPI and macOS VoiceProcessingIO; recording normally here."); let host = cpal::default_host(); let device = if opt.device == "default" { @@ -62,30 +65,18 @@ fn main() -> Result<(), anyhow::Error> { let stream_config = aec_stream_config(&config); let stream = match config.sample_format() { - cpal::SampleFormat::I8 => device.build_input_stream( - &stream_config, - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )?, - cpal::SampleFormat::I16 => device.build_input_stream( - &stream_config, - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )?, - cpal::SampleFormat::I32 => device.build_input_stream( - &stream_config, - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )?, - cpal::SampleFormat::F32 => device.build_input_stream( - &stream_config, - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )?, + cpal::SampleFormat::I8 => { + build_aec_recording_stream::(&device, &stream_config, writer_2, err_fn)? + } + cpal::SampleFormat::I16 => { + build_aec_recording_stream::(&device, &stream_config, writer_2, err_fn)? + } + cpal::SampleFormat::I32 => { + build_aec_recording_stream::(&device, &stream_config, writer_2, err_fn)? + } + cpal::SampleFormat::F32 => { + build_aec_recording_stream::(&device, &stream_config, writer_2, err_fn)? + } sample_format => { return Err(anyhow::anyhow!( "unsupported sample format '{sample_format}'" @@ -109,12 +100,16 @@ fn main() -> Result<(), anyhow::Error> { } fn aec_stream_config(config: &cpal::SupportedStreamConfig) -> cpal::StreamConfig { - let mut stream_config = config.config(); #[cfg(target_os = "windows")] { + let mut stream_config = config.config(); stream_config.windows_input_aec = true; + stream_config + } + #[cfg(not(target_os = "windows"))] + { + config.config() } - stream_config } fn sample_format(format: cpal::SampleFormat) -> hound::SampleFormat { @@ -136,6 +131,37 @@ fn wav_spec_from_config(config: &cpal::SupportedStreamConfig) -> hound::WavSpec type WavWriterHandle = Arc>>>>; +fn build_aec_recording_stream( + device: &cpal::Device, + config: &cpal::StreamConfig, + writer: WavWriterHandle, + err_fn: impl FnMut(cpal::StreamError) + Send + 'static, +) -> Result +where + T: Sample + cpal::SizedSample, + U: Sample + hound::Sample + FromSample, +{ + #[cfg(target_os = "macos")] + { + device.build_input_stream( + config, + move |data: &[T], _: &cpal::InputCallbackInfo| write_input_data::(data, &writer), + err_fn, + None, + Some(cpal::MacosVoiceProcessingInputConfig::screenpipe_aec()), + ) + } + #[cfg(not(target_os = "macos"))] + { + device.build_input_stream( + config, + move |data: &[T], _: &cpal::InputCallbackInfo| write_input_data::(data, &writer), + err_fn, + None, + ) + } +} + fn write_input_data(input: &[T], writer: &WavWriterHandle) where T: Sample, From bf174e566d5a45599e0f28d9f2d265a83d14e39e Mon Sep 17 00:00:00 2001 From: divanshu-go Date: Wed, 20 May 2026 21:58:21 +0530 Subject: [PATCH 4/4] fix(coreaudio): sample rate tolerance and continuous-range device support - configure_voice_processing_stream_format: widen sample rate match from f64::EPSILON to < 1.0 Hz; log actual vs requested rates on mismatch - set_sample_rate: use containment check (mMinimum <= rate <= mMaximum) instead of point equality so USB/virtual devices with continuous ranges (Focusrite, BlackHole, Loopback) no longer get StreamConfigNotSupported - AudioObjectSetPropertyData: pass target rate as bare f64 with sizeof(f64) instead of AudioValueRange struct (was correct by accident for discrete-point devices only) Co-Authored-By: Claude Sonnet 4.6 --- src/host/coreaudio/macos/mod.rs | 77 ++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/src/host/coreaudio/macos/mod.rs b/src/host/coreaudio/macos/mod.rs index 9ed1d8b..e8f11f3 100644 --- a/src/host/coreaudio/macos/mod.rs +++ b/src/host/coreaudio/macos/mod.rs @@ -965,7 +965,7 @@ fn build_voice_processing_audio_unit( config: &StreamConfig, sample_format: SampleFormat, voice_processing: &crate::MacosVoiceProcessingInputConfig, -) -> Result { +) -> Result { let mut audio_unit = AudioUnit::new(coreaudio::audio_unit::IOType::VoiceProcessingIO)?; configure_voice_processing_io(&mut audio_unit)?; @@ -977,17 +977,52 @@ fn build_voice_processing_audio_unit( )?; let asbd = asbd_from_config(config, sample_format); - // Some VoiceProcessingIO paths expose stream format as read-only; best-effort. - let _ = audio_unit.set_property( + configure_voice_processing_stream_format(&mut audio_unit, &asbd)?; + + apply_voice_processing_flags(&mut audio_unit, voice_processing); + + Ok(audio_unit) +} + +#[cfg(target_os = "macos")] +fn configure_voice_processing_stream_format( + audio_unit: &mut AudioUnit, + requested: &AudioStreamBasicDescription, +) -> Result<(), BuildStreamError> { + let set_result = audio_unit.set_property( kAudioUnitProperty_StreamFormat, Scope::Output, Element::Input, - Some(&asbd), + Some(requested), ); + let actual = audio_unit + .get_property::( + kAudioUnitProperty_StreamFormat, + Scope::Output, + Element::Input, + ) + .ok(); - apply_voice_processing_flags(&mut audio_unit, voice_processing); + if let Some(actual) = actual { + let sample_rate_matches = (actual.mSampleRate - requested.mSampleRate).abs() < 1.0; + let channels_match = actual.mChannelsPerFrame == requested.mChannelsPerFrame; - Ok(audio_unit) + if sample_rate_matches && channels_match { + return Ok(()); + } + + eprintln!( + "screenpipe/cpal: VoiceProcessingIO format mismatch — requested {:.1} Hz × {} ch, \ + unit accepted {:.1} Hz × {} ch", + requested.mSampleRate, + requested.mChannelsPerFrame, + actual.mSampleRate, + actual.mChannelsPerFrame, + ); + return Err(BuildStreamError::StreamConfigNotSupported); + } + + set_result.map_err(Into::into) } #[cfg(target_os = "macos")] @@ -1126,13 +1161,27 @@ fn set_sample_rate( let ranges: *mut AudioValueRange = ranges.as_mut_ptr() as *mut _; let ranges: &'static [AudioValueRange] = unsafe { slice::from_raw_parts(ranges, n_ranges) }; - // Now that we have the available ranges, pick the one matching the desired rate. + // Pick the first range that contains the desired rate. Devices that report + // discrete points have mMinimum == mMaximum; continuous-range devices + // (USB interfaces, virtual drivers like BlackHole/Loopback) have + // mMinimum < mMaximum. Both cases are covered by containment. let sample_rate = target_sample_rate.0; let maybe_index = ranges .iter() - .position(|r| r.mMinimum as u32 == sample_rate && r.mMaximum as u32 == sample_rate); + .position(|r| r.mMinimum as u32 <= sample_rate && sample_rate <= r.mMaximum as u32); let range_index = match maybe_index { - None => return Err(BuildStreamError::StreamConfigNotSupported), + None => { + eprintln!( + "screenpipe/cpal: set_sample_rate — {} Hz not in any available range; \ + ranges: {:?}", + sample_rate, + ranges + .iter() + .map(|r| format!("{:.0}–{:.0}", r.mMinimum, r.mMaximum)) + .collect::>(), + ); + return Err(BuildStreamError::StreamConfigNotSupported); + } Some(i) => i, }; @@ -1167,7 +1216,11 @@ fn set_sample_rate( sample_rate_handler, )?; - // Finally, set the sample rate. + // Finally, set the sample rate. Pass the target as a bare f64 — the device + // expects sizeof(Float64) bytes for kAudioDevicePropertyNominalSampleRate, + // not the AudioValueRange struct we used for the lookup. + let target_rate_f64: f64 = target_sample_rate.0 as f64; + let _ = range_index; // consumed by lookup; only needed to confirm support property_address.mSelector = kAudioDevicePropertyNominalSampleRate; let status = unsafe { AudioObjectSetPropertyData( @@ -1175,8 +1228,8 @@ fn set_sample_rate( &property_address as *const _, 0, null(), - data_size, - &ranges[range_index] as *const _ as *const _, + mem::size_of::() as u32, + &target_rate_f64 as *const _ as *const _, ) }; coreaudio::Error::from_os_status(status)?;