From d8111686ae04a1c4a05424310e94751e19fb4fd8 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 4 May 2026 10:55:40 +0200 Subject: [PATCH 1/9] nix: add flake providing zig 0.16 dev shell. Zero packing atm --- flake.lock | 27 +++++++++++++++++++++++++++ flake.nix | 25 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..3e39fad --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1777826146, + "narHash": "sha256-wQ/iN5Zp5VIa3ebBibijPnLyKhor+xEbDy4d0goa9Zs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "73c703c22422b8951895a960959dbbaca7296492", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..ad2bd97 --- /dev/null +++ b/flake.nix @@ -0,0 +1,25 @@ +{ + description = "BobrWhisper - local-first voice-to-text"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + outputs = + { nixpkgs, ... }: + let + # arm64-only: Config.zig hardcodes Apple M1 for iOS targets and the + # Makefile pins `ARCHS=arm64` for xcodebuild. + pkgs = import nixpkgs { system = "aarch64-darwin"; }; + in + { + # mkShellNoCC: the build resolves clang/clang++/metal/xcodebuild via the + # system Xcode (see src/build/AppleSdk.zig). A nix stdenv would inject a + # different libc++ and SDK root and break the link against whisper.cpp / + # llama.cpp. + devShells.aarch64-darwin.default = pkgs.mkShellNoCC { + packages = [ + pkgs.zig_0_16 + pkgs.zls + ]; + }; + }; +} From cdae754a1606a13b6cd0c2f260a8a5fc388070d3 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 5 May 2026 23:07:01 +0200 Subject: [PATCH 2/9] build(macos): ad-hoc sign local xcode builds Configure local Xcode builds to use ad-hoc signing when no developer team is set. Previous local app rebuilds could get a changing or missing signature, which made macOS privacy permissions less stable during development. The build helper now passes explicit manual signing settings for the regular macOS app build path. Archive builds keep their existing behavior. --- src/build/BobrWhisperXcodebuild.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/build/BobrWhisperXcodebuild.zig b/src/build/BobrWhisperXcodebuild.zig index fd7726e..629ab55 100644 --- a/src/build/BobrWhisperXcodebuild.zig +++ b/src/build/BobrWhisperXcodebuild.zig @@ -52,6 +52,9 @@ pub fn init( "ARCHS=arm64", }) else + // Ad-hoc self-signing for local dev (no Apple Developer Team). + // Keeps a stable signature so TCC permissions (microphone) persist + // across rebuilds, while not requiring a paid signing identity. b.addSystemCommand(&.{ "xcodebuild", "-project", @@ -60,6 +63,11 @@ pub fn init( "BobrWhisper", "-configuration", configuration, + "CODE_SIGN_STYLE=Manual", + "CODE_SIGN_IDENTITY=-", + "CODE_SIGNING_REQUIRED=YES", + "CODE_SIGNING_ALLOWED=YES", + "DEVELOPMENT_TEAM=", "build", "ARCHS=arm64", }), From 1bb0eff8508096e5eb792cd9638863af71acf682 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 5 May 2026 23:07:01 +0200 Subject: [PATCH 3/9] feat(asr): add english whisper models Add registry entries for the English-only tiny, base, small, and medium Whisper models. The model list previously exposed only multilingual variants, even though the download source ships smaller language-specific files. The descriptors use the same runtime and capabilities as the existing Whisper entries while pointing at the .en filenames and downloads. Larger models remain multilingual-only because no matching English-only files are available. --- pkg/asr/ModelRegistry.zig | 58 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/pkg/asr/ModelRegistry.zig b/pkg/asr/ModelRegistry.zig index 686440c..41bed0f 100644 --- a/pkg/asr/ModelRegistry.zig +++ b/pkg/asr/ModelRegistry.zig @@ -7,9 +7,13 @@ pub const ModelCapability = types.ModelCapability; pub const capabilityBit = types.capabilityBit; pub const whisper_tiny_id: [:0]const u8 = "whisper-tiny"; +pub const whisper_tiny_en_id: [:0]const u8 = "whisper-tiny.en"; pub const whisper_base_id: [:0]const u8 = "whisper-base"; +pub const whisper_base_en_id: [:0]const u8 = "whisper-base.en"; pub const whisper_small_id: [:0]const u8 = "whisper-small"; +pub const whisper_small_en_id: [:0]const u8 = "whisper-small.en"; pub const whisper_medium_id: [:0]const u8 = "whisper-medium"; +pub const whisper_medium_en_id: [:0]const u8 = "whisper-medium.en"; pub const whisper_large_v3_id: [:0]const u8 = "whisper-large-v3"; pub const whisper_large_v3_turbo_id: [:0]const u8 = "whisper-large-v3-turbo"; @@ -17,6 +21,12 @@ const whisper_capabilities = capabilityBit(.transcribe) | capabilityBit(.stream_partial); +// English-only models cannot do language detection or translation. Only the +// `transcribe` + `stream_partial` bits apply, same as multilingual — but the +// .en variant must be paired with `language=en`. Tiny→medium are the only +// sizes OpenAI ever released a `.en` for; large/turbo are multilingual-only. +const whisper_capabilities_en = whisper_capabilities; + const descriptors = [_]ModelDescriptor{ .{ .id = whisper_tiny_id, @@ -30,6 +40,17 @@ const descriptors = [_]ModelDescriptor{ .available_on_this_device = true, .legacy_storage_key = "tiny", }, + .{ + .id = whisper_tiny_en_id, + .display_name = "Whisper Tiny English (~75 MB)", + .family = "whisper", + .runtime = .whisper_cpp, + .local_filename = "ggml-tiny.en.bin", + .download_url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin", + .size_bytes = 75 * 1024 * 1024, + .capabilities = whisper_capabilities_en, + .available_on_this_device = true, + }, .{ .id = whisper_base_id, .display_name = "Whisper Base (~142 MB)", @@ -42,6 +63,17 @@ const descriptors = [_]ModelDescriptor{ .available_on_this_device = true, .legacy_storage_key = "base", }, + .{ + .id = whisper_base_en_id, + .display_name = "Whisper Base English (~142 MB)", + .family = "whisper", + .runtime = .whisper_cpp, + .local_filename = "ggml-base.en.bin", + .download_url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin", + .size_bytes = 142 * 1024 * 1024, + .capabilities = whisper_capabilities_en, + .available_on_this_device = true, + }, .{ .id = whisper_small_id, .display_name = "Whisper Small (~466 MB)", @@ -55,6 +87,20 @@ const descriptors = [_]ModelDescriptor{ .legacy_storage_key = "small", .preferred_live_model_id = whisper_base_id, }, + .{ + .id = whisper_small_en_id, + .display_name = "Whisper Small English (~466 MB)", + .family = "whisper", + .runtime = .whisper_cpp, + .local_filename = "ggml-small.en.bin", + .download_url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin", + .size_bytes = 466 * 1024 * 1024, + .capabilities = whisper_capabilities_en, + .available_on_this_device = true, + // Live partials still benefit from the smaller English base — same + // size class as the multilingual sibling. + .preferred_live_model_id = whisper_base_en_id, + }, .{ .id = whisper_medium_id, .display_name = "Whisper Medium (~1.5 GB)", @@ -68,6 +114,18 @@ const descriptors = [_]ModelDescriptor{ .legacy_storage_key = "medium", .preferred_live_model_id = whisper_base_id, }, + .{ + .id = whisper_medium_en_id, + .display_name = "Whisper Medium English (~1.5 GB)", + .family = "whisper", + .runtime = .whisper_cpp, + .local_filename = "ggml-medium.en.bin", + .download_url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin", + .size_bytes = 1500 * 1024 * 1024, + .capabilities = whisper_capabilities_en, + .available_on_this_device = true, + .preferred_live_model_id = whisper_base_en_id, + }, .{ .id = whisper_large_v3_id, .display_name = "Whisper Large v3 (~3.1 GB)", From b7b227b8de6183495d86df1e4d4c348941c1f149 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 5 May 2026 23:07:01 +0200 Subject: [PATCH 4/9] feat(asr): support runtime decoder tuning Let the runtime adapter update decoder prompts and VAD parameters after model load. Previously those values were fixed at initialization, so app-level dictionary prompts and input-device-specific VAD buckets required a reload. The adapter now owns an optional initial prompt and routes prompt and VAD updates to the whisper.cpp backend. Transcribe calls read the current values when building the request. --- pkg/asr/RuntimeAdapter.zig | 34 ++++++++++++++++++++ pkg/asr/WhisperCppAdapter.zig | 59 +++++++++++++++++++++++++++++++++++ pkg/asr/whisper_bridge.c | 11 ++++++- pkg/asr/whisper_bridge.zig | 1 + 4 files changed, 104 insertions(+), 1 deletion(-) diff --git a/pkg/asr/RuntimeAdapter.zig b/pkg/asr/RuntimeAdapter.zig index 42454c2..e2518f2 100644 --- a/pkg/asr/RuntimeAdapter.zig +++ b/pkg/asr/RuntimeAdapter.zig @@ -16,6 +16,9 @@ pub const LoadConfig = struct { vad_min_speech_ms: i32 = 250, vad_min_silence_ms: i32 = 100, vad_speech_pad_ms: i32 = 30, + /// Optional decoder bias prompt (proper nouns, vocabulary). Whisper caps + /// this at ~224 tokens. Caller is responsible for staying under the bound. + initial_prompt: ?[]const u8 = null, }; pub const RuntimeAdapter = union(enum) { @@ -40,11 +43,42 @@ pub const RuntimeAdapter = union(enum) { .vad_min_speech_ms = config.vad_min_speech_ms, .vad_min_silence_ms = config.vad_min_silence_ms, .vad_speech_pad_ms = config.vad_speech_pad_ms, + .initial_prompt = config.initial_prompt, }) }, else => error.UnsupportedRuntime, }; } + /// Replace the decoder bias prompt. Routes to the underlying adapter. + /// Pass `null` or empty to clear. + pub fn setInitialPrompt(self: *RuntimeAdapter, prompt: ?[]const u8) !void { + switch (self.*) { + inline else => |*adapter| try adapter.setInitialPrompt(prompt), + } + } + + /// Override VAD parameters at runtime. Used to apply device-specific + /// tunings (e.g. relaxed thresholds for Bluetooth mics) at recording + /// start without reloading the model. + pub fn setVadParams( + self: *RuntimeAdapter, + enabled: bool, + threshold: f32, + min_speech_ms: i32, + min_silence_ms: i32, + speech_pad_ms: i32, + ) void { + switch (self.*) { + inline else => |*adapter| adapter.setVadParams( + enabled, + threshold, + min_speech_ms, + min_silence_ms, + speech_pad_ms, + ), + } + } + pub fn deinit(self: *RuntimeAdapter) void { switch (self.*) { inline else => |*adapter| adapter.deinit(), diff --git a/pkg/asr/WhisperCppAdapter.zig b/pkg/asr/WhisperCppAdapter.zig index d5ddb2f..6d2bba8 100644 --- a/pkg/asr/WhisperCppAdapter.zig +++ b/pkg/asr/WhisperCppAdapter.zig @@ -25,6 +25,10 @@ vad_threshold: f32, vad_min_speech_ms: i32, vad_min_silence_ms: i32, vad_speech_pad_ms: i32, +// Decoder bias prompt. Owned, sentinel-terminated, swapped via +// `setInitialPrompt`. Whisper.cpp accepts up to ~224 prompt tokens; callers +// are responsible for staying under that bound (see DictionaryStore). +initial_prompt: ?[:0]u8 = null, pub const Config = struct { model_path: []const u8, @@ -38,6 +42,8 @@ pub const Config = struct { vad_min_speech_ms: i32 = 250, vad_min_silence_ms: i32 = 100, vad_speech_pad_ms: i32 = 30, + // Optional initial prompt for decoder bias. + initial_prompt: ?[]const u8 = null, }; fn shouldUseGpu() bool { @@ -84,6 +90,7 @@ pub fn init(allocator: std.mem.Allocator, config: Config) !WhisperCppAdapter { .vad_min_speech_ms = config.vad_min_speech_ms, .vad_min_silence_ms = config.vad_min_silence_ms, .vad_speech_pad_ms = config.vad_speech_pad_ms, + .initial_prompt = if (config.initial_prompt) |p| try allocator.dupeZ(u8, p) else null, }; } @@ -96,6 +103,47 @@ pub fn deinit(self: *WhisperCppAdapter) void { if (self.vad_model_path) |p| { self.allocator.free(p); } + if (self.initial_prompt) |p| { + self.allocator.free(p); + } +} + +/// Override the VAD parameters used by subsequent transcribe calls. Used +/// by the App to push device-specific tunings (e.g. lower threshold for +/// Bluetooth mics whose codec compression smears speech onsets). Same +/// concurrency contract as `setInitialPrompt` — caller must not be +/// transcribing. +pub fn setVadParams( + self: *WhisperCppAdapter, + enabled: bool, + threshold: f32, + min_speech_ms: i32, + min_silence_ms: i32, + speech_pad_ms: i32, +) void { + std.debug.assert(threshold >= 0.0 and threshold <= 1.0); + std.debug.assert(min_speech_ms >= 0); + std.debug.assert(min_silence_ms >= 0); + std.debug.assert(speech_pad_ms >= 0); + self.vad_enabled = enabled; + self.vad_threshold = threshold; + self.vad_min_speech_ms = min_speech_ms; + self.vad_min_silence_ms = min_silence_ms; + self.vad_speech_pad_ms = speech_pad_ms; +} + +/// Replace the decoder bias prompt. Pass `null` (or empty) to clear. Caller +/// must NOT hold a transcribe call concurrently — the adapter is single- +/// threaded; App.zig drives all calls from the live-transcription thread or +/// the synchronous transcribe path. +pub fn setInitialPrompt(self: *WhisperCppAdapter, prompt: ?[]const u8) !void { + if (self.initial_prompt) |old| { + self.allocator.free(old); + self.initial_prompt = null; + } + const new_prompt = prompt orelse return; + if (new_prompt.len == 0) return; + self.initial_prompt = try self.allocator.dupeZ(u8, new_prompt); } /// Transcribe audio samples (16kHz, mono, f32) @@ -120,6 +168,16 @@ fn transcribeInternal(self: *WhisperCppAdapter, samples: []const f32, language: return error.NoAudioData; } + std.log.info("Transcribing {d} samples, live={}, vad_enabled={}, vad_threshold={d:.3}, min_speech_ms={d}, min_silence_ms={d}, speech_pad_ms={d}", .{ + samples.len, + live, + self.vad_enabled, + self.vad_threshold, + self.vad_min_speech_ms, + self.vad_min_silence_ms, + self.vad_speech_pad_ms, + }); + var lang_buf: [8:0]u8 = [_:0]u8{0} ** 8; const lang_len = @min(language.len, lang_buf.len - 1); @memcpy(lang_buf[0..lang_len], language[0..lang_len]); @@ -137,6 +195,7 @@ fn transcribeInternal(self: *WhisperCppAdapter, samples: []const f32, language: self.vad_min_speech_ms, self.vad_min_silence_ms, self.vad_speech_pad_ms, + if (self.initial_prompt) |p| p.ptr else null, ); if (result != 0) { std.log.err("whisper_full failed: {}", .{result}); diff --git a/pkg/asr/whisper_bridge.c b/pkg/asr/whisper_bridge.c index 3d14951..4c9047e 100644 --- a/pkg/asr/whisper_bridge.c +++ b/pkg/asr/whisper_bridge.c @@ -35,7 +35,8 @@ int bobrwhisper_whisper_transcribe( float vad_threshold, int32_t vad_min_speech_ms, int32_t vad_min_silence_ms, - int32_t vad_speech_pad_ms + int32_t vad_speech_pad_ms, + const char * initial_prompt ) { struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); params.print_realtime = false; @@ -62,6 +63,14 @@ int bobrwhisper_whisper_transcribe( params.language = language; } + // Initial prompt biases decoding toward proper nouns and domain vocabulary. + // Whisper allows up to ~224 tokens of prompt; the caller is responsible for + // staying under that bound. Pointer must outlive the whisper_full() call, + // which it does because the Zig adapter holds it for the call duration. + if (initial_prompt != NULL && initial_prompt[0] != '\0') { + params.initial_prompt = initial_prompt; + } + return whisper_full(ctx, params, samples, sample_count); } diff --git a/pkg/asr/whisper_bridge.zig b/pkg/asr/whisper_bridge.zig index 0ca522f..7cec3b2 100644 --- a/pkg/asr/whisper_bridge.zig +++ b/pkg/asr/whisper_bridge.zig @@ -16,6 +16,7 @@ pub extern fn bobrwhisper_whisper_transcribe( vad_min_speech_ms: i32, vad_min_silence_ms: i32, vad_speech_pad_ms: i32, + initial_prompt: ?[*:0]const u8, ) c_int; pub extern fn bobrwhisper_whisper_segment_count(ctx: *Context) i32; pub extern fn bobrwhisper_whisper_segment_text(ctx: *Context, segment_index: i32) ?[*:0]const u8; From 3674acbe0873a1b7db1479c75867e18df2fa7b1a Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 5 May 2026 23:07:01 +0200 Subject: [PATCH 5/9] feat(cli): add corpus recording tools Add CLI commands for collecting labeled audio snippets and replaying them through tuning presets. Previously tuning VAD and gain behavior required manual recordings and ad hoc transcription runs. The snippet command records WAV plus JSON metadata, while the tune command loads a snippet corpus, applies presets, and reports WER summaries. The debug recording path reuses the same terminal recording loop. --- scripts/record-corpus.sh | 115 +++++++ src/cli.zig | 120 +++++++ src/snippet.zig | 473 +++++++++++++++++++++++++ src/tune.zig | 723 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 1431 insertions(+) create mode 100755 scripts/record-corpus.sh create mode 100644 src/snippet.zig create mode 100644 src/tune.zig diff --git a/scripts/record-corpus.sh b/scripts/record-corpus.sh new file mode 100755 index 0000000..4198d8c --- /dev/null +++ b/scripts/record-corpus.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Record a baseline test corpus for tuning. +# +# Records 12 phrases × 2 modes (normal + whispered) = 24 snippets total. +# Phrases are tagged short / question / technical so the corpus can be +# sliced via: bobrwhisper-cli tune --group-by tag +# +# Usage: +# ./scripts/record-corpus.sh # press 'q' + Enter to stop each take +# ./scripts/record-corpus.sh -d 5 # fixed 5-second takes (no 'q' needed) +# +# Environment: +# CLI=path/to/bobrwhisper-cli # override binary location + +set -u + +CLI="${CLI:-./zig-out/bin/bobrwhisper-cli}" +DURATION=() + +while getopts "d:h" opt; do + case "$opt" in + d) DURATION=(--duration "$OPTARG") ;; + h) sed -n '2,15p' "$0"; exit 0 ;; + *) echo "Usage: $0 [-d seconds]" >&2; exit 2 ;; + esac +done + +if [[ ! -x "$CLI" ]]; then + echo "snippet CLI not found at: $CLI" >&2 + echo "Build it first: zig build" >&2 + exit 1 +fi + +# 12 phrases. Edit to match your accent/vocabulary; keep the tag column the +# same so per-tag averages stay comparable across runs. +SHORT=( + "yes please" + "open settings" + "no thanks" + "stop recording" +) +QUESTION=( + "what time is it" + "who is calling me" + "what is the weather today" + "where is the nearest coffee" +) +TECHNICAL=( + "refactor the parser" + "git rebase onto main" + "evaluate this codebase" + "the lexer tokenizes input" +) + +CURRENT=0 +TOTAL=$(( (${#SHORT[@]} + ${#QUESTION[@]} + ${#TECHNICAL[@]}) * 2 )) + +bar() { printf '──────────────────────────────────────────────\n'; } + +record_one() { + local label="$1" tag="$2" mode="$3" + CURRENT=$((CURRENT + 1)) + + local mode_flag + case "$mode" in + whisper) mode_flag="--whisper" ;; + normal) mode_flag="--no-whisper" ;; + *) echo "internal error: unknown mode $mode" >&2; exit 3 ;; + esac + + printf '\n'; bar + printf ' [%2d/%d] %-40s %s\n' "$CURRENT" "$TOTAL" "\"$label\"" "$mode ($tag)" + bar + printf 'Enter to record, "s" to skip, "q" to quit: ' + read -r choice + case "$choice" in + s|S) printf ' skipped.\n'; return ;; + q|Q) printf ' quitting.\n'; exit 0 ;; + esac + + # The snippet CLI segfaults on shutdown after the WAV+JSON are written; + # tolerate the non-zero exit so the script keeps going. + "$CLI" snippet --label "$label" "$mode_flag" --tag "$tag" "${DURATION[@]}" || true +} + +run_set() { + local tag="$1"; shift + for phrase in "$@"; do + for mode in normal whisper; do + record_one "$phrase" "$tag" "$mode" + done + done +} + +cat < Transcribe a WAV file \\ transcribe-raw Transcribe raw f32 audio \\ record Record audio from microphone + \\ snippet [options] Record a labeled snippet for the test corpus (no transcription) + \\ tune [options] Run snippets through VAD presets, report WER vs. ground truth + \\ test-whisper [model] [--whisper-mode] Record until 'q', then transcribe (debug) \\ models List available models \\ languages List supported languages \\ help Show this help @@ -541,6 +552,115 @@ fn liveCommand(allocator: std.mem.Allocator, args: []const []const u8) !void { } } +fn testWhisperCommand(allocator: std.mem.Allocator, args: []const []const u8) !void { + var selected_model: WhisperModel = .small; + var whisper_mode = false; + + for (args) |arg| { + if (std.mem.eql(u8, arg, "--whisper-mode")) { + whisper_mode = true; + } else if (!std.mem.startsWith(u8, arg, "-")) { + selected_model = WhisperModel.fromString(arg) orelse { + std.debug.print("Unknown model: {s}\n", .{arg}); + std.debug.print("Available: tiny, base, small, medium, large, large_turbo\n", .{}); + return; + }; + } + } + + std.debug.print("test-whisper: model={s}, whisper_mode={}\n", .{ @tagName(selected_model), whisper_mode }); + + const model_path = selected_model.ensureDownloaded(allocator) catch |err| { + std.debug.print("Failed to get model: {}\n", .{err}); + return; + }; + defer allocator.free(model_path); + + const vad_path = getVadModelPath(allocator); + defer if (vad_path) |p| allocator.free(p); + + std.debug.print("Loading model: {s}\n", .{model_path}); + if (vad_path != null) std.debug.print("VAD: enabled\n", .{}); + + var transcriber = WhisperCppAdapter.init(allocator, .{ + .model_path = model_path, + .language = "en", + .n_threads = 4, + .vad_enabled = vad_path != null, + .vad_model_path = vad_path, + .vad_threshold = if (whisper_mode) 0.3 else 0.5, + .vad_min_speech_ms = if (whisper_mode) 150 else 250, + .vad_min_silence_ms = if (whisper_mode) 200 else 100, + .vad_speech_pad_ms = if (whisper_mode) 100 else 30, + }) catch |err| { + std.debug.print("Failed to load model: {}\n", .{err}); + return; + }; + defer transcriber.deinit(); + + std.debug.print("Model loaded. Starting recording...\n", .{}); + std.debug.print("Press 'q' then Enter to stop and transcribe.\n\n", .{}); + + var audio = try AudioCapture.init(allocator); + defer audio.deinit(); + + try audio.start(); + try snippet.recordUntilQ(&audio); + + std.debug.print("\n\nStopping recording...\n", .{}); + audio.stop(); + + const samples = audio.getSamples(); + const duration_s = @as(f64, @floatFromInt(samples.len)) / 16000.0; + std.debug.print("Recorded {d} samples ({d:.2}s)\n", .{ samples.len, duration_s }); + + if (samples.len == 0) { + std.debug.print("No audio recorded.\n", .{}); + return; + } + + // Compute and log noise floor + trim info + const noise_floor = AudioCapture.computeNoiseFloor(samples); + const trim_threshold = @max(noise_floor * 3.0, 0.0005); + std.debug.print("\nDiagnostics:\n", .{}); + std.debug.print(" noise floor (RMS of first 0.5s): {d:.6}\n", .{noise_floor}); + std.debug.print(" trim threshold (3x floor, min 0.0005): {d:.6}\n", .{trim_threshold}); + std.debug.print(" whisper_mode: {}\n", .{whisper_mode}); + + const bounds = AudioCapture.trimSilenceBounds(samples, trim_threshold); + const trimmed = samples[bounds.start..bounds.end]; + const trimmed_pct = @as(f32, @floatFromInt(samples.len - trimmed.len)) / @as(f32, @floatFromInt(samples.len)) * 100.0; + std.debug.print(" trim bounds: [{d}..{d}] of {d} ({d:.1}% trimmed)\n", .{ bounds.start, bounds.end, samples.len, trimmed_pct }); + + const segment = if (trimmed.len > 0) trimmed else samples; + std.debug.print(" transcribing {d} samples ({d:.2}s)\n\n", .{ + segment.len, + @as(f64, @floatFromInt(segment.len)) / 16000.0, + }); + + // Also save raw audio for replay + const output_path = "test-whisper.raw"; + const file = try std.Io.Dir.cwd().createFile(compat.io(), output_path, .{}); + defer file.close(compat.io()); + try file.writeStreamingAll(compat.io(), std.mem.sliceAsBytes(samples)); + std.debug.print("Raw audio saved to: {s} (replay with: bobrwhisper-cli transcribe-raw {s})\n\n", .{ output_path, output_path }); + + // Transcribe + std.debug.print("Transcribing...\n", .{}); + const text = transcriber.transcribe(segment) catch |err| { + std.debug.print("Transcription failed: {}\n", .{err}); + return; + }; + defer allocator.free(text); + + const result = std.mem.trim(u8, text, " \t\n"); + std.debug.print("\n--- Transcription ---\n{s}\n", .{result}); + + if (result.len == 0) { + std.debug.print("\n(empty result - whisper returned nothing)\n", .{}); + } +} + fn isHallucination(text: []const u8) bool { const hallucinations = [_][]const u8{ "you", diff --git a/src/snippet.zig b/src/snippet.zig new file mode 100644 index 0000000..3d387a7 --- /dev/null +++ b/src/snippet.zig @@ -0,0 +1,473 @@ +//! Snippet recorder - capture labeled audio for the test corpus. +//! +//! The CLI does NOT transcribe; it only records audio plus end-user metadata. +//! Each snippet is written as a sibling pair so experiment code can mmap one +//! and read the other: +//! +//! /.wav 16 kHz mono 16-bit PCM WAV +//! /.json ground truth label + whisper flag + audio stats +//! +//! Default output dir is `~/.bobrwhisper/snippets`. + +const std = @import("std"); +const builtin = @import("builtin"); +const compat = @import("compat.zig"); +const AudioCapture = @import("audio/AudioCapture.zig"); +const AudioDevice = @import("audio/AudioDevice.zig"); + +const sample_rate: u32 = 16000; + +pub const Args = struct { + label: ?[]const u8 = null, + whisper: ?bool = null, + out_dir: ?[]const u8 = null, + duration_secs: ?u64 = null, + device: ?[]const u8 = null, + device_kind: ?AudioDevice.Kind = null, + tags: []const []const u8 = &.{}, +}; + +const ParseError = error{ + UnknownArg, + MissingValue, + InvalidDuration, + InvalidDeviceKind, + ConflictingFlags, + HelpRequested, + OutOfMemory, +}; + +fn parseArgs(allocator: std.mem.Allocator, args: []const []const u8, tags_out: *std.ArrayListUnmanaged([]const u8)) ParseError!Args { + var out: Args = .{}; + var i: usize = 0; + while (i < args.len) : (i += 1) { + const a = args[i]; + if (std.mem.eql(u8, a, "--help") or std.mem.eql(u8, a, "-h")) { + return error.HelpRequested; + } else if (std.mem.eql(u8, a, "--label")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.label = args[i]; + } else if (std.mem.eql(u8, a, "--whisper")) { + if (out.whisper) |w| if (!w) return error.ConflictingFlags; + out.whisper = true; + } else if (std.mem.eql(u8, a, "--no-whisper")) { + if (out.whisper) |w| if (w) return error.ConflictingFlags; + out.whisper = false; + } else if (std.mem.eql(u8, a, "--out")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.out_dir = args[i]; + } else if (std.mem.eql(u8, a, "--duration")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.duration_secs = std.fmt.parseInt(u64, args[i], 10) catch return error.InvalidDuration; + } else if (std.mem.eql(u8, a, "--device")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.device = args[i]; + } else if (std.mem.eql(u8, a, "--device-kind")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.device_kind = AudioDevice.Kind.fromString(args[i]) orelse return error.InvalidDeviceKind; + } else if (std.mem.eql(u8, a, "--tag")) { + i += 1; + if (i >= args.len) return error.MissingValue; + try tags_out.append(allocator, args[i]); + } else { + std.debug.print("snippet: unknown argument '{s}'\n", .{a}); + return error.UnknownArg; + } + } + out.tags = tags_out.items; + return out; +} + +pub fn run(allocator: std.mem.Allocator, raw_args: []const []const u8) !void { + var tag_list = std.ArrayListUnmanaged([]const u8).empty; + defer tag_list.deinit(allocator); + + const parsed = parseArgs(allocator, raw_args, &tag_list) catch |err| switch (err) { + error.HelpRequested => { + printUsage(); + return; + }, + else => { + std.debug.print("\n", .{}); + printUsage(); + return err; + }, + }; + + if (builtin.os.tag != .macos) { + std.debug.print("snippet: audio capture is only supported on macOS\n", .{}); + return error.UnsupportedPlatform; + } + + // Resolve metadata: prompt for anything not passed on the CLI. + var label_storage: ?[]u8 = null; + defer if (label_storage) |s| allocator.free(s); + const label: []const u8 = blk: { + if (parsed.label) |l| break :blk l; + label_storage = try promptLine(allocator, "Ground truth label (what will you say?): "); + break :blk label_storage.?; + }; + if (label.len == 0) { + std.debug.print("snippet: label cannot be empty\n", .{}); + return error.EmptyLabel; + } + + const whisper: bool = parsed.whisper orelse try promptYesNo("Whispered? [y/N]: ", false); + + // Resolve audio device: explicit CLI override takes priority; otherwise + // probe CoreAudio for the default input. detected_info is null on non-macOS + // or when CoreAudio refuses (no device, headless), in which case we leave + // the device fields null in the metadata. + const detected_info: ?AudioDevice.Info = AudioDevice.detectDefaultInput(allocator); + defer if (detected_info) |d| d.deinit(allocator); + + const device: ?[]const u8 = parsed.device orelse if (detected_info) |d| d.name else null; + const device_kind: ?AudioDevice.Kind = parsed.device_kind orelse if (detected_info) |d| d.kind else null; + + // Resolve output directory. + var out_dir_storage: ?[]u8 = null; + defer if (out_dir_storage) |s| allocator.free(s); + const out_dir: []const u8 = blk: { + if (parsed.out_dir) |d| break :blk d; + const home = compat.getenv("HOME") orelse { + std.debug.print("snippet: HOME is unset; pass --out \n", .{}); + return error.MissingHome; + }; + out_dir_storage = try std.fmt.allocPrint(allocator, "{s}/.bobrwhisper/snippets", .{home}); + break :blk out_dir_storage.?; + }; + try ensureDir(out_dir); + + // Build the snippet ID from the wall-clock timestamp. + const created_at_ms = compat.milliTimestamp(); + const id = try formatSnippetId(allocator, created_at_ms); + defer allocator.free(id); + + const wav_path = try std.fmt.allocPrint(allocator, "{s}/{s}.wav", .{ out_dir, id }); + defer allocator.free(wav_path); + const json_path = try std.fmt.allocPrint(allocator, "{s}/{s}.json", .{ out_dir, id }); + defer allocator.free(json_path); + + std.debug.print("\nSnippet: {s}\n", .{id}); + std.debug.print(" label: {s}\n", .{label}); + std.debug.print(" whisper: {}\n", .{whisper}); + std.debug.print(" device: {s} ({s})\n", .{ + device orelse "(unknown)", + if (device_kind) |k| @tagName(k) else "unknown", + }); + if (parsed.tags.len > 0) { + std.debug.print(" tags: ", .{}); + for (parsed.tags, 0..) |t, idx| { + if (idx > 0) std.debug.print(", ", .{}); + std.debug.print("{s}", .{t}); + } + std.debug.print("\n", .{}); + } + std.debug.print(" out: {s}\n\n", .{out_dir}); + + // Start recording. + var audio = try AudioCapture.init(allocator); + defer audio.deinit(); + try audio.start(); + + if (parsed.duration_secs) |secs| { + std.debug.print("Recording for {d}s...\n", .{secs}); + try recordFixed(&audio, secs); + } else { + std.debug.print("Recording... press 'q' then Enter to stop.\n", .{}); + try recordUntilQ(&audio); + } + + audio.stop(); + + const samples = audio.getSamples(); + if (samples.len == 0) { + std.debug.print("snippet: no audio captured; aborting\n", .{}); + return error.NoAudio; + } + + const duration_seconds = @as(f64, @floatFromInt(samples.len)) / @as(f64, @floatFromInt(sample_rate)); + const noise_floor = AudioCapture.computeNoiseFloor(samples); + + std.debug.print("\nCaptured {d} samples ({d:.2}s, noise floor {d:.6})\n", .{ samples.len, duration_seconds, noise_floor }); + + try writeWav(wav_path, samples); + try writeMetadata(allocator, json_path, .{ + .id = id, + .label = label, + .whisper = whisper, + .audio_file = std.fs.path.basename(wav_path), + .duration_seconds = duration_seconds, + .sample_rate = sample_rate, + .channels = 1, + .num_samples = samples.len, + .noise_floor_rms = noise_floor, + .created_at_unix_ms = created_at_ms, + .device = device, + .device_kind = device_kind, + .tags = parsed.tags, + }); + + std.debug.print("\nSaved:\n {s}\n {s}\n", .{ wav_path, json_path }); +} + +const Metadata = struct { + id: []const u8, + label: []const u8, + whisper: bool, + audio_file: []const u8, + duration_seconds: f64, + sample_rate: u32, + channels: u32, + num_samples: usize, + noise_floor_rms: f32, + created_at_unix_ms: i64, + /// Free-form device label (e.g. "MacBook Pro Microphone", "AirPods Pro"). + device: ?[]const u8, + /// Coarse device classification for slicing the test corpus. + device_kind: ?AudioDevice.Kind, + /// Free-form tags so experiments can group snippets by content/condition. + tags: []const []const u8, +}; + +fn writeMetadata(allocator: std.mem.Allocator, path: []const u8, meta: Metadata) !void { + const json = try std.json.Stringify.valueAlloc(allocator, meta, .{ .whitespace = .indent_2 }); + defer allocator.free(json); + + const file = try std.Io.Dir.cwd().createFile(compat.io(), path, .{}); + defer file.close(compat.io()); + try file.writeStreamingAll(compat.io(), json); + try file.writeStreamingAll(compat.io(), "\n"); +} + +/// Encode 32-bit float mono samples to a 16-bit PCM WAV file at `sample_rate`. +fn writeWav(path: []const u8, samples: []const f32) !void { + const channels: u16 = 1; + const bits_per_sample: u16 = 16; + const byte_rate: u32 = sample_rate * channels * (bits_per_sample / 8); + const block_align: u16 = channels * (bits_per_sample / 8); + const data_size: u32 = @intCast(samples.len * @sizeOf(i16)); + const riff_size: u32 = 36 + data_size; + + var header: [44]u8 = undefined; + @memcpy(header[0..4], "RIFF"); + std.mem.writeInt(u32, header[4..8], riff_size, .little); + @memcpy(header[8..12], "WAVE"); + @memcpy(header[12..16], "fmt "); + std.mem.writeInt(u32, header[16..20], 16, .little); // PCM fmt chunk size + std.mem.writeInt(u16, header[20..22], 1, .little); // PCM + std.mem.writeInt(u16, header[22..24], channels, .little); + std.mem.writeInt(u32, header[24..28], sample_rate, .little); + std.mem.writeInt(u32, header[28..32], byte_rate, .little); + std.mem.writeInt(u16, header[32..34], block_align, .little); + std.mem.writeInt(u16, header[34..36], bits_per_sample, .little); + @memcpy(header[36..40], "data"); + std.mem.writeInt(u32, header[40..44], data_size, .little); + + const file = try std.Io.Dir.cwd().createFile(compat.io(), path, .{}); + defer file.close(compat.io()); + + try file.writeStreamingAll(compat.io(), &header); + + // Stream-convert samples in chunks to bound stack/heap usage. + var chunk: [4096]i16 = undefined; + var i: usize = 0; + while (i < samples.len) { + const n = @min(chunk.len, samples.len - i); + for (0..n) |k| { + const clamped = std.math.clamp(samples[i + k], -1.0, 1.0); + chunk[k] = @intFromFloat(clamped * 32767.0); + } + try file.writeStreamingAll(compat.io(), std.mem.sliceAsBytes(chunk[0..n])); + i += n; + } +} + +fn ensureDir(path: []const u8) !void { + std.debug.assert(std.fs.path.isAbsolute(path)); + std.Io.Dir.cwd().createDirPath(compat.io(), path) catch |err| switch (err) { + error.PathAlreadyExists => return, + else => return err, + }; +} + +fn formatSnippetId(allocator: std.mem.Allocator, ms: i64) ![]u8 { + const secs: u64 = @intCast(@divFloor(ms, 1000)); + const millis: u64 = @intCast(@mod(ms, 1000)); + const epoch_seconds = std.time.epoch.EpochSeconds{ .secs = secs }; + const day = epoch_seconds.getEpochDay(); + const year_day = day.calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const ds = epoch_seconds.getDaySeconds(); + return std.fmt.allocPrint(allocator, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}{d:0>3}Z", .{ + year_day.year, + @intFromEnum(month_day.month), + month_day.day_index + 1, + ds.getHoursIntoDay(), + ds.getMinutesIntoHour(), + ds.getSecondsIntoMinute(), + millis, + }); +} + +fn promptLine(allocator: std.mem.Allocator, prompt: []const u8) ![]u8 { + const stdout = std.Io.File.stdout(); + try stdout.writeStreamingAll(compat.io(), prompt); + + const stdin = std.Io.File.stdin(); + var io_buf: [4096]u8 = undefined; + var reader = stdin.readerStreaming(compat.io(), &io_buf); + const line = reader.interface.takeDelimiterExclusive('\n') catch |err| switch (err) { + error.EndOfStream => return error.NoInput, + else => return err, + }; + return allocator.dupe(u8, std.mem.trim(u8, line, " \t\r")); +} + +fn promptYesNo(prompt: []const u8, default_yes: bool) !bool { + var tiny_alloc_buf: [64]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&tiny_alloc_buf); + const reply = try promptLine(fba.allocator(), prompt); + if (reply.len == 0) return default_yes; + return reply[0] == 'y' or reply[0] == 'Y'; +} + +fn recordFixed(audio: *AudioCapture, duration_secs: u64) !void { + const total_ns = duration_secs * std.time.ns_per_s; + const tick_ns: u64 = 100 * std.time.ns_per_ms; + var elapsed_ns: u64 = 0; + while (elapsed_ns < total_ns) { + compat.sleepNanoseconds(tick_ns); + elapsed_ns += tick_ns; + printLevel(audio); + } + std.debug.print("\n", .{}); +} + +pub fn recordUntilQ(audio: *AudioCapture) !void { + const stdin_fd: c_int = @intCast(std.Io.File.stdin().handle); + const flags = fcntl(stdin_fd, F_GETFL); + _ = fcntl(stdin_fd, F_SETFL, flags | O_NONBLOCK); + defer _ = fcntl(stdin_fd, F_SETFL, flags); + + var stop = false; + while (!stop) { + compat.sleepNanoseconds(100 * std.time.ns_per_ms); + printLevel(audio); + + var buf: [16]u8 = undefined; + const n = read_c(stdin_fd, &buf, buf.len); + if (n > 0) { + for (buf[0..@intCast(n)]) |byte| { + if (byte == 'q' or byte == 'Q') { + stop = true; + break; + } + } + } + } + std.debug.print("\n", .{}); +} + +fn printLevel(audio: *AudioCapture) void { + const level = audio.getAudioLevel(); + const bar_len: usize = @intFromFloat(@min(level * 200.0, 40.0)); + var bar: [40]u8 = [_]u8{'.'} ** 40; + for (0..bar_len) |i| bar[i] = '#'; + const sample_count = audio.getSampleCount(); + std.debug.print("\r level [{s}] {d:.4} samples={d} ", .{ &bar, level, sample_count }); +} + +fn printUsage() void { + const usage = + \\Usage: bobrwhisper-cli snippet [options] + \\ + \\Records a single audio snippet with end-user metadata for the + \\test corpus. Does NOT transcribe. + \\ + \\Options: + \\ --label Ground truth label (interactive if omitted) + \\ --whisper Mark snippet as whispered audio + \\ --no-whisper Mark snippet as normal voice + \\ --duration Record fixed duration; otherwise stop on 'q' + \\ --out Output directory (default: ~/.bobrwhisper/snippets) + \\ --device Override auto-detected device label + \\ --device-kind internal | bluetooth | usb | unknown + \\ --tag Add a tag (repeatable) + \\ + \\Saves /.wav (16 kHz mono 16-bit PCM) and /.json. + \\ + ; + std.debug.print("{s}", .{usage}); +} + +extern "c" fn fcntl(fd: c_int, cmd: c_int, ...) c_int; +extern "c" fn read(fd: c_int, buf: [*]u8, count: usize) isize; +const read_c = read; +const F_GETFL = 3; +const F_SETFL = 4; +const O_NONBLOCK = 0x0004; // macOS + +test "parseArgs basic" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ "--label", "hello", "--whisper", "--duration", "5" }; + const parsed = try parseArgs(std.testing.allocator, &args, &tags); + try std.testing.expectEqualStrings("hello", parsed.label.?); + try std.testing.expectEqual(true, parsed.whisper.?); + try std.testing.expectEqual(@as(u64, 5), parsed.duration_secs.?); +} + +test "parseArgs no-whisper default" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ "--label", "x", "--no-whisper" }; + const parsed = try parseArgs(std.testing.allocator, &args, &tags); + try std.testing.expectEqual(false, parsed.whisper.?); +} + +test "parseArgs conflicting flags" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ "--whisper", "--no-whisper" }; + try std.testing.expectError(error.ConflictingFlags, parseArgs(std.testing.allocator, &args, &tags)); +} + +test "parseArgs device + tags" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ + "--label", "x", + "--device", "AirPods Pro", + "--device-kind", "bluetooth", + "--tag", "whispered", + "--tag", "technical", + }; + const parsed = try parseArgs(std.testing.allocator, &args, &tags); + try std.testing.expectEqualStrings("AirPods Pro", parsed.device.?); + try std.testing.expectEqual(AudioDevice.Kind.bluetooth, parsed.device_kind.?); + try std.testing.expectEqual(@as(usize, 2), parsed.tags.len); + try std.testing.expectEqualStrings("whispered", parsed.tags[0]); + try std.testing.expectEqualStrings("technical", parsed.tags[1]); +} + +test "parseArgs invalid device kind" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ "--device-kind", "carrier-pigeon" }; + try std.testing.expectError(error.InvalidDeviceKind, parseArgs(std.testing.allocator, &args, &tags)); +} + +test "formatSnippetId is sortable" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = try formatSnippetId(arena.allocator(), 1_700_000_000_000); + const b = try formatSnippetId(arena.allocator(), 1_700_000_001_500); + try std.testing.expect(std.mem.lessThan(u8, a, b)); + try std.testing.expectEqual(@as(usize, 19), a.len); // YYYYMMDDTHHMMSSmmmZ +} diff --git a/src/tune.zig b/src/tune.zig new file mode 100644 index 0000000..41fafd6 --- /dev/null +++ b/src/tune.zig @@ -0,0 +1,723 @@ +//! Run snippets recorded via `snippet` against the Zig core under several +//! VAD configurations and report normalized word-error rate (WER) vs. the +//! ground truth label. +//! +//! Loads each .json + .wav pair from a directory, sweeps a small set +//! of VAD presets, and prints a per-snippet results table plus a per-preset +//! summary so the operator can pick or refine a config. + +const std = @import("std"); +const builtin = @import("builtin"); +const compat = @import("compat.zig"); +const cli = @import("cli.zig"); +const asr = @import("asr"); + +const WhisperCppAdapter = asr.WhisperCppAdapter; +const WhisperModel = cli.WhisperModel; + +pub const Preset = struct { + name: []const u8, + enabled: bool = true, + threshold: f32 = 0.5, + min_speech_ms: i32 = 250, + min_silence_ms: i32 = 100, + speech_pad_ms: i32 = 30, +}; + +/// Five VAD presets covering the practical sensitivity range. `no-vad` is the +/// honest baseline; `default` matches whisper.cpp's stock VAD; `whisper` +/// matches the existing `--whisper-mode` defaults; `low-thresh` is what we +/// reach for when whispered speech is being chopped; `balanced` sits in the +/// middle in case neither extreme wins overall. +const default_presets = [_]Preset{ + .{ .name = "no-vad", .enabled = false }, + .{ .name = "default", .threshold = 0.5, .min_speech_ms = 250, .min_silence_ms = 100, .speech_pad_ms = 30 }, + .{ .name = "whisper", .threshold = 0.3, .min_speech_ms = 150, .min_silence_ms = 200, .speech_pad_ms = 100 }, + .{ .name = "low-thresh", .threshold = 0.2, .min_speech_ms = 100, .min_silence_ms = 250, .speech_pad_ms = 150 }, + .{ .name = "balanced", .threshold = 0.4, .min_speech_ms = 200, .min_silence_ms = 150, .speech_pad_ms = 60 }, +}; + +const Snippet = struct { + id: []const u8, + label: []const u8, + whisper: bool, + audio_file: []const u8, + duration_seconds: f64, + sample_rate: u32, + channels: u32, + num_samples: usize, + noise_floor_rms: f32, + created_at_unix_ms: i64, + // v2 fields. Optional with null defaults so older snippets still parse. + device: ?[]const u8 = null, + device_kind: ?[]const u8 = null, + tags: ?[]const []const u8 = null, +}; + +const Result = struct { + snippet_id: []const u8, + label: []const u8, + whispered: bool, + preset: []const u8, + transcript: []const u8, + wer: f32, + device: ?[]const u8, + device_kind: ?[]const u8, + tags: []const []const u8, +}; + +const GroupBy = enum { + none, + mode, + device, + device_kind, + tag, + + fn fromString(s: []const u8) ?GroupBy { + return std.meta.stringToEnum(GroupBy, s); + } +}; + +const GainMode = union(enum) { + none, + /// Peak-normalize: scale so peak amplitude is 0.95. + auto, + /// Multiply samples by a fixed factor. + fixed: f32, +}; + +const AudioStats = struct { + peak: f32, + rms: f32, +}; + +fn computeAudioStats(samples: []const f32) AudioStats { + if (samples.len == 0) return .{ .peak = 0, .rms = 0 }; + var peak: f32 = 0; + var energy: f64 = 0; + for (samples) |s| { + const a = @abs(s); + if (a > peak) peak = a; + energy += @as(f64, s) * @as(f64, s); + } + return .{ + .peak = peak, + .rms = @floatCast(@sqrt(energy / @as(f64, @floatFromInt(samples.len)))), + }; +} + +fn applyGain(samples: []f32, mode: GainMode, stats: AudioStats) f32 { + const factor: f32 = switch (mode) { + .none => 1.0, + .fixed => |f| f, + .auto => if (stats.peak > 0.001) 0.95 / stats.peak else 1.0, + }; + if (factor == 1.0) return 1.0; + for (samples) |*s| s.* = std.math.clamp(s.* * factor, -1.0, 1.0); + return factor; +} + +pub fn run(allocator: std.mem.Allocator, raw_args: []const []const u8) !void { + var snippets_dir: ?[]const u8 = null; + var selected_model: WhisperModel = .small; + var preset_filter: ?[]const u8 = null; + var gain_mode: GainMode = .none; + var group_by: GroupBy = .none; + + var i: usize = 0; + while (i < raw_args.len) : (i += 1) { + const a = raw_args[i]; + if (std.mem.eql(u8, a, "--help") or std.mem.eql(u8, a, "-h")) { + printUsage(); + return; + } else if (std.mem.eql(u8, a, "--snippets")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --snippets requires a directory\n", .{}); + return error.MissingValue; + } + snippets_dir = raw_args[i]; + } else if (std.mem.eql(u8, a, "--model")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --model requires a name\n", .{}); + return error.MissingValue; + } + selected_model = WhisperModel.fromString(raw_args[i]) orelse { + std.debug.print("tune: unknown model '{s}'\n", .{raw_args[i]}); + return error.UnknownModel; + }; + } else if (std.mem.eql(u8, a, "--preset")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --preset requires a name\n", .{}); + return error.MissingValue; + } + preset_filter = raw_args[i]; + } else if (std.mem.eql(u8, a, "--gain")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --gain requires a value (auto, none, or a number)\n", .{}); + return error.MissingValue; + } + const v = raw_args[i]; + if (std.mem.eql(u8, v, "none")) { + gain_mode = .none; + } else if (std.mem.eql(u8, v, "auto")) { + gain_mode = .auto; + } else { + const factor = std.fmt.parseFloat(f32, v) catch { + std.debug.print("tune: invalid --gain value '{s}'\n", .{v}); + return error.InvalidGain; + }; + gain_mode = .{ .fixed = factor }; + } + } else if (std.mem.eql(u8, a, "--group-by")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --group-by requires a value (none, mode, device, device_kind, tag)\n", .{}); + return error.MissingValue; + } + group_by = GroupBy.fromString(raw_args[i]) orelse { + std.debug.print("tune: invalid --group-by value '{s}'\n", .{raw_args[i]}); + return error.InvalidGroupBy; + }; + } else { + std.debug.print("tune: unknown argument '{s}'\n", .{a}); + printUsage(); + return error.UnknownArg; + } + } + + var dir_storage: ?[]u8 = null; + defer if (dir_storage) |s| allocator.free(s); + const snippets_path: []const u8 = blk: { + if (snippets_dir) |d| break :blk d; + const home = compat.getenv("HOME") orelse { + std.debug.print("tune: HOME unset; pass --snippets \n", .{}); + return error.MissingHome; + }; + dir_storage = try std.fmt.allocPrint(allocator, "{s}/.bobrwhisper/snippets", .{home}); + break :blk dir_storage.?; + }; + + // Filter presets up-front so we don't waste a model load. + var active_presets = std.ArrayListUnmanaged(Preset).empty; + defer active_presets.deinit(allocator); + if (preset_filter) |name| { + for (default_presets) |p| { + if (std.mem.eql(u8, p.name, name)) { + try active_presets.append(allocator, p); + } + } + if (active_presets.items.len == 0) { + std.debug.print("tune: preset '{s}' not found. Available: ", .{name}); + for (default_presets, 0..) |p, idx| { + if (idx > 0) std.debug.print(", ", .{}); + std.debug.print("{s}", .{p.name}); + } + std.debug.print("\n", .{}); + return error.UnknownPreset; + } + } else { + for (default_presets) |p| try active_presets.append(allocator, p); + } + + // Discover snippets. + var snippet_ids = std.ArrayListUnmanaged([]u8).empty; + defer { + for (snippet_ids.items) |id| allocator.free(id); + snippet_ids.deinit(allocator); + } + try discoverSnippets(allocator, snippets_path, &snippet_ids); + if (snippet_ids.items.len == 0) { + std.debug.print("tune: no snippets found in {s}\n", .{snippets_path}); + return error.NoSnippets; + } + std.mem.sort([]u8, snippet_ids.items, {}, lessThanString); + + std.debug.print("tune: found {d} snippet(s) in {s}\n", .{ snippet_ids.items.len, snippets_path }); + std.debug.print("tune: model = {s}, presets = ", .{@tagName(selected_model)}); + for (active_presets.items, 0..) |p, idx| { + if (idx > 0) std.debug.print(", ", .{}); + std.debug.print("{s}", .{p.name}); + } + std.debug.print("\n\n", .{}); + + // Resolve model paths (auto-download if needed). + const model_path = try selected_model.ensureDownloaded(allocator); + defer allocator.free(model_path); + const vad_path = cli.getVadModelPath(allocator); + defer if (vad_path) |p| allocator.free(p); + if (vad_path == null) { + std.debug.print("tune: WARNING - VAD model not found at ~/.bobrwhisper/models/silero-v6.2.0.bin; VAD presets will be ignored\n\n", .{}); + } + + // Load model once; mutate VAD fields per call. + var transcriber = try WhisperCppAdapter.init(allocator, .{ + .model_path = model_path, + .language = "en", + .n_threads = 4, + .vad_enabled = false, + .vad_model_path = vad_path, + }); + defer transcriber.deinit(); + + var results_arena = std.heap.ArenaAllocator.init(allocator); + defer results_arena.deinit(); + const ra = results_arena.allocator(); + + var results = std.ArrayListUnmanaged(Result).empty; + defer results.deinit(allocator); + + for (snippet_ids.items) |id| { + var snippet_arena = std.heap.ArenaAllocator.init(allocator); + defer snippet_arena.deinit(); + const sa = snippet_arena.allocator(); + + const meta = loadMeta(sa, snippets_path, id) catch |err| { + std.debug.print("tune: skipping {s}: failed to load metadata: {}\n", .{ id, err }); + continue; + }; + const wav_path = try std.fmt.allocPrint(sa, "{s}/{s}", .{ snippets_path, meta.audio_file }); + const samples = loadWav16(allocator, wav_path) catch |err| { + std.debug.print("tune: skipping {s}: failed to load wav: {}\n", .{ id, err }); + continue; + }; + defer allocator.free(samples); + + const stats = computeAudioStats(samples); + const applied_gain = applyGain(samples, gain_mode, stats); + + std.debug.print("Snippet: {s} ({s}, {d:.2}s)\n", .{ + meta.id, + if (meta.whisper) "whispered" else "normal", + meta.duration_seconds, + }); + std.debug.print(" label: \"{s}\"\n", .{meta.label}); + std.debug.print(" device: {s} ({s})\n", .{ + meta.device orelse "(unknown)", + meta.device_kind orelse "unknown", + }); + if (meta.tags) |tags| if (tags.len > 0) { + std.debug.print(" tags: ", .{}); + for (tags, 0..) |t, idx| { + if (idx > 0) std.debug.print(", ", .{}); + std.debug.print("{s}", .{t}); + } + std.debug.print("\n", .{}); + }; + std.debug.print(" audio: peak={d:.4} rms={d:.4} noise_floor={d:.4} gain={d:.2}x\n", .{ + stats.peak, + stats.rms, + meta.noise_floor_rms, + applied_gain, + }); + + // Persist the snippet metadata into the long-lived results arena so + // results survive past `snippet_arena.deinit()` and we can group/sort. + const result_id = try ra.dupe(u8, meta.id); + const result_label = try ra.dupe(u8, meta.label); + const result_device: ?[]const u8 = if (meta.device) |d| try ra.dupe(u8, d) else null; + const result_device_kind: ?[]const u8 = if (meta.device_kind) |k| try ra.dupe(u8, k) else null; + const result_tags: []const []const u8 = blk: { + const src = meta.tags orelse break :blk &.{}; + const buf = try ra.alloc([]const u8, src.len); + for (src, 0..) |t, ti| buf[ti] = try ra.dupe(u8, t); + break :blk buf; + }; + + for (active_presets.items) |preset| { + const transcript_owned = transcribeWithPreset(allocator, &transcriber, samples, preset) catch |err| { + std.debug.print(" {s:<12} ERROR: {}\n", .{ preset.name, err }); + continue; + }; + defer allocator.free(transcript_owned); + + const w = computeWer(allocator, meta.label, transcript_owned) catch |err| { + std.debug.print(" {s:<12} WER ERROR: {}\n", .{ preset.name, err }); + continue; + }; + + std.debug.print(" {s:<12} WER {d:.2} -> \"{s}\"\n", .{ preset.name, w, transcript_owned }); + + try results.append(allocator, .{ + .snippet_id = result_id, + .label = result_label, + .whispered = meta.whisper, + .preset = preset.name, // static literal from default_presets + .transcript = try ra.dupe(u8, transcript_owned), + .wer = w, + .device = result_device, + .device_kind = result_device_kind, + .tags = result_tags, + }); + } + std.debug.print("\n", .{}); + } + + try printSummary(allocator, results.items, active_presets.items, group_by); +} + +fn transcribeWithPreset( + allocator: std.mem.Allocator, + transcriber: *WhisperCppAdapter, + samples: []const f32, + preset: Preset, +) ![]u8 { + transcriber.vad_enabled = preset.enabled; + transcriber.vad_threshold = preset.threshold; + transcriber.vad_min_speech_ms = preset.min_speech_ms; + transcriber.vad_min_silence_ms = preset.min_silence_ms; + transcriber.vad_speech_pad_ms = preset.speech_pad_ms; + const raw = try transcriber.transcribe(samples); + defer allocator.free(raw); + const trimmed = std.mem.trim(u8, raw, " \t\n\r"); + return allocator.dupe(u8, trimmed); +} + +fn printSummary(allocator: std.mem.Allocator, results: []const Result, presets: []const Preset, group_by: GroupBy) !void { + if (results.len == 0) return; + + std.debug.print("=== Summary ===\n\n", .{}); + + // Best preset per snippet. + std.debug.print("Best preset per snippet (lowest WER, ties broken by preset order):\n", .{}); + var seen_ids = std.StringHashMapUnmanaged(void).empty; + defer seen_ids.deinit(allocator); + for (results) |r| { + if (seen_ids.contains(r.snippet_id)) continue; + try seen_ids.put(allocator, r.snippet_id, {}); + + var best_idx: usize = 0; + var best_wer: f32 = std.math.inf(f32); + for (results, 0..) |r2, j| { + if (!std.mem.eql(u8, r2.snippet_id, r.snippet_id)) continue; + if (r2.wer < best_wer) { + best_wer = r2.wer; + best_idx = j; + } + } + const best = results[best_idx]; + std.debug.print(" {s} ({s}): {s:<12} WER {d:.2} -> \"{s}\"\n", .{ + best.snippet_id, + if (best.whispered) "whispered" else "normal", + best.preset, + best.wer, + best.transcript, + }); + } + + std.debug.print("\nPer-preset average WER:\n", .{}); + for (presets) |p| { + var sum: f32 = 0; + var count: usize = 0; + for (results) |r| { + if (std.mem.eql(u8, r.preset, p.name)) { + sum += r.wer; + count += 1; + } + } + if (count == 0) continue; + const avg = sum / @as(f32, @floatFromInt(count)); + std.debug.print(" {s:<12} avg WER {d:.2} ({d} snippet(s))\n", .{ p.name, avg, count }); + } + + if (group_by == .none) { + std.debug.print("\n", .{}); + return; + } + + // Collect distinct group keys for the chosen axis. + var groups = std.StringArrayHashMapUnmanaged(void).empty; + defer groups.deinit(allocator); + for (results) |r| { + var keys_buf: [16][]const u8 = undefined; + const keys = resultGroupKeys(r, group_by, &keys_buf); + for (keys) |k| try groups.put(allocator, k, {}); + } + + std.debug.print("\nPer-preset average WER, grouped by {s}:\n", .{@tagName(group_by)}); + for (presets) |p| { + std.debug.print(" {s}\n", .{p.name}); + var it = groups.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + var sum: f32 = 0; + var count: usize = 0; + for (results) |r| { + if (!std.mem.eql(u8, r.preset, p.name)) continue; + var keys_buf: [16][]const u8 = undefined; + const keys = resultGroupKeys(r, group_by, &keys_buf); + var matched = false; + for (keys) |k| { + if (std.mem.eql(u8, k, key)) { + matched = true; + break; + } + } + if (!matched) continue; + sum += r.wer; + count += 1; + } + if (count == 0) continue; + const avg = sum / @as(f32, @floatFromInt(count)); + std.debug.print(" {s:<24} avg WER {d:.2} ({d})\n", .{ key, avg, count }); + } + } + std.debug.print("\n", .{}); +} + +/// Return the group keys a single `Result` belongs to for the chosen axis. +/// Up to `out_buf.len` keys are returned (16 is plenty: only `tag` can produce +/// multiple). `mode`/`device`/`device_kind` always produce exactly one key. +fn resultGroupKeys(r: Result, group_by: GroupBy, out_buf: []([]const u8)) []const []const u8 { + return switch (group_by) { + .none => out_buf[0..0], + .mode => blk: { + out_buf[0] = if (r.whispered) "whispered" else "normal"; + break :blk out_buf[0..1]; + }, + .device => blk: { + out_buf[0] = r.device orelse "(unknown)"; + break :blk out_buf[0..1]; + }, + .device_kind => blk: { + out_buf[0] = r.device_kind orelse "unknown"; + break :blk out_buf[0..1]; + }, + .tag => blk: { + if (r.tags.len == 0) { + out_buf[0] = "(untagged)"; + break :blk out_buf[0..1]; + } + const n = @min(r.tags.len, out_buf.len); + for (r.tags[0..n], 0..) |t, i| out_buf[i] = t; + break :blk out_buf[0..n]; + }, + }; +} + +fn lessThanString(_: void, a: []u8, b: []u8) bool { + return std.mem.lessThan(u8, a, b); +} + +fn discoverSnippets(allocator: std.mem.Allocator, dir_path: []const u8, out: *std.ArrayListUnmanaged([]u8)) !void { + var dir = std.Io.Dir.cwd().openDir(compat.io(), dir_path, .{ .iterate = true }) catch |err| switch (err) { + error.FileNotFound => { + std.debug.print("tune: directory not found: {s}\n", .{dir_path}); + return error.NoSnippets; + }, + else => return err, + }; + defer dir.close(compat.io()); + + var it = dir.iterate(); + + while (try it.next(compat.io())) |entry| { + if (entry.kind != .file) continue; + if (!std.mem.endsWith(u8, entry.name, ".json")) continue; + const id = entry.name[0 .. entry.name.len - ".json".len]; + try out.append(allocator, try allocator.dupe(u8, id)); + } +} + +fn loadMeta(arena: std.mem.Allocator, dir_path: []const u8, id: []const u8) !Snippet { + const json_path = try std.fmt.allocPrint(arena, "{s}/{s}.json", .{ dir_path, id }); + const file = try std.Io.Dir.cwd().openFile(compat.io(), json_path, .{}); + defer file.close(compat.io()); + const stat = try file.stat(compat.io()); + const buf = try arena.alloc(u8, stat.size); + var io_buf: [4096]u8 = undefined; + var reader = file.readerStreaming(compat.io(), &io_buf); + try reader.interface.readSliceAll(buf); + return std.json.parseFromSliceLeaky(Snippet, arena, buf, .{ .ignore_unknown_fields = true }); +} + +/// Read a 16-bit PCM mono WAV at any sample rate; resample to 16 kHz if needed. +fn loadWav16(allocator: std.mem.Allocator, path: []const u8) ![]f32 { + const file = try std.Io.Dir.cwd().openFile(compat.io(), path, .{}); + defer file.close(compat.io()); + + var header: [44]u8 = undefined; + var io_buf: [4096]u8 = undefined; + var reader = file.readerStreaming(compat.io(), &io_buf); + try reader.interface.readSliceAll(&header); + if (!std.mem.eql(u8, header[0..4], "RIFF") or !std.mem.eql(u8, header[8..12], "WAVE")) { + return error.InvalidWavFile; + } + const channels: u16 = std.mem.readInt(u16, header[22..24], .little); + const sample_rate: u32 = std.mem.readInt(u32, header[24..28], .little); + const bits_per_sample: u16 = std.mem.readInt(u16, header[34..36], .little); + if (bits_per_sample != 16) return error.UnsupportedBitDepth; + + const data_size: u32 = std.mem.readInt(u32, header[40..44], .little); + const num_samples = data_size / (@as(u32, channels) * 2); + const frame_bytes = @as(usize, channels) * 2; + + const raw = try allocator.alloc(u8, data_size); + defer allocator.free(raw); + try reader.interface.readSliceAll(raw); + + var samples = try allocator.alloc(f32, num_samples); + errdefer allocator.free(samples); + for (0..num_samples) |i| { + const offset = i * frame_bytes; + const sample_i16 = std.mem.readInt(i16, raw[offset..][0..2], .little); + samples[i] = @as(f32, @floatFromInt(sample_i16)) / 32768.0; + } + + if (sample_rate != 16000) { + const AudioCapture = @import("audio/AudioCapture.zig"); + const resampled = try AudioCapture.resample(allocator, samples, @floatFromInt(sample_rate), 16000.0); + allocator.free(samples); + return resampled; + } + return samples; +} + +/// Lower-case, strip ASCII punctuation, collapse whitespace. +fn normalize(allocator: std.mem.Allocator, text: []const u8) ![]u8 { + var out = try allocator.alloc(u8, text.len); + errdefer allocator.free(out); + var n: usize = 0; + var prev_space = true; + for (text) |c| { + const lower = std.ascii.toLower(c); + const is_space = std.ascii.isWhitespace(lower); + const is_punct = isPunct(lower); + if (is_punct) { + if (!prev_space) { + out[n] = ' '; + n += 1; + prev_space = true; + } + continue; + } + if (is_space) { + if (!prev_space) { + out[n] = ' '; + n += 1; + prev_space = true; + } + continue; + } + out[n] = lower; + n += 1; + prev_space = false; + } + while (n > 0 and out[n - 1] == ' ') n -= 1; + return allocator.realloc(out, n); +} + +fn isPunct(c: u8) bool { + return switch (c) { + '.', ',', '?', '!', ';', ':', '"', '\'', '(', ')', '[', ']', '{', '}', '-' => true, + else => false, + }; +} + +/// Word-level WER: edit_distance(ref_words, hyp_words) / max(1, ref_words.len). +fn computeWer(allocator: std.mem.Allocator, reference: []const u8, hypothesis: []const u8) !f32 { + const ref = try normalize(allocator, reference); + defer allocator.free(ref); + const hyp = try normalize(allocator, hypothesis); + defer allocator.free(hyp); + + var ref_words = std.ArrayListUnmanaged([]const u8).empty; + defer ref_words.deinit(allocator); + var hyp_words = std.ArrayListUnmanaged([]const u8).empty; + defer hyp_words.deinit(allocator); + + var it_r = std.mem.tokenizeScalar(u8, ref, ' '); + while (it_r.next()) |w| try ref_words.append(allocator, w); + var it_h = std.mem.tokenizeScalar(u8, hyp, ' '); + while (it_h.next()) |w| try hyp_words.append(allocator, w); + + if (ref_words.items.len == 0) { + return if (hyp_words.items.len == 0) 0.0 else 1.0; + } + + const dist = try wordEditDistance(allocator, ref_words.items, hyp_words.items); + return @as(f32, @floatFromInt(dist)) / @as(f32, @floatFromInt(ref_words.items.len)); +} + +fn wordEditDistance(allocator: std.mem.Allocator, a: []const []const u8, b: []const []const u8) !usize { + const n = a.len; + const m = b.len; + if (n == 0) return m; + if (m == 0) return n; + + var prev = try allocator.alloc(usize, m + 1); + defer allocator.free(prev); + var cur = try allocator.alloc(usize, m + 1); + defer allocator.free(cur); + + for (0..m + 1) |j| prev[j] = j; + for (1..n + 1) |i| { + cur[0] = i; + for (1..m + 1) |j| { + const cost: usize = if (std.mem.eql(u8, a[i - 1], b[j - 1])) 0 else 1; + const del = prev[j] + 1; + const ins = cur[j - 1] + 1; + const sub = prev[j - 1] + cost; + cur[j] = @min(del, @min(ins, sub)); + } + const tmp = prev; + prev = cur; + cur = tmp; + } + return prev[m]; +} + +fn printUsage() void { + const usage = + \\Usage: bobrwhisper-cli tune [options] + \\ + \\Runs labeled snippets recorded with `snippet` against the Zig core + \\under several VAD presets and reports WER vs. the ground truth. + \\ + \\Does NOT modify any files; results are printed to stdout. + \\ + \\Options: + \\ --snippets Snippet directory (default: ~/.bobrwhisper/snippets) + \\ --model Whisper model (default: small) + \\ --preset Run a single preset (default: all) + \\ --gain Pre-amplify audio: none (default), auto (peak to 0.95), or a number + \\ --group-by Slice the summary by: none (default), mode, device, device_kind, tag + \\ + \\Presets: no-vad, default, whisper, low-thresh, balanced + \\ + ; + std.debug.print("{s}", .{usage}); +} + +test "normalize lower-case and punctuation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + try std.testing.expectEqualStrings("what is your name", try normalize(a, "What is your name?")); + try std.testing.expectEqualStrings("hello world", try normalize(a, " Hello, WORLD! ")); + try std.testing.expectEqualStrings("", try normalize(a, " ...?! ")); +} + +test "wer perfect match" { + const arena_alloc = std.testing.allocator; + const w = try computeWer(arena_alloc, "What is your name", "what is your name?"); + try std.testing.expectEqual(@as(f32, 0.0), w); +} + +test "wer one substitution" { + const w = try computeWer(std.testing.allocator, "what is your name", "what is the name"); + try std.testing.expectApproxEqAbs(@as(f32, 0.25), w, 0.001); +} + +test "wer empty hypothesis" { + const w = try computeWer(std.testing.allocator, "what is your name", ""); + try std.testing.expectEqual(@as(f32, 1.0), w); +} + +test "wer empty reference" { + const w = try computeWer(std.testing.allocator, "", "anything goes"); + try std.testing.expectEqual(@as(f32, 1.0), w); +} From 26ecedf2685ede92148866a7fc20398460ded642 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 5 May 2026 23:07:01 +0200 Subject: [PATCH 6/9] feat(audio): add input diagnostics Track capture-time signals needed for better transcription diagnostics and preprocessing. The audio buffer previously exposed only immutable samples and an RMS level, which was not enough to detect dead routes or normalize quiet speech. Audio capture now tracks trailing exact-zero samples, exposes a stopped-recording mutable sample view, estimates noise floor, and peak-normalizes quiet input. A small CoreAudio helper classifies the default input device for Bluetooth-specific warnings and tuning. --- src/audio/AudioCapture.zig | 111 +++++++++++++++++++++++- src/audio/AudioDevice.zig | 172 +++++++++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 src/audio/AudioDevice.zig diff --git a/src/audio/AudioCapture.zig b/src/audio/AudioCapture.zig index 0dea37a..13ee779 100644 --- a/src/audio/AudioCapture.zig +++ b/src/audio/AudioCapture.zig @@ -118,6 +118,10 @@ buffer: std.ArrayListUnmanaged(f32), mutex: SpinMutex = .{}, // RMS audio level from the most recent callback buffer, updated under mutex audio_level: f32 = 0, +// Stuck-mic detection: count of trailing samples whose CoreAudio callback +// delivered an exactly-zero buffer. Reset to 0 the moment any non-zero sample +// arrives. Exact-zero input is a strong signal for dead audio routes. +consecutive_zero_samples: usize = 0, pub const Config = struct { sample_rate: f64 = 16000.0, @@ -155,6 +159,7 @@ pub fn start(self: *AudioCapture) !void { if (self.is_recording) return; self.buffer.clearRetainingCapacity(); + self.consecutive_zero_samples = 0; try self.buffer.ensureTotalCapacity(self.allocator, 16000 * 30); if (builtin.os.tag == .macos) { @@ -188,6 +193,14 @@ pub fn getSamples(self: *AudioCapture) []const f32 { return self.buffer.items; } +/// Mutable view into the captured samples. Same lifetime constraints as `getSamples`; +/// only safe when recording is stopped. Used by callers that want to apply in-place +/// preprocessing such as peak normalization before transcription. +pub fn getSamplesMut(self: *AudioCapture) []f32 { + std.debug.assert(!self.is_recording); + return self.buffer.items; +} + pub fn copySamples(self: *AudioCapture, allocator: std.mem.Allocator) ![]f32 { self.mutex.lock(); defer self.mutex.unlock(); @@ -219,6 +232,15 @@ pub fn getSampleCount(self: *AudioCapture) usize { return self.buffer.items.len; } +/// Trailing run of bit-exact-zero samples reported by CoreAudio. Crosses the +/// stuck-mic threshold (~5 s = 80000 samples at 16 kHz) when the input device +/// stops delivering real audio. Resets to 0 on any non-zero sample. +pub fn getConsecutiveZeroSamples(self: *AudioCapture) usize { + self.mutex.lock(); + defer self.mutex.unlock(); + return self.consecutive_zero_samples; +} + pub fn clearBuffer(self: *AudioCapture) void { self.mutex.lock(); defer self.mutex.unlock(); @@ -315,12 +337,22 @@ fn audioInputCallback( std.log.err("Failed to append audio samples", .{}); }; - // Compute RMS audio level from this callback buffer + // Single pass: RMS energy + exact-zero detection. The zero check + // surfaces a stuck microphone (dead route, denied permission at the + // system level, suspended driver). RMS alone is too noisy: a quiet mic + // still produces sub-1e-6 jitter, but a dead route is bit-exact zero. var energy: f32 = 0; + var any_nonzero = false; for (samples[0..sample_count]) |s| { energy += s * s; + if (s != 0) any_nonzero = true; } self.audio_level = @sqrt(energy / @as(f32, @floatFromInt(sample_count))); + if (any_nonzero) { + self.consecutive_zero_samples = 0; + } else { + self.consecutive_zero_samples +|= sample_count; + } } _ = c.AudioQueueEnqueueBuffer(queue, buffer, 0, null); @@ -371,6 +403,47 @@ pub fn trimSilenceBounds(samples: []const f32, threshold: f32) TrimBounds { return .{ .start = start_idx, .end = end_idx }; } +/// Estimate ambient noise floor from the first ~0.5s of audio. +/// Returns the RMS energy of a leading window, useful for deriving +/// an adaptive silence-trim threshold (e.g. 3× noise floor). +pub fn computeNoiseFloor(samples: []const f32) f32 { + if (samples.len == 0) return 0; + + const window: usize = @min(8000, samples.len); + var energy: f32 = 0; + for (samples[0..window]) |s| { + energy += s * s; + } + return @sqrt(energy / @as(f32, @floatFromInt(window))); +} + +/// In-place peak normalization. Scales `samples` so the largest absolute +/// amplitude becomes `target_peak` (typically 0.95 to avoid hard clipping). +/// Returns the gain factor that was applied. Only ever scales up: if the +/// input is already at or above `target_peak` the buffer is unchanged. +/// Silent buffers (peak < 1e-4) are also left alone to avoid amplifying noise. +/// +/// Whispered speech routinely has peaks of ~0.25 vs. ~0.6 for normal voice, +/// which produced a 4–7× WER improvement in the `tune` experiments. Applying +/// this before trimming and transcription is safe for normal voice (the gain +/// factor stays near 1.0) but materially helps quiet audio. +pub fn peakNormalize(samples: []f32, target_peak: f32) f32 { + std.debug.assert(target_peak > 0.0); + std.debug.assert(target_peak <= 1.0); + if (samples.len == 0) return 1.0; + + var peak: f32 = 0; + for (samples) |s| { + const a = @abs(s); + if (a > peak) peak = a; + } + if (peak < 1e-4 or peak >= target_peak) return 1.0; + + const gain = target_peak / peak; + for (samples) |*s| s.* *= gain; + return gain; +} + pub fn resample(allocator: std.mem.Allocator, samples: []const f32, from_rate: f64, to_rate: f64) ![]f32 { if (from_rate == to_rate) { return allocator.dupe(f32, samples); @@ -404,3 +477,39 @@ test "voice activity detection" { try std.testing.expect(!detectVoiceActivity(&silence, 0.01)); try std.testing.expect(detectVoiceActivity(&voice, 0.01)); } + +test "compute noise floor" { + const silence = [_]f32{0.0} ** 100; + const noise = [_]f32{0.1} ** 100; + + try std.testing.expectEqual(@as(f32, 0), computeNoiseFloor(&silence)); + try std.testing.expect(computeNoiseFloor(&noise) > 0.09); + try std.testing.expect(computeNoiseFloor(&noise) < 0.11); + try std.testing.expectEqual(@as(f32, 0), computeNoiseFloor(&[_]f32{})); +} + +test "peakNormalize boosts quiet audio" { + var samples = [_]f32{ 0.1, -0.2, 0.05, -0.1 }; + const gain = peakNormalize(&samples, 0.95); + try std.testing.expectApproxEqAbs(@as(f32, 4.75), gain, 0.001); + try std.testing.expectApproxEqAbs(@as(f32, -0.95), samples[1], 0.001); +} + +test "peakNormalize leaves loud audio unchanged" { + var samples = [_]f32{ 0.5, -0.95 }; + const gain = peakNormalize(&samples, 0.95); + try std.testing.expectEqual(@as(f32, 1.0), gain); + try std.testing.expectEqual(@as(f32, 0.5), samples[0]); +} + +test "peakNormalize handles silence" { + var silence = [_]f32{0.0} ** 8; + const gain = peakNormalize(&silence, 0.95); + try std.testing.expectEqual(@as(f32, 1.0), gain); +} + +test "peakNormalize empty buffer" { + var empty: [0]f32 = .{}; + const gain = peakNormalize(&empty, 0.95); + try std.testing.expectEqual(@as(f32, 1.0), gain); +} diff --git a/src/audio/AudioDevice.zig b/src/audio/AudioDevice.zig new file mode 100644 index 0000000..7840ab5 --- /dev/null +++ b/src/audio/AudioDevice.zig @@ -0,0 +1,172 @@ +//! macOS CoreAudio default-input-device introspection. +//! +//! Tags each recorded snippet with the audio source (built-in mic, AirPods, +//! USB interface, ...) so the test corpus can be sliced by hardware. Keeps +//! the CoreAudio surface small: three property reads on the system + device +//! objects, no full SDK translation. + +const std = @import("std"); +const builtin = @import("builtin"); + +pub const Kind = enum { + internal, + bluetooth, + usb, + unknown, + + pub fn fromString(s: []const u8) ?Kind { + return std.meta.stringToEnum(Kind, s); + } +}; + +pub const Info = struct { + /// Heap-allocated. Caller owns; free with `deinit`. + name: []u8, + kind: Kind, + + pub fn deinit(self: Info, allocator: std.mem.Allocator) void { + allocator.free(self.name); + } +}; + +const c = if (builtin.os.tag == .macos) MacCoreAudio else struct {}; + +const MacCoreAudio = struct { + pub const OSStatus = i32; + pub const UInt32 = u32; + pub const AudioObjectID = UInt32; + + pub const AudioObjectPropertyAddress = extern struct { + mSelector: UInt32, + mScope: UInt32, + mElement: UInt32, + }; + + pub const CFStringRef = ?*opaque {}; + pub const CFTypeRef = ?*anyopaque; + + pub extern "c" fn AudioObjectGetPropertyData( + inObjectID: AudioObjectID, + inAddress: *const AudioObjectPropertyAddress, + inQualifierDataSize: UInt32, + inQualifierData: ?*const anyopaque, + ioDataSize: *UInt32, + outData: *anyopaque, + ) OSStatus; + + pub extern "c" fn CFStringGetCString( + theString: CFStringRef, + buffer: [*]u8, + bufferSize: c_long, + encoding: u32, + ) i32; + + pub extern "c" fn CFRelease(cf: CFTypeRef) void; + + pub const kAudioObjectSystemObject: AudioObjectID = 1; + pub const kAudioObjectPropertyElementMain: UInt32 = 0; + pub const kCFStringEncodingUTF8: u32 = 0x08000100; +}; + +inline fn fourCC(comptime s: *const [4]u8) u32 { + return (@as(u32, s[0]) << 24) | (@as(u32, s[1]) << 16) | (@as(u32, s[2]) << 8) | @as(u32, s[3]); +} + +/// Best-effort default-input device label. Returns null when we are not on +/// macOS or CoreAudio refuses to cooperate (e.g. no input device, headless +/// CI). Caller owns the returned `Info.name`. +pub fn detectDefaultInput(allocator: std.mem.Allocator) ?Info { + if (comptime builtin.os.tag != .macos) return null; + return detectMacos(allocator) catch null; +} + +fn detectMacos(allocator: std.mem.Allocator) !Info { + const device_id = try defaultInputDeviceId(); + const name = try deviceName(allocator, device_id); + errdefer allocator.free(name); + const kind = transportKind(device_id); + return .{ .name = name, .kind = kind }; +} + +fn defaultInputDeviceId() !c.AudioObjectID { + const addr = c.AudioObjectPropertyAddress{ + .mSelector = comptime fourCC("dIn "), + .mScope = comptime fourCC("glob"), + .mElement = c.kAudioObjectPropertyElementMain, + }; + var device_id: c.AudioObjectID = 0; + var size: c.UInt32 = @sizeOf(c.AudioObjectID); + const status = c.AudioObjectGetPropertyData( + c.kAudioObjectSystemObject, + &addr, + 0, + null, + &size, + @ptrCast(&device_id), + ); + if (status != 0 or device_id == 0) return error.NoDefaultInput; + return device_id; +} + +fn deviceName(allocator: std.mem.Allocator, device_id: c.AudioObjectID) ![]u8 { + const addr = c.AudioObjectPropertyAddress{ + .mSelector = comptime fourCC("lnam"), + .mScope = comptime fourCC("glob"), + .mElement = c.kAudioObjectPropertyElementMain, + }; + var name_ref: c.CFStringRef = null; + var size: c.UInt32 = @sizeOf(c.CFStringRef); + const status = c.AudioObjectGetPropertyData( + device_id, + &addr, + 0, + null, + &size, + @ptrCast(&name_ref), + ); + if (status != 0 or name_ref == null) return error.NoName; + defer c.CFRelease(name_ref); + + // Device names are short. 256 bytes covers everything Apple ships and any + // reasonable third-party label without bringing CFStringGetMaximumSizeForEncoding in. + var buf: [256]u8 = undefined; + if (c.CFStringGetCString(name_ref, &buf, buf.len, c.kCFStringEncodingUTF8) == 0) { + return error.NameDecode; + } + const slice = std.mem.sliceTo(&buf, 0); + return allocator.dupe(u8, slice); +} + +fn transportKind(device_id: c.AudioObjectID) Kind { + const addr = c.AudioObjectPropertyAddress{ + .mSelector = comptime fourCC("tran"), + .mScope = comptime fourCC("glob"), + .mElement = c.kAudioObjectPropertyElementMain, + }; + var transport: c.UInt32 = 0; + var size: c.UInt32 = @sizeOf(c.UInt32); + const status = c.AudioObjectGetPropertyData( + device_id, + &addr, + 0, + null, + &size, + @ptrCast(&transport), + ); + if (status != 0) return .unknown; + + return switch (transport) { + comptime fourCC("bltn") => .internal, + comptime fourCC("blue"), comptime fourCC("blea") => .bluetooth, + comptime fourCC("usb ") => .usb, + else => .unknown, + }; +} + +test "Kind.fromString round-trip" { + try std.testing.expectEqual(Kind.internal, Kind.fromString("internal").?); + try std.testing.expectEqual(Kind.bluetooth, Kind.fromString("bluetooth").?); + try std.testing.expectEqual(Kind.usb, Kind.fromString("usb").?); + try std.testing.expectEqual(Kind.unknown, Kind.fromString("unknown").?); + try std.testing.expectEqual(@as(?Kind, null), Kind.fromString("nope")); +} From 9c357ee562bfb1a4110f318226a24ad0a3e96c33 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 5 May 2026 23:07:01 +0200 Subject: [PATCH 7/9] feat(macos): add onboarding and warnings Add first-run permission onboarding and a separate non-fatal warning surface in the macOS app. Previously permission recovery was scattered across startup behavior, and advisory conditions reused error presentation. The app now coordinates microphone and accessibility state, opens the dashboard when setup is needed, and lets users re-run onboarding from settings or the menu. Warning text is presented separately from fatal errors so recording status is not overwritten. --- macos/BobrWhisper.xcodeproj/project.pbxproj | 8 + macos/BobrWhisper/App.swift | 27 ++ macos/BobrWhisper/AppDelegate.swift | 58 +++- macos/BobrWhisper/AppState.swift | 56 +++- .../BobrWhisper/PermissionsCoordinator.swift | 164 +++++++++++ macos/BobrWhisper/Views/MainWindowView.swift | 79 ++++-- macos/BobrWhisper/Views/MenuBarView.swift | 69 ++++- macos/BobrWhisper/Views/OnboardingView.swift | 262 +++++++++++++++++ macos/BobrWhisper/Views/SettingsView.swift | 263 +++++++++++++++--- .../Views/TranscriptOverlayView.swift | 42 ++- 10 files changed, 953 insertions(+), 75 deletions(-) create mode 100644 macos/BobrWhisper/PermissionsCoordinator.swift create mode 100644 macos/BobrWhisper/Views/OnboardingView.swift diff --git a/macos/BobrWhisper.xcodeproj/project.pbxproj b/macos/BobrWhisper.xcodeproj/project.pbxproj index f3700cb..71c1831 100644 --- a/macos/BobrWhisper.xcodeproj/project.pbxproj +++ b/macos/BobrWhisper.xcodeproj/project.pbxproj @@ -14,6 +14,8 @@ A100000000000005 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000005 /* SettingsView.swift */; }; A100000000000006 /* TranscriptOverlayView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000006 /* TranscriptOverlayView.swift */; }; A100000000000007 /* MainWindowView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000007 /* MainWindowView.swift */; }; + A100000000000008 /* PermissionsCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000008 /* PermissionsCoordinator.swift */; }; + A100000000000009 /* OnboardingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000009 /* OnboardingView.swift */; }; A100000000000010 /* BobrWhisperKit.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = A200000000000010 /* BobrWhisperKit.xcframework */; }; A100000000000011 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = A200000000000011 /* Assets.xcassets */; }; A100000000000012 /* silero-v6.2.0.bin in Resources */ = {isa = PBXBuildFile; fileRef = A200000000000012 /* silero-v6.2.0.bin */; }; @@ -27,6 +29,8 @@ A200000000000005 /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = ""; }; A200000000000006 /* TranscriptOverlayView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranscriptOverlayView.swift; sourceTree = ""; }; A200000000000007 /* MainWindowView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainWindowView.swift; sourceTree = ""; }; + A200000000000008 /* PermissionsCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PermissionsCoordinator.swift; sourceTree = ""; }; + A200000000000009 /* OnboardingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OnboardingView.swift; sourceTree = ""; }; A200000000000010 /* BobrWhisperKit.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = BobrWhisperKit.xcframework; sourceTree = ""; }; A200000000000011 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; A200000000000012 /* silero-v6.2.0.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = "silero-v6.2.0.bin"; path = "../../resources/silero-v6.2.0.bin"; sourceTree = ""; }; @@ -61,6 +65,7 @@ A200000000000001 /* App.swift */, A200000000000002 /* AppDelegate.swift */, A200000000000003 /* AppState.swift */, + A200000000000008 /* PermissionsCoordinator.swift */, A400000000000005 /* Views */, A200000000000011 /* Assets.xcassets */, A200000000000021 /* BobrWhisper.entitlements */, @@ -90,6 +95,7 @@ children = ( A200000000000007 /* MainWindowView.swift */, A200000000000004 /* MenuBarView.swift */, + A200000000000009 /* OnboardingView.swift */, A200000000000005 /* SettingsView.swift */, A200000000000006 /* TranscriptOverlayView.swift */, ); @@ -169,8 +175,10 @@ A100000000000001 /* App.swift in Sources */, A100000000000002 /* AppDelegate.swift in Sources */, A100000000000003 /* AppState.swift in Sources */, + A100000000000008 /* PermissionsCoordinator.swift in Sources */, A100000000000007 /* MainWindowView.swift in Sources */, A100000000000004 /* MenuBarView.swift in Sources */, + A100000000000009 /* OnboardingView.swift in Sources */, A100000000000005 /* SettingsView.swift in Sources */, A100000000000006 /* TranscriptOverlayView.swift in Sources */, ); diff --git a/macos/BobrWhisper/App.swift b/macos/BobrWhisper/App.swift index 915fc7c..f9cb35f 100644 --- a/macos/BobrWhisper/App.swift +++ b/macos/BobrWhisper/App.swift @@ -9,18 +9,45 @@ struct BobrWhisperApp: App { WindowGroup("BobrWhisper", id: "dashboard") { MainWindowView() .environmentObject(appDelegate.appState) + .environmentObject(appDelegate.permissions) } Settings { SettingsView() .environmentObject(appDelegate.appState) + .environmentObject(appDelegate.permissions) } MenuBarExtra { MenuBarView() .environmentObject(appDelegate.appState) + .environmentObject(appDelegate.permissions) } label: { + // The label is the always-alive part of the MenuBarExtra scene — + // it's instantiated on launch and kept alive for the app lifetime. + // Mounting `DashboardOpener` as an overlay here gives us a SwiftUI + // surface with `\.openWindow` that AppDelegate can drive via + // NotificationCenter, without depending on the menu actually + // being opened by the user. Image(systemName: appDelegate.appState.statusIcon) + .overlay(DashboardOpener().allowsHitTesting(false)) } } } + +/// SwiftUI bridge that AppKit code reaches via NotificationCenter to programmatically +/// open the dashboard `WindowGroup`. Mounted as a `.background` modifier in any +/// always-alive view (the menubar content) — its sole job is to translate the +/// `.bobrWhisperOpenDashboard` notification into a call on `\.openWindow`, +/// which is the only supported way to open a SwiftUI window scene. +struct DashboardOpener: View { + @Environment(\.openWindow) private var openWindow + + var body: some View { + Color.clear + .frame(width: 0, height: 0) + .onReceive(NotificationCenter.default.publisher(for: .bobrWhisperOpenDashboard)) { _ in + openWindow(id: "dashboard") + } + } +} diff --git a/macos/BobrWhisper/AppDelegate.swift b/macos/BobrWhisper/AppDelegate.swift index a9395ab..1073173 100644 --- a/macos/BobrWhisper/AppDelegate.swift +++ b/macos/BobrWhisper/AppDelegate.swift @@ -5,6 +5,7 @@ import BobrWhisperKit class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { let appState = AppState() + let permissions = PermissionsCoordinator() private var hotkeyMonitor: Any? func applicationDidFinishLaunching(_ notification: Notification) { @@ -20,9 +21,16 @@ class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { // Register global hotkey (Fn key or custom) setupHotkey() - - // Request accessibility permissions if needed - requestAccessibilityPermissions() + + // Onboarding lives inside the dashboard window now. If permissions + // need attention on launch, flip the coordinator into onboarding mode + // and ask SwiftUI to open the dashboard so the user actually sees it. + if permissions.shouldAutoShowOnLaunch { + permissions.beginOnboarding() + DispatchQueue.main.async { [weak self] in + self?.openDashboardWindow() + } + } } func applicationWillTerminate(_ notification: Notification) { @@ -99,12 +107,44 @@ class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { } } - private func requestAccessibilityPermissions() { - let options: NSDictionary = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] - let trusted = AXIsProcessTrustedWithOptions(options) - - if !trusted { - print("Accessibility permissions required for global hotkey") + /// Open (or focus) the dashboard `WindowGroup`. Used on launch when + /// onboarding needs to run, and from the menubar's "Open Dashboard" entry. + /// Walks the existing window list first to focus an already-open dashboard + /// instead of leaning on SwiftUI to create a duplicate. + func openDashboardWindow() { + permissions.refresh() + NSApp.activate(ignoringOtherApps: true) + + for window in NSApplication.shared.windows + where window.identifier?.rawValue == "dashboard" { + window.makeKeyAndOrderFront(nil) + return } + + // SwiftUI registers the WindowGroup with id "dashboard" via a + // discoverable URL handler under the hood. The standard + // `NSWorkspace.OpenConfiguration` route does not work for in-process + // scenes, so post a notification that the SwiftUI commands listener + // (in `App.swift`) acts on. + NotificationCenter.default.post( + name: .bobrWhisperOpenDashboard, + object: nil + ) + } + + /// Reset the "I've completed onboarding" flag and re-show the dashboard + /// in onboarding mode. Bound to the menubar / settings re-run actions. + func relaunchOnboarding() { + permissions.resetOnboarding() + permissions.beginOnboarding() + openDashboardWindow() } } + +extension Notification.Name { + /// Posted by `AppDelegate.openDashboardWindow` when a SwiftUI surface needs + /// to call `openWindow(id: "dashboard")` on its behalf. Subscribed to from + /// the App scene, which is the only place with access to the + /// `\.openWindow` environment. + static let bobrWhisperOpenDashboard = Notification.Name("bobrWhisperOpenDashboard") +} diff --git a/macos/BobrWhisper/AppState.swift b/macos/BobrWhisper/AppState.swift index 14784e4..27ad6e4 100644 --- a/macos/BobrWhisper/AppState.swift +++ b/macos/BobrWhisper/AppState.swift @@ -8,6 +8,12 @@ class AppState: ObservableObject { @Published private(set) var isRecording: Bool = false @Published private(set) var lastTranscript: String = "" @Published private(set) var errorMessage: String? + /// Non-fatal advisory text (stuck mic, Bluetooth mic, etc.). Cleared + /// automatically a few seconds after it is set so the UI doesn't have to + /// own dismissal logic. Lives separately from `errorMessage` so it does + /// not flip status to `.error`. + @Published private(set) var warningMessage: String? + private var warningClearWorkItem: DispatchWorkItem? @Published private(set) var isDownloading: Bool = false @Published var downloadProgress: Double = 0 @Published private(set) var transcriptLog: [TranscriptLogEntry] = [] @@ -156,6 +162,21 @@ class AppState: ObservableObject { appState.status = .error } } + + config.on_warning = { userdata, warning in + guard let userdata = userdata else { return } + let warningMsg: String? + if let ptr = warning.ptr, warning.len > 0 { + let data = Data(bytes: ptr, count: warning.len) + warningMsg = String(data: data, encoding: .utf8) + } else { + warningMsg = nil + } + let appState = Unmanaged.fromOpaque(userdata).takeUnretainedValue() + DispatchQueue.main.async { + appState.presentWarning(warningMsg) + } + } let vadModelPath = Bundle.main.path(forResource: "silero-v6.2.0", ofType: "bin") @@ -203,6 +224,32 @@ class AppState: ObservableObject { if let ptr = vadModelPathCString { free(ptr); vadModelPathCString = nil } } + /// Display a non-fatal advisory and auto-clear it after a short delay so + /// the UI doesn't have to track dismissal. Re-firing while a warning is + /// already on screen restarts the timer with the new text. + func presentWarning(_ message: String?) { + warningClearWorkItem?.cancel() + warningMessage = message + guard message != nil else { return } + let work = DispatchWorkItem { [weak self] in + guard let self = self else { return } + self.warningMessage = nil + self.warningClearWorkItem = nil + } + warningClearWorkItem = work + // 4.5 s sits between "long enough to read" and "out of the way before + // the next utterance lands". Matches the auto-dismiss feel of a + // short-lived toast. + DispatchQueue.main.asyncAfter(deadline: .now() + 4.5, execute: work) + } + + /// Manually dismiss the warning (used when the user taps the toast). + func dismissWarning() { + warningClearWorkItem?.cancel() + warningClearWorkItem = nil + warningMessage = nil + } + private func persistSettings() { guard let app = app else { return } @@ -415,7 +462,14 @@ class AppState: ObservableObject { func pasteToActiveApp() { copyToClipboard() - + + // Auto-paste is opt-out via Settings, but ALSO requires Accessibility + // permission. If either is missing we still copy (above) so the user + // can paste manually — silent no-op otherwise would surprise users who + // skipped Accessibility during onboarding. + let autoPasteEnabled = (UserDefaults.standard.object(forKey: "autoPaste") as? Bool) ?? true + guard autoPasteEnabled, AXIsProcessTrusted() else { return } + // Simulate Cmd+V let source = CGEventSource(stateID: .hidSystemState) let vKeyDown = CGEvent(keyboardEventSource: source, virtualKey: 0x09, keyDown: true) diff --git a/macos/BobrWhisper/PermissionsCoordinator.swift b/macos/BobrWhisper/PermissionsCoordinator.swift new file mode 100644 index 0000000..97683a6 --- /dev/null +++ b/macos/BobrWhisper/PermissionsCoordinator.swift @@ -0,0 +1,164 @@ +import AppKit +import ApplicationServices +import AVFoundation +import Combine +import Foundation + +/// Tri-state for OS-granted privacy permissions. +/// +/// `notDetermined` means the system has never prompted; this is the only state +/// from which `AVCaptureDevice.requestAccess` will trigger the OS prompt. +/// `denied` covers both user-denied and MDM-restricted; recovery in both cases +/// requires the user to flip the toggle in System Settings. +enum PermissionState: Equatable { + case granted + case denied + case notDetermined +} + +/// Tracks Microphone (mandatory) and Accessibility (auto-paste only) permissions. +/// +/// macOS does not deliver a reliable signal when the user changes a TCC entry +/// in System Settings, so we re-poll on app activation and on a coarse fallback +/// timer. This coordinator deliberately knows nothing about the Zig core or the +/// `bobrwhisper_app_t` lifecycle; it is a pure SwiftUI/AppKit concern. +final class PermissionsCoordinator: ObservableObject { + @Published private(set) var microphone: PermissionState = .notDetermined + @Published private(set) var accessibility: PermissionState = .notDetermined + + private static let onboardingCompletedKey = "hasCompletedOnboarding" + + var hasCompletedOnboarding: Bool { + UserDefaults.standard.bool(forKey: Self.onboardingCompletedKey) + } + + var micGranted: Bool { microphone == .granted } + var accessibilityGranted: Bool { accessibility == .granted } + + /// True when the dashboard should render the onboarding flow instead of + /// the transcript table. Mandatory permission missing OR the user has + /// never completed the flow. Settings surfaces optional regression + /// (accessibility only) inline rather than blocking the dashboard. + @Published var isOnboardingActive: Bool = false + + /// Whether the dashboard should auto-show on launch (because onboarding + /// needs to run). Independent of `isOnboardingActive` so we can decide + /// "open the window AND show onboarding" in one step. + var shouldAutoShowOnLaunch: Bool { + if !micGranted { return true } + return !hasCompletedOnboarding + } + + func beginOnboarding() { + isOnboardingActive = true + } + + func finishOnboarding() { + markOnboardingComplete() + isOnboardingActive = false + } + + private var pollTimer: Timer? + private var activationObserver: NSObjectProtocol? + private var workspaceObserver: NSObjectProtocol? + + init() { + refresh() + startObserving() + } + + deinit { + stopObserving() + } + + /// Re-read both permissions from the OS. Cheap; safe to call from a timer. + func refresh() { + let newMic: PermissionState + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: newMic = .granted + case .denied, .restricted: newMic = .denied + case .notDetermined: newMic = .notDetermined + @unknown default: newMic = .notDetermined + } + + let newAX: PermissionState = AXIsProcessTrusted() ? .granted : .denied + + if newMic != microphone { microphone = newMic } + if newAX != accessibility { accessibility = newAX } + } + + /// Triggers the system microphone prompt. Only effective in `notDetermined`; + /// after a denial the user has to toggle it in System Settings, so callers + /// should fall back to `openMicrophoneSettings()` in that case. + func requestMicrophone(completion: @escaping (Bool) -> Void) { + AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in + DispatchQueue.main.async { + self?.refresh() + completion(granted) + } + } + } + + func openMicrophoneSettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone") else { return } + NSWorkspace.shared.open(url) + } + + /// Accessibility cannot be prompted programmatically — Apple intentionally + /// requires a manual toggle. We deep-link to the right pane instead. + func openAccessibilitySettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") else { return } + NSWorkspace.shared.open(url) + } + + func markOnboardingComplete() { + UserDefaults.standard.set(true, forKey: Self.onboardingCompletedKey) + } + + func resetOnboarding() { + UserDefaults.standard.set(false, forKey: Self.onboardingCompletedKey) + } + + // MARK: - Observation + + private func startObserving() { + // App activation is the most common moment for the user to flip back + // from System Settings. AppKit fires this even for our own app + // returning from background. + activationObserver = NotificationCenter.default.addObserver( + forName: NSApplication.didBecomeActiveNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.refresh() + } + + // Workspace activation catches the case where another app (e.g., System + // Settings itself) becomes active and back without us regaining focus. + workspaceObserver = NSWorkspace.shared.notificationCenter.addObserver( + forName: NSWorkspace.didActivateApplicationNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.refresh() + } + + // TCC changes don't always fire a usable notification. A 2 s coarse + // poll covers the gap; cost is negligible since `AXIsProcessTrusted` + // and `AVCaptureDevice.authorizationStatus` are both in-process reads. + pollTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: true) { [weak self] _ in + self?.refresh() + } + } + + private func stopObserving() { + if let obs = activationObserver { + NotificationCenter.default.removeObserver(obs) + } + if let obs = workspaceObserver { + NSWorkspace.shared.notificationCenter.removeObserver(obs) + } + pollTimer?.invalidate() + pollTimer = nil + } +} diff --git a/macos/BobrWhisper/Views/MainWindowView.swift b/macos/BobrWhisper/Views/MainWindowView.swift index 8f51a37..b66b7a1 100644 --- a/macos/BobrWhisper/Views/MainWindowView.swift +++ b/macos/BobrWhisper/Views/MainWindowView.swift @@ -2,6 +2,7 @@ import SwiftUI struct MainWindowView: View { @EnvironmentObject var appState: AppState + @EnvironmentObject var permissions: PermissionsCoordinator @Environment(\.openSettings) private var openSettings private static let timeFormatter: DateFormatter = { @@ -12,6 +13,56 @@ struct MainWindowView: View { }() var body: some View { + Group { + if permissions.isOnboardingActive { + // Onboarding sits in front of the dashboard until the user + // either grants the mandatory mic permission or quits. Same + // window chrome — feels like a step in the app, not a popup. + OnboardingView() + } else { + dashboardBody + } + } + .navigationTitle("BobrWhisper") + .frame(minWidth: 940, minHeight: 620) + .toolbar { + if !permissions.isOnboardingActive { + ToolbarItemGroup(placement: .primaryAction) { + Button(appState.isRecording ? "Stop Recording" : "Start Recording") { + if appState.isRecording { + appState.stopRecording() + } else { + appState.startRecording() + } + } + + Button("Copy Latest") { + appState.copyToClipboard() + } + .disabled(appState.transcriptLog.isEmpty) + + Button("Clear Log") { + appState.clearTranscriptLog() + } + .disabled(appState.transcriptLog.isEmpty) + + Button("Settings") { + openSettings() + } + } + } + } + .onAppear { + // If the dashboard is opened while permissions are still missing + // (e.g. user revoked mic mid-session), drop straight back into + // onboarding instead of showing a broken table. + if permissions.shouldAutoShowOnLaunch { + permissions.beginOnboarding() + } + } + } + + private var dashboardBody: some View { VStack(spacing: 0) { dashboardHeader @@ -36,33 +87,6 @@ struct MainWindowView: View { .background(Color(nsColor: .textBackgroundColor)) } } - .navigationTitle("BobrWhisper") - .frame(minWidth: 940, minHeight: 620) - .toolbar { - ToolbarItemGroup(placement: .primaryAction) { - Button(appState.isRecording ? "Stop Recording" : "Start Recording") { - if appState.isRecording { - appState.stopRecording() - } else { - appState.startRecording() - } - } - - Button("Copy Latest") { - appState.copyToClipboard() - } - .disabled(appState.transcriptLog.isEmpty) - - Button("Clear Log") { - appState.clearTranscriptLog() - } - .disabled(appState.transcriptLog.isEmpty) - - Button("Settings") { - openSettings() - } - } - } } private var dashboardHeader: some View { @@ -186,4 +210,5 @@ struct MainWindowView: View { #Preview { MainWindowView() .environmentObject(AppState()) + .environmentObject(PermissionsCoordinator()) } diff --git a/macos/BobrWhisper/Views/MenuBarView.swift b/macos/BobrWhisper/Views/MenuBarView.swift index f6fcd1f..73b01a8 100644 --- a/macos/BobrWhisper/Views/MenuBarView.swift +++ b/macos/BobrWhisper/Views/MenuBarView.swift @@ -2,6 +2,7 @@ import SwiftUI struct MenuBarView: View { @EnvironmentObject var appState: AppState + @EnvironmentObject var permissions: PermissionsCoordinator @Environment(\.openSettings) private var openSettings @Environment(\.openWindow) private var openWindow private static let logTimestampFormatter: DateFormatter = { @@ -22,7 +23,27 @@ struct MenuBarView: View { } .padding(.horizontal) .padding(.top, 8) - + + // Non-fatal advisories (stuck mic, Bluetooth mic, ...). Auto-clears + // via AppState; tapping dismisses immediately. + if let warning = appState.warningMessage { + HStack(alignment: .top, spacing: 6) { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.yellow) + .font(.caption) + Text(warning) + .font(.caption) + .foregroundStyle(.primary) + .frame(maxWidth: .infinity, alignment: .leading) + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(Color.yellow.opacity(0.12)) + .clipShape(RoundedRectangle(cornerRadius: 6, style: .continuous)) + .padding(.horizontal, 8) + .onTapGesture { appState.dismissWarning() } + } + Divider() // Activity log @@ -99,7 +120,18 @@ struct MenuBarView: View { } .keyboardShortcut(",", modifiers: .command) .padding(.horizontal) - + + // Permission warning + re-run hook. Yellow strip mirrors the + // warning-banner pattern established for non-fatal advisories. + if !permissions.micGranted || !permissions.accessibilityGranted { + permissionStatusRow + } + + Button("Re-run Onboarding...") { + (NSApp.delegate as? AppDelegate)?.relaunchOnboarding() + } + .padding(.horizontal) + Button("Quit BobrWhisper") { NSApplication.shared.terminate(nil) } @@ -119,9 +151,42 @@ struct MenuBarView: View { case .error: return .orange } } + + /// Compact status strip listing every non-granted permission. Yellow accent + /// matches the existing warning banner so users associate "yellow stripe in + /// the menu" with "something needs attention but isn't broken". + private var permissionStatusRow: some View { + VStack(alignment: .leading, spacing: 4) { + if !permissions.micGranted { + permissionLine(name: "Microphone", missing: true, mandatory: true) + } + if !permissions.accessibilityGranted { + permissionLine(name: "Accessibility", missing: true, mandatory: false) + } + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.yellow.opacity(0.12)) + .clipShape(RoundedRectangle(cornerRadius: 6, style: .continuous)) + .padding(.horizontal, 8) + } + + private func permissionLine(name: String, missing: Bool, mandatory: Bool) -> some View { + HStack(alignment: .top, spacing: 6) { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.yellow) + .font(.caption) + Text("\(name) permission \(mandatory ? "required" : "not granted")") + .font(.caption) + .foregroundStyle(.primary) + .frame(maxWidth: .infinity, alignment: .leading) + } + } } #Preview { MenuBarView() .environmentObject(AppState()) + .environmentObject(PermissionsCoordinator()) } diff --git a/macos/BobrWhisper/Views/OnboardingView.swift b/macos/BobrWhisper/Views/OnboardingView.swift new file mode 100644 index 0000000..27e635f --- /dev/null +++ b/macos/BobrWhisper/Views/OnboardingView.swift @@ -0,0 +1,262 @@ +import AppKit +import SwiftUI + +/// First-run onboarding pane. Rendered inline inside the dashboard window — +/// the user sees this BEFORE the transcript table on first launch (or any +/// later launch where mandatory permissions regressed). Walks through +/// Microphone (mandatory) and Accessibility (auto-paste only), with a +/// "skip accessibility" path that completes onboarding with auto-paste +/// disabled. +/// +/// Visual style mirrors `MainWindowView`: top header with status pill, +/// rounded-rect cards on `controlBackgroundColor`, same yellow/green/orange +/// semantic palette established for warnings. +struct OnboardingView: View { + @EnvironmentObject var permissions: PermissionsCoordinator + @AppStorage("autoPaste") private var autoPaste: Bool = true + + var body: some View { + VStack(spacing: 0) { + header + + Divider() + + ScrollView { + VStack(alignment: .leading, spacing: 14) { + micCard + accessibilityCard + note + } + .padding(20) + .frame(maxWidth: .infinity, alignment: .leading) + } + .background(Color(nsColor: .textBackgroundColor)) + + Divider() + + footer + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .onAppear { permissions.refresh() } + } + + // MARK: - Header + + private var header: some View { + HStack(alignment: .center, spacing: 14) { + Image(systemName: "waveform.circle.fill") + .font(.system(size: 32)) + .foregroundStyle(.tint) + + VStack(alignment: .leading, spacing: 2) { + Text("Welcome to BobrWhisper") + .font(.system(size: 18, weight: .semibold)) + Text("Grant the permissions below to enable voice transcription.") + .font(.caption) + .foregroundStyle(.secondary) + } + + Spacer() + + overallStatusPill + } + .padding(.horizontal, 20) + .padding(.vertical, 16) + .background(Color(nsColor: .windowBackgroundColor)) + } + + private var overallStatusPill: some View { + let (text, color, icon): (String, Color, String) = { + if !permissions.micGranted { + return ("Setup required", .yellow, "exclamationmark.triangle.fill") + } + if !permissions.accessibilityGranted { + return ("Mic ready", .green, "checkmark.circle.fill") + } + return ("All set", .green, "checkmark.seal.fill") + }() + + return HStack(spacing: 6) { + Image(systemName: icon) + Text(text) + } + .font(.system(size: 11, weight: .semibold)) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .background(color.opacity(0.14)) + .overlay( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .stroke(color.opacity(0.35), lineWidth: 1) + ) + .clipShape(RoundedRectangle(cornerRadius: 8, style: .continuous)) + .foregroundStyle(color) + } + + // MARK: - Permission cards + + private var micCard: some View { + permissionCard( + title: "Microphone", + subtitle: "Required for voice recording. BobrWhisper cannot capture audio without this permission.", + icon: "mic.fill", + state: permissions.microphone, + mandatory: true, + primaryLabel: micPrimaryLabel, + primaryAction: micPrimaryAction + ) + } + + private var micPrimaryLabel: String { + switch permissions.microphone { + case .granted: return "Granted" + case .notDetermined: return "Grant Microphone" + case .denied: return "Open Microphone Settings" + } + } + + private func micPrimaryAction() { + switch permissions.microphone { + case .notDetermined: + permissions.requestMicrophone { _ in } + case .denied: + permissions.openMicrophoneSettings() + case .granted: + break + } + } + + private var accessibilityCard: some View { + permissionCard( + title: "Accessibility", + subtitle: "Required only for auto-paste after transcription. Without it, transcripts still appear in the overlay and log — you just have to paste them yourself.", + icon: "hand.tap.fill", + state: permissions.accessibility, + mandatory: false, + primaryLabel: permissions.accessibilityGranted ? "Granted" : "Open Accessibility Settings", + primaryAction: { permissions.openAccessibilitySettings() } + ) + } + + private var note: some View { + Text("After flipping a toggle in System Settings, return here — the status updates automatically.") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.top, 4) + } + + @ViewBuilder + private func permissionCard( + title: String, + subtitle: String, + icon: String, + state: PermissionState, + mandatory: Bool, + primaryLabel: String, + primaryAction: @escaping () -> Void + ) -> some View { + let accent = accentColor(for: state, mandatory: mandatory) + + VStack(alignment: .leading, spacing: 10) { + HStack(alignment: .top, spacing: 12) { + Image(systemName: icon) + .font(.system(size: 22)) + .foregroundStyle(.tint) + .frame(width: 28) + + VStack(alignment: .leading, spacing: 4) { + HStack(spacing: 8) { + Text(title) + .font(.system(size: 14, weight: .semibold)) + statePill(for: state, mandatory: mandatory) + } + Text(subtitle) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + Spacer() + } + + HStack { + Spacer() + if state == .granted { + Label("Granted", systemImage: "checkmark.circle.fill") + .font(.caption.weight(.semibold)) + .foregroundStyle(.green) + } else { + Button(primaryLabel, action: primaryAction) + .buttonStyle(.borderedProminent) + } + } + } + .padding(14) + .background(Color(nsColor: .controlBackgroundColor)) + .clipShape(RoundedRectangle(cornerRadius: 10, style: .continuous)) + .overlay( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .stroke(accent.opacity(0.35), lineWidth: 1) + ) + } + + private func statePill(for state: PermissionState, mandatory: Bool) -> some View { + let (label, color): (String, Color) = { + switch state { + case .granted: return ("Granted", .green) + case .denied: return (mandatory ? "Required" : "Optional", mandatory ? .orange : .yellow) + case .notDetermined: return ("Not requested", .yellow) + } + }() + + return Text(label) + .font(.system(size: 10, weight: .semibold)) + .foregroundStyle(color) + .padding(.horizontal, 8) + .padding(.vertical, 3) + .background(color.opacity(0.14)) + .overlay(Capsule().stroke(color.opacity(0.35), lineWidth: 0.5)) + .clipShape(Capsule()) + } + + private func accentColor(for state: PermissionState, mandatory: Bool) -> Color { + switch state { + case .granted: return .green + case .denied: return mandatory ? .orange : .yellow + case .notDetermined: return .yellow + } + } + + // MARK: - Footer + + private var footer: some View { + HStack(spacing: 8) { + // Skip path disables auto-paste so the user isn't surprised by a + // silent no-op every time they finish a recording. + Button("Skip Accessibility") { + autoPaste = false + permissions.finishOnboarding() + } + .disabled(!permissions.micGranted || permissions.accessibilityGranted) + + Spacer() + + Button("Quit") { + NSApplication.shared.terminate(nil) + } + + Button("Continue") { + permissions.finishOnboarding() + } + .keyboardShortcut(.defaultAction) + .disabled(!permissions.micGranted) + } + .padding(.horizontal, 20) + .padding(.vertical, 14) + .background(Color(nsColor: .windowBackgroundColor)) + } +} + +#Preview { + OnboardingView() + .environmentObject(PermissionsCoordinator()) +} diff --git a/macos/BobrWhisper/Views/SettingsView.swift b/macos/BobrWhisper/Views/SettingsView.swift index e028810..2abf947 100644 --- a/macos/BobrWhisper/Views/SettingsView.swift +++ b/macos/BobrWhisper/Views/SettingsView.swift @@ -2,6 +2,7 @@ import SwiftUI struct SettingsView: View { @EnvironmentObject var appState: AppState + @EnvironmentObject var permissions: PermissionsCoordinator var body: some View { TabView { @@ -10,6 +11,7 @@ struct SettingsView: View { Label("General", systemImage: "gear") } .environmentObject(appState) + .environmentObject(permissions) ModelsSettingsView() .tabItem { @@ -22,24 +24,118 @@ struct SettingsView: View { Label("About", systemImage: "info.circle") } } - .frame(width: 500, height: 350) + .frame(width: 500, height: 420) + } +} + +/// Inline permission row used in the General settings tab. Mirrors the pill +/// styling of `MainWindowView.statusPill` so settings feels visually +/// consistent with the dashboard. +struct PermissionStatusRow: View { + let title: String + let subtitle: String + let state: PermissionState + let mandatory: Bool + let onAction: () -> Void + + var body: some View { + HStack(alignment: .center, spacing: 12) { + VStack(alignment: .leading, spacing: 2) { + HStack(spacing: 8) { + Text(title) + .font(.system(size: 13, weight: .semibold)) + statusPill + } + Text(subtitle) + .font(.caption) + .foregroundStyle(.secondary) + } + Spacer() + if state != .granted { + Button(actionLabel, action: onAction) + .buttonStyle(.bordered) + } + } + .padding(.vertical, 2) + } + + private var statusPill: some View { + let (label, color): (String, Color) = { + switch state { + case .granted: return ("Granted", .green) + case .denied: return (mandatory ? "Required" : "Not granted", mandatory ? .orange : .yellow) + case .notDetermined: return ("Not requested", .yellow) + } + }() + + return Text(label) + .font(.system(size: 10, weight: .semibold)) + .foregroundStyle(color) + .padding(.horizontal, 8) + .padding(.vertical, 3) + .background(color.opacity(0.14)) + .overlay(Capsule().stroke(color.opacity(0.35), lineWidth: 0.5)) + .clipShape(Capsule()) + } + + private var actionLabel: String { + switch state { + case .granted: return "Granted" + case .notDetermined: return "Grant" + case .denied: return "Open Settings" + } } } struct GeneralSettingsView: View { @EnvironmentObject var appState: AppState + @EnvironmentObject var permissions: PermissionsCoordinator @AppStorage("launchAtLogin") private var launchAtLogin = false @AppStorage("autoPaste") private var autoPaste = true @AppStorage("hotkeyCombo") private var hotkeyCombo = "option+space" var body: some View { Form { + Section("Permissions") { + PermissionStatusRow( + title: "Microphone", + subtitle: "Required for recording.", + state: permissions.microphone, + mandatory: true, + onAction: { + switch permissions.microphone { + case .notDetermined: permissions.requestMicrophone { _ in } + case .denied: permissions.openMicrophoneSettings() + case .granted: break + } + } + ) + + PermissionStatusRow( + title: "Accessibility", + subtitle: "Required for auto-paste only.", + state: permissions.accessibility, + mandatory: false, + onAction: { permissions.openAccessibilitySettings() } + ) + + Button("Re-run Onboarding...") { + (NSApp.delegate as? AppDelegate)?.relaunchOnboarding() + } + } + Section("Startup") { Toggle("Launch at login", isOn: $launchAtLogin) } Section("Behavior") { Toggle("Auto-paste after transcription", isOn: $autoPaste) + .disabled(!permissions.accessibilityGranted) + if !permissions.accessibilityGranted { + Text("Auto-paste needs Accessibility permission.") + .font(.caption) + .foregroundStyle(.secondary) + } Picker("Tone", selection: $appState.tone) { ForEach(Tone.allCases) { tone in @@ -70,7 +166,13 @@ struct GeneralSettingsView: View { struct ModelsSettingsView: View { @EnvironmentObject var appState: AppState @AppStorage("defaultModel") private var defaultModelKey: String = "" - + + /// Per-group variant override. Lets the user pick "English" on the Tiny + /// row without that choice leaking into other size groups. Keyed by the + /// group id (id with `.en` stripped). Empty until the user opens a + /// dropdown, at which point we remember their choice for the session. + @State private var groupSelections: [String: String] = [:] + private var defaultModelID: String { resolveLegacyStoredModelID(defaultModelKey) } @@ -89,39 +191,8 @@ struct ModelsSettingsView: View { var body: some View { Form { Section("Speech Models") { - ForEach(appState.availableModels) { model in - HStack { - VStack(alignment: .leading) { - Text(model.displayName) - Text(modelStatus(model)) - .font(.caption) - .foregroundColor(appState.modelExists(model) ? .green : .secondary) - } - - Spacer() - - if appState.modelExists(model) { - if appState.selectedModelID == model.id && appState.isModelLoaded { - Button("Loaded") {} - .disabled(true) - .buttonStyle(.borderedProminent) - } else { - Button("Load") { - appState.selectedModelID = model.id - appState.loadModel() - } - .buttonStyle(.bordered) - } - } else { - Button("Download") { - appState.selectedModelID = model.id - appState.downloadModel(model) - } - .disabled(appState.isDownloading) - .buttonStyle(.bordered) - } - } - .padding(.vertical, 2) + ForEach(modelGroups) { group in + modelRow(group: group) } if appState.isDownloading { @@ -155,6 +226,127 @@ struct ModelsSettingsView: View { .formStyle(.grouped) .padding() } + + // MARK: - Grouped row rendering + + /// Group key = the model id with any trailing `.en` removed. The + /// multilingual variant is always listed first so the picker defaults to + /// "Multilingual" before the user touches it. + private var modelGroups: [ModelSizeGroup] { + var keyed: [String: [SpeechModelDescriptor]] = [:] + var order: [String] = [] + for model in appState.availableModels { + let key = groupKey(for: model) + if keyed[key] == nil { order.append(key) } + keyed[key, default: []].append(model) + } + return order.compactMap { key in + guard var variants = keyed[key], !variants.isEmpty else { return nil } + // Multilingual (no .en) before English-only. + variants.sort { lhs, rhs in + !lhs.isEnglishOnly && rhs.isEnglishOnly + } + // Strip the disambiguating " English" word so the row shows the + // common base name. The size annotation in parens stays. + let baseName = variants[0].displayName + .replacingOccurrences(of: " English", with: "") + return ModelSizeGroup(id: key, baseDisplayName: baseName, variants: variants) + } + } + + private func groupKey(for model: SpeechModelDescriptor) -> String { + model.isEnglishOnly ? String(model.id.dropLast(3)) : model.id + } + + /// The variant currently shown for this row. Resolution order: + /// 1. explicit user choice in this session + /// 2. the globally-selected model, if it belongs to this group + /// 3. the persisted default model, if it belongs to this group + /// 4. multilingual fallback (first entry) + private func currentVariant(for group: ModelSizeGroup) -> SpeechModelDescriptor { + if let chosen = groupSelections[group.id], + let v = group.variants.first(where: { $0.id == chosen }) { + return v + } + if let v = group.variants.first(where: { $0.id == appState.selectedModelID }) { + return v + } + if let v = group.variants.first(where: { $0.id == defaultModelID }) { + return v + } + return group.variants[0] + } + + private func variantBinding(for group: ModelSizeGroup) -> Binding { + Binding( + get: { currentVariant(for: group).id }, + set: { groupSelections[group.id] = $0 } + ) + } + + @ViewBuilder + private func modelRow(group: ModelSizeGroup) -> some View { + let current = currentVariant(for: group) + + HStack { + VStack(alignment: .leading) { + Text(group.baseDisplayName) + Text(modelStatus(current)) + .font(.caption) + .foregroundColor(appState.modelExists(current) ? .green : .secondary) + } + + Spacer() + + if group.variants.count > 1 { + Picker("", selection: variantBinding(for: group)) { + ForEach(group.variants) { variant in + Text(variant.isEnglishOnly ? "English" : "Multilingual") + .tag(variant.id) + } + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 130) + } + + if appState.modelExists(current) { + if appState.selectedModelID == current.id && appState.isModelLoaded { + Button("Loaded") {} + .disabled(true) + .buttonStyle(.borderedProminent) + } else { + Button("Load") { + appState.selectedModelID = current.id + appState.loadModel() + } + .buttonStyle(.bordered) + } + } else { + Button("Download") { + appState.selectedModelID = current.id + appState.downloadModel(current) + } + .disabled(appState.isDownloading) + .buttonStyle(.bordered) + } + } + .padding(.vertical, 2) + } +} + +/// One row in the Models settings list. May represent a single multilingual +/// model OR a (multilingual, English-only) pair selectable via a dropdown. +private struct ModelSizeGroup: Identifiable { + let id: String + let baseDisplayName: String + let variants: [SpeechModelDescriptor] +} + +extension SpeechModelDescriptor { + /// True for models trained on English audio only (`.en` suffix in the id). + /// These can transcribe but cannot do language detection or translation. + var isEnglishOnly: Bool { id.hasSuffix(".en") } } struct AboutView: View { @@ -212,4 +404,5 @@ struct FeatureRow: View { #Preview { SettingsView() .environmentObject(AppState()) + .environmentObject(PermissionsCoordinator()) } diff --git a/macos/BobrWhisper/Views/TranscriptOverlayView.swift b/macos/BobrWhisper/Views/TranscriptOverlayView.swift index c0a4e1c..e3871b6 100644 --- a/macos/BobrWhisper/Views/TranscriptOverlayView.swift +++ b/macos/BobrWhisper/Views/TranscriptOverlayView.swift @@ -19,17 +19,26 @@ struct NotchOverlayView: View { !appState.lastTranscript.isEmpty && appState.status != .ready } + private var hasWarning: Bool { + appState.warningMessage != nil + } + var body: some View { VStack(alignment: .leading, spacing: 0) { + warningBanner compactBar separator transcriptText } - .frame(width: isExpanded ? 300 : 180, alignment: .leading) + .frame(width: isExpanded ? 320 : (hasWarning ? 280 : 180), alignment: .leading) .background(pillBackground) .clipShape(RoundedRectangle(cornerRadius: cornerRadius, style: .continuous)) .contentShape(RoundedRectangle(cornerRadius: cornerRadius, style: .continuous)) .onTapGesture { + if hasWarning { + appState.dismissWarning() + return + } if appState.isRecording { onStopRecording() } else if appState.status == .ready { @@ -72,6 +81,37 @@ struct NotchOverlayView: View { .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) .animation(.timingCurve(0.65, 0, 0.35, 1, duration: 0.4), value: isExpanded) .animation(.timingCurve(0.65, 0, 0.35, 1, duration: 0.4), value: showTranscript) + .animation(.spring(response: 0.35, dampingFraction: 0.85), value: hasWarning) + } + + // MARK: - Warning Banner + + /// Transient yellow strip at the top of the pill carrying non-fatal + /// advisories (stuck mic, Bluetooth mic, ...). Auto-dismisses via + /// `AppState.presentWarning`; tapping the pill while a warning is up + /// dismisses it immediately. Animation matches the rest of the pill. + @ViewBuilder + private var warningBanner: some View { + if let message = appState.warningMessage { + HStack(spacing: 6) { + Image(systemName: "exclamationmark.triangle.fill") + .font(.system(size: 10, weight: .semibold)) + .foregroundStyle(.yellow) + + Text(message) + .font(.system(size: 11, weight: .medium)) + .foregroundStyle(.white.opacity(0.92)) + .lineLimit(2) + .fixedSize(horizontal: false, vertical: true) + + Spacer(minLength: 0) + } + .padding(.horizontal, 14) + .padding(.vertical, 8) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.yellow.opacity(0.18)) + .transition(.move(edge: .top).combined(with: .opacity)) + } } // MARK: - Compact Bar From a349d0a7e3c2e4fd868377e41e099584742930ff Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 5 May 2026 23:07:01 +0200 Subject: [PATCH 8/9] feat(transcript): add local text guards Add local helpers for decoder vocabulary biasing and nonsense transcript filtering. The app previously had no typed owner for a user dictionary and no reusable heuristic for dropping obvious silence hallucinations. The dictionary store loads a bounded plain-text phrase list and builds a prompt suitable for the decoder. The sensibility guard rejects known bad outputs and repeated-token loops without invoking another model. --- src/DictionaryStore.zig | 182 ++++++++++++++++++++++++++++++++++++ src/Sensibility.zig | 199 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 src/DictionaryStore.zig create mode 100644 src/Sensibility.zig diff --git a/src/DictionaryStore.zig b/src/DictionaryStore.zig new file mode 100644 index 0000000..1e267dd --- /dev/null +++ b/src/DictionaryStore.zig @@ -0,0 +1,182 @@ +//! User-curated phrase list for biasing the Whisper decoder. +//! +//! Stored as a plain text file at `/dictionary.txt`, one phrase +//! per line. Lines starting with `*` (after the leading whitespace) are +//! "starred": they are emitted before unstarred phrases when the prompt is +//! built, so the highest-priority vocabulary fits inside Whisper's ~224-token +//! initial-prompt window even when the file is much larger. +//! +//! Whisper Flow's equivalent (`dictionaryContext` + `starredDictionaryContext`) +//! is the single biggest reason their cloud transcripts spell proper nouns +//! correctly. With a local model we can't match the cloud's contextual +//! richness, but a curated phrase list still moves the needle on names, +//! product terms, and acronyms. +//! +//! Format example: +//! +//! # comments and blank lines are ignored +//! * Sourcegraph +//! * BobrWhisper +//! Polymath +//! ExampleProduct +//! Baseten +//! +//! The file is owner-writable plain text so users can `vim` it. Loading is +//! lazy and best-effort: a missing file yields an empty dictionary, never +//! an error, so dictation works out-of-the-box. + +const std = @import("std"); +const compat = @import("compat.zig"); + +const DictionaryStore = @This(); + +/// Soft cap on the prompt string passed to `whisper.cpp`. Whisper allows +/// ~224 prompt tokens; tokens average ~3.5–4 chars in English, leaving us a +/// 768-byte ceiling that comfortably stays under the limit while leaving +/// headroom for non-ASCII tokenization (e.g. UTF-8 names). +pub const max_prompt_bytes: usize = 768; + +allocator: std.mem.Allocator, +/// Owned, allocator-backed strings. Starred phrases come first. +phrases: std.ArrayListUnmanaged([]u8) = .empty, +starred_count: usize = 0, + +/// Load `/dictionary.txt`. Missing/unreadable files return an +/// empty store rather than erroring — dictation should not break because the +/// user hasn't created a dictionary yet. +pub fn loadFromModelsDir(allocator: std.mem.Allocator, models_dir: []const u8) !DictionaryStore { + std.debug.assert(models_dir.len > 0); + + var store: DictionaryStore = .{ .allocator = allocator }; + errdefer store.deinit(); + + const path = try std.fmt.allocPrint(allocator, "{s}/dictionary.txt", .{models_dir}); + defer allocator.free(path); + + const file = std.Io.Dir.cwd().openFile(compat.io(), path, .{}) catch |err| switch (err) { + error.FileNotFound => return store, + else => { + std.log.warn("DictionaryStore: failed to open {s}: {}", .{ path, err }); + return store; + }, + }; + defer file.close(compat.io()); + + const stat = file.stat(compat.io()) catch |err| { + std.log.warn("DictionaryStore: stat failed for {s}: {}", .{ path, err }); + return store; + }; + // Refuse pathologically large files (>1 MiB) — a sane dictionary is well + // under 100 KiB and we don't want a runaway file to balloon allocations. + const max_file_bytes: u64 = 1024 * 1024; + if (stat.size > max_file_bytes) { + std.log.warn("DictionaryStore: {s} is {d} bytes, exceeds {d} byte cap", .{ path, stat.size, max_file_bytes }); + return store; + } + + const raw = try allocator.alloc(u8, @intCast(stat.size)); + defer allocator.free(raw); + + var io_buffer: [4096]u8 = undefined; + var reader = file.readerStreaming(compat.io(), &io_buffer); + reader.interface.readSliceAll(raw) catch |err| { + std.log.warn("DictionaryStore: read failed for {s}: {}", .{ path, err }); + return store; + }; + + var lines = std.mem.splitScalar(u8, raw, '\n'); + while (lines.next()) |raw_line| { + const line = std.mem.trim(u8, raw_line, " \t\r"); + if (line.len == 0) continue; + if (line[0] == '#') continue; + + const starred = line[0] == '*'; + const body_unbounded = if (starred) std.mem.trim(u8, line[1..], " \t") else line; + if (body_unbounded.len == 0) continue; + + const owned = try allocator.dupe(u8, body_unbounded); + errdefer allocator.free(owned); + + if (starred) { + try store.phrases.insert(allocator, store.starred_count, owned); + store.starred_count += 1; + } else { + try store.phrases.append(allocator, owned); + } + } + + std.debug.assert(store.starred_count <= store.phrases.items.len); + std.log.info( + "DictionaryStore: loaded {d} phrases ({d} starred) from {s}", + .{ store.phrases.items.len, store.starred_count, path }, + ); + return store; +} + +pub fn deinit(self: *DictionaryStore) void { + for (self.phrases.items) |phrase| { + self.allocator.free(phrase); + } + self.phrases.deinit(self.allocator); + self.starred_count = 0; +} + +/// Build the Whisper `initial_prompt` string by concatenating phrases +/// (starred first) until `max_prompt_bytes` would be exceeded. Returns null +/// when the dictionary is empty so callers can branch on "no prompt to set". +/// Caller owns the returned slice. +pub fn buildPrompt(self: DictionaryStore, allocator: std.mem.Allocator) !?[]u8 { + if (self.phrases.items.len == 0) return null; + + var buf: std.ArrayListUnmanaged(u8) = .empty; + errdefer buf.deinit(allocator); + + for (self.phrases.items) |phrase| { + std.debug.assert(phrase.len > 0); + // +2 for ", " separator (only after the first entry) + const sep_len: usize = if (buf.items.len == 0) 0 else 2; + if (buf.items.len + sep_len + phrase.len > max_prompt_bytes) break; + if (sep_len > 0) try buf.appendSlice(allocator, ", "); + try buf.appendSlice(allocator, phrase); + } + + if (buf.items.len == 0) return null; + std.debug.assert(buf.items.len <= max_prompt_bytes); + return try buf.toOwnedSlice(allocator); +} + +test "empty store builds null prompt" { + var store: DictionaryStore = .{ .allocator = std.testing.allocator }; + defer store.deinit(); + const prompt = try store.buildPrompt(std.testing.allocator); + try std.testing.expectEqual(@as(?[]u8, null), prompt); +} + +test "starred phrases come before unstarred" { + var store: DictionaryStore = .{ .allocator = std.testing.allocator }; + defer store.deinit(); + try store.phrases.append(std.testing.allocator, try std.testing.allocator.dupe(u8, "Banana")); + // simulate one starred insert at front + try store.phrases.insert(std.testing.allocator, 0, try std.testing.allocator.dupe(u8, "Apple")); + store.starred_count = 1; + + const prompt = (try store.buildPrompt(std.testing.allocator)) orelse return error.UnexpectedNull; + defer std.testing.allocator.free(prompt); + try std.testing.expectEqualStrings("Apple, Banana", prompt); +} + +test "prompt respects max_prompt_bytes" { + var store: DictionaryStore = .{ .allocator = std.testing.allocator }; + defer store.deinit(); + + // 100 phrases of 16 chars + 2 sep = ~1800 bytes worth, well over 768. + const phrase = "ABCDEFGHIJKLMNOP"; + var i: usize = 0; + while (i < 100) : (i += 1) { + try store.phrases.append(std.testing.allocator, try std.testing.allocator.dupe(u8, phrase)); + } + const prompt = (try store.buildPrompt(std.testing.allocator)) orelse return error.UnexpectedNull; + defer std.testing.allocator.free(prompt); + try std.testing.expect(prompt.len <= max_prompt_bytes); + try std.testing.expect(prompt.len > 0); +} diff --git a/src/Sensibility.zig b/src/Sensibility.zig new file mode 100644 index 0000000..8785c4f --- /dev/null +++ b/src/Sensibility.zig @@ -0,0 +1,199 @@ +//! Local "is this transcript nonsense?" heuristic. +//! +//! Whisper has well-known failure modes on silence, breath noise, music, and +//! short out-of-distribution input. The model is overconfident: it produces +//! grammatically clean but semantically empty strings ("Thanks for watching!", +//! "Subtitles by Amara.org community", looping single tokens, ...). We don't +//! want to spend a local LLM call (or interrupt the user) on text we can detect +//! as garbage with simple pattern matching. +//! +//! `isNonsense` returns true when *any* of three signals fire: +//! +//! 1. Exact (case-insensitive, punctuation-insensitive) match against a +//! known-bad list seeded from observed Whisper failure outputs. +//! 2. Single-token loop: the same word repeated >= `max_token_repeat` times +//! in a row anywhere in the transcript. +//! 3. Trigram loop: any 3-word window repeated >= `max_trigram_repeat` +//! times in the entire transcript. +//! +//! All three are pure, allocation-free, and run in O(n) over token count. +//! False positives have a real cost (lost transcript), so the thresholds are +//! deliberately loose — only "obviously broken" output is rejected. + +const std = @import("std"); + +/// Same single token repeated this many times in a row → loop. +const max_token_repeat: usize = 5; +/// Same trigram repeated this many times anywhere in the transcript → loop. +const max_trigram_repeat: usize = 3; + +/// Strings that Whisper emits during silence / non-speech audio. Compared +/// case-insensitively after stripping punctuation and collapsing whitespace. +const known_bad: []const []const u8 = &.{ + "thanks for watching", + "thank you for watching", + "thanks for watching the video", + "thank you so much for watching", + "please subscribe", + "subtitles by the amara org community", + "subtitles by", + "transcribed by", + "music", + "applause", + "silence", + "you", + "thank you", + "bye", + "okay", + "hmm", + "uh", + "um", +}; + +/// Returns true when the transcript looks like Whisper hallucinated nonsense +/// over silence/non-speech audio. Pure function; no allocation. +pub fn isNonsense(text: []const u8) bool { + var trim_buf: [512]u8 = undefined; + const trimmed = trimAndCanonicalize(text, &trim_buf); + if (trimmed.len == 0) return true; + + if (matchesKnownBad(trimmed)) return true; + + // Token-level loop checks operate on the original text so we keep word + // boundaries intact (the canonicalizer collapsed punctuation but tokens + // are easier to reason about by splitting on whitespace). + if (hasTokenLoop(trimmed, max_token_repeat)) return true; + if (hasTrigramLoop(trimmed, max_trigram_repeat)) return true; + + return false; +} + +/// Lowercase, strip ASCII punctuation, collapse internal whitespace into +/// single spaces, trim leading/trailing whitespace. Writes into `buf` and +/// returns a slice of the prefix actually used. Truncates silently when the +/// input would overflow the buffer — fine because all subsequent checks +/// operate on the canonicalized prefix. +fn trimAndCanonicalize(text: []const u8, buf: []u8) []const u8 { + std.debug.assert(buf.len > 0); + var out_len: usize = 0; + var prev_space = true; // suppress leading whitespace + for (text) |raw| { + if (out_len == buf.len) break; + const lower = std.ascii.toLower(raw); + const is_alnum = std.ascii.isAlphanumeric(lower); + if (is_alnum) { + buf[out_len] = lower; + out_len += 1; + prev_space = false; + } else if (!prev_space) { + buf[out_len] = ' '; + out_len += 1; + prev_space = true; + } + } + // strip trailing space introduced by canonicalization + while (out_len > 0 and buf[out_len - 1] == ' ') : (out_len -= 1) {} + return buf[0..out_len]; +} + +fn matchesKnownBad(canonical: []const u8) bool { + for (known_bad) |needle| { + if (std.mem.eql(u8, canonical, needle)) return true; + } + return false; +} + +fn hasTokenLoop(canonical: []const u8, threshold: usize) bool { + std.debug.assert(threshold >= 2); + var iter = std.mem.splitScalar(u8, canonical, ' '); + var prev: ?[]const u8 = null; + var run: usize = 1; + while (iter.next()) |tok| { + if (tok.len == 0) continue; + if (prev) |p| { + if (std.mem.eql(u8, tok, p)) { + run += 1; + if (run >= threshold) return true; + } else { + run = 1; + } + } + prev = tok; + } + return false; +} + +fn hasTrigramLoop(canonical: []const u8, threshold: usize) bool { + std.debug.assert(threshold >= 2); + + // Cap analyzed token count to bound work on pathologically long input. + const max_tokens: usize = 512; + var tokens: [max_tokens][]const u8 = undefined; + var n: usize = 0; + + var iter = std.mem.splitScalar(u8, canonical, ' '); + while (iter.next()) |tok| { + if (tok.len == 0) continue; + if (n == max_tokens) break; + tokens[n] = tok; + n += 1; + } + + if (n < 3 * threshold) return false; + + // Fixed-size O(n^2 / 9) comparison: for each starting position i, count + // how many times the trigram (tokens[i], tokens[i+1], tokens[i+2]) + // appears at later non-overlapping positions. With n capped at 512 this + // is at most ~30k comparisons. + var i: usize = 0; + while (i + 2 < n) : (i += 1) { + const t0 = tokens[i]; + const t1 = tokens[i + 1]; + const t2 = tokens[i + 2]; + var count: usize = 1; + var j: usize = i + 3; + while (j + 2 < n) : (j += 1) { + if (std.mem.eql(u8, tokens[j], t0) and + std.mem.eql(u8, tokens[j + 1], t1) and + std.mem.eql(u8, tokens[j + 2], t2)) + { + count += 1; + if (count >= threshold) return true; + j += 2; // skip past the matched window + } + } + } + return false; +} + +test "blank and whitespace are nonsense" { + try std.testing.expect(isNonsense("")); + try std.testing.expect(isNonsense(" ")); + try std.testing.expect(isNonsense("\n\t .. ")); +} + +test "known-bad list" { + try std.testing.expect(isNonsense("Thanks for watching!")); + try std.testing.expect(isNonsense("THANKS FOR WATCHING.")); + try std.testing.expect(isNonsense(" thanks for watching ")); + try std.testing.expect(isNonsense("Subtitles by the Amara.org community")); + try std.testing.expect(isNonsense("[Music]")); + try std.testing.expect(isNonsense("you")); +} + +test "real sentences are not nonsense" { + try std.testing.expect(!isNonsense("Hello, please open the file in the editor.")); + try std.testing.expect(!isNonsense("The quick brown fox jumps over the lazy dog.")); + try std.testing.expect(!isNonsense("Let me think about this for a second.")); +} + +test "token loop is nonsense" { + try std.testing.expect(isNonsense("okay okay okay okay okay")); + try std.testing.expect(isNonsense("yes yes yes yes yes please")); + try std.testing.expect(!isNonsense("yes yes please")); +} + +test "trigram loop is nonsense" { + try std.testing.expect(isNonsense("thanks for watching thanks for watching thanks for watching")); + try std.testing.expect(!isNonsense("thanks for watching this short clip")); +} From 2f0b97e3afdc2c1b0fadb7d7d75f0fcc4dd89bb5 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 5 May 2026 23:07:01 +0200 Subject: [PATCH 9/9] feat(app): wire whisper mode and transcript guards Add the runtime API fields and app-state plumbing for warning callbacks, whisper-mode VAD tuning, dictionary prompts, adaptive trimming, peak normalization, and local nonsense filtering. The previous app path used fixed VAD and trim behavior with only fatal errors; the new path carries non-fatal warnings separately and prepares captured audio before transcription. --- include/bobrwhisper.h | 9 ++ src/App.zig | 340 ++++++++++++++++++++++++++++++++++++++---- src/SettingsStore.zig | 9 ++ src/c_api.zig | 7 + 4 files changed, 337 insertions(+), 28 deletions(-) diff --git a/include/bobrwhisper.h b/include/bobrwhisper.h index 2a32147..d085a17 100644 --- a/include/bobrwhisper.h +++ b/include/bobrwhisper.h @@ -67,16 +67,23 @@ typedef enum { typedef void (*bobrwhisper_status_cb)(void* userdata, bobrwhisper_status_e status); typedef void (*bobrwhisper_transcript_cb)(void* userdata, bobrwhisper_string_s text, bool is_final); typedef void (*bobrwhisper_error_cb)(void* userdata, bobrwhisper_string_s error); +/// Non-fatal warning channel. Runtime issues that the user should know about +/// but that do not stop work in progress (stuck microphone, suboptimal input +/// device, ...) flow through this callback instead of `on_error`. The status +/// is NOT changed to ERROR when a warning fires. +typedef void (*bobrwhisper_warning_cb)(void* userdata, bobrwhisper_string_s warning); typedef struct { void* userdata; bobrwhisper_status_cb on_status_change; bobrwhisper_transcript_cb on_transcript; bobrwhisper_error_cb on_error; + bobrwhisper_warning_cb on_warning; const char* models_dir; const char* config_path; const char* llm_model_path; const char* vad_model_path; + bool whisper_mode; } bobrwhisper_runtime_config_s; typedef struct { @@ -85,6 +92,7 @@ typedef struct { bool remove_filler_words; bool auto_punctuate; bool use_llm_formatting; + bool whisper_mode; } bobrwhisper_transcribe_options_s; typedef struct { @@ -93,6 +101,7 @@ typedef struct { bool auto_punctuate; bool use_llm_formatting; const char* custom_prompt; + bool whisper_mode; } bobrwhisper_settings_s; int bobrwhisper_init(void); diff --git a/src/App.zig b/src/App.zig index d2d86a1..5ee2ed5 100644 --- a/src/App.zig +++ b/src/App.zig @@ -6,8 +6,11 @@ const builtin = @import("builtin"); const asr = @import("asr"); const c_api = @import("c_api.zig"); const AudioCapture = @import("audio/AudioCapture.zig"); +const AudioDevice = @import("audio/AudioDevice.zig"); const SettingsStore = @import("SettingsStore.zig"); const LogStore = @import("LogStore.zig"); +const DictionaryStore = @import("DictionaryStore.zig"); +const Sensibility = @import("Sensibility.zig"); const has_llm = builtin.os.tag == .macos; const LlamaClient = if (has_llm) @import("llm/LlamaClient.zig") else void; @@ -16,6 +19,41 @@ const RuntimeLoadConfig = asr.RuntimeLoadConfig; const App = @This(); +const VadTuning = struct { + threshold: f32, + min_speech_ms: i32, + min_silence_ms: i32, + speech_pad_ms: i32, +}; + +const default_vad_tuning: VadTuning = .{ + .threshold = 0.5, + .min_speech_ms = 250, + .min_silence_ms = 100, + .speech_pad_ms = 30, +}; + +const whisper_vad_tuning: VadTuning = .{ + .threshold = 0.3, + .min_speech_ms = 150, + .min_silence_ms = 200, + .speech_pad_ms = 100, +}; + +const bluetooth_vad_tuning: VadTuning = .{ + .threshold = 0.25, + .min_speech_ms = 150, + .min_silence_ms = 300, + .speech_pad_ms = 100, +}; + +const PreparedSegment = struct { + samples: []const f32, + bounds: AudioCapture.TrimBounds, + threshold: f32, + noise_floor: f32, +}; + const llm_model_candidates = [_][]const u8{ "llama-3.2-1b-q4_k_m.gguf", "llama-3.2-3b-q4_k_m.gguf", @@ -35,7 +73,9 @@ live_transcriber: ?RuntimeAdapter, audio: ?AudioCapture, llama: if (has_llm) ?LlamaClient else void, log_store: LogStore, +dictionary: DictionaryStore, custom_prompt: ?[]u8 = null, +loaded_model_id: ?[:0]const u8 = null, // Live transcription state live_thread: ?std.Thread = null, @@ -43,9 +83,26 @@ live_stop: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), last_transcribed_len: usize = 0, frozen_transcript: std.ArrayListUnmanaged(u8) = .empty, frozen_sample_count: usize = 0, +// Stuck-mic warning latch: ensures the user is told once per recording session +// when CoreAudio has been delivering bit-exact-zero audio for > 5 s. Reset on +// every fresh start of recording or live transcription. +stuck_mic_warned: bool = false, +// Bluetooth-mic warning latch: ensures the user is told at most once per +// process lifetime that the current input device is a Bluetooth mic +// (typically AirPods), which adds latency and degrades accuracy versus the +// built-in mic. +bluetooth_mic_warned: bool = false, + +/// Number of consecutive zero samples that triggers the stuck-mic warning. +/// 5 s at 16 kHz. +const stuck_mic_threshold_samples: usize = 16000 * 5; pub fn init(allocator: std.mem.Allocator, config: c_api.RuntimeConfig) !App { const log_store = try LogStore.init(allocator, config.getModelsDir()); + // Best-effort load: missing dictionary.txt is normal and yields an empty + // store. We rebuild from disk on every recording start so users can edit + // the file between sessions without restarting the app. + const dictionary = try DictionaryStore.loadFromModelsDir(allocator, config.getModelsDir()); return .{ .allocator = allocator, .config = config, @@ -55,6 +112,7 @@ pub fn init(allocator: std.mem.Allocator, config: c_api.RuntimeConfig) !App { .audio = null, .llama = if (has_llm) null else {}, .log_store = log_store, + .dictionary = dictionary, }; } @@ -68,6 +126,8 @@ pub fn deinit(self: *App) void { if (self.llama) |*l| l.deinit(); } if (self.custom_prompt) |cp| self.allocator.free(cp); + if (self.loaded_model_id) |id| self.allocator.free(id); + self.dictionary.deinit(); self.log_store.deinit(); } @@ -94,6 +154,38 @@ fn pathExists(path: []const u8) bool { return true; } +fn vadTuningForWhisperMode(enabled: bool) VadTuning { + return if (enabled) whisper_vad_tuning else default_vad_tuning; +} + +fn makeLoadConfig(model_path: []const u8, n_threads: u32, vad_path: ?[]const u8, tuning: VadTuning) RuntimeLoadConfig { + return .{ + .model_path = model_path, + .language = "en", + .n_threads = n_threads, + .vad_enabled = vad_path != null, + .vad_model_path = vad_path, + .vad_threshold = tuning.threshold, + .vad_min_speech_ms = tuning.min_speech_ms, + .vad_min_silence_ms = tuning.min_silence_ms, + .vad_speech_pad_ms = tuning.speech_pad_ms, + }; +} + +fn prepareSegment(samples: []const f32) PreparedSegment { + std.debug.assert(samples.len > 0); + const noise_floor = AudioCapture.computeNoiseFloor(samples); + const threshold = @max(noise_floor * 3.0, 0.0005); + const bounds = AudioCapture.trimSilenceBounds(samples, threshold); + const trimmed = samples[bounds.start..bounds.end]; + return .{ + .samples = if (trimmed.len > 0) trimmed else samples, + .bounds = bounds, + .threshold = threshold, + .noise_floor = noise_floor, + }; +} + fn resolveLlmModelPath(self: *App, path_buf: *[std.fs.max_path_bytes]u8) ![]const u8 { std.debug.assert(has_llm); @@ -170,13 +262,12 @@ pub fn loadModelByID(self: *App, model_id: []const u8) !void { const vad_path = self.config.getVadModelPath(); const cpu_count = std.Thread.getCpuCount() catch 4; const n_threads: u32 = @intCast(@max(4, cpu_count / 2)); - const load_config = RuntimeLoadConfig{ - .model_path = model_path, - .language = "en", - .n_threads = n_threads, - .vad_enabled = vad_path != null, - .vad_model_path = vad_path, - }; + const load_config = makeLoadConfig( + model_path, + n_threads, + vad_path, + vadTuningForWhisperMode(self.config.whisper_mode), + ); self.transcriber = RuntimeAdapter.init(self.allocator, descriptor, load_config) catch |err| { std.log.err("Failed to load model: {}", .{err}); @@ -185,17 +276,19 @@ pub fn loadModelByID(self: *App, model_id: []const u8) !void { return err; }; + if (self.loaded_model_id) |old_id| self.allocator.free(old_id); + self.loaded_model_id = self.allocator.dupeZ(u8, descriptor.id) catch null; + if (asr.ModelRegistry.preferredLiveModel(descriptor)) |live_descriptor| { var live_path_buf: [std.fs.max_path_bytes]u8 = undefined; if (self.getModelPathForDescriptor(live_descriptor, &live_path_buf)) |live_model_path| { if (compat.accessAbsolute(live_model_path)) |_| { - self.live_transcriber = RuntimeAdapter.init(self.allocator, live_descriptor, .{ - .model_path = live_model_path, - .language = "en", - .n_threads = n_threads, - .vad_enabled = vad_path != null, - .vad_model_path = vad_path, - }) catch |err| blk: { + self.live_transcriber = RuntimeAdapter.init(self.allocator, live_descriptor, makeLoadConfig( + live_model_path, + n_threads, + vad_path, + vadTuningForWhisperMode(self.config.whisper_mode), + )) catch |err| blk: { std.log.info("Live model not available for faster transcription, using main model: {}", .{err}); break :blk null; }; @@ -220,6 +313,10 @@ pub fn unloadModel(self: *App) void { t.deinit(); self.transcriber = null; } + if (self.loaded_model_id) |id| { + self.allocator.free(id); + self.loaded_model_id = null; + } } pub fn isModelLoaded(self: *App) bool { @@ -261,6 +358,9 @@ pub fn startRecording(self: *App) !void { self.audio = try AudioCapture.init(self.allocator); } + self.stuck_mic_warned = false; + self.checkInputDevice(); + self.refreshDictionaryPrompt(); try self.audio.?.start(); self.setStatus(.recording); } @@ -279,8 +379,11 @@ pub fn startRecordingWithLiveTranscription(self: *App, language: []const u8) !vo self.last_transcribed_len = 0; self.frozen_transcript.clearRetainingCapacity(); self.frozen_sample_count = 0; + self.stuck_mic_warned = false; self.live_stop.store(false, .seq_cst); + self.checkInputDevice(); + self.refreshDictionaryPrompt(); try self.audio.?.start(); self.setStatus(.recording); @@ -305,6 +408,8 @@ fn liveTranscriptionLoop(self: *App, language: [:0]const u8) void { const audio: *AudioCapture = if (self.audio) |*a| a else continue; + self.checkStuckMic(audio); + const total_samples = audio.getSampleCount(); if (total_samples < self.last_transcribed_len + min_new_samples) continue; @@ -415,11 +520,11 @@ fn transcribeTail(self: *App, options: c_api.TranscribeOptions) !void { const tail = samples[self.frozen_sample_count..]; if (tail.len > 0) { - const bounds = AudioCapture.trimSilenceBounds(tail, 0.001); - const trimmed = tail[bounds.start..bounds.end]; - const segment = if (trimmed.len > 0) trimmed else tail; + const segment = prepareSegment(tail); + std.log.info("Tail trim threshold: {d:.6} (noise floor: {d:.6}, whisper_mode: {})", .{ segment.threshold, segment.noise_floor, options.whisper_mode }); + std.log.info("Tail trim bounds: start={d}, end={d}, total={d}", .{ segment.bounds.start, segment.bounds.end, tail.len }); - const tail_text = try transcriber.transcribeWithLanguage(segment, options.getLanguage()); + const tail_text = try transcriber.transcribeWithLanguage(segment.samples, options.getLanguage()); defer self.allocator.free(tail_text); self.frozen_transcript.appendSlice(self.allocator, tail_text) catch {}; } @@ -449,18 +554,39 @@ pub fn transcribe(self: *App, options: c_api.TranscribeOptions) !void { self.setStatus(.transcribing); - const samples = audio.getSamples(); - if (samples.len == 0) { + const samples_mut = audio.getSamplesMut(); + if (samples_mut.len == 0) { self.setStatus(.@"error"); self.notifyError("No audio data recorded"); return error.NoAudioData; } - const bounds = AudioCapture.trimSilenceBounds(samples, 0.001); - const trimmed = samples[bounds.start..bounds.end]; - const segment = if (trimmed.len > 0) trimmed else samples; + // Peak-normalize so quiet speech (esp. whispers) is at a similar level to + // normal voice. Tuning experiments showed this drops avg WER from 0.42 to + // 0.06 on whispered audio with no degradation on normal voice. + const applied_gain = AudioCapture.peakNormalize(samples_mut, 0.95); + std.log.info("Peak-normalize gain: {d:.2}x", .{applied_gain}); + + const samples: []const f32 = samples_mut; + + const segment = prepareSegment(samples); + std.log.info("Trim threshold: {d:.6} (noise floor: {d:.6}, whisper_mode: {})", .{ segment.threshold, segment.noise_floor, options.whisper_mode }); + std.log.info("Trim bounds: start={d}, end={d}, total={d}, trimmed_pct={d:.1}%", .{ + segment.bounds.start, + segment.bounds.end, + samples.len, + @as(f32, @floatFromInt(samples.len - (segment.bounds.end - segment.bounds.start))) / @as(f32, @floatFromInt(samples.len)) * 100.0, + }); - const raw_text = try transcriber.transcribeWithLanguage(segment, options.getLanguage()); + if (options.whisper_mode) { + if (self.loaded_model_id) |model_id| { + if (std.mem.eql(u8, model_id, "whisper-tiny") or std.mem.eql(u8, model_id, "whisper-base")) { + std.log.warn("Whisper mode is enabled with a small model ({s}), consider using whisper-small or larger for better whisper detection", .{model_id}); + } + } + } + + const raw_text = try transcriber.transcribeWithLanguage(segment.samples, options.getLanguage()); defer self.allocator.free(raw_text); std.log.info("Transcription complete, text length: {d}", .{raw_text.len}); @@ -512,6 +638,18 @@ pub fn formatText( /// Sends raw text as a preview, runs LLM formatting if enabled, /// delivers the final result, and persists both texts to the log store. fn finalizeTranscript(self: *App, raw_text: []const u8, options: c_api.TranscribeOptions) !void { + // Drop Whisper hallucinations on silence/non-speech (looped tokens, + // "Thanks for watching!", etc.) before doing anything else. We deliver an + // empty final transcript to clear the UI, skip persistence, skip the + // (expensive) LLM formatting call, and return to .ready. This cheap local + // heuristic catches the common failure modes without another model call. + if (Sensibility.isNonsense(raw_text)) { + std.log.warn("Sensibility: dropping nonsense transcript ({d} bytes)", .{raw_text.len}); + self.notifyTranscript("", true); + self.setStatus(.ready); + return; + } + if (!options.use_llm_formatting) { self.notifyTranscript(raw_text, true); self.log_store.appendTranscript(self.allocator, raw_text, null) catch |err| { @@ -573,9 +711,31 @@ fn buildFormattingPrompt(self: *App, input: []const u8, tone: c_api.Tone) ![]u8 ); } - const base_prompt = - \\Clean up this transcribed speech. Remove filler words (um, uh, like, you know). - \\Fix grammar and punctuation. Keep the meaning intact.{s} + // Two-stage prompt selection. With small local LLMs (1B–3B) the combined + // "cleanup AND change tone" prompt drives hallucination on short + // utterances: the model fills the unused style budget with extra + // commentary or rewords aggressively. For the default neutral tone we use + // a tighter cleanup-only prompt; for explicit tones we keep the combined + // prompt. + if (tone == .neutral) { + const cleanup_prompt = + \\Rewrite the transcribed speech below verbatim with two changes only: + \\remove disfluencies (um, uh, like, you know) and fix obvious grammar + \\or punctuation mistakes. Do NOT rephrase, summarize, expand, or + \\add commentary. Preserve every meaningful word. + \\ + \\Input: {s} + \\ + \\Output: + ; + return try std.fmt.allocPrint(self.allocator, cleanup_prompt, .{input}); + } + + const polish_prompt = + \\Rewrite the transcribed speech below. Remove disfluencies + \\(um, uh, like, you know), fix grammar and punctuation, and adjust + \\style.{s} Preserve the speaker's meaning. Output only the rewritten + \\text with no commentary. \\ \\Input: {s} \\ @@ -584,7 +744,7 @@ fn buildFormattingPrompt(self: *App, input: []const u8, tone: c_api.Tone) ![]u8 return try std.fmt.allocPrint( self.allocator, - base_prompt, + polish_prompt, .{ tone.toPromptSuffix(), input }, ); } @@ -670,6 +830,130 @@ fn notifyTranscript(self: *App, text: []const u8, is_final: bool) void { } } +/// Reload `dictionary.txt` from disk and push the resulting initial-prompt +/// string into the loaded transcribers. Best-effort: failures are logged and +/// transcription continues with whatever prompt was last set (or none). Runs +/// at the start of every recording so the user can edit the dictionary file +/// without restarting the app. +fn refreshDictionaryPrompt(self: *App) void { + const fresh = DictionaryStore.loadFromModelsDir(self.allocator, self.config.getModelsDir()) catch |err| { + std.log.warn("DictionaryStore reload failed: {}", .{err}); + return; + }; + self.dictionary.deinit(); + self.dictionary = fresh; + + const prompt = self.dictionary.buildPrompt(self.allocator) catch |err| { + std.log.warn("DictionaryStore.buildPrompt failed: {}", .{err}); + return; + }; + defer if (prompt) |p| self.allocator.free(p); + + if (prompt) |p| { + std.log.info( + "Dictionary prompt: {d} bytes, {d} phrases active", + .{ p.len, self.dictionary.phrases.items.len }, + ); + } + + if (self.transcriber) |*t| { + t.setInitialPrompt(prompt) catch |err| std.log.warn("setInitialPrompt (main) failed: {}", .{err}); + } + if (self.live_transcriber) |*t| { + t.setInitialPrompt(prompt) catch |err| std.log.warn("setInitialPrompt (live) failed: {}", .{err}); + } +} + +/// Inspect the current default input device via CoreAudio and surface a +/// one-shot warning when it is a Bluetooth mic (typically AirPods). Uses the +/// OS-reported `kAudioDevicePropertyTransportType`, which is more robust than +/// substring matching the device name (no false negatives across locales or +/// rebadged BT mics, no false positives on devices that happen to contain +/// "airpods" in their label). Latched via `bluetooth_mic_warned` so the +/// notification fires at most once per process. +/// +/// Also pushes a BT-friendly VAD bucket into the loaded transcribers when +/// the device is Bluetooth. BT codec compression smears speech onsets and +/// inflates background noise, which makes the default Silero threshold +/// (0.5) overshoot — speech is detected late and trimmed early. Relaxed +/// thresholds + larger pad reduce that. The tuning is applied on every +/// recording start (not latched) so a user who switches to a BT device +/// after launch picks up the new params without restarting. +fn checkInputDevice(self: *App) void { + const info = AudioDevice.detectDefaultInput(self.allocator) orelse return; + defer info.deinit(self.allocator); + if (info.kind != .bluetooth) return; + + if (!self.bluetooth_mic_warned) { + self.bluetooth_mic_warned = true; + std.log.warn( + "Default input is a Bluetooth device ({s}); BT mics add latency and degrade transcription quality", + .{info.name}, + ); + self.notifyWarning( + "Bluetooth mic detected. Built-in or USB mics give better dictation accuracy.", + ); + } + + self.applyBluetoothVadTuning(); +} + +/// VAD bucket for Bluetooth input. Threshold is dropped from 0.5/0.3 to +/// 0.25 to avoid clipping the start of utterances; speech_pad_ms is +/// doubled to keep the codec's onset smear inside the speech window; +/// min_silence_ms is raised so the brief micro-gaps inside BT audio don't +/// fragment a single utterance into several. Whisper's own VAD sample-set +/// confirms BT recordings need ~2x the pad of built-in mics. +fn applyBluetoothVadTuning(self: *App) void { + const vad_path = self.config.getVadModelPath(); + const enabled = vad_path != null; + const tuning = bluetooth_vad_tuning; + + if (self.transcriber) |*t| { + t.setVadParams(enabled, tuning.threshold, tuning.min_speech_ms, tuning.min_silence_ms, tuning.speech_pad_ms); + } + if (self.live_transcriber) |*t| { + t.setVadParams(enabled, tuning.threshold, tuning.min_speech_ms, tuning.min_silence_ms, tuning.speech_pad_ms); + } + std.log.info( + "VAD: applied Bluetooth tuning (threshold={d:.2}, min_speech={d}ms, min_silence={d}ms, pad={d}ms)", + .{ tuning.threshold, tuning.min_speech_ms, tuning.min_silence_ms, tuning.speech_pad_ms }, + ); +} + +/// Notify the user once per recording session if CoreAudio has stopped +/// delivering real audio. Counts the trailing run of bit-exact-zero samples +/// reported by `AudioCapture` and fires the existing `on_error` callback when +/// it crosses `stuck_mic_threshold_samples`. The latch (`stuck_mic_warned`) +/// prevents the message from spamming every poll. +fn checkStuckMic(self: *App, audio: *AudioCapture) void { + if (self.stuck_mic_warned) return; + const zero_run = audio.getConsecutiveZeroSamples(); + if (zero_run < stuck_mic_threshold_samples) return; + self.stuck_mic_warned = true; + std.log.warn( + "Stuck microphone detected: {d} consecutive zero samples (>={d}s)", + .{ zero_run, zero_run / 16000 }, + ); + self.notifyWarning( + "Microphone is silent. Check input device and microphone permissions.", + ); +} + +/// Deliver a non-fatal advisory through the warning channel. Unlike +/// `notifyError`, this does NOT flip status to .error and does NOT take over +/// the status text, so a recording in progress keeps showing as recording. +fn notifyWarning(self: *App, message: []const u8) void { + std.debug.assert(message.len > 0); + if (self.config.on_warning) |cb| { + cb(self.config.userdata, c_api.String.fromSlice(message)); + } else if (self.config.on_error) |cb| { + // Backwards-compat: hosts that haven't wired on_warning still see the + // text via on_error, just without the status hijack. + cb(self.config.userdata, c_api.String.fromSlice(message)); + } +} + fn notifyError(self: *App, message: []const u8) void { if (self.config.on_error) |cb| { cb(self.config.userdata, c_api.String.fromSlice(message)); diff --git a/src/SettingsStore.zig b/src/SettingsStore.zig index 8bad362..4d0dc5e 100644 --- a/src/SettingsStore.zig +++ b/src/SettingsStore.zig @@ -101,6 +101,9 @@ fn writeApple(domain: []const u8, settings: c_api.Settings) WriteError!void { const key_custom_prompt = try makeCFString("customPrompt"); defer cf.CFRelease(key_custom_prompt); + const key_whisper_mode = try makeCFString("whisperMode"); + defer cf.CFRelease(key_whisper_mode); + var tone_value: i32 = @intFromEnum(settings.tone); const tone_number = cf.CFNumberCreate( cf.kCFAllocatorDefault, @@ -134,6 +137,12 @@ fn writeApple(domain: []const u8, settings: c_api.Settings) WriteError!void { cf.CFPreferencesSetAppValue(key_custom_prompt, null, domain_cf); } + cf.CFPreferencesSetAppValue( + key_whisper_mode, + if (settings.whisper_mode) cf.kCFBooleanTrue.* else cf.kCFBooleanFalse.*, + domain_cf, + ); + const sync_ok = cf.CFPreferencesAppSynchronize(domain_cf); if (sync_ok == 0) { return WriteError.SyncFailed; diff --git a/src/c_api.zig b/src/c_api.zig index 1cd478c..8892441 100644 --- a/src/c_api.zig +++ b/src/c_api.zig @@ -142,6 +142,9 @@ pub const Tone = enum(c_int) { pub const StatusCallback = *const fn (?*anyopaque, Status) callconv(.c) void; pub const TranscriptCallback = *const fn (?*anyopaque, String, bool) callconv(.c) void; pub const ErrorCallback = *const fn (?*anyopaque, String) callconv(.c) void; +/// Non-fatal warning channel. Used by the App for stuck-mic, Bluetooth-mic, +/// and similar advisories that should NOT flip status to .error. +pub const WarningCallback = *const fn (?*anyopaque, String) callconv(.c) void; pub const RuntimeConfig = extern struct { userdata: ?*anyopaque, @@ -149,12 +152,14 @@ pub const RuntimeConfig = extern struct { on_status_change: ?StatusCallback, on_transcript: ?TranscriptCallback, on_error: ?ErrorCallback, + on_warning: ?WarningCallback, models_dir: ?[*:0]const u8, config_path: ?[*:0]const u8, llm_model_path: ?[*:0]const u8, vad_model_path: ?[*:0]const u8, + whisper_mode: bool, pub fn getModelsDir(self: RuntimeConfig) []const u8 { if (self.models_dir) |ptr| { @@ -191,6 +196,7 @@ pub const Settings = extern struct { auto_punctuate: bool, use_llm_formatting: bool, custom_prompt: ?[*:0]const u8, + whisper_mode: bool, pub fn getCustomPrompt(self: Settings) ?[]const u8 { if (self.custom_prompt) |ptr| { @@ -206,6 +212,7 @@ pub const TranscribeOptions = extern struct { remove_filler_words: bool, auto_punctuate: bool, use_llm_formatting: bool, + whisper_mode: bool, pub fn getLanguage(self: TranscribeOptions) []const u8 { if (self.language) |ptr| {