diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..3e39fad --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1777826146, + "narHash": "sha256-wQ/iN5Zp5VIa3ebBibijPnLyKhor+xEbDy4d0goa9Zs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "73c703c22422b8951895a960959dbbaca7296492", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..ad2bd97 --- /dev/null +++ b/flake.nix @@ -0,0 +1,25 @@ +{ + description = "BobrWhisper - local-first voice-to-text"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + outputs = + { nixpkgs, ... }: + let + # arm64-only: Config.zig hardcodes Apple M1 for iOS targets and the + # Makefile pins `ARCHS=arm64` for xcodebuild. + pkgs = import nixpkgs { system = "aarch64-darwin"; }; + in + { + # mkShellNoCC: the build resolves clang/clang++/metal/xcodebuild via the + # system Xcode (see src/build/AppleSdk.zig). A nix stdenv would inject a + # different libc++ and SDK root and break the link against whisper.cpp / + # llama.cpp. + devShells.aarch64-darwin.default = pkgs.mkShellNoCC { + packages = [ + pkgs.zig_0_16 + pkgs.zls + ]; + }; + }; +} diff --git a/include/bobrwhisper.h b/include/bobrwhisper.h index 2a32147..d085a17 100644 --- a/include/bobrwhisper.h +++ b/include/bobrwhisper.h @@ -67,16 +67,23 @@ typedef enum { typedef void (*bobrwhisper_status_cb)(void* userdata, bobrwhisper_status_e status); typedef void (*bobrwhisper_transcript_cb)(void* userdata, bobrwhisper_string_s text, bool is_final); typedef void (*bobrwhisper_error_cb)(void* userdata, bobrwhisper_string_s error); +/// Non-fatal warning channel. Runtime issues that the user should know about +/// but that do not stop work in progress (stuck microphone, suboptimal input +/// device, ...) flow through this callback instead of `on_error`. The status +/// is NOT changed to ERROR when a warning fires. +typedef void (*bobrwhisper_warning_cb)(void* userdata, bobrwhisper_string_s warning); typedef struct { void* userdata; bobrwhisper_status_cb on_status_change; bobrwhisper_transcript_cb on_transcript; bobrwhisper_error_cb on_error; + bobrwhisper_warning_cb on_warning; const char* models_dir; const char* config_path; const char* llm_model_path; const char* vad_model_path; + bool whisper_mode; } bobrwhisper_runtime_config_s; typedef struct { @@ -85,6 +92,7 @@ typedef struct { bool remove_filler_words; bool auto_punctuate; bool use_llm_formatting; + bool whisper_mode; } bobrwhisper_transcribe_options_s; typedef struct { @@ -93,6 +101,7 @@ typedef struct { bool auto_punctuate; bool use_llm_formatting; const char* custom_prompt; + bool whisper_mode; } bobrwhisper_settings_s; int bobrwhisper_init(void); diff --git a/macos/BobrWhisper.xcodeproj/project.pbxproj b/macos/BobrWhisper.xcodeproj/project.pbxproj index f3700cb..71c1831 100644 --- a/macos/BobrWhisper.xcodeproj/project.pbxproj +++ b/macos/BobrWhisper.xcodeproj/project.pbxproj @@ -14,6 +14,8 @@ A100000000000005 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000005 /* SettingsView.swift */; }; A100000000000006 /* TranscriptOverlayView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000006 /* TranscriptOverlayView.swift */; }; A100000000000007 /* MainWindowView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000007 /* MainWindowView.swift */; }; + A100000000000008 /* PermissionsCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000008 /* PermissionsCoordinator.swift */; }; + A100000000000009 /* OnboardingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A200000000000009 /* OnboardingView.swift */; }; A100000000000010 /* BobrWhisperKit.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = A200000000000010 /* BobrWhisperKit.xcframework */; }; A100000000000011 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = A200000000000011 /* Assets.xcassets */; }; A100000000000012 /* silero-v6.2.0.bin in Resources */ = {isa = PBXBuildFile; fileRef = A200000000000012 /* silero-v6.2.0.bin */; }; @@ -27,6 +29,8 @@ A200000000000005 /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = ""; }; A200000000000006 /* TranscriptOverlayView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranscriptOverlayView.swift; sourceTree = ""; }; A200000000000007 /* MainWindowView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainWindowView.swift; sourceTree = ""; }; + A200000000000008 /* PermissionsCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PermissionsCoordinator.swift; sourceTree = ""; }; + A200000000000009 /* OnboardingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OnboardingView.swift; sourceTree = ""; }; A200000000000010 /* BobrWhisperKit.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = BobrWhisperKit.xcframework; sourceTree = ""; }; A200000000000011 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; A200000000000012 /* silero-v6.2.0.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = "silero-v6.2.0.bin"; path = "../../resources/silero-v6.2.0.bin"; sourceTree = ""; }; @@ -61,6 +65,7 @@ A200000000000001 /* App.swift */, A200000000000002 /* AppDelegate.swift */, A200000000000003 /* AppState.swift */, + A200000000000008 /* PermissionsCoordinator.swift */, A400000000000005 /* Views */, A200000000000011 /* Assets.xcassets */, A200000000000021 /* BobrWhisper.entitlements */, @@ -90,6 +95,7 @@ children = ( A200000000000007 /* MainWindowView.swift */, A200000000000004 /* MenuBarView.swift */, + A200000000000009 /* OnboardingView.swift */, A200000000000005 /* SettingsView.swift */, A200000000000006 /* TranscriptOverlayView.swift */, ); @@ -169,8 +175,10 @@ A100000000000001 /* App.swift in Sources */, A100000000000002 /* AppDelegate.swift in Sources */, A100000000000003 /* AppState.swift in Sources */, + A100000000000008 /* PermissionsCoordinator.swift in Sources */, A100000000000007 /* MainWindowView.swift in Sources */, A100000000000004 /* MenuBarView.swift in Sources */, + A100000000000009 /* OnboardingView.swift in Sources */, A100000000000005 /* SettingsView.swift in Sources */, A100000000000006 /* TranscriptOverlayView.swift in Sources */, ); diff --git a/macos/BobrWhisper/App.swift b/macos/BobrWhisper/App.swift index 915fc7c..f9cb35f 100644 --- a/macos/BobrWhisper/App.swift +++ b/macos/BobrWhisper/App.swift @@ -9,18 +9,45 @@ struct BobrWhisperApp: App { WindowGroup("BobrWhisper", id: "dashboard") { MainWindowView() .environmentObject(appDelegate.appState) + .environmentObject(appDelegate.permissions) } Settings { SettingsView() .environmentObject(appDelegate.appState) + .environmentObject(appDelegate.permissions) } MenuBarExtra { MenuBarView() .environmentObject(appDelegate.appState) + .environmentObject(appDelegate.permissions) } label: { + // The label is the always-alive part of the MenuBarExtra scene — + // it's instantiated on launch and kept alive for the app lifetime. + // Mounting `DashboardOpener` as an overlay here gives us a SwiftUI + // surface with `\.openWindow` that AppDelegate can drive via + // NotificationCenter, without depending on the menu actually + // being opened by the user. Image(systemName: appDelegate.appState.statusIcon) + .overlay(DashboardOpener().allowsHitTesting(false)) } } } + +/// SwiftUI bridge that AppKit code reaches via NotificationCenter to programmatically +/// open the dashboard `WindowGroup`. Mounted as a `.background` modifier in any +/// always-alive view (the menubar content) — its sole job is to translate the +/// `.bobrWhisperOpenDashboard` notification into a call on `\.openWindow`, +/// which is the only supported way to open a SwiftUI window scene. +struct DashboardOpener: View { + @Environment(\.openWindow) private var openWindow + + var body: some View { + Color.clear + .frame(width: 0, height: 0) + .onReceive(NotificationCenter.default.publisher(for: .bobrWhisperOpenDashboard)) { _ in + openWindow(id: "dashboard") + } + } +} diff --git a/macos/BobrWhisper/AppDelegate.swift b/macos/BobrWhisper/AppDelegate.swift index a9395ab..1073173 100644 --- a/macos/BobrWhisper/AppDelegate.swift +++ b/macos/BobrWhisper/AppDelegate.swift @@ -5,6 +5,7 @@ import BobrWhisperKit class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { let appState = AppState() + let permissions = PermissionsCoordinator() private var hotkeyMonitor: Any? func applicationDidFinishLaunching(_ notification: Notification) { @@ -20,9 +21,16 @@ class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { // Register global hotkey (Fn key or custom) setupHotkey() - - // Request accessibility permissions if needed - requestAccessibilityPermissions() + + // Onboarding lives inside the dashboard window now. If permissions + // need attention on launch, flip the coordinator into onboarding mode + // and ask SwiftUI to open the dashboard so the user actually sees it. + if permissions.shouldAutoShowOnLaunch { + permissions.beginOnboarding() + DispatchQueue.main.async { [weak self] in + self?.openDashboardWindow() + } + } } func applicationWillTerminate(_ notification: Notification) { @@ -99,12 +107,44 @@ class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { } } - private func requestAccessibilityPermissions() { - let options: NSDictionary = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] - let trusted = AXIsProcessTrustedWithOptions(options) - - if !trusted { - print("Accessibility permissions required for global hotkey") + /// Open (or focus) the dashboard `WindowGroup`. Used on launch when + /// onboarding needs to run, and from the menubar's "Open Dashboard" entry. + /// Walks the existing window list first to focus an already-open dashboard + /// instead of leaning on SwiftUI to create a duplicate. + func openDashboardWindow() { + permissions.refresh() + NSApp.activate(ignoringOtherApps: true) + + for window in NSApplication.shared.windows + where window.identifier?.rawValue == "dashboard" { + window.makeKeyAndOrderFront(nil) + return } + + // SwiftUI registers the WindowGroup with id "dashboard" via a + // discoverable URL handler under the hood. The standard + // `NSWorkspace.OpenConfiguration` route does not work for in-process + // scenes, so post a notification that the SwiftUI commands listener + // (in `App.swift`) acts on. + NotificationCenter.default.post( + name: .bobrWhisperOpenDashboard, + object: nil + ) + } + + /// Reset the "I've completed onboarding" flag and re-show the dashboard + /// in onboarding mode. Bound to the menubar / settings re-run actions. + func relaunchOnboarding() { + permissions.resetOnboarding() + permissions.beginOnboarding() + openDashboardWindow() } } + +extension Notification.Name { + /// Posted by `AppDelegate.openDashboardWindow` when a SwiftUI surface needs + /// to call `openWindow(id: "dashboard")` on its behalf. Subscribed to from + /// the App scene, which is the only place with access to the + /// `\.openWindow` environment. + static let bobrWhisperOpenDashboard = Notification.Name("bobrWhisperOpenDashboard") +} diff --git a/macos/BobrWhisper/AppState.swift b/macos/BobrWhisper/AppState.swift index 14784e4..27ad6e4 100644 --- a/macos/BobrWhisper/AppState.swift +++ b/macos/BobrWhisper/AppState.swift @@ -8,6 +8,12 @@ class AppState: ObservableObject { @Published private(set) var isRecording: Bool = false @Published private(set) var lastTranscript: String = "" @Published private(set) var errorMessage: String? + /// Non-fatal advisory text (stuck mic, Bluetooth mic, etc.). Cleared + /// automatically a few seconds after it is set so the UI doesn't have to + /// own dismissal logic. Lives separately from `errorMessage` so it does + /// not flip status to `.error`. + @Published private(set) var warningMessage: String? + private var warningClearWorkItem: DispatchWorkItem? @Published private(set) var isDownloading: Bool = false @Published var downloadProgress: Double = 0 @Published private(set) var transcriptLog: [TranscriptLogEntry] = [] @@ -156,6 +162,21 @@ class AppState: ObservableObject { appState.status = .error } } + + config.on_warning = { userdata, warning in + guard let userdata = userdata else { return } + let warningMsg: String? + if let ptr = warning.ptr, warning.len > 0 { + let data = Data(bytes: ptr, count: warning.len) + warningMsg = String(data: data, encoding: .utf8) + } else { + warningMsg = nil + } + let appState = Unmanaged.fromOpaque(userdata).takeUnretainedValue() + DispatchQueue.main.async { + appState.presentWarning(warningMsg) + } + } let vadModelPath = Bundle.main.path(forResource: "silero-v6.2.0", ofType: "bin") @@ -203,6 +224,32 @@ class AppState: ObservableObject { if let ptr = vadModelPathCString { free(ptr); vadModelPathCString = nil } } + /// Display a non-fatal advisory and auto-clear it after a short delay so + /// the UI doesn't have to track dismissal. Re-firing while a warning is + /// already on screen restarts the timer with the new text. + func presentWarning(_ message: String?) { + warningClearWorkItem?.cancel() + warningMessage = message + guard message != nil else { return } + let work = DispatchWorkItem { [weak self] in + guard let self = self else { return } + self.warningMessage = nil + self.warningClearWorkItem = nil + } + warningClearWorkItem = work + // 4.5 s sits between "long enough to read" and "out of the way before + // the next utterance lands". Matches the auto-dismiss feel of a + // short-lived toast. + DispatchQueue.main.asyncAfter(deadline: .now() + 4.5, execute: work) + } + + /// Manually dismiss the warning (used when the user taps the toast). + func dismissWarning() { + warningClearWorkItem?.cancel() + warningClearWorkItem = nil + warningMessage = nil + } + private func persistSettings() { guard let app = app else { return } @@ -415,7 +462,14 @@ class AppState: ObservableObject { func pasteToActiveApp() { copyToClipboard() - + + // Auto-paste is opt-out via Settings, but ALSO requires Accessibility + // permission. If either is missing we still copy (above) so the user + // can paste manually — silent no-op otherwise would surprise users who + // skipped Accessibility during onboarding. + let autoPasteEnabled = (UserDefaults.standard.object(forKey: "autoPaste") as? Bool) ?? true + guard autoPasteEnabled, AXIsProcessTrusted() else { return } + // Simulate Cmd+V let source = CGEventSource(stateID: .hidSystemState) let vKeyDown = CGEvent(keyboardEventSource: source, virtualKey: 0x09, keyDown: true) diff --git a/macos/BobrWhisper/PermissionsCoordinator.swift b/macos/BobrWhisper/PermissionsCoordinator.swift new file mode 100644 index 0000000..97683a6 --- /dev/null +++ b/macos/BobrWhisper/PermissionsCoordinator.swift @@ -0,0 +1,164 @@ +import AppKit +import ApplicationServices +import AVFoundation +import Combine +import Foundation + +/// Tri-state for OS-granted privacy permissions. +/// +/// `notDetermined` means the system has never prompted; this is the only state +/// from which `AVCaptureDevice.requestAccess` will trigger the OS prompt. +/// `denied` covers both user-denied and MDM-restricted; recovery in both cases +/// requires the user to flip the toggle in System Settings. +enum PermissionState: Equatable { + case granted + case denied + case notDetermined +} + +/// Tracks Microphone (mandatory) and Accessibility (auto-paste only) permissions. +/// +/// macOS does not deliver a reliable signal when the user changes a TCC entry +/// in System Settings, so we re-poll on app activation and on a coarse fallback +/// timer. This coordinator deliberately knows nothing about the Zig core or the +/// `bobrwhisper_app_t` lifecycle; it is a pure SwiftUI/AppKit concern. +final class PermissionsCoordinator: ObservableObject { + @Published private(set) var microphone: PermissionState = .notDetermined + @Published private(set) var accessibility: PermissionState = .notDetermined + + private static let onboardingCompletedKey = "hasCompletedOnboarding" + + var hasCompletedOnboarding: Bool { + UserDefaults.standard.bool(forKey: Self.onboardingCompletedKey) + } + + var micGranted: Bool { microphone == .granted } + var accessibilityGranted: Bool { accessibility == .granted } + + /// True when the dashboard should render the onboarding flow instead of + /// the transcript table. Mandatory permission missing OR the user has + /// never completed the flow. Settings surfaces optional regression + /// (accessibility only) inline rather than blocking the dashboard. + @Published var isOnboardingActive: Bool = false + + /// Whether the dashboard should auto-show on launch (because onboarding + /// needs to run). Independent of `isOnboardingActive` so we can decide + /// "open the window AND show onboarding" in one step. + var shouldAutoShowOnLaunch: Bool { + if !micGranted { return true } + return !hasCompletedOnboarding + } + + func beginOnboarding() { + isOnboardingActive = true + } + + func finishOnboarding() { + markOnboardingComplete() + isOnboardingActive = false + } + + private var pollTimer: Timer? + private var activationObserver: NSObjectProtocol? + private var workspaceObserver: NSObjectProtocol? + + init() { + refresh() + startObserving() + } + + deinit { + stopObserving() + } + + /// Re-read both permissions from the OS. Cheap; safe to call from a timer. + func refresh() { + let newMic: PermissionState + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: newMic = .granted + case .denied, .restricted: newMic = .denied + case .notDetermined: newMic = .notDetermined + @unknown default: newMic = .notDetermined + } + + let newAX: PermissionState = AXIsProcessTrusted() ? .granted : .denied + + if newMic != microphone { microphone = newMic } + if newAX != accessibility { accessibility = newAX } + } + + /// Triggers the system microphone prompt. Only effective in `notDetermined`; + /// after a denial the user has to toggle it in System Settings, so callers + /// should fall back to `openMicrophoneSettings()` in that case. + func requestMicrophone(completion: @escaping (Bool) -> Void) { + AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in + DispatchQueue.main.async { + self?.refresh() + completion(granted) + } + } + } + + func openMicrophoneSettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone") else { return } + NSWorkspace.shared.open(url) + } + + /// Accessibility cannot be prompted programmatically — Apple intentionally + /// requires a manual toggle. We deep-link to the right pane instead. + func openAccessibilitySettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") else { return } + NSWorkspace.shared.open(url) + } + + func markOnboardingComplete() { + UserDefaults.standard.set(true, forKey: Self.onboardingCompletedKey) + } + + func resetOnboarding() { + UserDefaults.standard.set(false, forKey: Self.onboardingCompletedKey) + } + + // MARK: - Observation + + private func startObserving() { + // App activation is the most common moment for the user to flip back + // from System Settings. AppKit fires this even for our own app + // returning from background. + activationObserver = NotificationCenter.default.addObserver( + forName: NSApplication.didBecomeActiveNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.refresh() + } + + // Workspace activation catches the case where another app (e.g., System + // Settings itself) becomes active and back without us regaining focus. + workspaceObserver = NSWorkspace.shared.notificationCenter.addObserver( + forName: NSWorkspace.didActivateApplicationNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.refresh() + } + + // TCC changes don't always fire a usable notification. A 2 s coarse + // poll covers the gap; cost is negligible since `AXIsProcessTrusted` + // and `AVCaptureDevice.authorizationStatus` are both in-process reads. + pollTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: true) { [weak self] _ in + self?.refresh() + } + } + + private func stopObserving() { + if let obs = activationObserver { + NotificationCenter.default.removeObserver(obs) + } + if let obs = workspaceObserver { + NSWorkspace.shared.notificationCenter.removeObserver(obs) + } + pollTimer?.invalidate() + pollTimer = nil + } +} diff --git a/macos/BobrWhisper/Views/MainWindowView.swift b/macos/BobrWhisper/Views/MainWindowView.swift index 8f51a37..b66b7a1 100644 --- a/macos/BobrWhisper/Views/MainWindowView.swift +++ b/macos/BobrWhisper/Views/MainWindowView.swift @@ -2,6 +2,7 @@ import SwiftUI struct MainWindowView: View { @EnvironmentObject var appState: AppState + @EnvironmentObject var permissions: PermissionsCoordinator @Environment(\.openSettings) private var openSettings private static let timeFormatter: DateFormatter = { @@ -12,6 +13,56 @@ struct MainWindowView: View { }() var body: some View { + Group { + if permissions.isOnboardingActive { + // Onboarding sits in front of the dashboard until the user + // either grants the mandatory mic permission or quits. Same + // window chrome — feels like a step in the app, not a popup. + OnboardingView() + } else { + dashboardBody + } + } + .navigationTitle("BobrWhisper") + .frame(minWidth: 940, minHeight: 620) + .toolbar { + if !permissions.isOnboardingActive { + ToolbarItemGroup(placement: .primaryAction) { + Button(appState.isRecording ? "Stop Recording" : "Start Recording") { + if appState.isRecording { + appState.stopRecording() + } else { + appState.startRecording() + } + } + + Button("Copy Latest") { + appState.copyToClipboard() + } + .disabled(appState.transcriptLog.isEmpty) + + Button("Clear Log") { + appState.clearTranscriptLog() + } + .disabled(appState.transcriptLog.isEmpty) + + Button("Settings") { + openSettings() + } + } + } + } + .onAppear { + // If the dashboard is opened while permissions are still missing + // (e.g. user revoked mic mid-session), drop straight back into + // onboarding instead of showing a broken table. + if permissions.shouldAutoShowOnLaunch { + permissions.beginOnboarding() + } + } + } + + private var dashboardBody: some View { VStack(spacing: 0) { dashboardHeader @@ -36,33 +87,6 @@ struct MainWindowView: View { .background(Color(nsColor: .textBackgroundColor)) } } - .navigationTitle("BobrWhisper") - .frame(minWidth: 940, minHeight: 620) - .toolbar { - ToolbarItemGroup(placement: .primaryAction) { - Button(appState.isRecording ? "Stop Recording" : "Start Recording") { - if appState.isRecording { - appState.stopRecording() - } else { - appState.startRecording() - } - } - - Button("Copy Latest") { - appState.copyToClipboard() - } - .disabled(appState.transcriptLog.isEmpty) - - Button("Clear Log") { - appState.clearTranscriptLog() - } - .disabled(appState.transcriptLog.isEmpty) - - Button("Settings") { - openSettings() - } - } - } } private var dashboardHeader: some View { @@ -186,4 +210,5 @@ struct MainWindowView: View { #Preview { MainWindowView() .environmentObject(AppState()) + .environmentObject(PermissionsCoordinator()) } diff --git a/macos/BobrWhisper/Views/MenuBarView.swift b/macos/BobrWhisper/Views/MenuBarView.swift index f6fcd1f..73b01a8 100644 --- a/macos/BobrWhisper/Views/MenuBarView.swift +++ b/macos/BobrWhisper/Views/MenuBarView.swift @@ -2,6 +2,7 @@ import SwiftUI struct MenuBarView: View { @EnvironmentObject var appState: AppState + @EnvironmentObject var permissions: PermissionsCoordinator @Environment(\.openSettings) private var openSettings @Environment(\.openWindow) private var openWindow private static let logTimestampFormatter: DateFormatter = { @@ -22,7 +23,27 @@ struct MenuBarView: View { } .padding(.horizontal) .padding(.top, 8) - + + // Non-fatal advisories (stuck mic, Bluetooth mic, ...). Auto-clears + // via AppState; tapping dismisses immediately. + if let warning = appState.warningMessage { + HStack(alignment: .top, spacing: 6) { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.yellow) + .font(.caption) + Text(warning) + .font(.caption) + .foregroundStyle(.primary) + .frame(maxWidth: .infinity, alignment: .leading) + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(Color.yellow.opacity(0.12)) + .clipShape(RoundedRectangle(cornerRadius: 6, style: .continuous)) + .padding(.horizontal, 8) + .onTapGesture { appState.dismissWarning() } + } + Divider() // Activity log @@ -99,7 +120,18 @@ struct MenuBarView: View { } .keyboardShortcut(",", modifiers: .command) .padding(.horizontal) - + + // Permission warning + re-run hook. Yellow strip mirrors the + // warning-banner pattern established for non-fatal advisories. + if !permissions.micGranted || !permissions.accessibilityGranted { + permissionStatusRow + } + + Button("Re-run Onboarding...") { + (NSApp.delegate as? AppDelegate)?.relaunchOnboarding() + } + .padding(.horizontal) + Button("Quit BobrWhisper") { NSApplication.shared.terminate(nil) } @@ -119,9 +151,42 @@ struct MenuBarView: View { case .error: return .orange } } + + /// Compact status strip listing every non-granted permission. Yellow accent + /// matches the existing warning banner so users associate "yellow stripe in + /// the menu" with "something needs attention but isn't broken". + private var permissionStatusRow: some View { + VStack(alignment: .leading, spacing: 4) { + if !permissions.micGranted { + permissionLine(name: "Microphone", missing: true, mandatory: true) + } + if !permissions.accessibilityGranted { + permissionLine(name: "Accessibility", missing: true, mandatory: false) + } + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.yellow.opacity(0.12)) + .clipShape(RoundedRectangle(cornerRadius: 6, style: .continuous)) + .padding(.horizontal, 8) + } + + private func permissionLine(name: String, missing: Bool, mandatory: Bool) -> some View { + HStack(alignment: .top, spacing: 6) { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.yellow) + .font(.caption) + Text("\(name) permission \(mandatory ? "required" : "not granted")") + .font(.caption) + .foregroundStyle(.primary) + .frame(maxWidth: .infinity, alignment: .leading) + } + } } #Preview { MenuBarView() .environmentObject(AppState()) + .environmentObject(PermissionsCoordinator()) } diff --git a/macos/BobrWhisper/Views/OnboardingView.swift b/macos/BobrWhisper/Views/OnboardingView.swift new file mode 100644 index 0000000..27e635f --- /dev/null +++ b/macos/BobrWhisper/Views/OnboardingView.swift @@ -0,0 +1,262 @@ +import AppKit +import SwiftUI + +/// First-run onboarding pane. Rendered inline inside the dashboard window — +/// the user sees this BEFORE the transcript table on first launch (or any +/// later launch where mandatory permissions regressed). Walks through +/// Microphone (mandatory) and Accessibility (auto-paste only), with a +/// "skip accessibility" path that completes onboarding with auto-paste +/// disabled. +/// +/// Visual style mirrors `MainWindowView`: top header with status pill, +/// rounded-rect cards on `controlBackgroundColor`, same yellow/green/orange +/// semantic palette established for warnings. +struct OnboardingView: View { + @EnvironmentObject var permissions: PermissionsCoordinator + @AppStorage("autoPaste") private var autoPaste: Bool = true + + var body: some View { + VStack(spacing: 0) { + header + + Divider() + + ScrollView { + VStack(alignment: .leading, spacing: 14) { + micCard + accessibilityCard + note + } + .padding(20) + .frame(maxWidth: .infinity, alignment: .leading) + } + .background(Color(nsColor: .textBackgroundColor)) + + Divider() + + footer + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .onAppear { permissions.refresh() } + } + + // MARK: - Header + + private var header: some View { + HStack(alignment: .center, spacing: 14) { + Image(systemName: "waveform.circle.fill") + .font(.system(size: 32)) + .foregroundStyle(.tint) + + VStack(alignment: .leading, spacing: 2) { + Text("Welcome to BobrWhisper") + .font(.system(size: 18, weight: .semibold)) + Text("Grant the permissions below to enable voice transcription.") + .font(.caption) + .foregroundStyle(.secondary) + } + + Spacer() + + overallStatusPill + } + .padding(.horizontal, 20) + .padding(.vertical, 16) + .background(Color(nsColor: .windowBackgroundColor)) + } + + private var overallStatusPill: some View { + let (text, color, icon): (String, Color, String) = { + if !permissions.micGranted { + return ("Setup required", .yellow, "exclamationmark.triangle.fill") + } + if !permissions.accessibilityGranted { + return ("Mic ready", .green, "checkmark.circle.fill") + } + return ("All set", .green, "checkmark.seal.fill") + }() + + return HStack(spacing: 6) { + Image(systemName: icon) + Text(text) + } + .font(.system(size: 11, weight: .semibold)) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .background(color.opacity(0.14)) + .overlay( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .stroke(color.opacity(0.35), lineWidth: 1) + ) + .clipShape(RoundedRectangle(cornerRadius: 8, style: .continuous)) + .foregroundStyle(color) + } + + // MARK: - Permission cards + + private var micCard: some View { + permissionCard( + title: "Microphone", + subtitle: "Required for voice recording. BobrWhisper cannot capture audio without this permission.", + icon: "mic.fill", + state: permissions.microphone, + mandatory: true, + primaryLabel: micPrimaryLabel, + primaryAction: micPrimaryAction + ) + } + + private var micPrimaryLabel: String { + switch permissions.microphone { + case .granted: return "Granted" + case .notDetermined: return "Grant Microphone" + case .denied: return "Open Microphone Settings" + } + } + + private func micPrimaryAction() { + switch permissions.microphone { + case .notDetermined: + permissions.requestMicrophone { _ in } + case .denied: + permissions.openMicrophoneSettings() + case .granted: + break + } + } + + private var accessibilityCard: some View { + permissionCard( + title: "Accessibility", + subtitle: "Required only for auto-paste after transcription. Without it, transcripts still appear in the overlay and log — you just have to paste them yourself.", + icon: "hand.tap.fill", + state: permissions.accessibility, + mandatory: false, + primaryLabel: permissions.accessibilityGranted ? "Granted" : "Open Accessibility Settings", + primaryAction: { permissions.openAccessibilitySettings() } + ) + } + + private var note: some View { + Text("After flipping a toggle in System Settings, return here — the status updates automatically.") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.top, 4) + } + + @ViewBuilder + private func permissionCard( + title: String, + subtitle: String, + icon: String, + state: PermissionState, + mandatory: Bool, + primaryLabel: String, + primaryAction: @escaping () -> Void + ) -> some View { + let accent = accentColor(for: state, mandatory: mandatory) + + VStack(alignment: .leading, spacing: 10) { + HStack(alignment: .top, spacing: 12) { + Image(systemName: icon) + .font(.system(size: 22)) + .foregroundStyle(.tint) + .frame(width: 28) + + VStack(alignment: .leading, spacing: 4) { + HStack(spacing: 8) { + Text(title) + .font(.system(size: 14, weight: .semibold)) + statePill(for: state, mandatory: mandatory) + } + Text(subtitle) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + Spacer() + } + + HStack { + Spacer() + if state == .granted { + Label("Granted", systemImage: "checkmark.circle.fill") + .font(.caption.weight(.semibold)) + .foregroundStyle(.green) + } else { + Button(primaryLabel, action: primaryAction) + .buttonStyle(.borderedProminent) + } + } + } + .padding(14) + .background(Color(nsColor: .controlBackgroundColor)) + .clipShape(RoundedRectangle(cornerRadius: 10, style: .continuous)) + .overlay( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .stroke(accent.opacity(0.35), lineWidth: 1) + ) + } + + private func statePill(for state: PermissionState, mandatory: Bool) -> some View { + let (label, color): (String, Color) = { + switch state { + case .granted: return ("Granted", .green) + case .denied: return (mandatory ? "Required" : "Optional", mandatory ? .orange : .yellow) + case .notDetermined: return ("Not requested", .yellow) + } + }() + + return Text(label) + .font(.system(size: 10, weight: .semibold)) + .foregroundStyle(color) + .padding(.horizontal, 8) + .padding(.vertical, 3) + .background(color.opacity(0.14)) + .overlay(Capsule().stroke(color.opacity(0.35), lineWidth: 0.5)) + .clipShape(Capsule()) + } + + private func accentColor(for state: PermissionState, mandatory: Bool) -> Color { + switch state { + case .granted: return .green + case .denied: return mandatory ? .orange : .yellow + case .notDetermined: return .yellow + } + } + + // MARK: - Footer + + private var footer: some View { + HStack(spacing: 8) { + // Skip path disables auto-paste so the user isn't surprised by a + // silent no-op every time they finish a recording. + Button("Skip Accessibility") { + autoPaste = false + permissions.finishOnboarding() + } + .disabled(!permissions.micGranted || permissions.accessibilityGranted) + + Spacer() + + Button("Quit") { + NSApplication.shared.terminate(nil) + } + + Button("Continue") { + permissions.finishOnboarding() + } + .keyboardShortcut(.defaultAction) + .disabled(!permissions.micGranted) + } + .padding(.horizontal, 20) + .padding(.vertical, 14) + .background(Color(nsColor: .windowBackgroundColor)) + } +} + +#Preview { + OnboardingView() + .environmentObject(PermissionsCoordinator()) +} diff --git a/macos/BobrWhisper/Views/SettingsView.swift b/macos/BobrWhisper/Views/SettingsView.swift index e028810..2abf947 100644 --- a/macos/BobrWhisper/Views/SettingsView.swift +++ b/macos/BobrWhisper/Views/SettingsView.swift @@ -2,6 +2,7 @@ import SwiftUI struct SettingsView: View { @EnvironmentObject var appState: AppState + @EnvironmentObject var permissions: PermissionsCoordinator var body: some View { TabView { @@ -10,6 +11,7 @@ struct SettingsView: View { Label("General", systemImage: "gear") } .environmentObject(appState) + .environmentObject(permissions) ModelsSettingsView() .tabItem { @@ -22,24 +24,118 @@ struct SettingsView: View { Label("About", systemImage: "info.circle") } } - .frame(width: 500, height: 350) + .frame(width: 500, height: 420) + } +} + +/// Inline permission row used in the General settings tab. Mirrors the pill +/// styling of `MainWindowView.statusPill` so settings feels visually +/// consistent with the dashboard. +struct PermissionStatusRow: View { + let title: String + let subtitle: String + let state: PermissionState + let mandatory: Bool + let onAction: () -> Void + + var body: some View { + HStack(alignment: .center, spacing: 12) { + VStack(alignment: .leading, spacing: 2) { + HStack(spacing: 8) { + Text(title) + .font(.system(size: 13, weight: .semibold)) + statusPill + } + Text(subtitle) + .font(.caption) + .foregroundStyle(.secondary) + } + Spacer() + if state != .granted { + Button(actionLabel, action: onAction) + .buttonStyle(.bordered) + } + } + .padding(.vertical, 2) + } + + private var statusPill: some View { + let (label, color): (String, Color) = { + switch state { + case .granted: return ("Granted", .green) + case .denied: return (mandatory ? "Required" : "Not granted", mandatory ? .orange : .yellow) + case .notDetermined: return ("Not requested", .yellow) + } + }() + + return Text(label) + .font(.system(size: 10, weight: .semibold)) + .foregroundStyle(color) + .padding(.horizontal, 8) + .padding(.vertical, 3) + .background(color.opacity(0.14)) + .overlay(Capsule().stroke(color.opacity(0.35), lineWidth: 0.5)) + .clipShape(Capsule()) + } + + private var actionLabel: String { + switch state { + case .granted: return "Granted" + case .notDetermined: return "Grant" + case .denied: return "Open Settings" + } } } struct GeneralSettingsView: View { @EnvironmentObject var appState: AppState + @EnvironmentObject var permissions: PermissionsCoordinator @AppStorage("launchAtLogin") private var launchAtLogin = false @AppStorage("autoPaste") private var autoPaste = true @AppStorage("hotkeyCombo") private var hotkeyCombo = "option+space" var body: some View { Form { + Section("Permissions") { + PermissionStatusRow( + title: "Microphone", + subtitle: "Required for recording.", + state: permissions.microphone, + mandatory: true, + onAction: { + switch permissions.microphone { + case .notDetermined: permissions.requestMicrophone { _ in } + case .denied: permissions.openMicrophoneSettings() + case .granted: break + } + } + ) + + PermissionStatusRow( + title: "Accessibility", + subtitle: "Required for auto-paste only.", + state: permissions.accessibility, + mandatory: false, + onAction: { permissions.openAccessibilitySettings() } + ) + + Button("Re-run Onboarding...") { + (NSApp.delegate as? AppDelegate)?.relaunchOnboarding() + } + } + Section("Startup") { Toggle("Launch at login", isOn: $launchAtLogin) } Section("Behavior") { Toggle("Auto-paste after transcription", isOn: $autoPaste) + .disabled(!permissions.accessibilityGranted) + if !permissions.accessibilityGranted { + Text("Auto-paste needs Accessibility permission.") + .font(.caption) + .foregroundStyle(.secondary) + } Picker("Tone", selection: $appState.tone) { ForEach(Tone.allCases) { tone in @@ -70,7 +166,13 @@ struct GeneralSettingsView: View { struct ModelsSettingsView: View { @EnvironmentObject var appState: AppState @AppStorage("defaultModel") private var defaultModelKey: String = "" - + + /// Per-group variant override. Lets the user pick "English" on the Tiny + /// row without that choice leaking into other size groups. Keyed by the + /// group id (id with `.en` stripped). Empty until the user opens a + /// dropdown, at which point we remember their choice for the session. + @State private var groupSelections: [String: String] = [:] + private var defaultModelID: String { resolveLegacyStoredModelID(defaultModelKey) } @@ -89,39 +191,8 @@ struct ModelsSettingsView: View { var body: some View { Form { Section("Speech Models") { - ForEach(appState.availableModels) { model in - HStack { - VStack(alignment: .leading) { - Text(model.displayName) - Text(modelStatus(model)) - .font(.caption) - .foregroundColor(appState.modelExists(model) ? .green : .secondary) - } - - Spacer() - - if appState.modelExists(model) { - if appState.selectedModelID == model.id && appState.isModelLoaded { - Button("Loaded") {} - .disabled(true) - .buttonStyle(.borderedProminent) - } else { - Button("Load") { - appState.selectedModelID = model.id - appState.loadModel() - } - .buttonStyle(.bordered) - } - } else { - Button("Download") { - appState.selectedModelID = model.id - appState.downloadModel(model) - } - .disabled(appState.isDownloading) - .buttonStyle(.bordered) - } - } - .padding(.vertical, 2) + ForEach(modelGroups) { group in + modelRow(group: group) } if appState.isDownloading { @@ -155,6 +226,127 @@ struct ModelsSettingsView: View { .formStyle(.grouped) .padding() } + + // MARK: - Grouped row rendering + + /// Group key = the model id with any trailing `.en` removed. The + /// multilingual variant is always listed first so the picker defaults to + /// "Multilingual" before the user touches it. + private var modelGroups: [ModelSizeGroup] { + var keyed: [String: [SpeechModelDescriptor]] = [:] + var order: [String] = [] + for model in appState.availableModels { + let key = groupKey(for: model) + if keyed[key] == nil { order.append(key) } + keyed[key, default: []].append(model) + } + return order.compactMap { key in + guard var variants = keyed[key], !variants.isEmpty else { return nil } + // Multilingual (no .en) before English-only. + variants.sort { lhs, rhs in + !lhs.isEnglishOnly && rhs.isEnglishOnly + } + // Strip the disambiguating " English" word so the row shows the + // common base name. The size annotation in parens stays. + let baseName = variants[0].displayName + .replacingOccurrences(of: " English", with: "") + return ModelSizeGroup(id: key, baseDisplayName: baseName, variants: variants) + } + } + + private func groupKey(for model: SpeechModelDescriptor) -> String { + model.isEnglishOnly ? String(model.id.dropLast(3)) : model.id + } + + /// The variant currently shown for this row. Resolution order: + /// 1. explicit user choice in this session + /// 2. the globally-selected model, if it belongs to this group + /// 3. the persisted default model, if it belongs to this group + /// 4. multilingual fallback (first entry) + private func currentVariant(for group: ModelSizeGroup) -> SpeechModelDescriptor { + if let chosen = groupSelections[group.id], + let v = group.variants.first(where: { $0.id == chosen }) { + return v + } + if let v = group.variants.first(where: { $0.id == appState.selectedModelID }) { + return v + } + if let v = group.variants.first(where: { $0.id == defaultModelID }) { + return v + } + return group.variants[0] + } + + private func variantBinding(for group: ModelSizeGroup) -> Binding { + Binding( + get: { currentVariant(for: group).id }, + set: { groupSelections[group.id] = $0 } + ) + } + + @ViewBuilder + private func modelRow(group: ModelSizeGroup) -> some View { + let current = currentVariant(for: group) + + HStack { + VStack(alignment: .leading) { + Text(group.baseDisplayName) + Text(modelStatus(current)) + .font(.caption) + .foregroundColor(appState.modelExists(current) ? .green : .secondary) + } + + Spacer() + + if group.variants.count > 1 { + Picker("", selection: variantBinding(for: group)) { + ForEach(group.variants) { variant in + Text(variant.isEnglishOnly ? "English" : "Multilingual") + .tag(variant.id) + } + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 130) + } + + if appState.modelExists(current) { + if appState.selectedModelID == current.id && appState.isModelLoaded { + Button("Loaded") {} + .disabled(true) + .buttonStyle(.borderedProminent) + } else { + Button("Load") { + appState.selectedModelID = current.id + appState.loadModel() + } + .buttonStyle(.bordered) + } + } else { + Button("Download") { + appState.selectedModelID = current.id + appState.downloadModel(current) + } + .disabled(appState.isDownloading) + .buttonStyle(.bordered) + } + } + .padding(.vertical, 2) + } +} + +/// One row in the Models settings list. May represent a single multilingual +/// model OR a (multilingual, English-only) pair selectable via a dropdown. +private struct ModelSizeGroup: Identifiable { + let id: String + let baseDisplayName: String + let variants: [SpeechModelDescriptor] +} + +extension SpeechModelDescriptor { + /// True for models trained on English audio only (`.en` suffix in the id). + /// These can transcribe but cannot do language detection or translation. + var isEnglishOnly: Bool { id.hasSuffix(".en") } } struct AboutView: View { @@ -212,4 +404,5 @@ struct FeatureRow: View { #Preview { SettingsView() .environmentObject(AppState()) + .environmentObject(PermissionsCoordinator()) } diff --git a/macos/BobrWhisper/Views/TranscriptOverlayView.swift b/macos/BobrWhisper/Views/TranscriptOverlayView.swift index c0a4e1c..e3871b6 100644 --- a/macos/BobrWhisper/Views/TranscriptOverlayView.swift +++ b/macos/BobrWhisper/Views/TranscriptOverlayView.swift @@ -19,17 +19,26 @@ struct NotchOverlayView: View { !appState.lastTranscript.isEmpty && appState.status != .ready } + private var hasWarning: Bool { + appState.warningMessage != nil + } + var body: some View { VStack(alignment: .leading, spacing: 0) { + warningBanner compactBar separator transcriptText } - .frame(width: isExpanded ? 300 : 180, alignment: .leading) + .frame(width: isExpanded ? 320 : (hasWarning ? 280 : 180), alignment: .leading) .background(pillBackground) .clipShape(RoundedRectangle(cornerRadius: cornerRadius, style: .continuous)) .contentShape(RoundedRectangle(cornerRadius: cornerRadius, style: .continuous)) .onTapGesture { + if hasWarning { + appState.dismissWarning() + return + } if appState.isRecording { onStopRecording() } else if appState.status == .ready { @@ -72,6 +81,37 @@ struct NotchOverlayView: View { .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) .animation(.timingCurve(0.65, 0, 0.35, 1, duration: 0.4), value: isExpanded) .animation(.timingCurve(0.65, 0, 0.35, 1, duration: 0.4), value: showTranscript) + .animation(.spring(response: 0.35, dampingFraction: 0.85), value: hasWarning) + } + + // MARK: - Warning Banner + + /// Transient yellow strip at the top of the pill carrying non-fatal + /// advisories (stuck mic, Bluetooth mic, ...). Auto-dismisses via + /// `AppState.presentWarning`; tapping the pill while a warning is up + /// dismisses it immediately. Animation matches the rest of the pill. + @ViewBuilder + private var warningBanner: some View { + if let message = appState.warningMessage { + HStack(spacing: 6) { + Image(systemName: "exclamationmark.triangle.fill") + .font(.system(size: 10, weight: .semibold)) + .foregroundStyle(.yellow) + + Text(message) + .font(.system(size: 11, weight: .medium)) + .foregroundStyle(.white.opacity(0.92)) + .lineLimit(2) + .fixedSize(horizontal: false, vertical: true) + + Spacer(minLength: 0) + } + .padding(.horizontal, 14) + .padding(.vertical, 8) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.yellow.opacity(0.18)) + .transition(.move(edge: .top).combined(with: .opacity)) + } } // MARK: - Compact Bar diff --git a/pkg/asr/ModelRegistry.zig b/pkg/asr/ModelRegistry.zig index 686440c..41bed0f 100644 --- a/pkg/asr/ModelRegistry.zig +++ b/pkg/asr/ModelRegistry.zig @@ -7,9 +7,13 @@ pub const ModelCapability = types.ModelCapability; pub const capabilityBit = types.capabilityBit; pub const whisper_tiny_id: [:0]const u8 = "whisper-tiny"; +pub const whisper_tiny_en_id: [:0]const u8 = "whisper-tiny.en"; pub const whisper_base_id: [:0]const u8 = "whisper-base"; +pub const whisper_base_en_id: [:0]const u8 = "whisper-base.en"; pub const whisper_small_id: [:0]const u8 = "whisper-small"; +pub const whisper_small_en_id: [:0]const u8 = "whisper-small.en"; pub const whisper_medium_id: [:0]const u8 = "whisper-medium"; +pub const whisper_medium_en_id: [:0]const u8 = "whisper-medium.en"; pub const whisper_large_v3_id: [:0]const u8 = "whisper-large-v3"; pub const whisper_large_v3_turbo_id: [:0]const u8 = "whisper-large-v3-turbo"; @@ -17,6 +21,12 @@ const whisper_capabilities = capabilityBit(.transcribe) | capabilityBit(.stream_partial); +// English-only models cannot do language detection or translation. Only the +// `transcribe` + `stream_partial` bits apply, same as multilingual — but the +// .en variant must be paired with `language=en`. Tiny→medium are the only +// sizes OpenAI ever released a `.en` for; large/turbo are multilingual-only. +const whisper_capabilities_en = whisper_capabilities; + const descriptors = [_]ModelDescriptor{ .{ .id = whisper_tiny_id, @@ -30,6 +40,17 @@ const descriptors = [_]ModelDescriptor{ .available_on_this_device = true, .legacy_storage_key = "tiny", }, + .{ + .id = whisper_tiny_en_id, + .display_name = "Whisper Tiny English (~75 MB)", + .family = "whisper", + .runtime = .whisper_cpp, + .local_filename = "ggml-tiny.en.bin", + .download_url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin", + .size_bytes = 75 * 1024 * 1024, + .capabilities = whisper_capabilities_en, + .available_on_this_device = true, + }, .{ .id = whisper_base_id, .display_name = "Whisper Base (~142 MB)", @@ -42,6 +63,17 @@ const descriptors = [_]ModelDescriptor{ .available_on_this_device = true, .legacy_storage_key = "base", }, + .{ + .id = whisper_base_en_id, + .display_name = "Whisper Base English (~142 MB)", + .family = "whisper", + .runtime = .whisper_cpp, + .local_filename = "ggml-base.en.bin", + .download_url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin", + .size_bytes = 142 * 1024 * 1024, + .capabilities = whisper_capabilities_en, + .available_on_this_device = true, + }, .{ .id = whisper_small_id, .display_name = "Whisper Small (~466 MB)", @@ -55,6 +87,20 @@ const descriptors = [_]ModelDescriptor{ .legacy_storage_key = "small", .preferred_live_model_id = whisper_base_id, }, + .{ + .id = whisper_small_en_id, + .display_name = "Whisper Small English (~466 MB)", + .family = "whisper", + .runtime = .whisper_cpp, + .local_filename = "ggml-small.en.bin", + .download_url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin", + .size_bytes = 466 * 1024 * 1024, + .capabilities = whisper_capabilities_en, + .available_on_this_device = true, + // Live partials still benefit from the smaller English base — same + // size class as the multilingual sibling. + .preferred_live_model_id = whisper_base_en_id, + }, .{ .id = whisper_medium_id, .display_name = "Whisper Medium (~1.5 GB)", @@ -68,6 +114,18 @@ const descriptors = [_]ModelDescriptor{ .legacy_storage_key = "medium", .preferred_live_model_id = whisper_base_id, }, + .{ + .id = whisper_medium_en_id, + .display_name = "Whisper Medium English (~1.5 GB)", + .family = "whisper", + .runtime = .whisper_cpp, + .local_filename = "ggml-medium.en.bin", + .download_url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin", + .size_bytes = 1500 * 1024 * 1024, + .capabilities = whisper_capabilities_en, + .available_on_this_device = true, + .preferred_live_model_id = whisper_base_en_id, + }, .{ .id = whisper_large_v3_id, .display_name = "Whisper Large v3 (~3.1 GB)", diff --git a/pkg/asr/RuntimeAdapter.zig b/pkg/asr/RuntimeAdapter.zig index 42454c2..e2518f2 100644 --- a/pkg/asr/RuntimeAdapter.zig +++ b/pkg/asr/RuntimeAdapter.zig @@ -16,6 +16,9 @@ pub const LoadConfig = struct { vad_min_speech_ms: i32 = 250, vad_min_silence_ms: i32 = 100, vad_speech_pad_ms: i32 = 30, + /// Optional decoder bias prompt (proper nouns, vocabulary). Whisper caps + /// this at ~224 tokens. Caller is responsible for staying under the bound. + initial_prompt: ?[]const u8 = null, }; pub const RuntimeAdapter = union(enum) { @@ -40,11 +43,42 @@ pub const RuntimeAdapter = union(enum) { .vad_min_speech_ms = config.vad_min_speech_ms, .vad_min_silence_ms = config.vad_min_silence_ms, .vad_speech_pad_ms = config.vad_speech_pad_ms, + .initial_prompt = config.initial_prompt, }) }, else => error.UnsupportedRuntime, }; } + /// Replace the decoder bias prompt. Routes to the underlying adapter. + /// Pass `null` or empty to clear. + pub fn setInitialPrompt(self: *RuntimeAdapter, prompt: ?[]const u8) !void { + switch (self.*) { + inline else => |*adapter| try adapter.setInitialPrompt(prompt), + } + } + + /// Override VAD parameters at runtime. Used to apply device-specific + /// tunings (e.g. relaxed thresholds for Bluetooth mics) at recording + /// start without reloading the model. + pub fn setVadParams( + self: *RuntimeAdapter, + enabled: bool, + threshold: f32, + min_speech_ms: i32, + min_silence_ms: i32, + speech_pad_ms: i32, + ) void { + switch (self.*) { + inline else => |*adapter| adapter.setVadParams( + enabled, + threshold, + min_speech_ms, + min_silence_ms, + speech_pad_ms, + ), + } + } + pub fn deinit(self: *RuntimeAdapter) void { switch (self.*) { inline else => |*adapter| adapter.deinit(), diff --git a/pkg/asr/WhisperCppAdapter.zig b/pkg/asr/WhisperCppAdapter.zig index d5ddb2f..6d2bba8 100644 --- a/pkg/asr/WhisperCppAdapter.zig +++ b/pkg/asr/WhisperCppAdapter.zig @@ -25,6 +25,10 @@ vad_threshold: f32, vad_min_speech_ms: i32, vad_min_silence_ms: i32, vad_speech_pad_ms: i32, +// Decoder bias prompt. Owned, sentinel-terminated, swapped via +// `setInitialPrompt`. Whisper.cpp accepts up to ~224 prompt tokens; callers +// are responsible for staying under that bound (see DictionaryStore). +initial_prompt: ?[:0]u8 = null, pub const Config = struct { model_path: []const u8, @@ -38,6 +42,8 @@ pub const Config = struct { vad_min_speech_ms: i32 = 250, vad_min_silence_ms: i32 = 100, vad_speech_pad_ms: i32 = 30, + // Optional initial prompt for decoder bias. + initial_prompt: ?[]const u8 = null, }; fn shouldUseGpu() bool { @@ -84,6 +90,7 @@ pub fn init(allocator: std.mem.Allocator, config: Config) !WhisperCppAdapter { .vad_min_speech_ms = config.vad_min_speech_ms, .vad_min_silence_ms = config.vad_min_silence_ms, .vad_speech_pad_ms = config.vad_speech_pad_ms, + .initial_prompt = if (config.initial_prompt) |p| try allocator.dupeZ(u8, p) else null, }; } @@ -96,6 +103,47 @@ pub fn deinit(self: *WhisperCppAdapter) void { if (self.vad_model_path) |p| { self.allocator.free(p); } + if (self.initial_prompt) |p| { + self.allocator.free(p); + } +} + +/// Override the VAD parameters used by subsequent transcribe calls. Used +/// by the App to push device-specific tunings (e.g. lower threshold for +/// Bluetooth mics whose codec compression smears speech onsets). Same +/// concurrency contract as `setInitialPrompt` — caller must not be +/// transcribing. +pub fn setVadParams( + self: *WhisperCppAdapter, + enabled: bool, + threshold: f32, + min_speech_ms: i32, + min_silence_ms: i32, + speech_pad_ms: i32, +) void { + std.debug.assert(threshold >= 0.0 and threshold <= 1.0); + std.debug.assert(min_speech_ms >= 0); + std.debug.assert(min_silence_ms >= 0); + std.debug.assert(speech_pad_ms >= 0); + self.vad_enabled = enabled; + self.vad_threshold = threshold; + self.vad_min_speech_ms = min_speech_ms; + self.vad_min_silence_ms = min_silence_ms; + self.vad_speech_pad_ms = speech_pad_ms; +} + +/// Replace the decoder bias prompt. Pass `null` (or empty) to clear. Caller +/// must NOT hold a transcribe call concurrently — the adapter is single- +/// threaded; App.zig drives all calls from the live-transcription thread or +/// the synchronous transcribe path. +pub fn setInitialPrompt(self: *WhisperCppAdapter, prompt: ?[]const u8) !void { + if (self.initial_prompt) |old| { + self.allocator.free(old); + self.initial_prompt = null; + } + const new_prompt = prompt orelse return; + if (new_prompt.len == 0) return; + self.initial_prompt = try self.allocator.dupeZ(u8, new_prompt); } /// Transcribe audio samples (16kHz, mono, f32) @@ -120,6 +168,16 @@ fn transcribeInternal(self: *WhisperCppAdapter, samples: []const f32, language: return error.NoAudioData; } + std.log.info("Transcribing {d} samples, live={}, vad_enabled={}, vad_threshold={d:.3}, min_speech_ms={d}, min_silence_ms={d}, speech_pad_ms={d}", .{ + samples.len, + live, + self.vad_enabled, + self.vad_threshold, + self.vad_min_speech_ms, + self.vad_min_silence_ms, + self.vad_speech_pad_ms, + }); + var lang_buf: [8:0]u8 = [_:0]u8{0} ** 8; const lang_len = @min(language.len, lang_buf.len - 1); @memcpy(lang_buf[0..lang_len], language[0..lang_len]); @@ -137,6 +195,7 @@ fn transcribeInternal(self: *WhisperCppAdapter, samples: []const f32, language: self.vad_min_speech_ms, self.vad_min_silence_ms, self.vad_speech_pad_ms, + if (self.initial_prompt) |p| p.ptr else null, ); if (result != 0) { std.log.err("whisper_full failed: {}", .{result}); diff --git a/pkg/asr/whisper_bridge.c b/pkg/asr/whisper_bridge.c index 3d14951..4c9047e 100644 --- a/pkg/asr/whisper_bridge.c +++ b/pkg/asr/whisper_bridge.c @@ -35,7 +35,8 @@ int bobrwhisper_whisper_transcribe( float vad_threshold, int32_t vad_min_speech_ms, int32_t vad_min_silence_ms, - int32_t vad_speech_pad_ms + int32_t vad_speech_pad_ms, + const char * initial_prompt ) { struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); params.print_realtime = false; @@ -62,6 +63,14 @@ int bobrwhisper_whisper_transcribe( params.language = language; } + // Initial prompt biases decoding toward proper nouns and domain vocabulary. + // Whisper allows up to ~224 tokens of prompt; the caller is responsible for + // staying under that bound. Pointer must outlive the whisper_full() call, + // which it does because the Zig adapter holds it for the call duration. + if (initial_prompt != NULL && initial_prompt[0] != '\0') { + params.initial_prompt = initial_prompt; + } + return whisper_full(ctx, params, samples, sample_count); } diff --git a/pkg/asr/whisper_bridge.zig b/pkg/asr/whisper_bridge.zig index 0ca522f..7cec3b2 100644 --- a/pkg/asr/whisper_bridge.zig +++ b/pkg/asr/whisper_bridge.zig @@ -16,6 +16,7 @@ pub extern fn bobrwhisper_whisper_transcribe( vad_min_speech_ms: i32, vad_min_silence_ms: i32, vad_speech_pad_ms: i32, + initial_prompt: ?[*:0]const u8, ) c_int; pub extern fn bobrwhisper_whisper_segment_count(ctx: *Context) i32; pub extern fn bobrwhisper_whisper_segment_text(ctx: *Context, segment_index: i32) ?[*:0]const u8; diff --git a/scripts/record-corpus.sh b/scripts/record-corpus.sh new file mode 100755 index 0000000..4198d8c --- /dev/null +++ b/scripts/record-corpus.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Record a baseline test corpus for tuning. +# +# Records 12 phrases × 2 modes (normal + whispered) = 24 snippets total. +# Phrases are tagged short / question / technical so the corpus can be +# sliced via: bobrwhisper-cli tune --group-by tag +# +# Usage: +# ./scripts/record-corpus.sh # press 'q' + Enter to stop each take +# ./scripts/record-corpus.sh -d 5 # fixed 5-second takes (no 'q' needed) +# +# Environment: +# CLI=path/to/bobrwhisper-cli # override binary location + +set -u + +CLI="${CLI:-./zig-out/bin/bobrwhisper-cli}" +DURATION=() + +while getopts "d:h" opt; do + case "$opt" in + d) DURATION=(--duration "$OPTARG") ;; + h) sed -n '2,15p' "$0"; exit 0 ;; + *) echo "Usage: $0 [-d seconds]" >&2; exit 2 ;; + esac +done + +if [[ ! -x "$CLI" ]]; then + echo "snippet CLI not found at: $CLI" >&2 + echo "Build it first: zig build" >&2 + exit 1 +fi + +# 12 phrases. Edit to match your accent/vocabulary; keep the tag column the +# same so per-tag averages stay comparable across runs. +SHORT=( + "yes please" + "open settings" + "no thanks" + "stop recording" +) +QUESTION=( + "what time is it" + "who is calling me" + "what is the weather today" + "where is the nearest coffee" +) +TECHNICAL=( + "refactor the parser" + "git rebase onto main" + "evaluate this codebase" + "the lexer tokenizes input" +) + +CURRENT=0 +TOTAL=$(( (${#SHORT[@]} + ${#QUESTION[@]} + ${#TECHNICAL[@]}) * 2 )) + +bar() { printf '──────────────────────────────────────────────\n'; } + +record_one() { + local label="$1" tag="$2" mode="$3" + CURRENT=$((CURRENT + 1)) + + local mode_flag + case "$mode" in + whisper) mode_flag="--whisper" ;; + normal) mode_flag="--no-whisper" ;; + *) echo "internal error: unknown mode $mode" >&2; exit 3 ;; + esac + + printf '\n'; bar + printf ' [%2d/%d] %-40s %s\n' "$CURRENT" "$TOTAL" "\"$label\"" "$mode ($tag)" + bar + printf 'Enter to record, "s" to skip, "q" to quit: ' + read -r choice + case "$choice" in + s|S) printf ' skipped.\n'; return ;; + q|Q) printf ' quitting.\n'; exit 0 ;; + esac + + # The snippet CLI segfaults on shutdown after the WAV+JSON are written; + # tolerate the non-zero exit so the script keeps going. + "$CLI" snippet --label "$label" "$mode_flag" --tag "$tag" "${DURATION[@]}" || true +} + +run_set() { + local tag="$1"; shift + for phrase in "$@"; do + for mode in normal whisper; do + record_one "$phrase" "$tag" "$mode" + done + done +} + +cat < 5 s. Reset on +// every fresh start of recording or live transcription. +stuck_mic_warned: bool = false, +// Bluetooth-mic warning latch: ensures the user is told at most once per +// process lifetime that the current input device is a Bluetooth mic +// (typically AirPods), which adds latency and degrades accuracy versus the +// built-in mic. +bluetooth_mic_warned: bool = false, + +/// Number of consecutive zero samples that triggers the stuck-mic warning. +/// 5 s at 16 kHz. +const stuck_mic_threshold_samples: usize = 16000 * 5; pub fn init(allocator: std.mem.Allocator, config: c_api.RuntimeConfig) !App { const log_store = try LogStore.init(allocator, config.getModelsDir()); + // Best-effort load: missing dictionary.txt is normal and yields an empty + // store. We rebuild from disk on every recording start so users can edit + // the file between sessions without restarting the app. + const dictionary = try DictionaryStore.loadFromModelsDir(allocator, config.getModelsDir()); return .{ .allocator = allocator, .config = config, @@ -55,6 +112,7 @@ pub fn init(allocator: std.mem.Allocator, config: c_api.RuntimeConfig) !App { .audio = null, .llama = if (has_llm) null else {}, .log_store = log_store, + .dictionary = dictionary, }; } @@ -68,6 +126,8 @@ pub fn deinit(self: *App) void { if (self.llama) |*l| l.deinit(); } if (self.custom_prompt) |cp| self.allocator.free(cp); + if (self.loaded_model_id) |id| self.allocator.free(id); + self.dictionary.deinit(); self.log_store.deinit(); } @@ -94,6 +154,38 @@ fn pathExists(path: []const u8) bool { return true; } +fn vadTuningForWhisperMode(enabled: bool) VadTuning { + return if (enabled) whisper_vad_tuning else default_vad_tuning; +} + +fn makeLoadConfig(model_path: []const u8, n_threads: u32, vad_path: ?[]const u8, tuning: VadTuning) RuntimeLoadConfig { + return .{ + .model_path = model_path, + .language = "en", + .n_threads = n_threads, + .vad_enabled = vad_path != null, + .vad_model_path = vad_path, + .vad_threshold = tuning.threshold, + .vad_min_speech_ms = tuning.min_speech_ms, + .vad_min_silence_ms = tuning.min_silence_ms, + .vad_speech_pad_ms = tuning.speech_pad_ms, + }; +} + +fn prepareSegment(samples: []const f32) PreparedSegment { + std.debug.assert(samples.len > 0); + const noise_floor = AudioCapture.computeNoiseFloor(samples); + const threshold = @max(noise_floor * 3.0, 0.0005); + const bounds = AudioCapture.trimSilenceBounds(samples, threshold); + const trimmed = samples[bounds.start..bounds.end]; + return .{ + .samples = if (trimmed.len > 0) trimmed else samples, + .bounds = bounds, + .threshold = threshold, + .noise_floor = noise_floor, + }; +} + fn resolveLlmModelPath(self: *App, path_buf: *[std.fs.max_path_bytes]u8) ![]const u8 { std.debug.assert(has_llm); @@ -170,13 +262,12 @@ pub fn loadModelByID(self: *App, model_id: []const u8) !void { const vad_path = self.config.getVadModelPath(); const cpu_count = std.Thread.getCpuCount() catch 4; const n_threads: u32 = @intCast(@max(4, cpu_count / 2)); - const load_config = RuntimeLoadConfig{ - .model_path = model_path, - .language = "en", - .n_threads = n_threads, - .vad_enabled = vad_path != null, - .vad_model_path = vad_path, - }; + const load_config = makeLoadConfig( + model_path, + n_threads, + vad_path, + vadTuningForWhisperMode(self.config.whisper_mode), + ); self.transcriber = RuntimeAdapter.init(self.allocator, descriptor, load_config) catch |err| { std.log.err("Failed to load model: {}", .{err}); @@ -185,17 +276,19 @@ pub fn loadModelByID(self: *App, model_id: []const u8) !void { return err; }; + if (self.loaded_model_id) |old_id| self.allocator.free(old_id); + self.loaded_model_id = self.allocator.dupeZ(u8, descriptor.id) catch null; + if (asr.ModelRegistry.preferredLiveModel(descriptor)) |live_descriptor| { var live_path_buf: [std.fs.max_path_bytes]u8 = undefined; if (self.getModelPathForDescriptor(live_descriptor, &live_path_buf)) |live_model_path| { if (compat.accessAbsolute(live_model_path)) |_| { - self.live_transcriber = RuntimeAdapter.init(self.allocator, live_descriptor, .{ - .model_path = live_model_path, - .language = "en", - .n_threads = n_threads, - .vad_enabled = vad_path != null, - .vad_model_path = vad_path, - }) catch |err| blk: { + self.live_transcriber = RuntimeAdapter.init(self.allocator, live_descriptor, makeLoadConfig( + live_model_path, + n_threads, + vad_path, + vadTuningForWhisperMode(self.config.whisper_mode), + )) catch |err| blk: { std.log.info("Live model not available for faster transcription, using main model: {}", .{err}); break :blk null; }; @@ -220,6 +313,10 @@ pub fn unloadModel(self: *App) void { t.deinit(); self.transcriber = null; } + if (self.loaded_model_id) |id| { + self.allocator.free(id); + self.loaded_model_id = null; + } } pub fn isModelLoaded(self: *App) bool { @@ -261,6 +358,9 @@ pub fn startRecording(self: *App) !void { self.audio = try AudioCapture.init(self.allocator); } + self.stuck_mic_warned = false; + self.checkInputDevice(); + self.refreshDictionaryPrompt(); try self.audio.?.start(); self.setStatus(.recording); } @@ -279,8 +379,11 @@ pub fn startRecordingWithLiveTranscription(self: *App, language: []const u8) !vo self.last_transcribed_len = 0; self.frozen_transcript.clearRetainingCapacity(); self.frozen_sample_count = 0; + self.stuck_mic_warned = false; self.live_stop.store(false, .seq_cst); + self.checkInputDevice(); + self.refreshDictionaryPrompt(); try self.audio.?.start(); self.setStatus(.recording); @@ -305,6 +408,8 @@ fn liveTranscriptionLoop(self: *App, language: [:0]const u8) void { const audio: *AudioCapture = if (self.audio) |*a| a else continue; + self.checkStuckMic(audio); + const total_samples = audio.getSampleCount(); if (total_samples < self.last_transcribed_len + min_new_samples) continue; @@ -415,11 +520,11 @@ fn transcribeTail(self: *App, options: c_api.TranscribeOptions) !void { const tail = samples[self.frozen_sample_count..]; if (tail.len > 0) { - const bounds = AudioCapture.trimSilenceBounds(tail, 0.001); - const trimmed = tail[bounds.start..bounds.end]; - const segment = if (trimmed.len > 0) trimmed else tail; + const segment = prepareSegment(tail); + std.log.info("Tail trim threshold: {d:.6} (noise floor: {d:.6}, whisper_mode: {})", .{ segment.threshold, segment.noise_floor, options.whisper_mode }); + std.log.info("Tail trim bounds: start={d}, end={d}, total={d}", .{ segment.bounds.start, segment.bounds.end, tail.len }); - const tail_text = try transcriber.transcribeWithLanguage(segment, options.getLanguage()); + const tail_text = try transcriber.transcribeWithLanguage(segment.samples, options.getLanguage()); defer self.allocator.free(tail_text); self.frozen_transcript.appendSlice(self.allocator, tail_text) catch {}; } @@ -449,18 +554,39 @@ pub fn transcribe(self: *App, options: c_api.TranscribeOptions) !void { self.setStatus(.transcribing); - const samples = audio.getSamples(); - if (samples.len == 0) { + const samples_mut = audio.getSamplesMut(); + if (samples_mut.len == 0) { self.setStatus(.@"error"); self.notifyError("No audio data recorded"); return error.NoAudioData; } - const bounds = AudioCapture.trimSilenceBounds(samples, 0.001); - const trimmed = samples[bounds.start..bounds.end]; - const segment = if (trimmed.len > 0) trimmed else samples; + // Peak-normalize so quiet speech (esp. whispers) is at a similar level to + // normal voice. Tuning experiments showed this drops avg WER from 0.42 to + // 0.06 on whispered audio with no degradation on normal voice. + const applied_gain = AudioCapture.peakNormalize(samples_mut, 0.95); + std.log.info("Peak-normalize gain: {d:.2}x", .{applied_gain}); + + const samples: []const f32 = samples_mut; + + const segment = prepareSegment(samples); + std.log.info("Trim threshold: {d:.6} (noise floor: {d:.6}, whisper_mode: {})", .{ segment.threshold, segment.noise_floor, options.whisper_mode }); + std.log.info("Trim bounds: start={d}, end={d}, total={d}, trimmed_pct={d:.1}%", .{ + segment.bounds.start, + segment.bounds.end, + samples.len, + @as(f32, @floatFromInt(samples.len - (segment.bounds.end - segment.bounds.start))) / @as(f32, @floatFromInt(samples.len)) * 100.0, + }); - const raw_text = try transcriber.transcribeWithLanguage(segment, options.getLanguage()); + if (options.whisper_mode) { + if (self.loaded_model_id) |model_id| { + if (std.mem.eql(u8, model_id, "whisper-tiny") or std.mem.eql(u8, model_id, "whisper-base")) { + std.log.warn("Whisper mode is enabled with a small model ({s}), consider using whisper-small or larger for better whisper detection", .{model_id}); + } + } + } + + const raw_text = try transcriber.transcribeWithLanguage(segment.samples, options.getLanguage()); defer self.allocator.free(raw_text); std.log.info("Transcription complete, text length: {d}", .{raw_text.len}); @@ -512,6 +638,18 @@ pub fn formatText( /// Sends raw text as a preview, runs LLM formatting if enabled, /// delivers the final result, and persists both texts to the log store. fn finalizeTranscript(self: *App, raw_text: []const u8, options: c_api.TranscribeOptions) !void { + // Drop Whisper hallucinations on silence/non-speech (looped tokens, + // "Thanks for watching!", etc.) before doing anything else. We deliver an + // empty final transcript to clear the UI, skip persistence, skip the + // (expensive) LLM formatting call, and return to .ready. This cheap local + // heuristic catches the common failure modes without another model call. + if (Sensibility.isNonsense(raw_text)) { + std.log.warn("Sensibility: dropping nonsense transcript ({d} bytes)", .{raw_text.len}); + self.notifyTranscript("", true); + self.setStatus(.ready); + return; + } + if (!options.use_llm_formatting) { self.notifyTranscript(raw_text, true); self.log_store.appendTranscript(self.allocator, raw_text, null) catch |err| { @@ -573,9 +711,31 @@ fn buildFormattingPrompt(self: *App, input: []const u8, tone: c_api.Tone) ![]u8 ); } - const base_prompt = - \\Clean up this transcribed speech. Remove filler words (um, uh, like, you know). - \\Fix grammar and punctuation. Keep the meaning intact.{s} + // Two-stage prompt selection. With small local LLMs (1B–3B) the combined + // "cleanup AND change tone" prompt drives hallucination on short + // utterances: the model fills the unused style budget with extra + // commentary or rewords aggressively. For the default neutral tone we use + // a tighter cleanup-only prompt; for explicit tones we keep the combined + // prompt. + if (tone == .neutral) { + const cleanup_prompt = + \\Rewrite the transcribed speech below verbatim with two changes only: + \\remove disfluencies (um, uh, like, you know) and fix obvious grammar + \\or punctuation mistakes. Do NOT rephrase, summarize, expand, or + \\add commentary. Preserve every meaningful word. + \\ + \\Input: {s} + \\ + \\Output: + ; + return try std.fmt.allocPrint(self.allocator, cleanup_prompt, .{input}); + } + + const polish_prompt = + \\Rewrite the transcribed speech below. Remove disfluencies + \\(um, uh, like, you know), fix grammar and punctuation, and adjust + \\style.{s} Preserve the speaker's meaning. Output only the rewritten + \\text with no commentary. \\ \\Input: {s} \\ @@ -584,7 +744,7 @@ fn buildFormattingPrompt(self: *App, input: []const u8, tone: c_api.Tone) ![]u8 return try std.fmt.allocPrint( self.allocator, - base_prompt, + polish_prompt, .{ tone.toPromptSuffix(), input }, ); } @@ -670,6 +830,130 @@ fn notifyTranscript(self: *App, text: []const u8, is_final: bool) void { } } +/// Reload `dictionary.txt` from disk and push the resulting initial-prompt +/// string into the loaded transcribers. Best-effort: failures are logged and +/// transcription continues with whatever prompt was last set (or none). Runs +/// at the start of every recording so the user can edit the dictionary file +/// without restarting the app. +fn refreshDictionaryPrompt(self: *App) void { + const fresh = DictionaryStore.loadFromModelsDir(self.allocator, self.config.getModelsDir()) catch |err| { + std.log.warn("DictionaryStore reload failed: {}", .{err}); + return; + }; + self.dictionary.deinit(); + self.dictionary = fresh; + + const prompt = self.dictionary.buildPrompt(self.allocator) catch |err| { + std.log.warn("DictionaryStore.buildPrompt failed: {}", .{err}); + return; + }; + defer if (prompt) |p| self.allocator.free(p); + + if (prompt) |p| { + std.log.info( + "Dictionary prompt: {d} bytes, {d} phrases active", + .{ p.len, self.dictionary.phrases.items.len }, + ); + } + + if (self.transcriber) |*t| { + t.setInitialPrompt(prompt) catch |err| std.log.warn("setInitialPrompt (main) failed: {}", .{err}); + } + if (self.live_transcriber) |*t| { + t.setInitialPrompt(prompt) catch |err| std.log.warn("setInitialPrompt (live) failed: {}", .{err}); + } +} + +/// Inspect the current default input device via CoreAudio and surface a +/// one-shot warning when it is a Bluetooth mic (typically AirPods). Uses the +/// OS-reported `kAudioDevicePropertyTransportType`, which is more robust than +/// substring matching the device name (no false negatives across locales or +/// rebadged BT mics, no false positives on devices that happen to contain +/// "airpods" in their label). Latched via `bluetooth_mic_warned` so the +/// notification fires at most once per process. +/// +/// Also pushes a BT-friendly VAD bucket into the loaded transcribers when +/// the device is Bluetooth. BT codec compression smears speech onsets and +/// inflates background noise, which makes the default Silero threshold +/// (0.5) overshoot — speech is detected late and trimmed early. Relaxed +/// thresholds + larger pad reduce that. The tuning is applied on every +/// recording start (not latched) so a user who switches to a BT device +/// after launch picks up the new params without restarting. +fn checkInputDevice(self: *App) void { + const info = AudioDevice.detectDefaultInput(self.allocator) orelse return; + defer info.deinit(self.allocator); + if (info.kind != .bluetooth) return; + + if (!self.bluetooth_mic_warned) { + self.bluetooth_mic_warned = true; + std.log.warn( + "Default input is a Bluetooth device ({s}); BT mics add latency and degrade transcription quality", + .{info.name}, + ); + self.notifyWarning( + "Bluetooth mic detected. Built-in or USB mics give better dictation accuracy.", + ); + } + + self.applyBluetoothVadTuning(); +} + +/// VAD bucket for Bluetooth input. Threshold is dropped from 0.5/0.3 to +/// 0.25 to avoid clipping the start of utterances; speech_pad_ms is +/// doubled to keep the codec's onset smear inside the speech window; +/// min_silence_ms is raised so the brief micro-gaps inside BT audio don't +/// fragment a single utterance into several. Whisper's own VAD sample-set +/// confirms BT recordings need ~2x the pad of built-in mics. +fn applyBluetoothVadTuning(self: *App) void { + const vad_path = self.config.getVadModelPath(); + const enabled = vad_path != null; + const tuning = bluetooth_vad_tuning; + + if (self.transcriber) |*t| { + t.setVadParams(enabled, tuning.threshold, tuning.min_speech_ms, tuning.min_silence_ms, tuning.speech_pad_ms); + } + if (self.live_transcriber) |*t| { + t.setVadParams(enabled, tuning.threshold, tuning.min_speech_ms, tuning.min_silence_ms, tuning.speech_pad_ms); + } + std.log.info( + "VAD: applied Bluetooth tuning (threshold={d:.2}, min_speech={d}ms, min_silence={d}ms, pad={d}ms)", + .{ tuning.threshold, tuning.min_speech_ms, tuning.min_silence_ms, tuning.speech_pad_ms }, + ); +} + +/// Notify the user once per recording session if CoreAudio has stopped +/// delivering real audio. Counts the trailing run of bit-exact-zero samples +/// reported by `AudioCapture` and fires the existing `on_error` callback when +/// it crosses `stuck_mic_threshold_samples`. The latch (`stuck_mic_warned`) +/// prevents the message from spamming every poll. +fn checkStuckMic(self: *App, audio: *AudioCapture) void { + if (self.stuck_mic_warned) return; + const zero_run = audio.getConsecutiveZeroSamples(); + if (zero_run < stuck_mic_threshold_samples) return; + self.stuck_mic_warned = true; + std.log.warn( + "Stuck microphone detected: {d} consecutive zero samples (>={d}s)", + .{ zero_run, zero_run / 16000 }, + ); + self.notifyWarning( + "Microphone is silent. Check input device and microphone permissions.", + ); +} + +/// Deliver a non-fatal advisory through the warning channel. Unlike +/// `notifyError`, this does NOT flip status to .error and does NOT take over +/// the status text, so a recording in progress keeps showing as recording. +fn notifyWarning(self: *App, message: []const u8) void { + std.debug.assert(message.len > 0); + if (self.config.on_warning) |cb| { + cb(self.config.userdata, c_api.String.fromSlice(message)); + } else if (self.config.on_error) |cb| { + // Backwards-compat: hosts that haven't wired on_warning still see the + // text via on_error, just without the status hijack. + cb(self.config.userdata, c_api.String.fromSlice(message)); + } +} + fn notifyError(self: *App, message: []const u8) void { if (self.config.on_error) |cb| { cb(self.config.userdata, c_api.String.fromSlice(message)); diff --git a/src/DictionaryStore.zig b/src/DictionaryStore.zig new file mode 100644 index 0000000..1e267dd --- /dev/null +++ b/src/DictionaryStore.zig @@ -0,0 +1,182 @@ +//! User-curated phrase list for biasing the Whisper decoder. +//! +//! Stored as a plain text file at `/dictionary.txt`, one phrase +//! per line. Lines starting with `*` (after the leading whitespace) are +//! "starred": they are emitted before unstarred phrases when the prompt is +//! built, so the highest-priority vocabulary fits inside Whisper's ~224-token +//! initial-prompt window even when the file is much larger. +//! +//! Whisper Flow's equivalent (`dictionaryContext` + `starredDictionaryContext`) +//! is the single biggest reason their cloud transcripts spell proper nouns +//! correctly. With a local model we can't match the cloud's contextual +//! richness, but a curated phrase list still moves the needle on names, +//! product terms, and acronyms. +//! +//! Format example: +//! +//! # comments and blank lines are ignored +//! * Sourcegraph +//! * BobrWhisper +//! Polymath +//! ExampleProduct +//! Baseten +//! +//! The file is owner-writable plain text so users can `vim` it. Loading is +//! lazy and best-effort: a missing file yields an empty dictionary, never +//! an error, so dictation works out-of-the-box. + +const std = @import("std"); +const compat = @import("compat.zig"); + +const DictionaryStore = @This(); + +/// Soft cap on the prompt string passed to `whisper.cpp`. Whisper allows +/// ~224 prompt tokens; tokens average ~3.5–4 chars in English, leaving us a +/// 768-byte ceiling that comfortably stays under the limit while leaving +/// headroom for non-ASCII tokenization (e.g. UTF-8 names). +pub const max_prompt_bytes: usize = 768; + +allocator: std.mem.Allocator, +/// Owned, allocator-backed strings. Starred phrases come first. +phrases: std.ArrayListUnmanaged([]u8) = .empty, +starred_count: usize = 0, + +/// Load `/dictionary.txt`. Missing/unreadable files return an +/// empty store rather than erroring — dictation should not break because the +/// user hasn't created a dictionary yet. +pub fn loadFromModelsDir(allocator: std.mem.Allocator, models_dir: []const u8) !DictionaryStore { + std.debug.assert(models_dir.len > 0); + + var store: DictionaryStore = .{ .allocator = allocator }; + errdefer store.deinit(); + + const path = try std.fmt.allocPrint(allocator, "{s}/dictionary.txt", .{models_dir}); + defer allocator.free(path); + + const file = std.Io.Dir.cwd().openFile(compat.io(), path, .{}) catch |err| switch (err) { + error.FileNotFound => return store, + else => { + std.log.warn("DictionaryStore: failed to open {s}: {}", .{ path, err }); + return store; + }, + }; + defer file.close(compat.io()); + + const stat = file.stat(compat.io()) catch |err| { + std.log.warn("DictionaryStore: stat failed for {s}: {}", .{ path, err }); + return store; + }; + // Refuse pathologically large files (>1 MiB) — a sane dictionary is well + // under 100 KiB and we don't want a runaway file to balloon allocations. + const max_file_bytes: u64 = 1024 * 1024; + if (stat.size > max_file_bytes) { + std.log.warn("DictionaryStore: {s} is {d} bytes, exceeds {d} byte cap", .{ path, stat.size, max_file_bytes }); + return store; + } + + const raw = try allocator.alloc(u8, @intCast(stat.size)); + defer allocator.free(raw); + + var io_buffer: [4096]u8 = undefined; + var reader = file.readerStreaming(compat.io(), &io_buffer); + reader.interface.readSliceAll(raw) catch |err| { + std.log.warn("DictionaryStore: read failed for {s}: {}", .{ path, err }); + return store; + }; + + var lines = std.mem.splitScalar(u8, raw, '\n'); + while (lines.next()) |raw_line| { + const line = std.mem.trim(u8, raw_line, " \t\r"); + if (line.len == 0) continue; + if (line[0] == '#') continue; + + const starred = line[0] == '*'; + const body_unbounded = if (starred) std.mem.trim(u8, line[1..], " \t") else line; + if (body_unbounded.len == 0) continue; + + const owned = try allocator.dupe(u8, body_unbounded); + errdefer allocator.free(owned); + + if (starred) { + try store.phrases.insert(allocator, store.starred_count, owned); + store.starred_count += 1; + } else { + try store.phrases.append(allocator, owned); + } + } + + std.debug.assert(store.starred_count <= store.phrases.items.len); + std.log.info( + "DictionaryStore: loaded {d} phrases ({d} starred) from {s}", + .{ store.phrases.items.len, store.starred_count, path }, + ); + return store; +} + +pub fn deinit(self: *DictionaryStore) void { + for (self.phrases.items) |phrase| { + self.allocator.free(phrase); + } + self.phrases.deinit(self.allocator); + self.starred_count = 0; +} + +/// Build the Whisper `initial_prompt` string by concatenating phrases +/// (starred first) until `max_prompt_bytes` would be exceeded. Returns null +/// when the dictionary is empty so callers can branch on "no prompt to set". +/// Caller owns the returned slice. +pub fn buildPrompt(self: DictionaryStore, allocator: std.mem.Allocator) !?[]u8 { + if (self.phrases.items.len == 0) return null; + + var buf: std.ArrayListUnmanaged(u8) = .empty; + errdefer buf.deinit(allocator); + + for (self.phrases.items) |phrase| { + std.debug.assert(phrase.len > 0); + // +2 for ", " separator (only after the first entry) + const sep_len: usize = if (buf.items.len == 0) 0 else 2; + if (buf.items.len + sep_len + phrase.len > max_prompt_bytes) break; + if (sep_len > 0) try buf.appendSlice(allocator, ", "); + try buf.appendSlice(allocator, phrase); + } + + if (buf.items.len == 0) return null; + std.debug.assert(buf.items.len <= max_prompt_bytes); + return try buf.toOwnedSlice(allocator); +} + +test "empty store builds null prompt" { + var store: DictionaryStore = .{ .allocator = std.testing.allocator }; + defer store.deinit(); + const prompt = try store.buildPrompt(std.testing.allocator); + try std.testing.expectEqual(@as(?[]u8, null), prompt); +} + +test "starred phrases come before unstarred" { + var store: DictionaryStore = .{ .allocator = std.testing.allocator }; + defer store.deinit(); + try store.phrases.append(std.testing.allocator, try std.testing.allocator.dupe(u8, "Banana")); + // simulate one starred insert at front + try store.phrases.insert(std.testing.allocator, 0, try std.testing.allocator.dupe(u8, "Apple")); + store.starred_count = 1; + + const prompt = (try store.buildPrompt(std.testing.allocator)) orelse return error.UnexpectedNull; + defer std.testing.allocator.free(prompt); + try std.testing.expectEqualStrings("Apple, Banana", prompt); +} + +test "prompt respects max_prompt_bytes" { + var store: DictionaryStore = .{ .allocator = std.testing.allocator }; + defer store.deinit(); + + // 100 phrases of 16 chars + 2 sep = ~1800 bytes worth, well over 768. + const phrase = "ABCDEFGHIJKLMNOP"; + var i: usize = 0; + while (i < 100) : (i += 1) { + try store.phrases.append(std.testing.allocator, try std.testing.allocator.dupe(u8, phrase)); + } + const prompt = (try store.buildPrompt(std.testing.allocator)) orelse return error.UnexpectedNull; + defer std.testing.allocator.free(prompt); + try std.testing.expect(prompt.len <= max_prompt_bytes); + try std.testing.expect(prompt.len > 0); +} diff --git a/src/Sensibility.zig b/src/Sensibility.zig new file mode 100644 index 0000000..8785c4f --- /dev/null +++ b/src/Sensibility.zig @@ -0,0 +1,199 @@ +//! Local "is this transcript nonsense?" heuristic. +//! +//! Whisper has well-known failure modes on silence, breath noise, music, and +//! short out-of-distribution input. The model is overconfident: it produces +//! grammatically clean but semantically empty strings ("Thanks for watching!", +//! "Subtitles by Amara.org community", looping single tokens, ...). We don't +//! want to spend a local LLM call (or interrupt the user) on text we can detect +//! as garbage with simple pattern matching. +//! +//! `isNonsense` returns true when *any* of three signals fire: +//! +//! 1. Exact (case-insensitive, punctuation-insensitive) match against a +//! known-bad list seeded from observed Whisper failure outputs. +//! 2. Single-token loop: the same word repeated >= `max_token_repeat` times +//! in a row anywhere in the transcript. +//! 3. Trigram loop: any 3-word window repeated >= `max_trigram_repeat` +//! times in the entire transcript. +//! +//! All three are pure, allocation-free, and run in O(n) over token count. +//! False positives have a real cost (lost transcript), so the thresholds are +//! deliberately loose — only "obviously broken" output is rejected. + +const std = @import("std"); + +/// Same single token repeated this many times in a row → loop. +const max_token_repeat: usize = 5; +/// Same trigram repeated this many times anywhere in the transcript → loop. +const max_trigram_repeat: usize = 3; + +/// Strings that Whisper emits during silence / non-speech audio. Compared +/// case-insensitively after stripping punctuation and collapsing whitespace. +const known_bad: []const []const u8 = &.{ + "thanks for watching", + "thank you for watching", + "thanks for watching the video", + "thank you so much for watching", + "please subscribe", + "subtitles by the amara org community", + "subtitles by", + "transcribed by", + "music", + "applause", + "silence", + "you", + "thank you", + "bye", + "okay", + "hmm", + "uh", + "um", +}; + +/// Returns true when the transcript looks like Whisper hallucinated nonsense +/// over silence/non-speech audio. Pure function; no allocation. +pub fn isNonsense(text: []const u8) bool { + var trim_buf: [512]u8 = undefined; + const trimmed = trimAndCanonicalize(text, &trim_buf); + if (trimmed.len == 0) return true; + + if (matchesKnownBad(trimmed)) return true; + + // Token-level loop checks operate on the original text so we keep word + // boundaries intact (the canonicalizer collapsed punctuation but tokens + // are easier to reason about by splitting on whitespace). + if (hasTokenLoop(trimmed, max_token_repeat)) return true; + if (hasTrigramLoop(trimmed, max_trigram_repeat)) return true; + + return false; +} + +/// Lowercase, strip ASCII punctuation, collapse internal whitespace into +/// single spaces, trim leading/trailing whitespace. Writes into `buf` and +/// returns a slice of the prefix actually used. Truncates silently when the +/// input would overflow the buffer — fine because all subsequent checks +/// operate on the canonicalized prefix. +fn trimAndCanonicalize(text: []const u8, buf: []u8) []const u8 { + std.debug.assert(buf.len > 0); + var out_len: usize = 0; + var prev_space = true; // suppress leading whitespace + for (text) |raw| { + if (out_len == buf.len) break; + const lower = std.ascii.toLower(raw); + const is_alnum = std.ascii.isAlphanumeric(lower); + if (is_alnum) { + buf[out_len] = lower; + out_len += 1; + prev_space = false; + } else if (!prev_space) { + buf[out_len] = ' '; + out_len += 1; + prev_space = true; + } + } + // strip trailing space introduced by canonicalization + while (out_len > 0 and buf[out_len - 1] == ' ') : (out_len -= 1) {} + return buf[0..out_len]; +} + +fn matchesKnownBad(canonical: []const u8) bool { + for (known_bad) |needle| { + if (std.mem.eql(u8, canonical, needle)) return true; + } + return false; +} + +fn hasTokenLoop(canonical: []const u8, threshold: usize) bool { + std.debug.assert(threshold >= 2); + var iter = std.mem.splitScalar(u8, canonical, ' '); + var prev: ?[]const u8 = null; + var run: usize = 1; + while (iter.next()) |tok| { + if (tok.len == 0) continue; + if (prev) |p| { + if (std.mem.eql(u8, tok, p)) { + run += 1; + if (run >= threshold) return true; + } else { + run = 1; + } + } + prev = tok; + } + return false; +} + +fn hasTrigramLoop(canonical: []const u8, threshold: usize) bool { + std.debug.assert(threshold >= 2); + + // Cap analyzed token count to bound work on pathologically long input. + const max_tokens: usize = 512; + var tokens: [max_tokens][]const u8 = undefined; + var n: usize = 0; + + var iter = std.mem.splitScalar(u8, canonical, ' '); + while (iter.next()) |tok| { + if (tok.len == 0) continue; + if (n == max_tokens) break; + tokens[n] = tok; + n += 1; + } + + if (n < 3 * threshold) return false; + + // Fixed-size O(n^2 / 9) comparison: for each starting position i, count + // how many times the trigram (tokens[i], tokens[i+1], tokens[i+2]) + // appears at later non-overlapping positions. With n capped at 512 this + // is at most ~30k comparisons. + var i: usize = 0; + while (i + 2 < n) : (i += 1) { + const t0 = tokens[i]; + const t1 = tokens[i + 1]; + const t2 = tokens[i + 2]; + var count: usize = 1; + var j: usize = i + 3; + while (j + 2 < n) : (j += 1) { + if (std.mem.eql(u8, tokens[j], t0) and + std.mem.eql(u8, tokens[j + 1], t1) and + std.mem.eql(u8, tokens[j + 2], t2)) + { + count += 1; + if (count >= threshold) return true; + j += 2; // skip past the matched window + } + } + } + return false; +} + +test "blank and whitespace are nonsense" { + try std.testing.expect(isNonsense("")); + try std.testing.expect(isNonsense(" ")); + try std.testing.expect(isNonsense("\n\t .. ")); +} + +test "known-bad list" { + try std.testing.expect(isNonsense("Thanks for watching!")); + try std.testing.expect(isNonsense("THANKS FOR WATCHING.")); + try std.testing.expect(isNonsense(" thanks for watching ")); + try std.testing.expect(isNonsense("Subtitles by the Amara.org community")); + try std.testing.expect(isNonsense("[Music]")); + try std.testing.expect(isNonsense("you")); +} + +test "real sentences are not nonsense" { + try std.testing.expect(!isNonsense("Hello, please open the file in the editor.")); + try std.testing.expect(!isNonsense("The quick brown fox jumps over the lazy dog.")); + try std.testing.expect(!isNonsense("Let me think about this for a second.")); +} + +test "token loop is nonsense" { + try std.testing.expect(isNonsense("okay okay okay okay okay")); + try std.testing.expect(isNonsense("yes yes yes yes yes please")); + try std.testing.expect(!isNonsense("yes yes please")); +} + +test "trigram loop is nonsense" { + try std.testing.expect(isNonsense("thanks for watching thanks for watching thanks for watching")); + try std.testing.expect(!isNonsense("thanks for watching this short clip")); +} diff --git a/src/SettingsStore.zig b/src/SettingsStore.zig index 8bad362..4d0dc5e 100644 --- a/src/SettingsStore.zig +++ b/src/SettingsStore.zig @@ -101,6 +101,9 @@ fn writeApple(domain: []const u8, settings: c_api.Settings) WriteError!void { const key_custom_prompt = try makeCFString("customPrompt"); defer cf.CFRelease(key_custom_prompt); + const key_whisper_mode = try makeCFString("whisperMode"); + defer cf.CFRelease(key_whisper_mode); + var tone_value: i32 = @intFromEnum(settings.tone); const tone_number = cf.CFNumberCreate( cf.kCFAllocatorDefault, @@ -134,6 +137,12 @@ fn writeApple(domain: []const u8, settings: c_api.Settings) WriteError!void { cf.CFPreferencesSetAppValue(key_custom_prompt, null, domain_cf); } + cf.CFPreferencesSetAppValue( + key_whisper_mode, + if (settings.whisper_mode) cf.kCFBooleanTrue.* else cf.kCFBooleanFalse.*, + domain_cf, + ); + const sync_ok = cf.CFPreferencesAppSynchronize(domain_cf); if (sync_ok == 0) { return WriteError.SyncFailed; diff --git a/src/audio/AudioCapture.zig b/src/audio/AudioCapture.zig index 0dea37a..13ee779 100644 --- a/src/audio/AudioCapture.zig +++ b/src/audio/AudioCapture.zig @@ -118,6 +118,10 @@ buffer: std.ArrayListUnmanaged(f32), mutex: SpinMutex = .{}, // RMS audio level from the most recent callback buffer, updated under mutex audio_level: f32 = 0, +// Stuck-mic detection: count of trailing samples whose CoreAudio callback +// delivered an exactly-zero buffer. Reset to 0 the moment any non-zero sample +// arrives. Exact-zero input is a strong signal for dead audio routes. +consecutive_zero_samples: usize = 0, pub const Config = struct { sample_rate: f64 = 16000.0, @@ -155,6 +159,7 @@ pub fn start(self: *AudioCapture) !void { if (self.is_recording) return; self.buffer.clearRetainingCapacity(); + self.consecutive_zero_samples = 0; try self.buffer.ensureTotalCapacity(self.allocator, 16000 * 30); if (builtin.os.tag == .macos) { @@ -188,6 +193,14 @@ pub fn getSamples(self: *AudioCapture) []const f32 { return self.buffer.items; } +/// Mutable view into the captured samples. Same lifetime constraints as `getSamples`; +/// only safe when recording is stopped. Used by callers that want to apply in-place +/// preprocessing such as peak normalization before transcription. +pub fn getSamplesMut(self: *AudioCapture) []f32 { + std.debug.assert(!self.is_recording); + return self.buffer.items; +} + pub fn copySamples(self: *AudioCapture, allocator: std.mem.Allocator) ![]f32 { self.mutex.lock(); defer self.mutex.unlock(); @@ -219,6 +232,15 @@ pub fn getSampleCount(self: *AudioCapture) usize { return self.buffer.items.len; } +/// Trailing run of bit-exact-zero samples reported by CoreAudio. Crosses the +/// stuck-mic threshold (~5 s = 80000 samples at 16 kHz) when the input device +/// stops delivering real audio. Resets to 0 on any non-zero sample. +pub fn getConsecutiveZeroSamples(self: *AudioCapture) usize { + self.mutex.lock(); + defer self.mutex.unlock(); + return self.consecutive_zero_samples; +} + pub fn clearBuffer(self: *AudioCapture) void { self.mutex.lock(); defer self.mutex.unlock(); @@ -315,12 +337,22 @@ fn audioInputCallback( std.log.err("Failed to append audio samples", .{}); }; - // Compute RMS audio level from this callback buffer + // Single pass: RMS energy + exact-zero detection. The zero check + // surfaces a stuck microphone (dead route, denied permission at the + // system level, suspended driver). RMS alone is too noisy: a quiet mic + // still produces sub-1e-6 jitter, but a dead route is bit-exact zero. var energy: f32 = 0; + var any_nonzero = false; for (samples[0..sample_count]) |s| { energy += s * s; + if (s != 0) any_nonzero = true; } self.audio_level = @sqrt(energy / @as(f32, @floatFromInt(sample_count))); + if (any_nonzero) { + self.consecutive_zero_samples = 0; + } else { + self.consecutive_zero_samples +|= sample_count; + } } _ = c.AudioQueueEnqueueBuffer(queue, buffer, 0, null); @@ -371,6 +403,47 @@ pub fn trimSilenceBounds(samples: []const f32, threshold: f32) TrimBounds { return .{ .start = start_idx, .end = end_idx }; } +/// Estimate ambient noise floor from the first ~0.5s of audio. +/// Returns the RMS energy of a leading window, useful for deriving +/// an adaptive silence-trim threshold (e.g. 3× noise floor). +pub fn computeNoiseFloor(samples: []const f32) f32 { + if (samples.len == 0) return 0; + + const window: usize = @min(8000, samples.len); + var energy: f32 = 0; + for (samples[0..window]) |s| { + energy += s * s; + } + return @sqrt(energy / @as(f32, @floatFromInt(window))); +} + +/// In-place peak normalization. Scales `samples` so the largest absolute +/// amplitude becomes `target_peak` (typically 0.95 to avoid hard clipping). +/// Returns the gain factor that was applied. Only ever scales up: if the +/// input is already at or above `target_peak` the buffer is unchanged. +/// Silent buffers (peak < 1e-4) are also left alone to avoid amplifying noise. +/// +/// Whispered speech routinely has peaks of ~0.25 vs. ~0.6 for normal voice, +/// which produced a 4–7× WER improvement in the `tune` experiments. Applying +/// this before trimming and transcription is safe for normal voice (the gain +/// factor stays near 1.0) but materially helps quiet audio. +pub fn peakNormalize(samples: []f32, target_peak: f32) f32 { + std.debug.assert(target_peak > 0.0); + std.debug.assert(target_peak <= 1.0); + if (samples.len == 0) return 1.0; + + var peak: f32 = 0; + for (samples) |s| { + const a = @abs(s); + if (a > peak) peak = a; + } + if (peak < 1e-4 or peak >= target_peak) return 1.0; + + const gain = target_peak / peak; + for (samples) |*s| s.* *= gain; + return gain; +} + pub fn resample(allocator: std.mem.Allocator, samples: []const f32, from_rate: f64, to_rate: f64) ![]f32 { if (from_rate == to_rate) { return allocator.dupe(f32, samples); @@ -404,3 +477,39 @@ test "voice activity detection" { try std.testing.expect(!detectVoiceActivity(&silence, 0.01)); try std.testing.expect(detectVoiceActivity(&voice, 0.01)); } + +test "compute noise floor" { + const silence = [_]f32{0.0} ** 100; + const noise = [_]f32{0.1} ** 100; + + try std.testing.expectEqual(@as(f32, 0), computeNoiseFloor(&silence)); + try std.testing.expect(computeNoiseFloor(&noise) > 0.09); + try std.testing.expect(computeNoiseFloor(&noise) < 0.11); + try std.testing.expectEqual(@as(f32, 0), computeNoiseFloor(&[_]f32{})); +} + +test "peakNormalize boosts quiet audio" { + var samples = [_]f32{ 0.1, -0.2, 0.05, -0.1 }; + const gain = peakNormalize(&samples, 0.95); + try std.testing.expectApproxEqAbs(@as(f32, 4.75), gain, 0.001); + try std.testing.expectApproxEqAbs(@as(f32, -0.95), samples[1], 0.001); +} + +test "peakNormalize leaves loud audio unchanged" { + var samples = [_]f32{ 0.5, -0.95 }; + const gain = peakNormalize(&samples, 0.95); + try std.testing.expectEqual(@as(f32, 1.0), gain); + try std.testing.expectEqual(@as(f32, 0.5), samples[0]); +} + +test "peakNormalize handles silence" { + var silence = [_]f32{0.0} ** 8; + const gain = peakNormalize(&silence, 0.95); + try std.testing.expectEqual(@as(f32, 1.0), gain); +} + +test "peakNormalize empty buffer" { + var empty: [0]f32 = .{}; + const gain = peakNormalize(&empty, 0.95); + try std.testing.expectEqual(@as(f32, 1.0), gain); +} diff --git a/src/audio/AudioDevice.zig b/src/audio/AudioDevice.zig new file mode 100644 index 0000000..7840ab5 --- /dev/null +++ b/src/audio/AudioDevice.zig @@ -0,0 +1,172 @@ +//! macOS CoreAudio default-input-device introspection. +//! +//! Tags each recorded snippet with the audio source (built-in mic, AirPods, +//! USB interface, ...) so the test corpus can be sliced by hardware. Keeps +//! the CoreAudio surface small: three property reads on the system + device +//! objects, no full SDK translation. + +const std = @import("std"); +const builtin = @import("builtin"); + +pub const Kind = enum { + internal, + bluetooth, + usb, + unknown, + + pub fn fromString(s: []const u8) ?Kind { + return std.meta.stringToEnum(Kind, s); + } +}; + +pub const Info = struct { + /// Heap-allocated. Caller owns; free with `deinit`. + name: []u8, + kind: Kind, + + pub fn deinit(self: Info, allocator: std.mem.Allocator) void { + allocator.free(self.name); + } +}; + +const c = if (builtin.os.tag == .macos) MacCoreAudio else struct {}; + +const MacCoreAudio = struct { + pub const OSStatus = i32; + pub const UInt32 = u32; + pub const AudioObjectID = UInt32; + + pub const AudioObjectPropertyAddress = extern struct { + mSelector: UInt32, + mScope: UInt32, + mElement: UInt32, + }; + + pub const CFStringRef = ?*opaque {}; + pub const CFTypeRef = ?*anyopaque; + + pub extern "c" fn AudioObjectGetPropertyData( + inObjectID: AudioObjectID, + inAddress: *const AudioObjectPropertyAddress, + inQualifierDataSize: UInt32, + inQualifierData: ?*const anyopaque, + ioDataSize: *UInt32, + outData: *anyopaque, + ) OSStatus; + + pub extern "c" fn CFStringGetCString( + theString: CFStringRef, + buffer: [*]u8, + bufferSize: c_long, + encoding: u32, + ) i32; + + pub extern "c" fn CFRelease(cf: CFTypeRef) void; + + pub const kAudioObjectSystemObject: AudioObjectID = 1; + pub const kAudioObjectPropertyElementMain: UInt32 = 0; + pub const kCFStringEncodingUTF8: u32 = 0x08000100; +}; + +inline fn fourCC(comptime s: *const [4]u8) u32 { + return (@as(u32, s[0]) << 24) | (@as(u32, s[1]) << 16) | (@as(u32, s[2]) << 8) | @as(u32, s[3]); +} + +/// Best-effort default-input device label. Returns null when we are not on +/// macOS or CoreAudio refuses to cooperate (e.g. no input device, headless +/// CI). Caller owns the returned `Info.name`. +pub fn detectDefaultInput(allocator: std.mem.Allocator) ?Info { + if (comptime builtin.os.tag != .macos) return null; + return detectMacos(allocator) catch null; +} + +fn detectMacos(allocator: std.mem.Allocator) !Info { + const device_id = try defaultInputDeviceId(); + const name = try deviceName(allocator, device_id); + errdefer allocator.free(name); + const kind = transportKind(device_id); + return .{ .name = name, .kind = kind }; +} + +fn defaultInputDeviceId() !c.AudioObjectID { + const addr = c.AudioObjectPropertyAddress{ + .mSelector = comptime fourCC("dIn "), + .mScope = comptime fourCC("glob"), + .mElement = c.kAudioObjectPropertyElementMain, + }; + var device_id: c.AudioObjectID = 0; + var size: c.UInt32 = @sizeOf(c.AudioObjectID); + const status = c.AudioObjectGetPropertyData( + c.kAudioObjectSystemObject, + &addr, + 0, + null, + &size, + @ptrCast(&device_id), + ); + if (status != 0 or device_id == 0) return error.NoDefaultInput; + return device_id; +} + +fn deviceName(allocator: std.mem.Allocator, device_id: c.AudioObjectID) ![]u8 { + const addr = c.AudioObjectPropertyAddress{ + .mSelector = comptime fourCC("lnam"), + .mScope = comptime fourCC("glob"), + .mElement = c.kAudioObjectPropertyElementMain, + }; + var name_ref: c.CFStringRef = null; + var size: c.UInt32 = @sizeOf(c.CFStringRef); + const status = c.AudioObjectGetPropertyData( + device_id, + &addr, + 0, + null, + &size, + @ptrCast(&name_ref), + ); + if (status != 0 or name_ref == null) return error.NoName; + defer c.CFRelease(name_ref); + + // Device names are short. 256 bytes covers everything Apple ships and any + // reasonable third-party label without bringing CFStringGetMaximumSizeForEncoding in. + var buf: [256]u8 = undefined; + if (c.CFStringGetCString(name_ref, &buf, buf.len, c.kCFStringEncodingUTF8) == 0) { + return error.NameDecode; + } + const slice = std.mem.sliceTo(&buf, 0); + return allocator.dupe(u8, slice); +} + +fn transportKind(device_id: c.AudioObjectID) Kind { + const addr = c.AudioObjectPropertyAddress{ + .mSelector = comptime fourCC("tran"), + .mScope = comptime fourCC("glob"), + .mElement = c.kAudioObjectPropertyElementMain, + }; + var transport: c.UInt32 = 0; + var size: c.UInt32 = @sizeOf(c.UInt32); + const status = c.AudioObjectGetPropertyData( + device_id, + &addr, + 0, + null, + &size, + @ptrCast(&transport), + ); + if (status != 0) return .unknown; + + return switch (transport) { + comptime fourCC("bltn") => .internal, + comptime fourCC("blue"), comptime fourCC("blea") => .bluetooth, + comptime fourCC("usb ") => .usb, + else => .unknown, + }; +} + +test "Kind.fromString round-trip" { + try std.testing.expectEqual(Kind.internal, Kind.fromString("internal").?); + try std.testing.expectEqual(Kind.bluetooth, Kind.fromString("bluetooth").?); + try std.testing.expectEqual(Kind.usb, Kind.fromString("usb").?); + try std.testing.expectEqual(Kind.unknown, Kind.fromString("unknown").?); + try std.testing.expectEqual(@as(?Kind, null), Kind.fromString("nope")); +} diff --git a/src/build/BobrWhisperXcodebuild.zig b/src/build/BobrWhisperXcodebuild.zig index fd7726e..629ab55 100644 --- a/src/build/BobrWhisperXcodebuild.zig +++ b/src/build/BobrWhisperXcodebuild.zig @@ -52,6 +52,9 @@ pub fn init( "ARCHS=arm64", }) else + // Ad-hoc self-signing for local dev (no Apple Developer Team). + // Keeps a stable signature so TCC permissions (microphone) persist + // across rebuilds, while not requiring a paid signing identity. b.addSystemCommand(&.{ "xcodebuild", "-project", @@ -60,6 +63,11 @@ pub fn init( "BobrWhisper", "-configuration", configuration, + "CODE_SIGN_STYLE=Manual", + "CODE_SIGN_IDENTITY=-", + "CODE_SIGNING_REQUIRED=YES", + "CODE_SIGNING_ALLOWED=YES", + "DEVELOPMENT_TEAM=", "build", "ARCHS=arm64", }), diff --git a/src/c_api.zig b/src/c_api.zig index 1cd478c..8892441 100644 --- a/src/c_api.zig +++ b/src/c_api.zig @@ -142,6 +142,9 @@ pub const Tone = enum(c_int) { pub const StatusCallback = *const fn (?*anyopaque, Status) callconv(.c) void; pub const TranscriptCallback = *const fn (?*anyopaque, String, bool) callconv(.c) void; pub const ErrorCallback = *const fn (?*anyopaque, String) callconv(.c) void; +/// Non-fatal warning channel. Used by the App for stuck-mic, Bluetooth-mic, +/// and similar advisories that should NOT flip status to .error. +pub const WarningCallback = *const fn (?*anyopaque, String) callconv(.c) void; pub const RuntimeConfig = extern struct { userdata: ?*anyopaque, @@ -149,12 +152,14 @@ pub const RuntimeConfig = extern struct { on_status_change: ?StatusCallback, on_transcript: ?TranscriptCallback, on_error: ?ErrorCallback, + on_warning: ?WarningCallback, models_dir: ?[*:0]const u8, config_path: ?[*:0]const u8, llm_model_path: ?[*:0]const u8, vad_model_path: ?[*:0]const u8, + whisper_mode: bool, pub fn getModelsDir(self: RuntimeConfig) []const u8 { if (self.models_dir) |ptr| { @@ -191,6 +196,7 @@ pub const Settings = extern struct { auto_punctuate: bool, use_llm_formatting: bool, custom_prompt: ?[*:0]const u8, + whisper_mode: bool, pub fn getCustomPrompt(self: Settings) ?[]const u8 { if (self.custom_prompt) |ptr| { @@ -206,6 +212,7 @@ pub const TranscribeOptions = extern struct { remove_filler_words: bool, auto_punctuate: bool, use_llm_formatting: bool, + whisper_mode: bool, pub fn getLanguage(self: TranscribeOptions) []const u8 { if (self.language) |ptr| { diff --git a/src/cli.zig b/src/cli.zig index 44ca83c..b1be49a 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -5,6 +5,8 @@ const compat = @import("compat.zig"); const builtin = @import("builtin"); const asr = @import("asr"); const AudioCapture = @import("audio/AudioCapture.zig"); +const snippet = @import("snippet.zig"); +const tune = @import("tune.zig"); const has_llm = builtin.os.tag == .macos; const LlamaClient = if (has_llm) @import("llm/LlamaClient.zig") else void; @@ -223,12 +225,18 @@ pub fn main(init: std.process.Init) !void { try transcribeRawCommand(allocator, args[2..]); } else if (std.mem.eql(u8, command, "record")) { try recordCommand(allocator, args[2..]); + } else if (std.mem.eql(u8, command, "snippet")) { + try snippet.run(allocator, args[2..]); + } else if (std.mem.eql(u8, command, "tune")) { + try tune.run(allocator, args[2..]); } else if (std.mem.eql(u8, command, "live")) { try liveCommand(allocator, args[2..]); } else if (std.mem.eql(u8, command, "models")) { try modelsCommand(allocator); } else if (std.mem.eql(u8, command, "languages")) { languagesCommand(); + } else if (std.mem.eql(u8, command, "test-whisper")) { + try testWhisperCommand(allocator, args[2..]); } else if (std.mem.eql(u8, command, "help")) { printUsage(); } else { @@ -248,6 +256,9 @@ fn printUsage() void { \\ transcribe Transcribe a WAV file \\ transcribe-raw Transcribe raw f32 audio \\ record Record audio from microphone + \\ snippet [options] Record a labeled snippet for the test corpus (no transcription) + \\ tune [options] Run snippets through VAD presets, report WER vs. ground truth + \\ test-whisper [model] [--whisper-mode] Record until 'q', then transcribe (debug) \\ models List available models \\ languages List supported languages \\ help Show this help @@ -541,6 +552,115 @@ fn liveCommand(allocator: std.mem.Allocator, args: []const []const u8) !void { } } +fn testWhisperCommand(allocator: std.mem.Allocator, args: []const []const u8) !void { + var selected_model: WhisperModel = .small; + var whisper_mode = false; + + for (args) |arg| { + if (std.mem.eql(u8, arg, "--whisper-mode")) { + whisper_mode = true; + } else if (!std.mem.startsWith(u8, arg, "-")) { + selected_model = WhisperModel.fromString(arg) orelse { + std.debug.print("Unknown model: {s}\n", .{arg}); + std.debug.print("Available: tiny, base, small, medium, large, large_turbo\n", .{}); + return; + }; + } + } + + std.debug.print("test-whisper: model={s}, whisper_mode={}\n", .{ @tagName(selected_model), whisper_mode }); + + const model_path = selected_model.ensureDownloaded(allocator) catch |err| { + std.debug.print("Failed to get model: {}\n", .{err}); + return; + }; + defer allocator.free(model_path); + + const vad_path = getVadModelPath(allocator); + defer if (vad_path) |p| allocator.free(p); + + std.debug.print("Loading model: {s}\n", .{model_path}); + if (vad_path != null) std.debug.print("VAD: enabled\n", .{}); + + var transcriber = WhisperCppAdapter.init(allocator, .{ + .model_path = model_path, + .language = "en", + .n_threads = 4, + .vad_enabled = vad_path != null, + .vad_model_path = vad_path, + .vad_threshold = if (whisper_mode) 0.3 else 0.5, + .vad_min_speech_ms = if (whisper_mode) 150 else 250, + .vad_min_silence_ms = if (whisper_mode) 200 else 100, + .vad_speech_pad_ms = if (whisper_mode) 100 else 30, + }) catch |err| { + std.debug.print("Failed to load model: {}\n", .{err}); + return; + }; + defer transcriber.deinit(); + + std.debug.print("Model loaded. Starting recording...\n", .{}); + std.debug.print("Press 'q' then Enter to stop and transcribe.\n\n", .{}); + + var audio = try AudioCapture.init(allocator); + defer audio.deinit(); + + try audio.start(); + try snippet.recordUntilQ(&audio); + + std.debug.print("\n\nStopping recording...\n", .{}); + audio.stop(); + + const samples = audio.getSamples(); + const duration_s = @as(f64, @floatFromInt(samples.len)) / 16000.0; + std.debug.print("Recorded {d} samples ({d:.2}s)\n", .{ samples.len, duration_s }); + + if (samples.len == 0) { + std.debug.print("No audio recorded.\n", .{}); + return; + } + + // Compute and log noise floor + trim info + const noise_floor = AudioCapture.computeNoiseFloor(samples); + const trim_threshold = @max(noise_floor * 3.0, 0.0005); + std.debug.print("\nDiagnostics:\n", .{}); + std.debug.print(" noise floor (RMS of first 0.5s): {d:.6}\n", .{noise_floor}); + std.debug.print(" trim threshold (3x floor, min 0.0005): {d:.6}\n", .{trim_threshold}); + std.debug.print(" whisper_mode: {}\n", .{whisper_mode}); + + const bounds = AudioCapture.trimSilenceBounds(samples, trim_threshold); + const trimmed = samples[bounds.start..bounds.end]; + const trimmed_pct = @as(f32, @floatFromInt(samples.len - trimmed.len)) / @as(f32, @floatFromInt(samples.len)) * 100.0; + std.debug.print(" trim bounds: [{d}..{d}] of {d} ({d:.1}% trimmed)\n", .{ bounds.start, bounds.end, samples.len, trimmed_pct }); + + const segment = if (trimmed.len > 0) trimmed else samples; + std.debug.print(" transcribing {d} samples ({d:.2}s)\n\n", .{ + segment.len, + @as(f64, @floatFromInt(segment.len)) / 16000.0, + }); + + // Also save raw audio for replay + const output_path = "test-whisper.raw"; + const file = try std.Io.Dir.cwd().createFile(compat.io(), output_path, .{}); + defer file.close(compat.io()); + try file.writeStreamingAll(compat.io(), std.mem.sliceAsBytes(samples)); + std.debug.print("Raw audio saved to: {s} (replay with: bobrwhisper-cli transcribe-raw {s})\n\n", .{ output_path, output_path }); + + // Transcribe + std.debug.print("Transcribing...\n", .{}); + const text = transcriber.transcribe(segment) catch |err| { + std.debug.print("Transcription failed: {}\n", .{err}); + return; + }; + defer allocator.free(text); + + const result = std.mem.trim(u8, text, " \t\n"); + std.debug.print("\n--- Transcription ---\n{s}\n", .{result}); + + if (result.len == 0) { + std.debug.print("\n(empty result - whisper returned nothing)\n", .{}); + } +} + fn isHallucination(text: []const u8) bool { const hallucinations = [_][]const u8{ "you", diff --git a/src/snippet.zig b/src/snippet.zig new file mode 100644 index 0000000..3d387a7 --- /dev/null +++ b/src/snippet.zig @@ -0,0 +1,473 @@ +//! Snippet recorder - capture labeled audio for the test corpus. +//! +//! The CLI does NOT transcribe; it only records audio plus end-user metadata. +//! Each snippet is written as a sibling pair so experiment code can mmap one +//! and read the other: +//! +//! /.wav 16 kHz mono 16-bit PCM WAV +//! /.json ground truth label + whisper flag + audio stats +//! +//! Default output dir is `~/.bobrwhisper/snippets`. + +const std = @import("std"); +const builtin = @import("builtin"); +const compat = @import("compat.zig"); +const AudioCapture = @import("audio/AudioCapture.zig"); +const AudioDevice = @import("audio/AudioDevice.zig"); + +const sample_rate: u32 = 16000; + +pub const Args = struct { + label: ?[]const u8 = null, + whisper: ?bool = null, + out_dir: ?[]const u8 = null, + duration_secs: ?u64 = null, + device: ?[]const u8 = null, + device_kind: ?AudioDevice.Kind = null, + tags: []const []const u8 = &.{}, +}; + +const ParseError = error{ + UnknownArg, + MissingValue, + InvalidDuration, + InvalidDeviceKind, + ConflictingFlags, + HelpRequested, + OutOfMemory, +}; + +fn parseArgs(allocator: std.mem.Allocator, args: []const []const u8, tags_out: *std.ArrayListUnmanaged([]const u8)) ParseError!Args { + var out: Args = .{}; + var i: usize = 0; + while (i < args.len) : (i += 1) { + const a = args[i]; + if (std.mem.eql(u8, a, "--help") or std.mem.eql(u8, a, "-h")) { + return error.HelpRequested; + } else if (std.mem.eql(u8, a, "--label")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.label = args[i]; + } else if (std.mem.eql(u8, a, "--whisper")) { + if (out.whisper) |w| if (!w) return error.ConflictingFlags; + out.whisper = true; + } else if (std.mem.eql(u8, a, "--no-whisper")) { + if (out.whisper) |w| if (w) return error.ConflictingFlags; + out.whisper = false; + } else if (std.mem.eql(u8, a, "--out")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.out_dir = args[i]; + } else if (std.mem.eql(u8, a, "--duration")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.duration_secs = std.fmt.parseInt(u64, args[i], 10) catch return error.InvalidDuration; + } else if (std.mem.eql(u8, a, "--device")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.device = args[i]; + } else if (std.mem.eql(u8, a, "--device-kind")) { + i += 1; + if (i >= args.len) return error.MissingValue; + out.device_kind = AudioDevice.Kind.fromString(args[i]) orelse return error.InvalidDeviceKind; + } else if (std.mem.eql(u8, a, "--tag")) { + i += 1; + if (i >= args.len) return error.MissingValue; + try tags_out.append(allocator, args[i]); + } else { + std.debug.print("snippet: unknown argument '{s}'\n", .{a}); + return error.UnknownArg; + } + } + out.tags = tags_out.items; + return out; +} + +pub fn run(allocator: std.mem.Allocator, raw_args: []const []const u8) !void { + var tag_list = std.ArrayListUnmanaged([]const u8).empty; + defer tag_list.deinit(allocator); + + const parsed = parseArgs(allocator, raw_args, &tag_list) catch |err| switch (err) { + error.HelpRequested => { + printUsage(); + return; + }, + else => { + std.debug.print("\n", .{}); + printUsage(); + return err; + }, + }; + + if (builtin.os.tag != .macos) { + std.debug.print("snippet: audio capture is only supported on macOS\n", .{}); + return error.UnsupportedPlatform; + } + + // Resolve metadata: prompt for anything not passed on the CLI. + var label_storage: ?[]u8 = null; + defer if (label_storage) |s| allocator.free(s); + const label: []const u8 = blk: { + if (parsed.label) |l| break :blk l; + label_storage = try promptLine(allocator, "Ground truth label (what will you say?): "); + break :blk label_storage.?; + }; + if (label.len == 0) { + std.debug.print("snippet: label cannot be empty\n", .{}); + return error.EmptyLabel; + } + + const whisper: bool = parsed.whisper orelse try promptYesNo("Whispered? [y/N]: ", false); + + // Resolve audio device: explicit CLI override takes priority; otherwise + // probe CoreAudio for the default input. detected_info is null on non-macOS + // or when CoreAudio refuses (no device, headless), in which case we leave + // the device fields null in the metadata. + const detected_info: ?AudioDevice.Info = AudioDevice.detectDefaultInput(allocator); + defer if (detected_info) |d| d.deinit(allocator); + + const device: ?[]const u8 = parsed.device orelse if (detected_info) |d| d.name else null; + const device_kind: ?AudioDevice.Kind = parsed.device_kind orelse if (detected_info) |d| d.kind else null; + + // Resolve output directory. + var out_dir_storage: ?[]u8 = null; + defer if (out_dir_storage) |s| allocator.free(s); + const out_dir: []const u8 = blk: { + if (parsed.out_dir) |d| break :blk d; + const home = compat.getenv("HOME") orelse { + std.debug.print("snippet: HOME is unset; pass --out \n", .{}); + return error.MissingHome; + }; + out_dir_storage = try std.fmt.allocPrint(allocator, "{s}/.bobrwhisper/snippets", .{home}); + break :blk out_dir_storage.?; + }; + try ensureDir(out_dir); + + // Build the snippet ID from the wall-clock timestamp. + const created_at_ms = compat.milliTimestamp(); + const id = try formatSnippetId(allocator, created_at_ms); + defer allocator.free(id); + + const wav_path = try std.fmt.allocPrint(allocator, "{s}/{s}.wav", .{ out_dir, id }); + defer allocator.free(wav_path); + const json_path = try std.fmt.allocPrint(allocator, "{s}/{s}.json", .{ out_dir, id }); + defer allocator.free(json_path); + + std.debug.print("\nSnippet: {s}\n", .{id}); + std.debug.print(" label: {s}\n", .{label}); + std.debug.print(" whisper: {}\n", .{whisper}); + std.debug.print(" device: {s} ({s})\n", .{ + device orelse "(unknown)", + if (device_kind) |k| @tagName(k) else "unknown", + }); + if (parsed.tags.len > 0) { + std.debug.print(" tags: ", .{}); + for (parsed.tags, 0..) |t, idx| { + if (idx > 0) std.debug.print(", ", .{}); + std.debug.print("{s}", .{t}); + } + std.debug.print("\n", .{}); + } + std.debug.print(" out: {s}\n\n", .{out_dir}); + + // Start recording. + var audio = try AudioCapture.init(allocator); + defer audio.deinit(); + try audio.start(); + + if (parsed.duration_secs) |secs| { + std.debug.print("Recording for {d}s...\n", .{secs}); + try recordFixed(&audio, secs); + } else { + std.debug.print("Recording... press 'q' then Enter to stop.\n", .{}); + try recordUntilQ(&audio); + } + + audio.stop(); + + const samples = audio.getSamples(); + if (samples.len == 0) { + std.debug.print("snippet: no audio captured; aborting\n", .{}); + return error.NoAudio; + } + + const duration_seconds = @as(f64, @floatFromInt(samples.len)) / @as(f64, @floatFromInt(sample_rate)); + const noise_floor = AudioCapture.computeNoiseFloor(samples); + + std.debug.print("\nCaptured {d} samples ({d:.2}s, noise floor {d:.6})\n", .{ samples.len, duration_seconds, noise_floor }); + + try writeWav(wav_path, samples); + try writeMetadata(allocator, json_path, .{ + .id = id, + .label = label, + .whisper = whisper, + .audio_file = std.fs.path.basename(wav_path), + .duration_seconds = duration_seconds, + .sample_rate = sample_rate, + .channels = 1, + .num_samples = samples.len, + .noise_floor_rms = noise_floor, + .created_at_unix_ms = created_at_ms, + .device = device, + .device_kind = device_kind, + .tags = parsed.tags, + }); + + std.debug.print("\nSaved:\n {s}\n {s}\n", .{ wav_path, json_path }); +} + +const Metadata = struct { + id: []const u8, + label: []const u8, + whisper: bool, + audio_file: []const u8, + duration_seconds: f64, + sample_rate: u32, + channels: u32, + num_samples: usize, + noise_floor_rms: f32, + created_at_unix_ms: i64, + /// Free-form device label (e.g. "MacBook Pro Microphone", "AirPods Pro"). + device: ?[]const u8, + /// Coarse device classification for slicing the test corpus. + device_kind: ?AudioDevice.Kind, + /// Free-form tags so experiments can group snippets by content/condition. + tags: []const []const u8, +}; + +fn writeMetadata(allocator: std.mem.Allocator, path: []const u8, meta: Metadata) !void { + const json = try std.json.Stringify.valueAlloc(allocator, meta, .{ .whitespace = .indent_2 }); + defer allocator.free(json); + + const file = try std.Io.Dir.cwd().createFile(compat.io(), path, .{}); + defer file.close(compat.io()); + try file.writeStreamingAll(compat.io(), json); + try file.writeStreamingAll(compat.io(), "\n"); +} + +/// Encode 32-bit float mono samples to a 16-bit PCM WAV file at `sample_rate`. +fn writeWav(path: []const u8, samples: []const f32) !void { + const channels: u16 = 1; + const bits_per_sample: u16 = 16; + const byte_rate: u32 = sample_rate * channels * (bits_per_sample / 8); + const block_align: u16 = channels * (bits_per_sample / 8); + const data_size: u32 = @intCast(samples.len * @sizeOf(i16)); + const riff_size: u32 = 36 + data_size; + + var header: [44]u8 = undefined; + @memcpy(header[0..4], "RIFF"); + std.mem.writeInt(u32, header[4..8], riff_size, .little); + @memcpy(header[8..12], "WAVE"); + @memcpy(header[12..16], "fmt "); + std.mem.writeInt(u32, header[16..20], 16, .little); // PCM fmt chunk size + std.mem.writeInt(u16, header[20..22], 1, .little); // PCM + std.mem.writeInt(u16, header[22..24], channels, .little); + std.mem.writeInt(u32, header[24..28], sample_rate, .little); + std.mem.writeInt(u32, header[28..32], byte_rate, .little); + std.mem.writeInt(u16, header[32..34], block_align, .little); + std.mem.writeInt(u16, header[34..36], bits_per_sample, .little); + @memcpy(header[36..40], "data"); + std.mem.writeInt(u32, header[40..44], data_size, .little); + + const file = try std.Io.Dir.cwd().createFile(compat.io(), path, .{}); + defer file.close(compat.io()); + + try file.writeStreamingAll(compat.io(), &header); + + // Stream-convert samples in chunks to bound stack/heap usage. + var chunk: [4096]i16 = undefined; + var i: usize = 0; + while (i < samples.len) { + const n = @min(chunk.len, samples.len - i); + for (0..n) |k| { + const clamped = std.math.clamp(samples[i + k], -1.0, 1.0); + chunk[k] = @intFromFloat(clamped * 32767.0); + } + try file.writeStreamingAll(compat.io(), std.mem.sliceAsBytes(chunk[0..n])); + i += n; + } +} + +fn ensureDir(path: []const u8) !void { + std.debug.assert(std.fs.path.isAbsolute(path)); + std.Io.Dir.cwd().createDirPath(compat.io(), path) catch |err| switch (err) { + error.PathAlreadyExists => return, + else => return err, + }; +} + +fn formatSnippetId(allocator: std.mem.Allocator, ms: i64) ![]u8 { + const secs: u64 = @intCast(@divFloor(ms, 1000)); + const millis: u64 = @intCast(@mod(ms, 1000)); + const epoch_seconds = std.time.epoch.EpochSeconds{ .secs = secs }; + const day = epoch_seconds.getEpochDay(); + const year_day = day.calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const ds = epoch_seconds.getDaySeconds(); + return std.fmt.allocPrint(allocator, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}{d:0>3}Z", .{ + year_day.year, + @intFromEnum(month_day.month), + month_day.day_index + 1, + ds.getHoursIntoDay(), + ds.getMinutesIntoHour(), + ds.getSecondsIntoMinute(), + millis, + }); +} + +fn promptLine(allocator: std.mem.Allocator, prompt: []const u8) ![]u8 { + const stdout = std.Io.File.stdout(); + try stdout.writeStreamingAll(compat.io(), prompt); + + const stdin = std.Io.File.stdin(); + var io_buf: [4096]u8 = undefined; + var reader = stdin.readerStreaming(compat.io(), &io_buf); + const line = reader.interface.takeDelimiterExclusive('\n') catch |err| switch (err) { + error.EndOfStream => return error.NoInput, + else => return err, + }; + return allocator.dupe(u8, std.mem.trim(u8, line, " \t\r")); +} + +fn promptYesNo(prompt: []const u8, default_yes: bool) !bool { + var tiny_alloc_buf: [64]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&tiny_alloc_buf); + const reply = try promptLine(fba.allocator(), prompt); + if (reply.len == 0) return default_yes; + return reply[0] == 'y' or reply[0] == 'Y'; +} + +fn recordFixed(audio: *AudioCapture, duration_secs: u64) !void { + const total_ns = duration_secs * std.time.ns_per_s; + const tick_ns: u64 = 100 * std.time.ns_per_ms; + var elapsed_ns: u64 = 0; + while (elapsed_ns < total_ns) { + compat.sleepNanoseconds(tick_ns); + elapsed_ns += tick_ns; + printLevel(audio); + } + std.debug.print("\n", .{}); +} + +pub fn recordUntilQ(audio: *AudioCapture) !void { + const stdin_fd: c_int = @intCast(std.Io.File.stdin().handle); + const flags = fcntl(stdin_fd, F_GETFL); + _ = fcntl(stdin_fd, F_SETFL, flags | O_NONBLOCK); + defer _ = fcntl(stdin_fd, F_SETFL, flags); + + var stop = false; + while (!stop) { + compat.sleepNanoseconds(100 * std.time.ns_per_ms); + printLevel(audio); + + var buf: [16]u8 = undefined; + const n = read_c(stdin_fd, &buf, buf.len); + if (n > 0) { + for (buf[0..@intCast(n)]) |byte| { + if (byte == 'q' or byte == 'Q') { + stop = true; + break; + } + } + } + } + std.debug.print("\n", .{}); +} + +fn printLevel(audio: *AudioCapture) void { + const level = audio.getAudioLevel(); + const bar_len: usize = @intFromFloat(@min(level * 200.0, 40.0)); + var bar: [40]u8 = [_]u8{'.'} ** 40; + for (0..bar_len) |i| bar[i] = '#'; + const sample_count = audio.getSampleCount(); + std.debug.print("\r level [{s}] {d:.4} samples={d} ", .{ &bar, level, sample_count }); +} + +fn printUsage() void { + const usage = + \\Usage: bobrwhisper-cli snippet [options] + \\ + \\Records a single audio snippet with end-user metadata for the + \\test corpus. Does NOT transcribe. + \\ + \\Options: + \\ --label Ground truth label (interactive if omitted) + \\ --whisper Mark snippet as whispered audio + \\ --no-whisper Mark snippet as normal voice + \\ --duration Record fixed duration; otherwise stop on 'q' + \\ --out Output directory (default: ~/.bobrwhisper/snippets) + \\ --device Override auto-detected device label + \\ --device-kind internal | bluetooth | usb | unknown + \\ --tag Add a tag (repeatable) + \\ + \\Saves /.wav (16 kHz mono 16-bit PCM) and /.json. + \\ + ; + std.debug.print("{s}", .{usage}); +} + +extern "c" fn fcntl(fd: c_int, cmd: c_int, ...) c_int; +extern "c" fn read(fd: c_int, buf: [*]u8, count: usize) isize; +const read_c = read; +const F_GETFL = 3; +const F_SETFL = 4; +const O_NONBLOCK = 0x0004; // macOS + +test "parseArgs basic" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ "--label", "hello", "--whisper", "--duration", "5" }; + const parsed = try parseArgs(std.testing.allocator, &args, &tags); + try std.testing.expectEqualStrings("hello", parsed.label.?); + try std.testing.expectEqual(true, parsed.whisper.?); + try std.testing.expectEqual(@as(u64, 5), parsed.duration_secs.?); +} + +test "parseArgs no-whisper default" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ "--label", "x", "--no-whisper" }; + const parsed = try parseArgs(std.testing.allocator, &args, &tags); + try std.testing.expectEqual(false, parsed.whisper.?); +} + +test "parseArgs conflicting flags" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ "--whisper", "--no-whisper" }; + try std.testing.expectError(error.ConflictingFlags, parseArgs(std.testing.allocator, &args, &tags)); +} + +test "parseArgs device + tags" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ + "--label", "x", + "--device", "AirPods Pro", + "--device-kind", "bluetooth", + "--tag", "whispered", + "--tag", "technical", + }; + const parsed = try parseArgs(std.testing.allocator, &args, &tags); + try std.testing.expectEqualStrings("AirPods Pro", parsed.device.?); + try std.testing.expectEqual(AudioDevice.Kind.bluetooth, parsed.device_kind.?); + try std.testing.expectEqual(@as(usize, 2), parsed.tags.len); + try std.testing.expectEqualStrings("whispered", parsed.tags[0]); + try std.testing.expectEqualStrings("technical", parsed.tags[1]); +} + +test "parseArgs invalid device kind" { + var tags = std.ArrayListUnmanaged([]const u8).empty; + defer tags.deinit(std.testing.allocator); + const args = [_][]const u8{ "--device-kind", "carrier-pigeon" }; + try std.testing.expectError(error.InvalidDeviceKind, parseArgs(std.testing.allocator, &args, &tags)); +} + +test "formatSnippetId is sortable" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = try formatSnippetId(arena.allocator(), 1_700_000_000_000); + const b = try formatSnippetId(arena.allocator(), 1_700_000_001_500); + try std.testing.expect(std.mem.lessThan(u8, a, b)); + try std.testing.expectEqual(@as(usize, 19), a.len); // YYYYMMDDTHHMMSSmmmZ +} diff --git a/src/tune.zig b/src/tune.zig new file mode 100644 index 0000000..41fafd6 --- /dev/null +++ b/src/tune.zig @@ -0,0 +1,723 @@ +//! Run snippets recorded via `snippet` against the Zig core under several +//! VAD configurations and report normalized word-error rate (WER) vs. the +//! ground truth label. +//! +//! Loads each .json + .wav pair from a directory, sweeps a small set +//! of VAD presets, and prints a per-snippet results table plus a per-preset +//! summary so the operator can pick or refine a config. + +const std = @import("std"); +const builtin = @import("builtin"); +const compat = @import("compat.zig"); +const cli = @import("cli.zig"); +const asr = @import("asr"); + +const WhisperCppAdapter = asr.WhisperCppAdapter; +const WhisperModel = cli.WhisperModel; + +pub const Preset = struct { + name: []const u8, + enabled: bool = true, + threshold: f32 = 0.5, + min_speech_ms: i32 = 250, + min_silence_ms: i32 = 100, + speech_pad_ms: i32 = 30, +}; + +/// Five VAD presets covering the practical sensitivity range. `no-vad` is the +/// honest baseline; `default` matches whisper.cpp's stock VAD; `whisper` +/// matches the existing `--whisper-mode` defaults; `low-thresh` is what we +/// reach for when whispered speech is being chopped; `balanced` sits in the +/// middle in case neither extreme wins overall. +const default_presets = [_]Preset{ + .{ .name = "no-vad", .enabled = false }, + .{ .name = "default", .threshold = 0.5, .min_speech_ms = 250, .min_silence_ms = 100, .speech_pad_ms = 30 }, + .{ .name = "whisper", .threshold = 0.3, .min_speech_ms = 150, .min_silence_ms = 200, .speech_pad_ms = 100 }, + .{ .name = "low-thresh", .threshold = 0.2, .min_speech_ms = 100, .min_silence_ms = 250, .speech_pad_ms = 150 }, + .{ .name = "balanced", .threshold = 0.4, .min_speech_ms = 200, .min_silence_ms = 150, .speech_pad_ms = 60 }, +}; + +const Snippet = struct { + id: []const u8, + label: []const u8, + whisper: bool, + audio_file: []const u8, + duration_seconds: f64, + sample_rate: u32, + channels: u32, + num_samples: usize, + noise_floor_rms: f32, + created_at_unix_ms: i64, + // v2 fields. Optional with null defaults so older snippets still parse. + device: ?[]const u8 = null, + device_kind: ?[]const u8 = null, + tags: ?[]const []const u8 = null, +}; + +const Result = struct { + snippet_id: []const u8, + label: []const u8, + whispered: bool, + preset: []const u8, + transcript: []const u8, + wer: f32, + device: ?[]const u8, + device_kind: ?[]const u8, + tags: []const []const u8, +}; + +const GroupBy = enum { + none, + mode, + device, + device_kind, + tag, + + fn fromString(s: []const u8) ?GroupBy { + return std.meta.stringToEnum(GroupBy, s); + } +}; + +const GainMode = union(enum) { + none, + /// Peak-normalize: scale so peak amplitude is 0.95. + auto, + /// Multiply samples by a fixed factor. + fixed: f32, +}; + +const AudioStats = struct { + peak: f32, + rms: f32, +}; + +fn computeAudioStats(samples: []const f32) AudioStats { + if (samples.len == 0) return .{ .peak = 0, .rms = 0 }; + var peak: f32 = 0; + var energy: f64 = 0; + for (samples) |s| { + const a = @abs(s); + if (a > peak) peak = a; + energy += @as(f64, s) * @as(f64, s); + } + return .{ + .peak = peak, + .rms = @floatCast(@sqrt(energy / @as(f64, @floatFromInt(samples.len)))), + }; +} + +fn applyGain(samples: []f32, mode: GainMode, stats: AudioStats) f32 { + const factor: f32 = switch (mode) { + .none => 1.0, + .fixed => |f| f, + .auto => if (stats.peak > 0.001) 0.95 / stats.peak else 1.0, + }; + if (factor == 1.0) return 1.0; + for (samples) |*s| s.* = std.math.clamp(s.* * factor, -1.0, 1.0); + return factor; +} + +pub fn run(allocator: std.mem.Allocator, raw_args: []const []const u8) !void { + var snippets_dir: ?[]const u8 = null; + var selected_model: WhisperModel = .small; + var preset_filter: ?[]const u8 = null; + var gain_mode: GainMode = .none; + var group_by: GroupBy = .none; + + var i: usize = 0; + while (i < raw_args.len) : (i += 1) { + const a = raw_args[i]; + if (std.mem.eql(u8, a, "--help") or std.mem.eql(u8, a, "-h")) { + printUsage(); + return; + } else if (std.mem.eql(u8, a, "--snippets")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --snippets requires a directory\n", .{}); + return error.MissingValue; + } + snippets_dir = raw_args[i]; + } else if (std.mem.eql(u8, a, "--model")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --model requires a name\n", .{}); + return error.MissingValue; + } + selected_model = WhisperModel.fromString(raw_args[i]) orelse { + std.debug.print("tune: unknown model '{s}'\n", .{raw_args[i]}); + return error.UnknownModel; + }; + } else if (std.mem.eql(u8, a, "--preset")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --preset requires a name\n", .{}); + return error.MissingValue; + } + preset_filter = raw_args[i]; + } else if (std.mem.eql(u8, a, "--gain")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --gain requires a value (auto, none, or a number)\n", .{}); + return error.MissingValue; + } + const v = raw_args[i]; + if (std.mem.eql(u8, v, "none")) { + gain_mode = .none; + } else if (std.mem.eql(u8, v, "auto")) { + gain_mode = .auto; + } else { + const factor = std.fmt.parseFloat(f32, v) catch { + std.debug.print("tune: invalid --gain value '{s}'\n", .{v}); + return error.InvalidGain; + }; + gain_mode = .{ .fixed = factor }; + } + } else if (std.mem.eql(u8, a, "--group-by")) { + i += 1; + if (i >= raw_args.len) { + std.debug.print("tune: --group-by requires a value (none, mode, device, device_kind, tag)\n", .{}); + return error.MissingValue; + } + group_by = GroupBy.fromString(raw_args[i]) orelse { + std.debug.print("tune: invalid --group-by value '{s}'\n", .{raw_args[i]}); + return error.InvalidGroupBy; + }; + } else { + std.debug.print("tune: unknown argument '{s}'\n", .{a}); + printUsage(); + return error.UnknownArg; + } + } + + var dir_storage: ?[]u8 = null; + defer if (dir_storage) |s| allocator.free(s); + const snippets_path: []const u8 = blk: { + if (snippets_dir) |d| break :blk d; + const home = compat.getenv("HOME") orelse { + std.debug.print("tune: HOME unset; pass --snippets \n", .{}); + return error.MissingHome; + }; + dir_storage = try std.fmt.allocPrint(allocator, "{s}/.bobrwhisper/snippets", .{home}); + break :blk dir_storage.?; + }; + + // Filter presets up-front so we don't waste a model load. + var active_presets = std.ArrayListUnmanaged(Preset).empty; + defer active_presets.deinit(allocator); + if (preset_filter) |name| { + for (default_presets) |p| { + if (std.mem.eql(u8, p.name, name)) { + try active_presets.append(allocator, p); + } + } + if (active_presets.items.len == 0) { + std.debug.print("tune: preset '{s}' not found. Available: ", .{name}); + for (default_presets, 0..) |p, idx| { + if (idx > 0) std.debug.print(", ", .{}); + std.debug.print("{s}", .{p.name}); + } + std.debug.print("\n", .{}); + return error.UnknownPreset; + } + } else { + for (default_presets) |p| try active_presets.append(allocator, p); + } + + // Discover snippets. + var snippet_ids = std.ArrayListUnmanaged([]u8).empty; + defer { + for (snippet_ids.items) |id| allocator.free(id); + snippet_ids.deinit(allocator); + } + try discoverSnippets(allocator, snippets_path, &snippet_ids); + if (snippet_ids.items.len == 0) { + std.debug.print("tune: no snippets found in {s}\n", .{snippets_path}); + return error.NoSnippets; + } + std.mem.sort([]u8, snippet_ids.items, {}, lessThanString); + + std.debug.print("tune: found {d} snippet(s) in {s}\n", .{ snippet_ids.items.len, snippets_path }); + std.debug.print("tune: model = {s}, presets = ", .{@tagName(selected_model)}); + for (active_presets.items, 0..) |p, idx| { + if (idx > 0) std.debug.print(", ", .{}); + std.debug.print("{s}", .{p.name}); + } + std.debug.print("\n\n", .{}); + + // Resolve model paths (auto-download if needed). + const model_path = try selected_model.ensureDownloaded(allocator); + defer allocator.free(model_path); + const vad_path = cli.getVadModelPath(allocator); + defer if (vad_path) |p| allocator.free(p); + if (vad_path == null) { + std.debug.print("tune: WARNING - VAD model not found at ~/.bobrwhisper/models/silero-v6.2.0.bin; VAD presets will be ignored\n\n", .{}); + } + + // Load model once; mutate VAD fields per call. + var transcriber = try WhisperCppAdapter.init(allocator, .{ + .model_path = model_path, + .language = "en", + .n_threads = 4, + .vad_enabled = false, + .vad_model_path = vad_path, + }); + defer transcriber.deinit(); + + var results_arena = std.heap.ArenaAllocator.init(allocator); + defer results_arena.deinit(); + const ra = results_arena.allocator(); + + var results = std.ArrayListUnmanaged(Result).empty; + defer results.deinit(allocator); + + for (snippet_ids.items) |id| { + var snippet_arena = std.heap.ArenaAllocator.init(allocator); + defer snippet_arena.deinit(); + const sa = snippet_arena.allocator(); + + const meta = loadMeta(sa, snippets_path, id) catch |err| { + std.debug.print("tune: skipping {s}: failed to load metadata: {}\n", .{ id, err }); + continue; + }; + const wav_path = try std.fmt.allocPrint(sa, "{s}/{s}", .{ snippets_path, meta.audio_file }); + const samples = loadWav16(allocator, wav_path) catch |err| { + std.debug.print("tune: skipping {s}: failed to load wav: {}\n", .{ id, err }); + continue; + }; + defer allocator.free(samples); + + const stats = computeAudioStats(samples); + const applied_gain = applyGain(samples, gain_mode, stats); + + std.debug.print("Snippet: {s} ({s}, {d:.2}s)\n", .{ + meta.id, + if (meta.whisper) "whispered" else "normal", + meta.duration_seconds, + }); + std.debug.print(" label: \"{s}\"\n", .{meta.label}); + std.debug.print(" device: {s} ({s})\n", .{ + meta.device orelse "(unknown)", + meta.device_kind orelse "unknown", + }); + if (meta.tags) |tags| if (tags.len > 0) { + std.debug.print(" tags: ", .{}); + for (tags, 0..) |t, idx| { + if (idx > 0) std.debug.print(", ", .{}); + std.debug.print("{s}", .{t}); + } + std.debug.print("\n", .{}); + }; + std.debug.print(" audio: peak={d:.4} rms={d:.4} noise_floor={d:.4} gain={d:.2}x\n", .{ + stats.peak, + stats.rms, + meta.noise_floor_rms, + applied_gain, + }); + + // Persist the snippet metadata into the long-lived results arena so + // results survive past `snippet_arena.deinit()` and we can group/sort. + const result_id = try ra.dupe(u8, meta.id); + const result_label = try ra.dupe(u8, meta.label); + const result_device: ?[]const u8 = if (meta.device) |d| try ra.dupe(u8, d) else null; + const result_device_kind: ?[]const u8 = if (meta.device_kind) |k| try ra.dupe(u8, k) else null; + const result_tags: []const []const u8 = blk: { + const src = meta.tags orelse break :blk &.{}; + const buf = try ra.alloc([]const u8, src.len); + for (src, 0..) |t, ti| buf[ti] = try ra.dupe(u8, t); + break :blk buf; + }; + + for (active_presets.items) |preset| { + const transcript_owned = transcribeWithPreset(allocator, &transcriber, samples, preset) catch |err| { + std.debug.print(" {s:<12} ERROR: {}\n", .{ preset.name, err }); + continue; + }; + defer allocator.free(transcript_owned); + + const w = computeWer(allocator, meta.label, transcript_owned) catch |err| { + std.debug.print(" {s:<12} WER ERROR: {}\n", .{ preset.name, err }); + continue; + }; + + std.debug.print(" {s:<12} WER {d:.2} -> \"{s}\"\n", .{ preset.name, w, transcript_owned }); + + try results.append(allocator, .{ + .snippet_id = result_id, + .label = result_label, + .whispered = meta.whisper, + .preset = preset.name, // static literal from default_presets + .transcript = try ra.dupe(u8, transcript_owned), + .wer = w, + .device = result_device, + .device_kind = result_device_kind, + .tags = result_tags, + }); + } + std.debug.print("\n", .{}); + } + + try printSummary(allocator, results.items, active_presets.items, group_by); +} + +fn transcribeWithPreset( + allocator: std.mem.Allocator, + transcriber: *WhisperCppAdapter, + samples: []const f32, + preset: Preset, +) ![]u8 { + transcriber.vad_enabled = preset.enabled; + transcriber.vad_threshold = preset.threshold; + transcriber.vad_min_speech_ms = preset.min_speech_ms; + transcriber.vad_min_silence_ms = preset.min_silence_ms; + transcriber.vad_speech_pad_ms = preset.speech_pad_ms; + const raw = try transcriber.transcribe(samples); + defer allocator.free(raw); + const trimmed = std.mem.trim(u8, raw, " \t\n\r"); + return allocator.dupe(u8, trimmed); +} + +fn printSummary(allocator: std.mem.Allocator, results: []const Result, presets: []const Preset, group_by: GroupBy) !void { + if (results.len == 0) return; + + std.debug.print("=== Summary ===\n\n", .{}); + + // Best preset per snippet. + std.debug.print("Best preset per snippet (lowest WER, ties broken by preset order):\n", .{}); + var seen_ids = std.StringHashMapUnmanaged(void).empty; + defer seen_ids.deinit(allocator); + for (results) |r| { + if (seen_ids.contains(r.snippet_id)) continue; + try seen_ids.put(allocator, r.snippet_id, {}); + + var best_idx: usize = 0; + var best_wer: f32 = std.math.inf(f32); + for (results, 0..) |r2, j| { + if (!std.mem.eql(u8, r2.snippet_id, r.snippet_id)) continue; + if (r2.wer < best_wer) { + best_wer = r2.wer; + best_idx = j; + } + } + const best = results[best_idx]; + std.debug.print(" {s} ({s}): {s:<12} WER {d:.2} -> \"{s}\"\n", .{ + best.snippet_id, + if (best.whispered) "whispered" else "normal", + best.preset, + best.wer, + best.transcript, + }); + } + + std.debug.print("\nPer-preset average WER:\n", .{}); + for (presets) |p| { + var sum: f32 = 0; + var count: usize = 0; + for (results) |r| { + if (std.mem.eql(u8, r.preset, p.name)) { + sum += r.wer; + count += 1; + } + } + if (count == 0) continue; + const avg = sum / @as(f32, @floatFromInt(count)); + std.debug.print(" {s:<12} avg WER {d:.2} ({d} snippet(s))\n", .{ p.name, avg, count }); + } + + if (group_by == .none) { + std.debug.print("\n", .{}); + return; + } + + // Collect distinct group keys for the chosen axis. + var groups = std.StringArrayHashMapUnmanaged(void).empty; + defer groups.deinit(allocator); + for (results) |r| { + var keys_buf: [16][]const u8 = undefined; + const keys = resultGroupKeys(r, group_by, &keys_buf); + for (keys) |k| try groups.put(allocator, k, {}); + } + + std.debug.print("\nPer-preset average WER, grouped by {s}:\n", .{@tagName(group_by)}); + for (presets) |p| { + std.debug.print(" {s}\n", .{p.name}); + var it = groups.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + var sum: f32 = 0; + var count: usize = 0; + for (results) |r| { + if (!std.mem.eql(u8, r.preset, p.name)) continue; + var keys_buf: [16][]const u8 = undefined; + const keys = resultGroupKeys(r, group_by, &keys_buf); + var matched = false; + for (keys) |k| { + if (std.mem.eql(u8, k, key)) { + matched = true; + break; + } + } + if (!matched) continue; + sum += r.wer; + count += 1; + } + if (count == 0) continue; + const avg = sum / @as(f32, @floatFromInt(count)); + std.debug.print(" {s:<24} avg WER {d:.2} ({d})\n", .{ key, avg, count }); + } + } + std.debug.print("\n", .{}); +} + +/// Return the group keys a single `Result` belongs to for the chosen axis. +/// Up to `out_buf.len` keys are returned (16 is plenty: only `tag` can produce +/// multiple). `mode`/`device`/`device_kind` always produce exactly one key. +fn resultGroupKeys(r: Result, group_by: GroupBy, out_buf: []([]const u8)) []const []const u8 { + return switch (group_by) { + .none => out_buf[0..0], + .mode => blk: { + out_buf[0] = if (r.whispered) "whispered" else "normal"; + break :blk out_buf[0..1]; + }, + .device => blk: { + out_buf[0] = r.device orelse "(unknown)"; + break :blk out_buf[0..1]; + }, + .device_kind => blk: { + out_buf[0] = r.device_kind orelse "unknown"; + break :blk out_buf[0..1]; + }, + .tag => blk: { + if (r.tags.len == 0) { + out_buf[0] = "(untagged)"; + break :blk out_buf[0..1]; + } + const n = @min(r.tags.len, out_buf.len); + for (r.tags[0..n], 0..) |t, i| out_buf[i] = t; + break :blk out_buf[0..n]; + }, + }; +} + +fn lessThanString(_: void, a: []u8, b: []u8) bool { + return std.mem.lessThan(u8, a, b); +} + +fn discoverSnippets(allocator: std.mem.Allocator, dir_path: []const u8, out: *std.ArrayListUnmanaged([]u8)) !void { + var dir = std.Io.Dir.cwd().openDir(compat.io(), dir_path, .{ .iterate = true }) catch |err| switch (err) { + error.FileNotFound => { + std.debug.print("tune: directory not found: {s}\n", .{dir_path}); + return error.NoSnippets; + }, + else => return err, + }; + defer dir.close(compat.io()); + + var it = dir.iterate(); + + while (try it.next(compat.io())) |entry| { + if (entry.kind != .file) continue; + if (!std.mem.endsWith(u8, entry.name, ".json")) continue; + const id = entry.name[0 .. entry.name.len - ".json".len]; + try out.append(allocator, try allocator.dupe(u8, id)); + } +} + +fn loadMeta(arena: std.mem.Allocator, dir_path: []const u8, id: []const u8) !Snippet { + const json_path = try std.fmt.allocPrint(arena, "{s}/{s}.json", .{ dir_path, id }); + const file = try std.Io.Dir.cwd().openFile(compat.io(), json_path, .{}); + defer file.close(compat.io()); + const stat = try file.stat(compat.io()); + const buf = try arena.alloc(u8, stat.size); + var io_buf: [4096]u8 = undefined; + var reader = file.readerStreaming(compat.io(), &io_buf); + try reader.interface.readSliceAll(buf); + return std.json.parseFromSliceLeaky(Snippet, arena, buf, .{ .ignore_unknown_fields = true }); +} + +/// Read a 16-bit PCM mono WAV at any sample rate; resample to 16 kHz if needed. +fn loadWav16(allocator: std.mem.Allocator, path: []const u8) ![]f32 { + const file = try std.Io.Dir.cwd().openFile(compat.io(), path, .{}); + defer file.close(compat.io()); + + var header: [44]u8 = undefined; + var io_buf: [4096]u8 = undefined; + var reader = file.readerStreaming(compat.io(), &io_buf); + try reader.interface.readSliceAll(&header); + if (!std.mem.eql(u8, header[0..4], "RIFF") or !std.mem.eql(u8, header[8..12], "WAVE")) { + return error.InvalidWavFile; + } + const channels: u16 = std.mem.readInt(u16, header[22..24], .little); + const sample_rate: u32 = std.mem.readInt(u32, header[24..28], .little); + const bits_per_sample: u16 = std.mem.readInt(u16, header[34..36], .little); + if (bits_per_sample != 16) return error.UnsupportedBitDepth; + + const data_size: u32 = std.mem.readInt(u32, header[40..44], .little); + const num_samples = data_size / (@as(u32, channels) * 2); + const frame_bytes = @as(usize, channels) * 2; + + const raw = try allocator.alloc(u8, data_size); + defer allocator.free(raw); + try reader.interface.readSliceAll(raw); + + var samples = try allocator.alloc(f32, num_samples); + errdefer allocator.free(samples); + for (0..num_samples) |i| { + const offset = i * frame_bytes; + const sample_i16 = std.mem.readInt(i16, raw[offset..][0..2], .little); + samples[i] = @as(f32, @floatFromInt(sample_i16)) / 32768.0; + } + + if (sample_rate != 16000) { + const AudioCapture = @import("audio/AudioCapture.zig"); + const resampled = try AudioCapture.resample(allocator, samples, @floatFromInt(sample_rate), 16000.0); + allocator.free(samples); + return resampled; + } + return samples; +} + +/// Lower-case, strip ASCII punctuation, collapse whitespace. +fn normalize(allocator: std.mem.Allocator, text: []const u8) ![]u8 { + var out = try allocator.alloc(u8, text.len); + errdefer allocator.free(out); + var n: usize = 0; + var prev_space = true; + for (text) |c| { + const lower = std.ascii.toLower(c); + const is_space = std.ascii.isWhitespace(lower); + const is_punct = isPunct(lower); + if (is_punct) { + if (!prev_space) { + out[n] = ' '; + n += 1; + prev_space = true; + } + continue; + } + if (is_space) { + if (!prev_space) { + out[n] = ' '; + n += 1; + prev_space = true; + } + continue; + } + out[n] = lower; + n += 1; + prev_space = false; + } + while (n > 0 and out[n - 1] == ' ') n -= 1; + return allocator.realloc(out, n); +} + +fn isPunct(c: u8) bool { + return switch (c) { + '.', ',', '?', '!', ';', ':', '"', '\'', '(', ')', '[', ']', '{', '}', '-' => true, + else => false, + }; +} + +/// Word-level WER: edit_distance(ref_words, hyp_words) / max(1, ref_words.len). +fn computeWer(allocator: std.mem.Allocator, reference: []const u8, hypothesis: []const u8) !f32 { + const ref = try normalize(allocator, reference); + defer allocator.free(ref); + const hyp = try normalize(allocator, hypothesis); + defer allocator.free(hyp); + + var ref_words = std.ArrayListUnmanaged([]const u8).empty; + defer ref_words.deinit(allocator); + var hyp_words = std.ArrayListUnmanaged([]const u8).empty; + defer hyp_words.deinit(allocator); + + var it_r = std.mem.tokenizeScalar(u8, ref, ' '); + while (it_r.next()) |w| try ref_words.append(allocator, w); + var it_h = std.mem.tokenizeScalar(u8, hyp, ' '); + while (it_h.next()) |w| try hyp_words.append(allocator, w); + + if (ref_words.items.len == 0) { + return if (hyp_words.items.len == 0) 0.0 else 1.0; + } + + const dist = try wordEditDistance(allocator, ref_words.items, hyp_words.items); + return @as(f32, @floatFromInt(dist)) / @as(f32, @floatFromInt(ref_words.items.len)); +} + +fn wordEditDistance(allocator: std.mem.Allocator, a: []const []const u8, b: []const []const u8) !usize { + const n = a.len; + const m = b.len; + if (n == 0) return m; + if (m == 0) return n; + + var prev = try allocator.alloc(usize, m + 1); + defer allocator.free(prev); + var cur = try allocator.alloc(usize, m + 1); + defer allocator.free(cur); + + for (0..m + 1) |j| prev[j] = j; + for (1..n + 1) |i| { + cur[0] = i; + for (1..m + 1) |j| { + const cost: usize = if (std.mem.eql(u8, a[i - 1], b[j - 1])) 0 else 1; + const del = prev[j] + 1; + const ins = cur[j - 1] + 1; + const sub = prev[j - 1] + cost; + cur[j] = @min(del, @min(ins, sub)); + } + const tmp = prev; + prev = cur; + cur = tmp; + } + return prev[m]; +} + +fn printUsage() void { + const usage = + \\Usage: bobrwhisper-cli tune [options] + \\ + \\Runs labeled snippets recorded with `snippet` against the Zig core + \\under several VAD presets and reports WER vs. the ground truth. + \\ + \\Does NOT modify any files; results are printed to stdout. + \\ + \\Options: + \\ --snippets Snippet directory (default: ~/.bobrwhisper/snippets) + \\ --model Whisper model (default: small) + \\ --preset Run a single preset (default: all) + \\ --gain Pre-amplify audio: none (default), auto (peak to 0.95), or a number + \\ --group-by Slice the summary by: none (default), mode, device, device_kind, tag + \\ + \\Presets: no-vad, default, whisper, low-thresh, balanced + \\ + ; + std.debug.print("{s}", .{usage}); +} + +test "normalize lower-case and punctuation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + try std.testing.expectEqualStrings("what is your name", try normalize(a, "What is your name?")); + try std.testing.expectEqualStrings("hello world", try normalize(a, " Hello, WORLD! ")); + try std.testing.expectEqualStrings("", try normalize(a, " ...?! ")); +} + +test "wer perfect match" { + const arena_alloc = std.testing.allocator; + const w = try computeWer(arena_alloc, "What is your name", "what is your name?"); + try std.testing.expectEqual(@as(f32, 0.0), w); +} + +test "wer one substitution" { + const w = try computeWer(std.testing.allocator, "what is your name", "what is the name"); + try std.testing.expectApproxEqAbs(@as(f32, 0.25), w, 0.001); +} + +test "wer empty hypothesis" { + const w = try computeWer(std.testing.allocator, "what is your name", ""); + try std.testing.expectEqual(@as(f32, 1.0), w); +} + +test "wer empty reference" { + const w = try computeWer(std.testing.allocator, "", "anything goes"); + try std.testing.expectEqual(@as(f32, 1.0), w); +}