From 94b3a5728d8b3f176266df6b85d48ce681d374aa Mon Sep 17 00:00:00 2001 From: Kevin McKee Date: Wed, 5 Feb 2025 13:15:50 -0800 Subject: [PATCH 1/5] Public init for downstream modules --- Sources/VimAssistant/Views/VimAssistantView.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/VimAssistant/Views/VimAssistantView.swift b/Sources/VimAssistant/Views/VimAssistantView.swift index 27459a1..c5b51f2 100644 --- a/Sources/VimAssistant/Views/VimAssistantView.swift +++ b/Sources/VimAssistant/Views/VimAssistantView.swift @@ -20,7 +20,7 @@ public struct VimAssistantView: View { /// Initializer. /// - Parameter enabled: flag indicating if the assistant should be enabled or not - init?(vim: Vim, _ enabled: Bool = false) { + public init?(vim: Vim, _ enabled: Bool = false) { if !enabled { return nil } self.vim = vim } From 2c2f798618a52de2cc503bc914f8f0bd6f7f3656 Mon Sep 17 00:00:00 2001 From: Kevin McKee Date: Wed, 5 Feb 2025 13:33:51 -0800 Subject: [PATCH 2/5] WIP --- .../VimAssistant/Views/VimAssistantView.swift | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/Sources/VimAssistant/Views/VimAssistantView.swift b/Sources/VimAssistant/Views/VimAssistantView.swift index c5b51f2..6c8396c 100644 --- a/Sources/VimAssistant/Views/VimAssistantView.swift +++ b/Sources/VimAssistant/Views/VimAssistantView.swift @@ -18,6 +18,13 @@ public struct VimAssistantView: View { @State var inputText: String = .empty + private var gradient = Gradient( + colors: [ + Color(.teal), + Color(.purple) + ] + ) + /// Initializer. /// - Parameter enabled: flag indicating if the assistant should be enabled or not public init?(vim: Vim, _ enabled: Bool = false) { @@ -27,8 +34,9 @@ public struct VimAssistantView: View { public var body: some View { - HStack { + HStack(spacing: 4) { Image(systemName: "apple.intelligence") + .padding() .symbolRenderingMode(.palette) .foregroundStyle( .angularGradient( @@ -37,14 +45,28 @@ public struct VimAssistantView: View { ) ) - TextField(text: $inputText, prompt: Text("Type here to use the assistant.")) { - Image(systemName: "microphone") + TextField(text: $inputText, prompt: Text("Type or tap microphone to use the AI assistant.")) { + Image(systemName: "microphone") + } + .textFieldStyle(.plain) - } - .textFieldStyle(.plain) microPhoneButton + .padding() + } + .background(Color.black.opacity(0.65)) + .cornerRadius(8) + .overlay{ + RoundedRectangle(cornerRadius: 8) + .stroke( + LinearGradient( + gradient: gradient, + startPoint: .leading, + endPoint: .trailing + ), + lineWidth: 4 + ) } - .padding() + .padding(24) } var microPhoneButton: some View { From 752ffd412cee87a3ee7c8427bc939c2a875898d9 Mon Sep 17 00:00:00 2001 From: Kevin McKee Date: Wed, 5 Feb 2025 16:11:21 -0800 Subject: [PATCH 3/5] WIP --- .../SpeechRecognizer/SpeechRecognizer.swift | 23 +++- .../VimAssistant/Views/VimAssistantView.swift | 113 ++++++++++++++---- 2 files changed, 109 insertions(+), 27 deletions(-) diff --git a/Sources/VimAssistant/SpeechRecognizer/SpeechRecognizer.swift b/Sources/VimAssistant/SpeechRecognizer/SpeechRecognizer.swift index 5d2870d..aa59794 100644 --- a/Sources/VimAssistant/SpeechRecognizer/SpeechRecognizer.swift +++ b/Sources/VimAssistant/SpeechRecognizer/SpeechRecognizer.swift @@ -10,6 +10,9 @@ import Foundation import Speech import SwiftUI +private let bus: AVAudioNodeBus = 0 +private let bufferSize: AVAudioFrameCount = 1024 + public actor SpeechRecognizer: ObservableObject { enum RecognizerError: Error { @@ -33,6 +36,18 @@ public actor SpeechRecognizer: ObservableObject { @MainActor public var transcript: String = .empty + @MainActor + public var run: Bool = false { + didSet { + if run { + resetTranscript() + startTranscribing() + } else { + stopTranscribing() + } + } + } + private var audioEngine: AVAudioEngine? private var request: SFSpeechAudioBufferRecognitionRequest? private var task: SFSpeechRecognitionTask? @@ -98,7 +113,7 @@ public actor SpeechRecognizer: ObservableObject { if receivedFinalResult || receivedError { audioEngine.stop() - audioEngine.inputNode.removeTap(onBus: 0) + audioEngine.inputNode.removeTap(onBus: bus) } if let result { @@ -130,12 +145,14 @@ public actor SpeechRecognizer: ObservableObject { #endif let inputNode = audioEngine.inputNode + let recordingFormat = inputNode.outputFormat(forBus: bus) + let inputFormat = inputNode.inputFormat(forBus: bus) - let recordingFormat = inputNode.outputFormat(forBus: 0) - inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { (buffer: AVAudioPCMBuffer, when: AVAudioTime) in + inputNode.installTap(onBus: bus, bufferSize: bufferSize, format: recordingFormat) { (buffer: AVAudioPCMBuffer, when: AVAudioTime) in request.append(buffer) } audioEngine.prepare() + try audioEngine.start() return (audioEngine, request) diff --git a/Sources/VimAssistant/Views/VimAssistantView.swift b/Sources/VimAssistant/Views/VimAssistantView.swift index 6c8396c..8198467 100644 --- a/Sources/VimAssistant/Views/VimAssistantView.swift +++ b/Sources/VimAssistant/Views/VimAssistantView.swift @@ -18,12 +18,16 @@ public struct VimAssistantView: View { @State var inputText: String = .empty - private var gradient = Gradient( - colors: [ - Color(.teal), - Color(.purple) - ] - ) + @State + private var animateGradient = false + + private var animation: Animation { + if animateGradient { + .easeOut(duration: 2).repeatForever() + } else { + .easeOut(duration: 2) + } + } /// Initializer. /// - Parameter enabled: flag indicating if the assistant should be enabled or not @@ -33,8 +37,15 @@ public struct VimAssistantView: View { } public var body: some View { + VStack { + inputView + responseView + } + } + + var inputView: some View { - HStack(spacing: 4) { + HStack { Image(systemName: "apple.intelligence") .padding() .symbolRenderingMode(.palette) @@ -45,38 +56,92 @@ public struct VimAssistantView: View { ) ) - TextField(text: $inputText, prompt: Text("Type or tap microphone to use the AI assistant.")) { - Image(systemName: "microphone") - } + TextField(text: $inputText, prompt: Text("Type or tap microphone to use the AI assistant.")) { + Image(systemName: "microphone") + } .textFieldStyle(.plain) - microPhoneButton + microphoneButton .padding() } .background(Color.black.opacity(0.65)) .cornerRadius(8) - .overlay{ - RoundedRectangle(cornerRadius: 8) - .stroke( - LinearGradient( - gradient: gradient, - startPoint: .leading, - endPoint: .trailing - ), - lineWidth: 4 - ) + .overlay { + overlayView } - .padding(24) + .padding() } - var microPhoneButton: some View { - Button(action: { + // The stroke gradient + private var gradient: Gradient { + .init(colors: animateGradient ? [.red, .orange] : [.teal, .purple]) + } + + // The gradient style + private var gradientStyle: some ShapeStyle { + LinearGradient( + gradient: gradient, + startPoint: .leading, + endPoint: .trailing + ) + } + + // The overlay view of the text box that animates the stroke + private var overlayView: some View { + RoundedRectangle(cornerRadius: 8) + .stroke(gradientStyle, lineWidth: 4) + .hueRotation(.degrees(animateGradient ? 45 : 0)) + .animation(animation, value: animateGradient) + } + + private var microphoneButton: some View { + Button(action: { + animateGradient.toggle() + speechRecognizer.run.toggle() }) { Image(systemName: "microphone") } .buttonStyle(.plain) + } + + var responseView: some View { + + VStack(spacing: 4) { + if speechRecognizer.transcript.isNotEmpty { + HStack { + Text(speechRecognizer.transcript) + .font(.title2) + Spacer() + } + .padding(.leading) + + HStack { + goodResponseButton + badResponseButton + Spacer() + } + .padding([.leading]) + } + } + .padding(.bottom) + + } + var goodResponseButton: some View { + Button(action: { + // TODO: Report a good response + }) { + Image(systemName: "hand.thumbsup") + } + } + + var badResponseButton: some View { + Button(action: { + // TODO: Report a bad response + }) { + Image(systemName: "hand.thumbsdown") + } } } From 1c77c481f8bbc943a693bd9f4804d2903faf43b0 Mon Sep 17 00:00:00 2001 From: Kevin McKee Date: Wed, 5 Feb 2025 16:36:02 -0800 Subject: [PATCH 4/5] WIP --- .../VimAssistant/Views/VimAssistantView.swift | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/Sources/VimAssistant/Views/VimAssistantView.swift b/Sources/VimAssistant/Views/VimAssistantView.swift index 8198467..451df91 100644 --- a/Sources/VimAssistant/Views/VimAssistantView.swift +++ b/Sources/VimAssistant/Views/VimAssistantView.swift @@ -21,6 +21,10 @@ public struct VimAssistantView: View { @State private var animateGradient = false + private var displayResponse: Bool { + speechRecognizer.transcript.isNotEmpty + } + private var animation: Animation { if animateGradient { .easeOut(duration: 2).repeatForever() @@ -55,6 +59,7 @@ public struct VimAssistantView: View { center: .center, startAngle: .zero, endAngle: .degrees(360) ) ) + .font(.title) TextField(text: $inputText, prompt: Text("Type or tap microphone to use the AI assistant.")) { Image(systemName: "microphone") @@ -69,7 +74,7 @@ public struct VimAssistantView: View { .overlay { overlayView } - .padding() + .padding([.leading, .top, .trailing]) } @@ -101,6 +106,7 @@ public struct VimAssistantView: View { speechRecognizer.run.toggle() }) { Image(systemName: "microphone") + .font(.title) } .buttonStyle(.plain) } @@ -108,23 +114,22 @@ public struct VimAssistantView: View { var responseView: some View { VStack(spacing: 4) { - if speechRecognizer.transcript.isNotEmpty { - HStack { - Text(speechRecognizer.transcript) + if displayResponse { + Text(speechRecognizer.transcript) + .frame(maxWidth: .infinity, alignment: .leading) .font(.title2) - Spacer() - } - .padding(.leading) - + .padding() HStack { + Spacer() goodResponseButton badResponseButton - Spacer() } - .padding([.leading]) + .padding([.bottom, .trailing]) } } - .padding(.bottom) + .background(Color.black.opacity(0.65)) + .cornerRadius(8) + .padding([.leading, .bottom, .trailing]) } @@ -133,6 +138,7 @@ public struct VimAssistantView: View { // TODO: Report a good response }) { Image(systemName: "hand.thumbsup") + .buttonStyle(.plain) } } From 17269041b5c5e02a9e45ee27cfd4e036636de0a2 Mon Sep 17 00:00:00 2001 From: Kevin McKee Date: Wed, 5 Feb 2025 19:01:39 -0800 Subject: [PATCH 5/5] WIP --- .../VimAssistant/SpeechRecognizer/SpeechRecognizer.swift | 6 ++++++ Sources/VimAssistant/Views/VimAssistantView.swift | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Sources/VimAssistant/SpeechRecognizer/SpeechRecognizer.swift b/Sources/VimAssistant/SpeechRecognizer/SpeechRecognizer.swift index aa59794..4cca357 100644 --- a/Sources/VimAssistant/SpeechRecognizer/SpeechRecognizer.swift +++ b/Sources/VimAssistant/SpeechRecognizer/SpeechRecognizer.swift @@ -33,6 +33,7 @@ public actor SpeechRecognizer: ObservableObject { } } + /// The speech recognition transcript result. @MainActor public var transcript: String = .empty @@ -107,6 +108,11 @@ public actor SpeechRecognizer: ObservableObject { } } + /// Handles speech recognition results. + /// - Parameters: + /// - audioEngine: the audio engine that processed the task + /// - result: the speech recognition result + /// - error: errors that could have occurred during recognition nonisolated private func recognitionHandler(audioEngine: AVAudioEngine, result: SFSpeechRecognitionResult?, error: Error?) { let receivedFinalResult = result?.isFinal ?? false let receivedError = error != nil diff --git a/Sources/VimAssistant/Views/VimAssistantView.swift b/Sources/VimAssistant/Views/VimAssistantView.swift index 451df91..3b28ad1 100644 --- a/Sources/VimAssistant/Views/VimAssistantView.swift +++ b/Sources/VimAssistant/Views/VimAssistantView.swift @@ -96,7 +96,7 @@ public struct VimAssistantView: View { private var overlayView: some View { RoundedRectangle(cornerRadius: 8) .stroke(gradientStyle, lineWidth: 4) - .hueRotation(.degrees(animateGradient ? 45 : 0)) + .hueRotation(.degrees(animateGradient ? 90 : 0)) .animation(animation, value: animateGradient) }