Fix WebSocket connection issues and modernize audio processing

examples/speech-to-speech/Dockerfile

@@ -26,6 +26,7 @@ COPY pyproject.toml uv.lock \
 COPY src/ ./src/
 COPY examples/static/ ./examples/static/
 COPY examples/speech-to-speech/ ./examples/speech-to-speech/
+COPY examples/static/ /app/static/
 
 # Example app directory
 WORKDIR /app/examples/speech-to-speech

examples/speech-to-speech/bot.py

@@ -56,22 +56,22 @@ async def create_pipeline_task(pipeline_metadata: PipelineMetadata):
     )
 
     llm = NvidiaLLMService(
-        api_key=os.getenv("NVIDIA_API_KEY"),
+        api_key=os.getenv("NVIDIA_API_KEY", "not-needed-for-local-nim"),
         base_url=os.getenv("NVIDIA_LLM_URL", "https://integrate.api.nvidia.com/v1"),
         model=os.getenv("NVIDIA_LLM_MODEL", "meta/llama-3.1-8b-instruct"),
     )
 
     stt = RivaASRService(
         server=os.getenv("RIVA_ASR_URL", "localhost:50051"),
-        api_key=os.getenv("NVIDIA_API_KEY"),
+        api_key=os.getenv("NVIDIA_API_KEY", "not-needed-for-local-nim"),
         language=os.getenv("RIVA_ASR_LANGUAGE", "en-US"),
         sample_rate=16000,
         model=os.getenv("RIVA_ASR_MODEL", "parakeet-1.1b-en-US-asr-streaming-silero-vad-asr-bls-ensemble"),
     )
 
     tts = RivaTTSService(
         server=os.getenv("RIVA_TTS_URL", "localhost:50051"),
-        api_key=os.getenv("NVIDIA_API_KEY"),
+        api_key=os.getenv("NVIDIA_API_KEY", "not-needed-for-local-nim"),
         voice_id=os.getenv("RIVA_TTS_VOICE_ID", "Magpie-Multilingual.EN-US.Sofia"),
         model=os.getenv("RIVA_TTS_MODEL", "magpie_tts_ensemble-Magpie-Multilingual"),
         language=os.getenv("RIVA_TTS_LANGUAGE", "en-US"),

examples/static/audio-processor.js

@@ -0,0 +1,34 @@
+class AudioProcessor extends AudioWorkletProcessor {
+    constructor(options) {
+        super();
+        this.sampleRate = options.processorOptions.sampleRate || 16000;
+        this.numChannels = options.processorOptions.numChannels || 1;
+    }
+
+    process(inputs, outputs, parameters) {
+        const input = inputs[0];
+
+        if (input && input.length > 0 && input[0]) {
+            const audioData = input[0]; // Get the first channel
+
+            // Convert Float32Array to Int16Array (PCM S16)
+            const pcmS16Array = new Int16Array(audioData.length);
+            for (let i = 0; i < audioData.length; i++) {
+                const sample = Math.max(-1, Math.min(1, audioData[i]));
+                pcmS16Array[i] = sample < 0 ? sample * 0x8000 : sample * 0x7FFF;
+            }
+
+            // Send the processed audio data to the main thread
+            this.port.postMessage({
+                type: 'audioData',
+                data: pcmS16Array.buffer,
+                sampleRate: this.sampleRate,
+                numChannels: this.numChannels
+            });
+        }
+
+        return true; // Keep the processor alive
+    }
+}
+
+registerProcessor('audio-processor', AudioProcessor);