From 1c06ec07f55bc69f681ab6c22a35b74f58c23ec0 Mon Sep 17 00:00:00 2001 From: Alfonso Date: Thu, 29 Jan 2026 10:04:32 +0100 Subject: [PATCH 1/5] push to my fork --- .gitignore | 1 + .vscode/settings.json | 1 + backend/main.py | 15 +++++++++------ backend/nova_sonic_simple.py | 22 ++++++++++++++++++---- 4 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 .gitignore create mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36d43b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +**/*/node_modules diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/backend/main.py b/backend/main.py index 1754be9..f16e7ff 100644 --- a/backend/main.py +++ b/backend/main.py @@ -185,14 +185,14 @@ async def handle_tool_use(self, event_data): if self.active_connection: await self.active_connection.send_text(json.dumps(error_result)) - async def connect(self, websocket: WebSocket): + async def connect(self, websocket: WebSocket, model_id: str = "amazon.nova-sonic-v1:0", language: str = "en"): await websocket.accept() self.active_connection = websocket - logger.info("WebSocket connection accepted") + logger.info(f"WebSocket connection accepted with model: {model_id}, language: {language}") - self.nova_client = SimpleNovaSonic() + self.nova_client = SimpleNovaSonic(model_id=model_id, language=language) await self.nova_client.start_session() - logger.info("Nova Sonic session started") + logger.info(f"Nova Sonic session started with model: {model_id}, language: {language}") # --- Send conversation history after system prompt --- history = self.get_history() @@ -454,8 +454,11 @@ async def handle_ui_interaction(self, interaction_data): @app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket): - logger.info("New WebSocket connection request") - await manager.connect(websocket) + # Get model and language from query parameters + model_id = websocket.query_params.get("model", "amazon.nova-sonic-v1:0") + language = websocket.query_params.get("language", "en") + logger.info(f"New WebSocket connection request with model: {model_id}, language: {language}") + await manager.connect(websocket, model_id=model_id, language=language) # Send tool configurations tool_configs = manager.nova_client.tool_manager.get_tool_configs() if manager.nova_client else [] diff --git a/backend/nova_sonic_simple.py b/backend/nova_sonic_simple.py index 492abba..19dfd58 100644 --- a/backend/nova_sonic_simple.py +++ b/backend/nova_sonic_simple.py @@ -46,9 +46,10 @@ def get_aws_credentials_resolver(): return EnvironmentCredentialsResolver() class SimpleNovaSonic: - def __init__(self, model_id='amazon.nova-sonic-v1:0', region='us-east-1'): + def __init__(self, model_id='amazon.nova-sonic-v1:0', region='us-east-1', language='en'): self.model_id = model_id self.region = region + self.language = language self.client = None self.stream = None self.response = None @@ -156,9 +157,22 @@ async def start_session(self): ''' await self.send_event(text_content_start) - system_prompt = "You are a friendly assistant. The user and you will engage in a spoken dialog " \ - "exchanging the transcripts of a natural real-time conversation. Keep your responses short, " \ - "generally two or three sentences for chatty scenarios. Important:If you are using a tool, mention that you are gathering information." + # Language-specific instructions + language_names = { + 'en': 'English', + 'es': 'Spanish', + 'de': 'German', + 'fr': 'French', + 'it': 'Italian', + 'pt': 'Portuguese', + 'hi': 'Hindi' + } + language_name = language_names.get(self.language, 'English') + + system_prompt = f"You are a friendly assistant. The user and you will engage in a spoken dialog " \ + f"exchanging the transcripts of a natural real-time conversation. Keep your responses short, " \ + f"generally two or three sentences for chatty scenarios. Important: If you are using a tool, mention that you are gathering information. " \ + f"IMPORTANT: You MUST respond in {language_name} language only. The user will speak in {language_name} and you must reply in {language_name}." text_input = f''' {{ From 66c3c35555c9cc6c2438c4c3872bd75cb0a6a0b7 Mon Sep 17 00:00:00 2001 From: Alfonso Date: Thu, 29 Jan 2026 10:05:13 +0100 Subject: [PATCH 2/5] changes --- frontend/app/page.tsx | 153 ++++++++++++++++++++-------- frontend/components/ui/select.tsx | 159 ++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 39 deletions(-) create mode 100644 frontend/components/ui/select.tsx diff --git a/frontend/app/page.tsx b/frontend/app/page.tsx index fa94081..86e522a 100644 --- a/frontend/app/page.tsx +++ b/frontend/app/page.tsx @@ -5,7 +5,14 @@ import { Button } from '@/components/ui/button'; import { Card } from '@/components/ui/card'; import AudioCapture from '@/components/audio-capture'; import { Avatar, AvatarFallback, AvatarImage } from '@/components/ui/avatar'; -import { User, Bot, Mic, MicOff, Power, PowerOff, X } from 'lucide-react'; +import { User, Bot, Mic, MicOff, Power, PowerOff, X, ChevronDown } from 'lucide-react'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import AudioCaptureMediaRecorder from '@/components/audio-capture-mediarecorder'; import { ToolOutput } from '@/components/tool-outputs/ToolOutput'; import type { ToolOutput as ToolOutputType } from '@/components/tool-outputs/types'; @@ -17,15 +24,49 @@ interface TextMessage { role: string; } +interface ModelOption { + id: string; + name: string; + description: string; +} + +const AVAILABLE_MODELS: ModelOption[] = [ + { + id: 'amazon.nova-sonic-v1:0', + name: 'Nova Sonic', + description: 'Speech-to-speech model for conversational AI' + }, + { + id: 'amazon.nova-2-sonic-v1:0', + name: 'Nova 2 Sonic', + description: 'Enhanced multilingual speech-to-speech model' + } +]; + +interface LanguageOption { + code: string; + name: string; +} + +const AVAILABLE_LANGUAGES: LanguageOption[] = [ + { code: 'en', name: 'English' }, + { code: 'es', name: 'Spanish' }, + { code: 'de', name: 'German' }, + { code: 'fr', name: 'French' }, + { code: 'it', name: 'Italian' }, + { code: 'pt', name: 'Portuguese' }, + { code: 'hi', name: 'Hindi' }, +]; + // Add these message animation variants before the Home component const messageVariants = { - initial: { - opacity: 0, + initial: { + opacity: 0, y: 20, scale: 0.95 }, - animate: { - opacity: 1, + animate: { + opacity: 1, y: 0, scale: 1, transition: { @@ -34,8 +75,8 @@ const messageVariants = { damping: 20 } }, - exit: { - opacity: 0, + exit: { + opacity: 0, scale: 0.95, transition: { duration: 0.2 } } @@ -43,7 +84,7 @@ const messageVariants = { const avatarVariants = { initial: { scale: 0 }, - animate: { + animate: { scale: 1, transition: { type: "spring", @@ -59,11 +100,13 @@ export default function Home() { const [recording, setRecording] = useState(false); const [textOutputs, setTextOutputs] = useState([]); const [toolUiOutput, setToolUiOutput] = useState(null); - const [currentTool, setCurrentTool] = useState<{name: string, content: string} | null>(null); + const [currentTool, setCurrentTool] = useState<{ name: string, content: string } | null>(null); const [waitingForTool, setWaitingForTool] = useState(false); const [wsKey, setWsKey] = useState(0); const [isThinking, setIsThinking] = useState(false); const [toolConfigs, setToolConfigs] = useState([]); + const [selectedModel, setSelectedModel] = useState(AVAILABLE_MODELS[0].id); + const [selectedLanguage, setSelectedLanguage] = useState(AVAILABLE_LANGUAGES[0].code); // Generate typing sound programmatically instead of using MP3 const [audioContext] = useState(() => typeof window !== 'undefined' ? new (window.AudioContext || (window as any).webkitAudioContext)() : null); const typingSoundInterval = useRef(null); @@ -84,10 +127,10 @@ export default function Home() { const connect = () => { if (wsRef.current) wsRef.current.close(); - wsRef.current = new WebSocket('ws://localhost:8000/ws'); + wsRef.current = new WebSocket(`ws://localhost:8000/ws?model=${encodeURIComponent(selectedModel)}&language=${encodeURIComponent(selectedLanguage)}`); setStatus('Connecting...'); setWsKey(k => k + 1); - + wsRef.current.onopen = () => setStatus('Connected'); wsRef.current.onclose = () => { setStatus('Disconnected'); @@ -99,7 +142,7 @@ export default function Home() { try { const msg = JSON.parse(event.data); console.log('[WS MESSAGE]', msg); - + if (msg.event) handleEventMessage(msg.event); } catch (e) { console.error('Error parsing message:', e); @@ -123,29 +166,29 @@ export default function Home() { } else { console.log('[SKIPPING DUPLICATE]', message); } - }, []); + }, []); // Function to generate a typing sound programmatically const playTypingBeep = useCallback(() => { if (!audioContext) return; - + try { // Create a short beep sound (like a keyboard click) const oscillator = audioContext.createOscillator(); const gainNode = audioContext.createGain(); - + oscillator.connect(gainNode); gainNode.connect(audioContext.destination); - + // Set frequency for a subtle click sound oscillator.frequency.setValueAtTime(800, audioContext.currentTime); oscillator.frequency.exponentialRampToValueAtTime(400, audioContext.currentTime + 0.1); - + // Set volume envelope for a quick click gainNode.gain.setValueAtTime(0, audioContext.currentTime); gainNode.gain.linearRampToValueAtTime(0.1, audioContext.currentTime + 0.01); gainNode.gain.exponentialRampToValueAtTime(0.001, audioContext.currentTime + 0.1); - + oscillator.start(audioContext.currentTime); oscillator.stop(audioContext.currentTime + 0.1); } catch (err) { @@ -182,13 +225,13 @@ export default function Home() { const handleEventMessage = (event: any) => { console.log('[WS EVENT]', event); - + if (event.init) { // Store tool configurations setToolConfigs(event.init.toolConfigs); return; } - + if (event.contentStart && event.contentStart.type === 'TEXT') { const contentId = event.contentStart.contentId; let stage = 'FINAL'; @@ -211,7 +254,7 @@ export default function Home() { console.log('[TOOL USE]', event.toolUse); setWaitingForTool(true); const toolName = event.toolUse.toolName; - + // Find the tool configuration and get its short description const toolConfig = toolConfigs.find((t: any) => t.name === toolName); setCurrentTool({ @@ -235,7 +278,7 @@ export default function Home() { } else if (event.toolUiOutput) { // Handle tool UI output console.log('[TOOL UI OUTPUT]', event.toolUiOutput); - + // Handle barge-in events if (event.toolUiOutput.type === 'barge_in') { console.log('[BARGE IN] Stopping audio playback'); @@ -243,7 +286,7 @@ export default function Home() { playbackServiceRef.current.stop(); } } - + // Handle tool execution progress events if (event.toolUiOutput.type === 'tool_exec_progress') { const status = event.toolUiOutput.content.status; @@ -268,7 +311,7 @@ export default function Home() { if (playbackServiceRef.current) { playbackServiceRef.current.playPCM(audioBytes); } - + // If there's pending text for this contentId, show it const contentId = event.audioOutput.contentId || 'default'; if (contentId && pendingTexts.current[contentId]) { @@ -281,7 +324,7 @@ export default function Home() { const text = event.textOutput.content; const role = event.textOutput.role || 'ASSISTANT'; const contentId = event.textOutput.contentId; - + console.log('[TEXT EVENT]', { text, role, @@ -303,7 +346,7 @@ export default function Home() { } else if (role === 'ASSISTANT') { // For assistant messages, check if we have corresponding audio const hasAudioOutput = event.audioOutput && event.audioOutput.contentId === contentId; - + // Show text immediately if no audio is expected if (!contentId || !hasAudioOutput) { console.log('[SHOWING ASSISTANT TEXT IMMEDIATELY]', { text, role }); @@ -380,7 +423,7 @@ export default function Home() { const base64LPCM = (base64String: string): string => { const byteCharacters = atob(base64String); const byteArrays = new Uint8Array(byteCharacters.length); - + for (let i = 0; i < byteCharacters.length; i++) { byteArrays[i] = byteCharacters.charCodeAt(i); } @@ -391,7 +434,7 @@ export default function Home() { const byteRate = sampleRate * numChannels * (bitsPerSample / 8); const blockAlign = numChannels * (bitsPerSample / 8); const wavSize = byteArrays.length + 36; - + const wavHeader = new Uint8Array(44); const view = new DataView(wavHeader.buffer); @@ -511,7 +554,41 @@ export default function Home() { {/* Right chat panel */}