diff --git a/packages/talkio/src/agent/actors/llm.ts b/packages/talkio/src/agent/actors/llm.ts index 6823f2e..9a5dfe8 100644 --- a/packages/talkio/src/agent/actors/llm.ts +++ b/packages/talkio/src/agent/actors/llm.ts @@ -38,6 +38,16 @@ export const llmActor = fromCallback< const debug = config.debug ?? false; let isAborted = false; + let isSettled = false; + let timeoutId: ReturnType | null = null; + + const finishWithError = (error: Error) => { + if (isAborted || isSettled) return; + isSettled = true; + isAborted = true; + if (timeoutId) clearTimeout(timeoutId); + sendBack({ type: "_llm:error", error, timestamp: Date.now() }); + }; const handleAbort = () => { isAborted = true; @@ -53,42 +63,33 @@ export const llmActor = fromCallback< }; } - // Set up timeout const timeoutMs = config.timeout?.llmMs; - let timeoutId: ReturnType | null = null; if (timeoutMs && timeoutMs > 0) { timeoutId = setTimeout(() => { - if (!isAborted) { - isAborted = true; - if (debug) console.error("[llm-actor] Timeout after", timeoutMs, "ms"); - sendBack({ - type: "_llm:error", - error: new Error(`LLM timeout after ${timeoutMs}ms`), - timestamp: Date.now(), - }); - } + if (debug) console.error("[llm-actor] Timeout after", timeoutMs, "ms"); + finishWithError(new Error(`LLM timeout after ${timeoutMs}ms`)); }, timeoutMs); } const ctx = { messages, token: (token: string) => { - if (isAborted) return; + if (isAborted || isSettled) return; sendBack({ type: "_llm:token", token, timestamp: Date.now() }); }, sentence: (sentence: string, index: number) => { - if (isAborted) return; + if (isAborted || isSettled) return; sendBack({ type: "_llm:sentence", sentence, index, timestamp: Date.now() }); }, complete: (fullText: string) => { - if (isAborted) return; + if (isAborted || isSettled) return; + isSettled = true; if (timeoutId) clearTimeout(timeoutId); sendBack({ type: "_llm:complete", fullText, timestamp: Date.now() }); }, error: (error: Error) => { - if (isAborted) return; - sendBack({ type: "_llm:error", error, timestamp: Date.now() }); + finishWithError(error); }, say: sayFn, interrupt: interruptFn, @@ -103,14 +104,8 @@ export const llmActor = fromCallback< config.llm(ctx); } } catch (error) { - if (!isAborted) { - if (debug) console.error("[llm-actor] Error starting generation:", error); - sendBack({ - type: "_llm:error", - error: error instanceof Error ? error : new Error(String(error)), - timestamp: Date.now(), - }); - } + if (debug) console.error("[llm-actor] Error starting generation:", error); + finishWithError(error instanceof Error ? error : new Error(String(error))); } return () => { diff --git a/packages/talkio/src/agent/actors/stt.ts b/packages/talkio/src/agent/actors/stt.ts index 6dc9b28..a8da9ef 100644 --- a/packages/talkio/src/agent/actors/stt.ts +++ b/packages/talkio/src/agent/actors/stt.ts @@ -42,6 +42,13 @@ export const sttActor = fromCallback< abortSignal.addEventListener("abort", handleAbort); + if (abortSignal.aborted) { + isAborted = true; + return () => { + abortSignal.removeEventListener("abort", handleAbort); + }; + } + if (debug) console.log("[stt-actor] Starting STT provider with format:", inputFormat); try { diff --git a/packages/talkio/src/agent/actors/tts.ts b/packages/talkio/src/agent/actors/tts.ts index ba39051..0f22352 100644 --- a/packages/talkio/src/agent/actors/tts.ts +++ b/packages/talkio/src/agent/actors/tts.ts @@ -27,15 +27,26 @@ export const ttsActor = fromCallback< { config: NormalizedAgentConfig; text: string; + requestId: string; abortSignal: AbortSignal; } >(({ sendBack, input }) => { - const { config, text, abortSignal } = input; + const { config, text, requestId, abortSignal } = input; const provider = config.tts; const outputFormat = config.audio.output; const debug = config.debug ?? false; let isAborted = false; + let isSettled = false; + let timeoutId: ReturnType | null = null; + + const finishWithError = (error: Error) => { + if (isAborted || isSettled) return; + isSettled = true; + isAborted = true; + if (timeoutId) clearTimeout(timeoutId); + sendBack({ type: "_tts:error", requestId, error, timestamp: Date.now() }); + }; const handleAbort = () => { isAborted = true; @@ -51,21 +62,12 @@ export const ttsActor = fromCallback< }; } - // Set up timeout const timeoutMs = config.timeout?.ttsMs; - let timeoutId: ReturnType | null = null; if (timeoutMs && timeoutMs > 0) { timeoutId = setTimeout(() => { - if (!isAborted) { - isAborted = true; - if (debug) console.error("[tts-actor] Timeout after", timeoutMs, "ms"); - sendBack({ - type: "_tts:error", - error: new Error(`TTS timeout after ${timeoutMs}ms`), - timestamp: Date.now(), - }); - } + if (debug) console.error("[tts-actor] Timeout after", timeoutMs, "ms"); + finishWithError(new Error(`TTS timeout after ${timeoutMs}ms`)); }, timeoutMs); } @@ -73,29 +75,23 @@ export const ttsActor = fromCallback< provider.synthesize(text, { audioFormat: outputFormat, audioChunk: (audio) => { - if (isAborted) return; - sendBack({ type: "_tts:chunk", audio, timestamp: Date.now() }); + if (isAborted || isSettled) return; + sendBack({ type: "_tts:chunk", requestId, audio, timestamp: Date.now() }); }, complete: () => { - if (isAborted) return; + if (isAborted || isSettled) return; + isSettled = true; if (timeoutId) clearTimeout(timeoutId); - sendBack({ type: "_tts:complete", timestamp: Date.now() }); + sendBack({ type: "_tts:complete", requestId, timestamp: Date.now() }); }, error: (error) => { - if (isAborted) return; - sendBack({ type: "_tts:error", error, timestamp: Date.now() }); + finishWithError(error); }, signal: abortSignal, }); } catch (error) { - if (!isAborted) { - if (debug) console.error("[tts-actor] Error starting synthesis:", error); - sendBack({ - type: "_tts:error", - error: error instanceof Error ? error : new Error(String(error)), - timestamp: Date.now(), - }); - } + if (debug) console.error("[tts-actor] Error starting synthesis:", error); + finishWithError(error instanceof Error ? error : new Error(String(error))); } return () => { diff --git a/packages/talkio/src/agent/actors/turn-detector.ts b/packages/talkio/src/agent/actors/turn-detector.ts index 4af4bd9..ac96cfa 100644 --- a/packages/talkio/src/agent/actors/turn-detector.ts +++ b/packages/talkio/src/agent/actors/turn-detector.ts @@ -45,6 +45,13 @@ export const turnDetectorActor = fromCallback< abortSignal.addEventListener("abort", handleAbort); + if (abortSignal.aborted) { + isAborted = true; + return () => { + abortSignal.removeEventListener("abort", handleAbort); + }; + } + try { provider.start({ turnEnd: (transcript) => { diff --git a/packages/talkio/src/agent/actors/vad.ts b/packages/talkio/src/agent/actors/vad.ts index 79ea5aa..300e527 100644 --- a/packages/talkio/src/agent/actors/vad.ts +++ b/packages/talkio/src/agent/actors/vad.ts @@ -44,6 +44,13 @@ export const vadActor = fromCallback< abortSignal.addEventListener("abort", handleAbort); + if (abortSignal.aborted) { + isAborted = true; + return () => { + abortSignal.removeEventListener("abort", handleAbort); + }; + } + try { provider.start({ speechStart: () => { @@ -58,6 +65,10 @@ export const vadActor = fromCallback< if (isAborted) return; sendBack({ type: "_vad:probability", value, timestamp: Date.now() }); }, + error: (error) => { + if (isAborted) return; + sendBack({ type: "_vad:error", error, timestamp: Date.now() }); + }, signal: abortSignal, }); } catch (error) { @@ -77,8 +88,13 @@ export const vadActor = fromCallback< try { provider.processAudio(event.audio); } catch (error) { - if (!isAborted && debug) { - console.error("[vad-actor] Error processing audio:", error); + if (!isAborted) { + if (debug) console.error("[vad-actor] Error processing audio:", error); + sendBack({ + type: "_vad:error", + error: error instanceof Error ? error : new Error(String(error)), + timestamp: Date.now(), + }); } } } diff --git a/packages/talkio/src/agent/context.ts b/packages/talkio/src/agent/context.ts index 3eb021f..ee56bf5 100644 --- a/packages/talkio/src/agent/context.ts +++ b/packages/talkio/src/agent/context.ts @@ -38,6 +38,7 @@ export interface AgentMachineContext { llmRef: ActorRefFrom | null; ttsRef: ActorRefFrom | null; + activeTTSRequestId: string | null; sentenceQueue: string[]; vadSource: "adapter" | "stt"; turnSource: "adapter" | "stt"; @@ -68,6 +69,7 @@ export function createInitialContext< turnAbortController: null, llmRef: null, ttsRef: null, + activeTTSRequestId: null, sentenceQueue: [], vadSource: config.vad ? "adapter" : "stt", turnSource: config.turnDetector ? "adapter" : "stt", diff --git a/packages/talkio/src/agent/machine.ts b/packages/talkio/src/agent/machine.ts index 4540507..277c1ff 100644 --- a/packages/talkio/src/agent/machine.ts +++ b/packages/talkio/src/agent/machine.ts @@ -98,6 +98,14 @@ const agentMachineSetup = setup({ hasPendingSentences: ({ context }) => context.sentenceQueue.length > 0, hasPendingTTS: ({ context }) => context.pendingTTSCount > 0, noPendingTTS: ({ context }) => context.pendingTTSCount === 0, + matchesActiveTTSRequest: ({ context, event }) => + (event.type === "_tts:chunk" || + event.type === "_tts:complete" || + event.type === "_tts:error") && + context.activeTTSRequestId !== null && + event.requestId === context.activeTTSRequestId, + speakingCurrentTTS: and(["isSpeaking", "matchesActiveTTSRequest"]), + hasPendingSentencesForCurrentTTS: and(["hasPendingSentences", "matchesActiveTTSRequest"]), // Turn state guards aiTurnHadNoAudio: ({ context }) => !context.aiTurnHadAudio, @@ -122,6 +130,9 @@ const agentMachineSetup = setup({ } else if (event.type === "_tts:error") { error = event.error; source = "tts"; + } else if (event.type === "_vad:error") { + error = event.error; + source = "vad"; } else { error = new Error("Unknown error"); source = "stt"; @@ -285,12 +296,14 @@ const agentMachineSetup = setup({ clearDynamicActorRefs: assign({ llmRef: () => null, ttsRef: () => null, + activeTTSRequestId: () => null, sentenceQueue: () => [], pendingTTSCount: () => 0, }), clearLLMRef: assign({ llmRef: () => null }), clearFillerTTS: assign({ ttsRef: () => null, + activeTTSRequestId: () => null, sentenceQueue: () => [], pendingTTSCount: () => 0, }), @@ -308,7 +321,7 @@ const agentMachineSetup = setup({ } return { type: "_audio:output-chunk" as const, - audio: new Float32Array(0), + audio: new ArrayBuffer(0), timestamp: Date.now(), }; }), @@ -347,41 +360,49 @@ const agentMachineSetup = setup({ pendingTTSCount: ({ context }) => context.pendingTTSCount + 1, }), - spawnTTSFromQueue: assign({ - ttsRef: ({ context, spawn }) => { - const text = getTTSText("queue", context.sentenceQueue, { type: "" }); - if (!text) return null; - if (!context.turnAbortController) { - throw new Error("[machine] Cannot spawn TTS: turnAbortController is null"); - } - const signal = context.turnAbortController.signal; - return spawn("tts", { - systemId: `currentTTS-${nanoid()}`, - input: { config: context.config, text, abortSignal: signal }, - }); - }, - sentenceQueue: ({ context }) => context.sentenceQueue.slice(1), + spawnTTSFromQueue: assign(({ context, spawn }) => { + const text = getTTSText("queue", context.sentenceQueue, { type: "" }); + if (!text) { + return { ttsRef: null, activeTTSRequestId: null }; + } + if (!context.turnAbortController) { + throw new Error("[machine] Cannot spawn TTS: turnAbortController is null"); + } + const signal = context.turnAbortController.signal; + const requestId = nanoid(); + return { + ttsRef: spawn("tts", { + systemId: `currentTTS-${requestId}`, + input: { config: context.config, text, requestId, abortSignal: signal }, + }), + activeTTSRequestId: requestId, + sentenceQueue: context.sentenceQueue.slice(1), + }; }), - clearTTSRef: assign({ ttsRef: () => null }), + clearTTSRef: assign({ ttsRef: () => null, activeTTSRequestId: () => null }), decrementPendingTTS: assign({ pendingTTSCount: ({ context }) => Math.max(0, context.pendingTTSCount - 1), }), - spawnTTSFromFiller: assign({ - ttsRef: ({ context, spawn, event }) => { - const text = getTTSText("filler", [], event); - if (!text) return null; - if (!context.turnAbortController) { - throw new Error("[machine] Cannot spawn TTS from filler: turnAbortController is null"); - } - const signal = context.turnAbortController.signal; - return spawn("tts", { - systemId: `currentTTS-${nanoid()}`, - input: { config: context.config, text, abortSignal: signal }, - }); - }, + spawnTTSFromFiller: assign(({ context, spawn, event }) => { + const text = getTTSText("filler", [], event); + if (!text) { + return { ttsRef: null, activeTTSRequestId: null }; + } + if (!context.turnAbortController) { + throw new Error("[machine] Cannot spawn TTS from filler: turnAbortController is null"); + } + const signal = context.turnAbortController.signal; + const requestId = nanoid(); + return { + ttsRef: spawn("tts", { + systemId: `currentTTS-${requestId}`, + input: { config: context.config, text, requestId, abortSignal: signal }, + }), + activeTTSRequestId: requestId, + }; }), recordSessionStart: assign({ metrics: ({ context }) => ({ @@ -560,6 +581,7 @@ const agentMachineSetup = setup({ let source: "stt" | "llm" | "tts" | "vad" = "stt"; if (event.type === "_llm:error") source = "llm"; else if (event.type === "_tts:error") source = "tts"; + else if (event.type === "_vad:error") source = "vad"; return { ...context.metrics, @@ -701,6 +723,9 @@ export const agentMachine = agentMachineSetup.createMachine({ { type: "warnSTTDegraded" }, ], }, + "_vad:error": { + actions: [{ type: "debugLogEvent" }, { type: "recordError" }, { type: "emitAgentError" }], + }, "_llm:error": { actions: [ { type: "debugLogEvent" }, @@ -715,6 +740,7 @@ export const agentMachine = agentMachineSetup.createMachine({ ], }, "_tts:error": { + guard: "matchesActiveTTSRequest", actions: [{ type: "debugLogEvent" }, { type: "recordError" }, { type: "emitAgentError" }], }, "_filler:say": { @@ -1083,7 +1109,7 @@ export const agentMachine = agentMachineSetup.createMachine({ silent: { on: { "_tts:chunk": { - guard: "isSpeaking", + guard: "speakingCurrentTTS", target: "streaming", actions: [ { type: "setAITurnHadAudio" }, @@ -1095,7 +1121,7 @@ export const agentMachine = agentMachineSetup.createMachine({ }, "_tts:complete": [ { - guard: "hasPendingSentences", + guard: "hasPendingSentencesForCurrentTTS", actions: [ { type: "clearTTSRef" }, { type: "decrementPendingTTS" }, @@ -1103,6 +1129,7 @@ export const agentMachine = agentMachineSetup.createMachine({ ], }, { + guard: "matchesActiveTTSRequest", actions: [ { type: "clearTTSRef" }, { type: "decrementPendingTTS" }, @@ -1112,7 +1139,7 @@ export const agentMachine = agentMachineSetup.createMachine({ ], "_tts:error": [ { - guard: "hasPendingSentences", + guard: "hasPendingSentencesForCurrentTTS", actions: [ { type: "debugLogEvent" }, { type: "recordError" }, @@ -1123,6 +1150,7 @@ export const agentMachine = agentMachineSetup.createMachine({ ], }, { + guard: "matchesActiveTTSRequest", actions: [ { type: "debugLogEvent" }, { type: "recordError" }, @@ -1138,7 +1166,7 @@ export const agentMachine = agentMachineSetup.createMachine({ streaming: { on: { "_tts:chunk": { - guard: "isSpeaking", + guard: "speakingCurrentTTS", actions: [ { type: "recordAudioChunk" }, { type: "emitAITurnAudio" }, @@ -1147,7 +1175,7 @@ export const agentMachine = agentMachineSetup.createMachine({ }, "_tts:complete": [ { - guard: "hasPendingSentences", + guard: "hasPendingSentencesForCurrentTTS", actions: [ { type: "clearTTSRef" }, { type: "decrementPendingTTS" }, @@ -1155,6 +1183,7 @@ export const agentMachine = agentMachineSetup.createMachine({ ], }, { + guard: "matchesActiveTTSRequest", target: "silent", actions: [ { type: "clearTTSRef" }, @@ -1168,7 +1197,7 @@ export const agentMachine = agentMachineSetup.createMachine({ ], "_tts:error": [ { - guard: "hasPendingSentences", + guard: "hasPendingSentencesForCurrentTTS", actions: [ { type: "debugLogEvent" }, { type: "recordError" }, @@ -1179,6 +1208,7 @@ export const agentMachine = agentMachineSetup.createMachine({ ], }, { + guard: "matchesActiveTTSRequest", target: "silent", actions: [ { type: "debugLogEvent" }, diff --git a/packages/talkio/src/providers/types.ts b/packages/talkio/src/providers/types.ts index 671ac63..97370a7 100644 --- a/packages/talkio/src/providers/types.ts +++ b/packages/talkio/src/providers/types.ts @@ -511,6 +511,11 @@ export interface VADContext { */ speechProbability(probability: number): void; + /** + * Report an error. + */ + error(error: Error): void; + /** * Abort signal for cancellation. */ @@ -720,6 +725,7 @@ export interface CreateCustomVADProviderOptions { * - `ctx.speechStart()` when speech begins * - `ctx.speechEnd(duration)` when speech ends * - `ctx.speechProbability(prob)` for probability scores (optional) + * - `ctx.error(error)` to report errors * * @param audio - Audio data (ArrayBuffer) in the configured input format */ diff --git a/packages/talkio/src/types/events.ts b/packages/talkio/src/types/events.ts index 404ab32..b72f04c 100644 --- a/packages/talkio/src/types/events.ts +++ b/packages/talkio/src/types/events.ts @@ -305,10 +305,17 @@ export interface InternalVADProbabilityEvent { timestamp: number; } +export interface InternalVADErrorEvent { + type: "_vad:error"; + error: Error; + timestamp: number; +} + export type InternalVADEvent = | InternalVADSpeechStartEvent | InternalVADSpeechEndEvent - | InternalVADProbabilityEvent; + | InternalVADProbabilityEvent + | InternalVADErrorEvent; export interface InternalTurnEndEvent { type: "_turn:end"; @@ -357,17 +364,20 @@ export type InternalLLMEvent = export interface InternalTTSChunkEvent { type: "_tts:chunk"; + requestId: string; audio: ArrayBuffer; timestamp: number; } export interface InternalTTSCompleteEvent { type: "_tts:complete"; + requestId: string; timestamp: number; } export interface InternalTTSErrorEvent { type: "_tts:error"; + requestId: string; error: Error; timestamp: number; } diff --git a/packages/talkio/test/agent/error-recovery.test.ts b/packages/talkio/test/agent/error-recovery.test.ts index e0ac23f..6aa105c 100644 --- a/packages/talkio/test/agent/error-recovery.test.ts +++ b/packages/talkio/test/agent/error-recovery.test.ts @@ -7,6 +7,24 @@ function makeAudioChunk(value: number): ArrayBuffer { } describe("error recovery", () => { + it("emits VAD error and continues running", async () => { + const harness = createAgentHarness({ useVAD: true }); + const { agent, events, vad } = harness; + + agent.start(); + await events.waitForEvent("agent:started"); + + vad?.emitError(new Error("vad failed")); + const errorEvent = await events.waitForEvent("agent:error"); + expect(errorEvent.source).toBe("vad"); + + const snapshot = agent.getSnapshot(); + expect(snapshot.isRunning).toBe(true); + + agent.stop(); + await events.waitForEvent("agent:stopped"); + }); + it("emits STT error and continues running", async () => { const harness = createAgentHarness(); const { agent, events, stt } = harness; @@ -77,7 +95,7 @@ describe("error recovery", () => { vi.setSystemTime(0); const harness = createAgentHarness({ timeout: { llmMs: 10 } }); - const { agent, events, stt } = harness; + const { agent, events, stt, llm } = harness; agent.start(); await events.waitForEvent("agent:started"); @@ -90,6 +108,9 @@ describe("error recovery", () => { const errorEvent = await events.waitForEvent("agent:error"); expect(errorEvent.source).toBe("llm"); + llm.error(new Error("late llm error")); + await drainMicrotasks(); + expect(events.byType("agent:error")).toHaveLength(1); agent.stop(); await events.waitForEvent("agent:stopped"); @@ -103,7 +124,7 @@ describe("error recovery", () => { vi.setSystemTime(0); const harness = createAgentHarness({ timeout: { ttsMs: 10 } }); - const { agent, events, stt, llm } = harness; + const { agent, events, stt, llm, tts } = harness; agent.start(); await events.waitForEvent("agent:started"); @@ -118,6 +139,9 @@ describe("error recovery", () => { const errorEvent = await events.waitForEvent("agent:error"); expect(errorEvent.source).toBe("tts"); + tts.error(new Error("late tts error")); + await drainMicrotasks(); + expect(events.byType("agent:error")).toHaveLength(1); agent.stop(); await events.waitForEvent("agent:stopped"); diff --git a/packages/talkio/test/agent/interruption.test.ts b/packages/talkio/test/agent/interruption.test.ts index 46d87a7..2afac1b 100644 --- a/packages/talkio/test/agent/interruption.test.ts +++ b/packages/talkio/test/agent/interruption.test.ts @@ -63,6 +63,41 @@ describe("interruptions", () => { await events.waitForEvent("agent:stopped"); }); + it("ignores stale filler audio after interrupt", async () => { + const harness = createAgentHarness(); + const { agent, events, stt, llm, tts } = harness; + + agent.start(); + await events.waitForEvent("agent:started"); + + stt.emitTranscript("hello", true); + await events.waitForEvent("ai-turn:started"); + + llm.say("Hmm..."); + await drainMicrotasks(); + + llm.interrupt(); + await drainMicrotasks(); + + llm.emitSentence("Real answer.", 0); + await drainMicrotasks(); + + tts.emitAudio(makeAudioChunk(9)); + tts.complete(); + tts.emitAudio(makeAudioChunk(7)); + tts.complete(); + llm.complete("Real answer."); + + await events.waitForEvent("ai-turn:ended"); + + const audioEvents = events.byType("ai-turn:audio"); + expect(audioEvents).toHaveLength(1); + expect(new Uint8Array(audioEvents[0].audio)).toEqual(new Uint8Array([7, 7])); + + agent.stop(); + await events.waitForEvent("agent:stopped"); + }); + it("does not interrupt for short STT speech below threshold", async () => { vi.useFakeTimers(); vi.setSystemTime(0); diff --git a/packages/talkio/test/helpers/fake-providers.ts b/packages/talkio/test/helpers/fake-providers.ts index f8df823..640c1dd 100644 --- a/packages/talkio/test/helpers/fake-providers.ts +++ b/packages/talkio/test/helpers/fake-providers.ts @@ -84,6 +84,7 @@ type FakeVAD = { emitSpeechStart: () => void; emitSpeechEnd: (duration: number) => void; emitProbability: (value: number) => void; + emitError: (error: Error) => void; }; type FakeTurnDetector = { @@ -395,6 +396,12 @@ export function createFakeVAD(): FakeVAD { } ctx.speechProbability(value); }, + emitError: (error) => { + if (!ctx) { + throw new Error("VAD context not initialized"); + } + ctx.error(error); + }, }; }