diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 2a8e0d5ee..ebd192138 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -1,24 +1,24 @@ -/// - +/// + declare namespace NodeJS { - interface ProcessEnv { - /** - * The built directory structure - * - * ```tree - * ├─┬─┬ dist - * │ │ └── index.html - * │ │ - * │ ├─┬ dist-electron - * │ │ ├── main.js - * │ │ └── preload.js - * │ - * ``` - */ - APP_ROOT: string - /** /dist/ or /public/ */ - VITE_PUBLIC: string - } + interface ProcessEnv { + /** + * The built directory structure + * + * ```tree + * ├─┬─┬ dist + * │ │ └── index.html + * │ │ + * │ ├─┬ dist-electron + * │ │ ├── main.js + * │ │ └── preload.js + * │ + * ``` + */ + APP_ROOT: string + /** /dist/ or /public/ */ + VITE_PUBLIC: string + } } type CursorTrackMetadata = { @@ -148,7 +148,7 @@ interface Window { openPermissionSettings: (target: PermissionSettingsTarget) => Promise<{ success: boolean; message?: string }> openPermissionChecker: () => Promise<{ success: boolean }> switchToEditor: () => Promise - openSourceSelector: () => Promise + openSourceSelector: () => Promise selectSource: (source: unknown) => Promise getSelectedSource: () => Promise storeRecordedVideo: ( @@ -226,7 +226,8 @@ interface Window { path?: string metadata?: { frameRate?: number; width?: number; height?: number; mimeType?: string; capturedAt?: number; systemCursorMode?: 'always' | 'never'; hasMicrophoneAudio?: boolean; cursorTrack?: CursorTrackMetadata } }> - clearCurrentVideoPath: () => Promise<{ success: boolean }> + clearCurrentVideoPath: () => Promise<{ success: boolean }> + readCurrentVideoBytes: () => Promise<{ success: boolean; buffer?: ArrayBuffer; error?: string }> getPlatform: () => Promise startVideoAnalysis: (options?: { videoPath?: string @@ -269,7 +270,7 @@ interface Window { hudOverlayClose: () => void; } } - + interface ProcessedDesktopSource { id: string name: string diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 1a406f3b1..10cf6e46b 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1878,6 +1878,22 @@ export function registerIpcHandlers( return { success: true }; }); + ipcMain.handle('read-current-video-bytes', async () => { + if (!currentVideoPath) { + return { success: false, error: 'No current video path is set.' }; + } + try { + const data = await fs.readFile(currentVideoPath); + const buffer = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength); + return { success: true, buffer }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } + }); + ipcMain.handle('get-platform', () => { return process.platform; }); diff --git a/electron/preload.ts b/electron/preload.ts index 43f4cff6c..54bfd09f8 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -180,6 +180,9 @@ contextBridge.exposeInMainWorld('electronAPI', { clearCurrentVideoPath: () => { return ipcRenderer.invoke('clear-current-video-path') }, + readCurrentVideoBytes: () => { + return ipcRenderer.invoke('read-current-video-bytes') + }, getPlatform: () => { return ipcRenderer.invoke('get-platform') }, diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 5838474b7..6fa68bc9e 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -13,7 +13,7 @@ import { type AudioEnergyStats, type NormalizedExportAudioProcessingConfig, } from '@/lib/audio/exportAudioProcessing'; -import { ALL_FORMATS, AudioBufferSink, BlobSource, Input, UrlSource, type InputAudioTrack } from 'mediabunny'; +import { ALL_FORMATS, AudioBufferSink, BlobSource, BufferSource, Input, UrlSource, type InputAudioTrack } from 'mediabunny'; interface VideoExporterConfig extends ExportConfig { videoUrl: string; @@ -348,40 +348,73 @@ export class VideoExporter { this.sourceAudioTrack = null; } + private async openSourceInputFromNativeBytes(): Promise { + const readCurrentVideoBytes = window.electronAPI?.readCurrentVideoBytes; + if (typeof readCurrentVideoBytes !== 'function') { + return null; + } + + const result = await readCurrentVideoBytes(); + if (!result.success || !result.buffer) { + throw new Error(`Failed to read source media bytes: ${result.error ?? 'unknown error'}`); + } + + return new Input({ + formats: ALL_FORMATS, + source: new BufferSource(result.buffer), + }); + } + private async resolveSourceAudioTrack(): Promise { this.disposeSourceAudioInput(); if (this.config.audioEnabled === false) { return false; } - try { - const input = await this.openSourceInputFromUrl(); - const audioTrack = await input.getPrimaryAudioTrack(); - if (audioTrack) { - this.sourceAudioInput = input; - this.sourceAudioTrack = audioTrack; - return true; + const attempts: Array<{ label: string; open: () => Promise }> = [ + { label: 'UrlSource', open: () => this.openSourceInputFromUrl() }, + { label: 'BlobSource', open: () => this.openSourceInputFromBlob() }, + { label: 'NativeBytes', open: () => this.openSourceInputFromNativeBytes() }, + ]; + + const errors: unknown[] = []; + + for (const attempt of attempts) { + let input: Input | null = null; + try { + input = await attempt.open(); + if (!input) { + continue; + } + const audioTrack = await input.getPrimaryAudioTrack(); + if (audioTrack) { + this.sourceAudioInput = input; + this.sourceAudioTrack = audioTrack; + return true; + } + // The container parsed successfully and genuinely has no audio track. + input.dispose(); + return false; + } catch (error) { + errors.push(error); + console.warn(`[VideoExporter] Unable to read source audio via ${attempt.label}.`, error); + if (input) { + try { + input.dispose(); + } catch { /* already failed */ } + } } - input.dispose(); - } catch (urlError) { - console.warn('[VideoExporter] Unable to read source audio via UrlSource. Retrying with BlobSource.', urlError); } - try { - const input = await this.openSourceInputFromBlob(); - const audioTrack = await input.getPrimaryAudioTrack(); - if (audioTrack) { - this.sourceAudioInput = input; - this.sourceAudioTrack = audioTrack; - return true; - } - input.dispose(); - return false; - } catch (blobError) { - console.warn('[VideoExporter] Audio track extraction failed; continuing with video-only export.', blobError); - this.disposeSourceAudioInput(); - return false; + this.disposeSourceAudioInput(); + if (errors.length > 0) { + const detail = errors[errors.length - 1]; + throw new Error( + `Source audio track could not be read (${errors.length} attempts failed). ` + + `Last error: ${detail instanceof Error ? detail.message : String(detail)}`, + ); } + return false; } private addWarning(warningKey: string): void { diff --git a/src/lib/exporter/videoExporterAudio.test.ts b/src/lib/exporter/videoExporterAudio.test.ts index 83f42f7d7..2f853e9cf 100644 --- a/src/lib/exporter/videoExporterAudio.test.ts +++ b/src/lib/exporter/videoExporterAudio.test.ts @@ -150,3 +150,58 @@ describe('VideoExporter createAudioSlice', () => { expect(result!.getChannelData(1)[0]).toBeCloseTo(0.6, 6); }); }); + +describe('VideoExporter resolveSourceAudioTrack', () => { + function makeInput(audioTrack: unknown): any { + return { + getPrimaryAudioTrack: async () => audioTrack, + dispose: () => {}, + }; + } + + it('returns false without attempting extraction when audio is disabled', async () => { + const exporter = makeExporter(); + exporter.config.audioEnabled = false; + exporter.openSourceInputFromUrl = async () => { throw new Error('should not be called'); }; + exporter.openSourceInputFromBlob = async () => { throw new Error('should not be called'); }; + exporter.openSourceInputFromNativeBytes = async () => { throw new Error('should not be called'); }; + + await expect(exporter.resolveSourceAudioTrack()).resolves.toBe(false); + }); + + it('falls back to the next source when an earlier one fails', async () => { + const exporter = makeExporter(); + const track = { codec: 'aac' }; + exporter.openSourceInputFromUrl = async () => { throw new Error('url read failed'); }; + exporter.openSourceInputFromBlob = async () => { throw new Error('blob read failed'); }; + exporter.openSourceInputFromNativeBytes = async () => makeInput(track); + + await expect(exporter.resolveSourceAudioTrack()).resolves.toBe(true); + expect(exporter.sourceAudioTrack).toBe(track); + }); + + it('returns false when the container parses but has no audio track', async () => { + const exporter = makeExporter(); + exporter.openSourceInputFromUrl = async () => makeInput(null); + + await expect(exporter.resolveSourceAudioTrack()).resolves.toBe(false); + }); + + it('throws instead of silently degrading when every source fails', async () => { + const exporter = makeExporter(); + exporter.openSourceInputFromUrl = async () => { throw new Error('url read failed'); }; + exporter.openSourceInputFromBlob = async () => { throw new Error('blob read failed'); }; + exporter.openSourceInputFromNativeBytes = async () => { throw new Error('ipc read failed'); }; + + await expect(exporter.resolveSourceAudioTrack()).rejects.toThrow(/Source audio track could not be read/); + }); + + it('skips the native-bytes source when the bridge is unavailable', async () => { + const exporter = makeExporter(); + exporter.openSourceInputFromUrl = async () => { throw new Error('url read failed'); }; + exporter.openSourceInputFromBlob = async () => { throw new Error('blob read failed'); }; + exporter.openSourceInputFromNativeBytes = async () => null; + + await expect(exporter.resolveSourceAudioTrack()).rejects.toThrow(/2 attempts failed/); + }); +}); diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts index 021b5d8d8..4c1a68698 100644 --- a/src/vite-env.d.ts +++ b/src/vite-env.d.ts @@ -1,6 +1,6 @@ -/// -/// - +/// +/// + interface ProcessedDesktopSource { id: string; name: string; @@ -123,7 +123,7 @@ type VideoAnalysisMetadata = { subtitleCues: SubtitleCueMetadata[]; roughCutSuggestions: RoughCutSuggestionMetadata[]; }; - + interface Window { electronAPI: { getSources: (opts: Electron.SourcesOptions) => Promise @@ -137,7 +137,7 @@ interface Window { openPermissionSettings: (target: PermissionSettingsTarget) => Promise<{ success: boolean; message?: string }> openPermissionChecker: () => Promise<{ success: boolean }> switchToEditor: () => Promise - openSourceSelector: () => Promise + openSourceSelector: () => Promise selectSource: (source: unknown) => Promise getSelectedSource: () => Promise storeRecordedVideo: ( @@ -233,6 +233,7 @@ interface Window { metadata?: { frameRate?: number; width?: number; height?: number; mimeType?: string; capturedAt?: number; systemCursorMode?: 'always' | 'never'; hasMicrophoneAudio?: boolean; cursorTrack?: CursorTrackMetadata } }> clearCurrentVideoPath: () => Promise<{ success: boolean }> + readCurrentVideoBytes: () => Promise<{ success: boolean; buffer?: ArrayBuffer; error?: string }> getPlatform: () => Promise startVideoAnalysis: (options?: { videoPath?: string