From 7851aa9aa2f39eefd743dd2a7544867507410d76 Mon Sep 17 00:00:00 2001 From: Aditya Gupta Date: Sun, 5 Jul 2026 00:33:57 +0530 Subject: [PATCH 1/2] added models to the tauri frontend --- .env.example | 2 + .gitignore | 3 + README.md | 2 +- src-tauri/src/lib.rs | 26 +++++- src-tauri/src/llm.rs | 155 ++++++++++++++++++++++++++++++++++ src-tauri/src/models.rs | 2 + src/main.tsx | 180 +++++++++++++++++++++++++++++++++++----- 7 files changed, 346 insertions(+), 24 deletions(-) diff --git a/.env.example b/.env.example index 6b442a1..eb58a3a 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,5 @@ DEEPGRAM_API_KEY=your_deepgram_key_here ANTHROPIC_API_KEY=your_anthropic_key_here LLM_PROVIDER=deepseek DEEPSEEK_API_KEY= +OPENAI_API_KEY= +GEMINI_API_KEY= diff --git a/.gitignore b/.gitignore index b6c81a6..5a9af55 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ node_modules dist +bun.lock +src-tauri/gen/schemas/* +package-lock.json target src-tauri/target .env diff --git a/README.md b/README.md index af94aed..96dce43 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This repository implements the desktop app foundation using **Tauri 2 + React + ## Key Features -- **Dynamic Multi-LLM Support**: Supports both **DeepSeek** (default) and **Claude** (anthropic) for viral moment detection and hooks analysis. +- **Dynamic Multi-LLM Support**: Supports **DeepSeek** (default), **Claude** (anthropic), **OpenAI**, and **Google Gemini** for cloud-based viral moment detection and hooks analysis, plus **Ollama** for fully-local inference. Pick your provider and model in API Settings. - **Automated Pipeline**: Imports media, extracts audio, transcribes using Deepgram, and automatically analyzes and ranks moments in a single automated chain. - **Local SQLite Storage**: Saves transcripts, candidates, custom names, and rendering data locally. - **Native Project Manager**: Create, open, rename, and delete projects from the dashboard. diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index eed4fa1..e510317 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -55,6 +55,8 @@ async fn environment_status(state: tauri::State<'_, AppState>) -> Result { + let key = api_key + .or_else(|| std::env::var("OPENAI_API_KEY").ok()) + .ok_or_else(|| "Set OPENAI_API_KEY or supply OpenAI API Key to generate candidates.".to_string())?; + let model = model_name + .or_else(|| std::env::var("OPENAI_MODEL").ok()) + .unwrap_or_else(|| "gpt-4o-mini".to_string()); + llm::detect_candidates_with_openai(&normalized, &key, &model) + .await + .map_err(to_command_error)? + } + "gemini" => { + let key = api_key + .or_else(|| std::env::var("GEMINI_API_KEY").ok()) + .ok_or_else(|| "Set GEMINI_API_KEY or supply Gemini API Key to generate candidates.".to_string())?; + let model = model_name + .or_else(|| std::env::var("GEMINI_MODEL").ok()) + .unwrap_or_else(|| "gemini-1.5-flash".to_string()); + llm::detect_candidates_with_gemini(&normalized, &key, &model) + .await + .map_err(to_command_error)? + } _ => { let key = api_key .or_else(|| std::env::var("DEEPSEEK_API_KEY").ok()) @@ -874,7 +898,7 @@ fn build_drawtext_filters( let mut font_option = String::new(); for path in &font_paths { if std::path::Path::new(path).exists() { - font_option = format!("fontfile='{}':", path.replace(':', "\:")); + font_option = format!("fontfile='{}':", path.replace(':', r"\:")); break; } } diff --git a/src-tauri/src/llm.rs b/src-tauri/src/llm.rs index 27907a3..ffef99b 100644 --- a/src-tauri/src/llm.rs +++ b/src-tauri/src/llm.rs @@ -86,6 +86,161 @@ Avoid rambling setup, context-dependent references, and pure filler. Return up t parse_candidate_json(&text, min_duration) } +#[derive(Debug, Deserialize)] +struct OpenAiMessage { + content: String, +} + +#[derive(Debug, Deserialize)] +struct OpenAiChoice { + message: OpenAiMessage, +} + +#[derive(Debug, Deserialize)] +struct OpenAiResponse { + choices: Vec, +} + +pub async fn detect_candidates_with_openai( + transcript: &NormalizedTranscript, + api_key: &str, + model: &str, +) -> Result> { + let segments = compact_segments(&transcript.segments); + let prompt = format!( + "You are identifying the most viral moments and strongest short-form clip candidates from a long-form transcript. \ +For each candidate, the clip must be self-contained, starting with an extremely engaging hook within the first 3 seconds (to capture immediate attention on social feeds), \ +30-90 seconds long, and cut at clean sentence/thought boundaries. Favor highly shareable content: concrete stories, \ +strong opinions, emotional turns, surprising or counter-intuitive claims, clear payoffs, and high-energy/dramatic peaks. \ +Avoid rambling setup, context-dependent references, and pure filler. Return up to 10 candidates as JSON matching this schema: \ +{{\"candidates\":[{{\"start\":0.0,\"end\":0.0,\"score\":0.0,\"hook\":\"...\",\"rationale\":\"...\"}}]}}\n\nTranscript:\n{segments}" + ); + + let response = reqwest::Client::new() + .post("https://api.openai.com/v1/chat/completions") + .header("Authorization", format!("Bearer {api_key}")) + .json(&json!({ + "model": model, + "messages": [ + { + "role": "user", + "content": prompt, + } + ], + "temperature": 0.2, + "response_format": { + "type": "json_object" + } + })) + .send() + .await + .context("calling OpenAI")?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow!("OpenAI request failed ({status}): {body}")); + } + + let res_body: OpenAiResponse = response.json().await.context("parsing OpenAI response")?; + let text = res_body + .choices + .first() + .map(|c| c.message.content.clone()) + .ok_or_else(|| anyhow!("OpenAI response did not include choices content"))?; + + let min_duration = if transcript.duration < 60.0 { + (transcript.duration * 0.5).max(5.0) + } else { + 30.0 + }; + parse_candidate_json(&text, min_duration) +} + +#[derive(Debug, Deserialize)] +struct GeminiPart { + text: Option, +} + +#[derive(Debug, Deserialize)] +struct GeminiContent { + parts: Vec, +} + +#[derive(Debug, Deserialize)] +struct GeminiCandidate { + content: GeminiContent, +} + +#[derive(Debug, Deserialize)] +struct GeminiResponse { + candidates: Vec, +} + +pub async fn detect_candidates_with_gemini( + transcript: &NormalizedTranscript, + api_key: &str, + model: &str, +) -> Result> { + let segments = compact_segments(&transcript.segments); + let prompt = format!( + "You are identifying the most viral moments and strongest short-form clip candidates from a long-form transcript. \ +For each candidate, the clip must be self-contained, starting with an extremely engaging hook within the first 3 seconds (to capture immediate attention on social feeds), \ +30-90 seconds long, and cut at clean sentence/thought boundaries. Favor highly shareable content: concrete stories, \ +strong opinions, emotional turns, surprising or counter-intuitive claims, clear payoffs, and high-energy/dramatic peaks. \ +Avoid rambling setup, context-dependent references, and pure filler. Return up to 10 candidates as JSON matching this schema: \ +{{\"candidates\":[{{\"start\":0.0,\"end\":0.0,\"score\":0.0,\"hook\":\"...\",\"rationale\":\"...\"}}]}}\n\nTranscript:\n{segments}" + ); + + let url = format!("https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent"); + + let response = reqwest::Client::new() + .post(&url) + .header("x-goog-api-key", api_key) + .json(&json!({ + "contents": [ + { + "parts": [ + { "text": prompt } + ] + } + ], + "generationConfig": { + "temperature": 0.2, + "responseMimeType": "application/json" + } + })) + .send() + .await + .context("calling Gemini")?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow!("Gemini request failed ({status}): {body}")); + } + + let res_body: GeminiResponse = response.json().await.context("parsing Gemini response")?; + let text = res_body + .candidates + .into_iter() + .find_map(|candidate| { + candidate + .content + .parts + .into_iter() + .find_map(|part| part.text) + }) + .ok_or_else(|| anyhow!("Gemini response did not include text content"))?; + + let min_duration = if transcript.duration < 60.0 { + (transcript.duration * 0.5).max(5.0) + } else { + 30.0 + }; + parse_candidate_json(&text, min_duration) +} + #[derive(Debug, Serialize)] struct ClaudeMessage<'a> { role: &'a str, diff --git a/src-tauri/src/models.rs b/src-tauri/src/models.rs index c6c1e46..5353a7a 100644 --- a/src-tauri/src/models.rs +++ b/src-tauri/src/models.rs @@ -9,6 +9,8 @@ pub struct EnvironmentStatus { pub has_deepgram_key: bool, pub has_anthropic_key: bool, pub has_deepseek_key: bool, + pub has_openai_key: bool, + pub has_gemini_key: bool, pub llm_provider: String, pub has_local_whisper_model: bool, pub has_ollama: bool, diff --git a/src/main.tsx b/src/main.tsx index 4e25abe..561ec36 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -32,6 +32,8 @@ type EnvironmentStatus = { hasDeepgramKey: boolean; hasAnthropicKey: boolean; hasDeepseekKey: boolean; + hasOpenaiKey: boolean; + hasGeminiKey: boolean; llmProvider: string; hasLocalWhisperModel: boolean; hasOllama: boolean; @@ -108,6 +110,14 @@ type BusyState = | "clipCount" | "cut"; +function cloudLlmLabel(engine: string): string { + if (engine === "claude") return "Claude"; + if (engine === "deepseek") return "DeepSeek"; + if (engine === "openai") return "OpenAI"; + if (engine === "gemini") return "Gemini"; + return engine; +} + function App() { const [environment, setEnvironment] = useState(null); const [projects, setProjects] = useState([]); @@ -125,8 +135,8 @@ function App() { const [transcriptionEngine, setTranscriptionEngine] = useState<"deepgram" | "local">(() => { return (localStorage.getItem("autoshorts_transcription_engine") as "deepgram" | "local") || "local"; }); - const [llmEngine, setLlmEngine] = useState<"claude" | "deepseek" | "local">(() => { - return (localStorage.getItem("autoshorts_llm_engine") as "claude" | "deepseek" | "local") || "local"; + const [llmEngine, setLlmEngine] = useState<"claude" | "deepseek" | "local" | "openai" | "gemini">(() => { + return (localStorage.getItem("autoshorts_llm_engine") as "claude" | "deepseek" | "local" | "openai" | "gemini") || "local"; }); const [localLlmModel, setLocalLlmModel] = useState(() => { return localStorage.getItem("autoshorts_local_llm_model") || "llama3.2"; @@ -140,6 +150,18 @@ function App() { const [deepseekKey, setDeepseekKey] = useState(() => { return localStorage.getItem("autoshorts_deepseek_key") || ""; }); + const [openaiKey, setOpenaiKey] = useState(() => { + return localStorage.getItem("autoshorts_openai_key") || ""; + }); + const [geminiKey, setGeminiKey] = useState(() => { + return localStorage.getItem("autoshorts_gemini_key") || ""; + }); + const [openaiModel, setOpenaiModel] = useState(() => { + return localStorage.getItem("autoshorts_openai_model") || "gpt-4o-mini"; + }); + const [geminiModel, setGeminiModel] = useState(() => { + return localStorage.getItem("autoshorts_gemini_model") || "gemini-1.5-flash"; + }); const [downloadingModelName, setDownloadingModelName] = useState(null); const [modelDownloadStatus, setModelDownloadStatus] = useState(""); @@ -170,6 +192,8 @@ function App() { const canUseCloudKey = environment?.hasDeepgramKey || deepgramKey.trim().length > 0; const canUseClaude = environment?.hasAnthropicKey || anthropicKey.trim().length > 0; const canUseDeepseek = environment?.hasDeepseekKey || deepseekKey.trim().length > 0; + const canUseOpenai = environment?.hasOpenaiKey || openaiKey.trim().length > 0; + const canUseGemini = environment?.hasGeminiKey || geminiKey.trim().length > 0; const canTranscribe = transcriptionEngine === "local" ? Boolean(environment?.hasLocalWhisperModel) @@ -177,7 +201,10 @@ function App() { const canUseActiveLlm = llmEngine === "local" ? Boolean(environment?.hasOllama) - : (llmEngine === "claude" ? canUseClaude : canUseDeepseek); + : llmEngine === "claude" ? canUseClaude + : llmEngine === "deepseek" ? canUseDeepseek + : llmEngine === "openai" ? canUseOpenai + : canUseGemini; useEffect(() => { void refresh(); @@ -213,6 +240,22 @@ function App() { localStorage.setItem("autoshorts_deepseek_key", deepseekKey); }, [deepseekKey]); + useEffect(() => { + localStorage.setItem("autoshorts_openai_key", openaiKey); + }, [openaiKey]); + + useEffect(() => { + localStorage.setItem("autoshorts_gemini_key", geminiKey); + }, [geminiKey]); + + useEffect(() => { + localStorage.setItem("autoshorts_openai_model", openaiModel); + }, [openaiModel]); + + useEffect(() => { + localStorage.setItem("autoshorts_gemini_model", geminiModel); + }, [geminiModel]); + const pullModelDirectly = async (modelName: string) => { setDownloadingModelName(modelName); setModelDownloadProgress(0); @@ -331,12 +374,13 @@ function App() { return; } } else { - const activeKey = llmEngine === "claude" ? anthropicKey : deepseekKey; - const hasActiveKey = llmEngine === "claude" - ? (env.hasAnthropicKey || activeKey.trim().length > 0) - : (env.hasDeepseekKey || activeKey.trim().length > 0); + const hasActiveKey = + llmEngine === "claude" ? (env.hasAnthropicKey || anthropicKey.trim().length > 0) + : llmEngine === "deepseek" ? (env.hasDeepseekKey || deepseekKey.trim().length > 0) + : llmEngine === "openai" ? (env.hasOpenaiKey || openaiKey.trim().length > 0) + : (env.hasGeminiKey || geminiKey.trim().length > 0); if (!hasActiveKey) { - setError(`Transcription complete. ${llmEngine === "claude" ? "Claude" : "DeepSeek"} API Key is missing. Please add it in settings to analyze viral moments.`); + setError(`Transcription complete. ${cloudLlmLabel(llmEngine)} API Key is missing. Please add it in settings to analyze viral moments.`); return; } } @@ -359,12 +403,12 @@ function App() { // 2. LLM Moments try { setBusy("moments"); - const activeKey = llmEngine === "claude" ? anthropicKey.trim() : (llmEngine === "deepseek" ? deepseekKey.trim() : ""); + const activeKey = llmEngine === "claude" ? anthropicKey.trim() : llmEngine === "deepseek" ? deepseekKey.trim() : llmEngine === "openai" ? openaiKey.trim() : llmEngine === "gemini" ? geminiKey.trim() : ""; await invoke("generate_candidates", { projectId, apiKey: activeKey || null, provider: llmEngine, - modelName: llmEngine === "local" ? localLlmModel.trim() : null, + modelName: llmEngine === "local" ? localLlmModel.trim() : llmEngine === "openai" ? (openaiModel.trim() || null) : llmEngine === "gemini" ? (geminiModel.trim() || null) : null, allowDemo: false, }); await refresh(projectId); @@ -440,13 +484,13 @@ function App() { async function moments(allowDemo: boolean) { if (!detail) return; await run("moments", async () => { - const activeKey = llmEngine === "claude" ? anthropicKey.trim() : (llmEngine === "deepseek" ? deepseekKey.trim() : ""); + const activeKey = llmEngine === "claude" ? anthropicKey.trim() : llmEngine === "deepseek" ? deepseekKey.trim() : llmEngine === "openai" ? openaiKey.trim() : llmEngine === "gemini" ? geminiKey.trim() : ""; try { await invoke("generate_candidates", { projectId: detail.project.id, apiKey: activeKey || null, provider: llmEngine, - modelName: llmEngine === "local" ? localLlmModel.trim() : null, + modelName: llmEngine === "local" ? localLlmModel.trim() : llmEngine === "openai" ? (openaiModel.trim() || null) : llmEngine === "gemini" ? (geminiModel.trim() || null) : null, allowDemo, }); await refresh(detail.project.id); @@ -531,9 +575,13 @@ function App() { setDeepgramKey={setDeepgramKey} setAnthropicKey={setAnthropicKey} setDeepseekKey={setDeepseekKey} + setOpenaiKey={setOpenaiKey} + setGeminiKey={setGeminiKey} deepgramKey={deepgramKey} anthropicKey={anthropicKey} deepseekKey={deepseekKey} + openaiKey={openaiKey} + geminiKey={geminiKey} refreshEnv={() => refresh()} /> ); @@ -627,11 +675,13 @@ function App() { LLM Engine {transcriptionEngine === "deepgram" && ( @@ -667,6 +717,50 @@ function App() { /> )} + {llmEngine === "openai" && ( + <> + + + + )} + {llmEngine === "gemini" && ( + <> + + + + )} {llmEngine === "local" && (