diff --git a/CLAUDE.md b/CLAUDE.md index ca7d0b0..db67a3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,9 @@ vertical video out via headless Chrome + ffmpeg + ElevenLabs). single-shot `--screenshot`, it hangs on macOS). - `crates/videoeditor-media` — all ffmpeg/ffprobe invocations + assembly. - `crates/videoeditor-voice` — ElevenLabs TTS/STT (`ELEVENLABS_API_KEY`). +- `crates/videoeditor-genai` — typed image-generation clients: xAI Grok + Imagine (`XAI_API_KEY`, reference images) + Google Imagen (`AI_STUDIO`); + Veo/Grok video is the planned next tenant. - `examples/hello-bench` — smallest end-to-end episode; keep it rendering. ## Commands diff --git a/Cargo.lock b/Cargo.lock index 471dca2..82626ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -819,6 +819,7 @@ dependencies = [ "serde", "serde_json", "videoeditor-chrome", + "videoeditor-genai", "videoeditor-media", "videoeditor-timeline", "videoeditor-voice", @@ -835,6 +836,17 @@ dependencies = [ "ureq", ] +[[package]] +name = "videoeditor-genai" +version = "0.1.1" +dependencies = [ + "anyhow", + "base64", + "serde", + "serde_json", + "ureq", +] + [[package]] name = "videoeditor-media" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index d34fa40..c199862 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ tungstenite = "0.24" ureq = { version = "2", features = ["json"] } videoeditor-chrome = { version = "0.1.1", path = "crates/videoeditor-chrome" } +videoeditor-genai = { version = "0.1.1", path = "crates/videoeditor-genai" } videoeditor-media = { version = "0.1.1", path = "crates/videoeditor-media" } videoeditor-timeline = { version = "0.1.1", path = "crates/videoeditor-timeline" } videoeditor-voice = { version = "0.1.1", path = "crates/videoeditor-voice" } diff --git a/crates/videoeditor-genai/Cargo.toml b/crates/videoeditor-genai/Cargo.toml new file mode 100644 index 0000000..72f44d3 --- /dev/null +++ b/crates/videoeditor-genai/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "videoeditor-genai" +description = "Typed generative-asset clients for videoeditor: xAI Grok Imagine and Google Imagen stills (reference-image conditioning included)" +keywords = ["image-generation", "grok", "imagen", "genai", "video"] +categories = ["multimedia::images", "api-bindings"] +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +anyhow.workspace = true +base64.workspace = true +serde.workspace = true +serde_json.workspace = true +ureq.workspace = true diff --git a/crates/videoeditor-genai/src/google.rs b/crates/videoeditor-genai/src/google.rs new file mode 100644 index 0000000..6ab7ab3 --- /dev/null +++ b/crates/videoeditor-genai/src/google.rs @@ -0,0 +1,167 @@ +//! Google Imagen stills via the Gemini API `:predict` endpoint. +//! +//! Safe-stills provider: honors a real aspect/size param, but takes NO +//! reference images, and its safety filter rejects named real people and +//! branded characters — route those requests to [`crate::xai`]. An empty +//! `predictions` array almost always means the filter fired, so the error +//! says so instead of "no images". +//! +//! Veo (video) rides the same API family and is this module's planned +//! second resident. + +use crate::{AspectRatio, GeneratedImage, ImageRequest}; +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; + +pub const DEFAULT_MODEL: &str = "imagen-4.0-generate-001"; +pub const MAX_IMAGES_PER_REQUEST: u8 = 4; + +/// The subset of [`AspectRatio`] Imagen accepts. +pub const SUPPORTED_ASPECTS: [AspectRatio; 5] = [ + AspectRatio::R1x1, + AspectRatio::R3x4, + AspectRatio::R4x3, + AspectRatio::R9x16, + AspectRatio::R16x9, +]; + +pub fn api_key() -> Result { + std::env::var("AI_STUDIO") + .or_else(|_| std::env::var("GEMINI_API_KEY")) + .context("set AI_STUDIO or GEMINI_API_KEY (a Google AI Studio key)") +} + +#[derive(Serialize)] +struct PredictPayload<'a> { + instances: [Instance<'a>; 1], + parameters: Parameters<'a>, +} + +#[derive(Serialize)] +struct Instance<'a> { + prompt: &'a str, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Parameters<'a> { + sample_count: u8, + #[serde(skip_serializing_if = "Option::is_none")] + aspect_ratio: Option<&'a str>, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct PredictResponse { + #[serde(default)] + predictions: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Prediction { + bytes_base64_encoded: Option, +} + +/// Generate `req.n` stills. Rejects reference images (Imagen has no such +/// input — use grok) and aspect ratios outside [`SUPPORTED_ASPECTS`]. +pub fn generate(req: &ImageRequest) -> Result> { + if req.n == 0 || req.n > MAX_IMAGES_PER_REQUEST { + bail!( + "imagen generates 1..={MAX_IMAGES_PER_REQUEST} images per request, got {}", + req.n + ); + } + if !req.reference_images.is_empty() { + bail!( + "imagen takes no reference images — use the grok provider for reference-conditioned stills" + ); + } + if req.aspect != AspectRatio::Auto && !SUPPORTED_ASPECTS.contains(&req.aspect) { + bail!( + "imagen supports aspect ratios 1:1, 3:4, 4:3, 9:16, 16:9 (got {})", + req.aspect + ); + } + + let model = req.model.as_deref().unwrap_or(DEFAULT_MODEL); + let payload = PredictPayload { + instances: [Instance { + prompt: &req.prompt, + }], + parameters: Parameters { + sample_count: req.n, + aspect_ratio: match req.aspect { + AspectRatio::Auto => None, + a => Some(a.as_str()), + }, + }, + }; + + let url = format!("https://generativelanguage.googleapis.com/v1beta/models/{model}:predict"); + let resp = ureq::post(&url) + .set("x-goog-api-key", &api_key()?) + .timeout(std::time::Duration::from_secs(300)) + .send_json(serde_json::to_value(&payload)?); + let resp = match resp { + Ok(r) => r, + Err(ureq::Error::Status(code, r)) => { + bail!("imagen {code}: {}", r.into_string().unwrap_or_default()) + } + Err(e) => return Err(e.into()), + }; + let parsed: PredictResponse = resp.into_json().context("parsing imagen response")?; + if parsed.predictions.is_empty() { + bail!( + "imagen returned no images — likely a safety-filter rejection \ + (named real people, branded characters, explicit content). \ + Rewrite the prompt or switch to the grok provider." + ); + } + + parsed + .predictions + .into_iter() + .map(|p| { + let b64 = p + .bytes_base64_encoded + .context("imagen prediction had no image bytes")?; + Ok(GeneratedImage { + bytes: crate::decode_b64(&b64)?, + revised_prompt: None, + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn payload_uses_camel_case_fields() { + let payload = PredictPayload { + instances: [Instance { prompt: "a bun" }], + parameters: Parameters { + sample_count: 2, + aspect_ratio: Some(AspectRatio::R9x16.as_str()), + }, + }; + let v = serde_json::to_value(&payload).unwrap(); + assert_eq!(v["parameters"]["sampleCount"], 2); + assert_eq!(v["parameters"]["aspectRatio"], "9:16"); + assert_eq!(v["instances"][0]["prompt"], "a bun"); + } + + #[test] + fn refs_are_rejected() { + let req = ImageRequest { + prompt: "x".into(), + model: None, + n: 1, + aspect: AspectRatio::Auto, + reference_images: vec!["a.png".into()], + }; + assert!(generate(&req).unwrap_err().to_string().contains("grok")); + } +} diff --git a/crates/videoeditor-genai/src/lib.rs b/crates/videoeditor-genai/src/lib.rs new file mode 100644 index 0000000..522b656 --- /dev/null +++ b/crates/videoeditor-genai/src/lib.rs @@ -0,0 +1,171 @@ +//! Generative-asset clients: typed bindings for the image APIs of xAI +//! (Grok Imagine) and Google (Imagen), so episodes can optionally generate +//! stills — memes, logos-in-costume, backdrops — straight from the CLI. +//! Video generation (Grok Imagine video, Google Veo) is the planned next +//! tenant of this crate; the module split anticipates it. +//! +//! Provider fit (learned in production, keep in mind when routing): +//! - **Grok** accepts up to [`xai::MAX_REFERENCE_IMAGES`] reference images +//! (data-URL encoded) and is lenient about brand mascots and named people. +//! - **Imagen** honors a real aspect/size API param but takes NO reference +//! images, and its safety filter rejects named real people — requests +//! that need either belong to Grok. + +pub mod google; +pub mod xai; + +use anyhow::{Context, Result, bail}; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +/// One still-image request, provider-agnostic. Each backend consumes the +/// fields it supports and rejects what it can't honor (no silent drops). +#[derive(Clone, Debug)] +pub struct ImageRequest { + pub prompt: String, + /// Explicit model id; `None` = the provider's default. + pub model: Option, + /// Variants to generate (grok ≤ 10, imagen ≤ 4). + pub n: u8, + pub aspect: AspectRatio, + /// Local images the model should condition on (grok only). + pub reference_images: Vec, +} + +/// A generated still plus whatever the provider said about it. +#[derive(Debug)] +pub struct GeneratedImage { + pub bytes: Vec, + /// Grok rewrites prompts before rendering; surfaced for prompt QC. + pub revised_prompt: Option, +} + +/// Aspect ratios the xAI image endpoint enumerates (probed 2026-07-07 via +/// its 400 on an unknown variant). Imagen accepts the subset noted on +/// [`google::SUPPORTED_ASPECTS`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[allow(non_camel_case_types)] +pub enum AspectRatio { + R1x1, + R3x4, + R4x3, + R9x16, + R16x9, + R2x3, + R3x2, + R9x19_5, + R19_5x9, + R9x20, + R20x9, + R1x2, + R2x1, + /// Let the model pick framing from the prompt. + Auto, +} + +impl AspectRatio { + pub fn as_str(&self) -> &'static str { + match self { + Self::R1x1 => "1:1", + Self::R3x4 => "3:4", + Self::R4x3 => "4:3", + Self::R9x16 => "9:16", + Self::R16x9 => "16:9", + Self::R2x3 => "2:3", + Self::R3x2 => "3:2", + Self::R9x19_5 => "9:19.5", + Self::R19_5x9 => "19.5:9", + Self::R9x20 => "9:20", + Self::R20x9 => "20:9", + Self::R1x2 => "1:2", + Self::R2x1 => "2:1", + Self::Auto => "auto", + } + } +} + +impl fmt::Display for AspectRatio { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for AspectRatio { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(match s { + "1:1" => Self::R1x1, + "3:4" => Self::R3x4, + "4:3" => Self::R4x3, + "9:16" => Self::R9x16, + "16:9" => Self::R16x9, + "2:3" => Self::R2x3, + "3:2" => Self::R3x2, + "9:19.5" => Self::R9x19_5, + "19.5:9" => Self::R19_5x9, + "9:20" => Self::R9x20, + "20:9" => Self::R20x9, + "1:2" => Self::R1x2, + "2:1" => Self::R2x1, + "auto" => Self::Auto, + other => bail!( + "unknown aspect ratio {other:?} (expected one of 1:1, 3:4, 4:3, \ + 9:16, 16:9, 2:3, 3:2, 9:19.5, 19.5:9, 9:20, 20:9, 1:2, 2:1, auto)" + ), + }) + } +} + +/// Encode a local image as a `data:` URL for a JSON body. +pub(crate) fn data_url(image: &Path) -> Result { + use base64::Engine as _; + let bytes = std::fs::read(image).with_context(|| format!("reading {}", image.display()))?; + let mime = match image + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_ascii_lowercase()) + .as_deref() + { + Some("jpg") | Some("jpeg") => "jpeg", + Some("webp") => "webp", + _ => "png", + }; + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + Ok(format!("data:image/{mime};base64,{b64}")) +} + +/// Download a provider-hosted asset. xAI's CDN 403s UA-less requests +/// (observed June 2026), so send a browser-ish User-Agent. +pub(crate) fn download(url: &str) -> Result> { + let resp = ureq::get(url) + .set("user-agent", "Mozilla/5.0 (videoeditor)") + .timeout(std::time::Duration::from_secs(300)) + .call() + .with_context(|| format!("downloading {url}"))?; + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut resp.into_reader(), &mut bytes)?; + Ok(bytes) +} + +pub(crate) fn decode_b64(b64: &str) -> Result> { + use base64::Engine as _; + Ok(base64::engine::general_purpose::STANDARD.decode(b64)?) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aspect_ratio_roundtrips() { + for s in [ + "1:1", "3:4", "4:3", "9:16", "16:9", "2:3", "3:2", "9:19.5", "19.5:9", "9:20", "20:9", + "1:2", "2:1", "auto", + ] { + assert_eq!(s.parse::().unwrap().as_str(), s); + } + assert!("21:9".parse::().is_err()); + } +} diff --git a/crates/videoeditor-genai/src/xai.rs b/crates/videoeditor-genai/src/xai.rs new file mode 100644 index 0000000..d3bae9d --- /dev/null +++ b/crates/videoeditor-genai/src/xai.rs @@ -0,0 +1,150 @@ +//! xAI Grok Imagine images — the OpenAI-compatible +//! `POST /v1/images/generations` endpoint, with multi-reference editing. +//! +//! Gotchas baked in from production use: +//! - Request `b64_json`: xAI's asset CDN has started 403-ing plain URL +//! downloads; the `url` branch survives only as a fallback (with a +//! browser-ish User-Agent). +//! - `reference_images` is an array of data-URL *strings* (unlike the +//! video endpoint, which wraps each in `{"url": ...}`); at most +//! [`MAX_REFERENCE_IMAGES`] are honored. + +use crate::{GeneratedImage, ImageRequest}; +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; + +pub const DEFAULT_MODEL: &str = "grok-imagine-image-quality"; +pub const MAX_REFERENCE_IMAGES: usize = 7; +pub const MAX_IMAGES_PER_REQUEST: u8 = 10; + +const IMAGES_URL: &str = "https://api.x.ai/v1/images/generations"; + +pub fn api_key() -> Result { + std::env::var("XAI_API_KEY").context("set XAI_API_KEY (your xAI API key)") +} + +#[derive(Serialize)] +struct ImagesPayload<'a> { + model: &'a str, + prompt: &'a str, + n: u8, + response_format: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + aspect_ratio: Option<&'a str>, + #[serde(skip_serializing_if = "Vec::is_empty")] + reference_images: Vec, +} + +#[derive(Deserialize)] +struct ImagesResponse { + #[serde(default)] + data: Vec, +} + +#[derive(Deserialize)] +struct ImageDatum { + url: Option, + b64_json: Option, + revised_prompt: Option, +} + +/// Generate `req.n` stills, optionally conditioned on reference images. +pub fn generate(req: &ImageRequest) -> Result> { + if req.n == 0 || req.n > MAX_IMAGES_PER_REQUEST { + bail!( + "grok generates 1..={MAX_IMAGES_PER_REQUEST} images per request, got {}", + req.n + ); + } + if req.reference_images.len() > MAX_REFERENCE_IMAGES { + bail!( + "grok honors at most {MAX_REFERENCE_IMAGES} reference images, got {}", + req.reference_images.len() + ); + } + let refs = req + .reference_images + .iter() + .map(|p| crate::data_url(p)) + .collect::>>()?; + + let payload = ImagesPayload { + model: req.model.as_deref().unwrap_or(DEFAULT_MODEL), + prompt: &req.prompt, + n: req.n, + response_format: "b64_json", + aspect_ratio: match req.aspect { + crate::AspectRatio::Auto => None, + a => Some(a.as_str()), + }, + reference_images: refs, + }; + + let resp = ureq::post(IMAGES_URL) + .set("authorization", &format!("Bearer {}", api_key()?)) + .timeout(std::time::Duration::from_secs(300)) + .send_json(serde_json::to_value(&payload)?); + let resp = match resp { + Ok(r) => r, + Err(ureq::Error::Status(code, r)) => { + bail!("grok image {code}: {}", r.into_string().unwrap_or_default()) + } + Err(e) => return Err(e.into()), + }; + let parsed: ImagesResponse = resp.into_json().context("parsing grok image response")?; + if parsed.data.is_empty() { + bail!("grok returned no images"); + } + + parsed + .data + .into_iter() + .map(|d| { + let bytes = match (&d.b64_json, &d.url) { + (Some(b64), _) => crate::decode_b64(b64)?, + (None, Some(url)) => crate::download(url)?, + (None, None) => bail!("grok image had neither b64_json nor url"), + }; + Ok(GeneratedImage { + bytes, + revised_prompt: d.revised_prompt, + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::AspectRatio; + + #[test] + fn payload_omits_empty_optionals() { + let payload = ImagesPayload { + model: DEFAULT_MODEL, + prompt: "a bun", + n: 1, + response_format: "b64_json", + aspect_ratio: None, + reference_images: vec![], + }; + let v = serde_json::to_value(&payload).unwrap(); + assert!(v.get("aspect_ratio").is_none()); + assert!(v.get("reference_images").is_none()); + } + + #[test] + fn payload_carries_refs_as_plain_strings() { + let payload = ImagesPayload { + model: DEFAULT_MODEL, + prompt: "a bun", + n: 2, + response_format: "b64_json", + aspect_ratio: Some(AspectRatio::R1x1.as_str()), + reference_images: vec!["data:image/png;base64,AAAA".into()], + }; + let v = serde_json::to_value(&payload).unwrap(); + assert_eq!(v["aspect_ratio"], "1:1"); + assert_eq!(v["reference_images"][0], "data:image/png;base64,AAAA"); + } +} diff --git a/crates/videoeditor/Cargo.toml b/crates/videoeditor/Cargo.toml index 6c80f46..0a986a9 100644 --- a/crates/videoeditor/Cargo.toml +++ b/crates/videoeditor/Cargo.toml @@ -19,6 +19,7 @@ include_dir.workspace = true serde.workspace = true serde_json.workspace = true videoeditor-chrome.workspace = true +videoeditor-genai.workspace = true videoeditor-media.workspace = true videoeditor-timeline.workspace = true videoeditor-voice.workspace = true diff --git a/crates/videoeditor/guide.md b/crates/videoeditor/guide.md index a01ecc6..3d48051 100644 --- a/crates/videoeditor/guide.md +++ b/crates/videoeditor/guide.md @@ -15,6 +15,7 @@ videoeditor render [--scene name] headless-Chrome frames → scene mp videoeditor assemble concat + narration@offsets + music → build/final.mp4 videoeditor build tts + render + assemble videoeditor analyze transcript + scene-cut timing table of a reference +videoeditor image "prompt" -o AI still for assets/ (grok|imagen; --ref conditions on an image) ``` Discovery: `videoeditor templates [dir]` lists every scene template visible @@ -133,6 +134,9 @@ compose the blocks. ## Env - `ELEVENLABS_API_KEY` — required for `tts`/`analyze` only. +- `XAI_API_KEY` — `image --provider grok` (the default; takes `--ref` images). +- `AI_STUDIO` (or `GEMINI_API_KEY`) — `image --provider imagen` (safety-filtered, + no references; rejections say so — reroute those prompts to grok). - `CHROME_BIN` — override the render browser. - `VIDEOEDITOR_ROOT` — use a checkout's templates instead of the embedded ones. - `VIDEOEDITOR_PACK_PATH` — colon-separated machine-wide template packs. diff --git a/crates/videoeditor/src/main.rs b/crates/videoeditor/src/main.rs index 40f3011..5c0871f 100644 --- a/crates/videoeditor/src/main.rs +++ b/crates/videoeditor/src/main.rs @@ -90,6 +90,34 @@ enum Cmd { /// Print the embedded director's guide: production workflow, script.md /// grammar, template authoring — written for AI agents and humans alike Guide, + /// Generate a still image with a generative model — xAI Grok Imagine + /// (XAI_API_KEY; accepts reference images) or Google Imagen + /// (AI_STUDIO/GEMINI_API_KEY; safety-filtered, no references) + Image { + /// What to render. With --ref, describe how the referenced + /// subject should appear + prompt: String, + /// Output PNG path (with -n > 1: name_v1.png ... name_vN.png) + #[arg(short, long)] + out: PathBuf, + /// Provider to use + #[arg(long, value_enum, default_value_t = ImageProvider::Grok)] + provider: ImageProvider, + /// Reference image to condition on (repeatable, grok only, ≤ 7) + #[arg(long = "ref")] + refs: Vec, + /// Number of variants (grok ≤ 10, imagen ≤ 4) + #[arg(short = 'n', long, default_value_t = 1)] + count: u8, + /// Aspect ratio: 1:1, 3:4, 4:3, 9:16, 16:9, 2:3, 3:2, 1:2, 2:1, + /// auto, ... (imagen: first five only) + #[arg(long, default_value = "auto")] + aspect: String, + /// Model id override (defaults: grok-imagine-image-quality / + /// imagen-4.0-generate-001) + #[arg(long)] + model: Option, + }, /// Research helper: fetch a URL through YOUR running Chrome (logged-in /// sessions bypass bot walls) and print the page text Grab { @@ -106,6 +134,14 @@ enum Cmd { }, } +#[derive(Clone, Copy, clap::ValueEnum)] +enum ImageProvider { + /// xAI Grok Imagine — reference images, lenient with mascots/people + Grok, + /// Google Imagen — aspect-true, safety-filtered, no reference images + Imagen, +} + #[derive(Subcommand)] enum PackCmd { /// Scaffold a self-contained template pack (vendors the scene runtime) @@ -196,6 +232,44 @@ fn main() -> Result<()> { &out, )?; } + Cmd::Image { + prompt, + out, + provider, + refs, + count, + aspect, + model, + } => { + let req = videoeditor_genai::ImageRequest { + prompt, + model, + n: count, + aspect: aspect.parse()?, + reference_images: refs, + }; + let images = match provider { + ImageProvider::Grok => videoeditor_genai::xai::generate(&req)?, + ImageProvider::Imagen => videoeditor_genai::google::generate(&req)?, + }; + for (i, img) in images.iter().enumerate() { + let path = if images.len() == 1 { + out.clone() + } else { + let stem = out.file_stem().unwrap_or_default().to_string_lossy(); + let ext = out.extension().unwrap_or_default().to_string_lossy(); + out.with_file_name(format!("{stem}_v{}.{ext}", i + 1)) + }; + if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) { + std::fs::create_dir_all(dir)?; + } + std::fs::write(&path, &img.bytes)?; + println!("image: wrote {}", path.display()); + if let Some(rp) = &img.revised_prompt { + println!("image: revised prompt: {rp}"); + } + } + } Cmd::Grab { url, port,