feat(media): video_understand — watch a video (video→text) (v0.140.0)

CHANGELOG.md

@@ -8,8 +8,19 @@ new version heading in the same commit.
 
 ## [Unreleased]
 
-## [0.139.0] — 2026-07-13
-### Added
+## [0.140.0] — 2026-07-13
+### Added
+- **`video_understand`: agents can now "watch" a video (video → text).** Claude can't see video natively;
+  this new governed MCP tool delegates to an Atlas **multimodal LLM** (the ~10 catalog models with video
+  input — qwen3.5, glm-5v, kimi-k2…) via the OpenAI-compatible chat endpoint and returns a **text**
+  answer directly (no artifact). Pass `video` (a Library artifact id — e.g. one `video_generate` just made
+  — a working-folder file written *or* terminal-uploaded, or an http(s) URL) and an optional `prompt`
+  ("summarise", "transcribe on-screen text", "what happens at the end?"); omit it for a general
+  description. Local files are inlined as base64 (no hosting needed). Also handles stills with
+  `kind:"image"`. Governed like the other media calls: classified `video.understand` with a cost estimate
+  (money-cap applies), audited `video.understood`. New backend `src/edge/media-understand.ts`, route
+  `POST /api/agent/video/understand`, default model `qwen/qwen3.5-27b`; exposed whenever Atlas is
+  configured (`VIDEO_UNDERSTAND`). Verified live (correctly described a test clip).
 - **`image_edit` gains a `remove-background` preset.** Alongside prompt-guided edit and upscale, agents can
   now cut out an image's subject with `image_edit({ image, operation: "remove-background" })` — no prompt
   needed. It returns a **transparent PNG** saved as a new Library image (source untouched), via Atlas's

package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-os",
-  "version": "0.139.0",
+  "version": "0.140.0",
   "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.",
   "license": "MIT",
   "type": "commonjs",

src/edge/media-understand.ts

@@ -0,0 +1,59 @@
+/**
+ * Media UNDERSTANDING (video/image → text) — the inverse of the generation tools. Claude can't natively
+ * watch a video, so this delegates to one of Atlas's video-capable multimodal LLMs (input modalities
+ * text+image+video) via the OpenAI-compatible chat endpoint: a message carrying a `video_url` content
+ * part (a URL or a base64 `data:` URL) + a text question returns a text answer the agent reads directly.
+ *
+ * Verified live: `{type:'video_url', video_url:{url}}` is the content shape Atlas's video LLMs accept
+ * (qwen3.5, glm-5v, kimi-k2…); a `{type:'video'}` shape is silently ignored by the model.
+ */
+
+/** A solid default: mid-size, reliably describes motion/content. Overridable per call. */
+export const DEFAULT_VIDEO_UNDERSTAND_MODEL = 'qwen/qwen3.5-27b';
+
+export interface UnderstandRequest {
+  atlasKey: string;
+  baseUrl?: string;
+  model?: string;
+  /** A URL or base64 data: URL for the media. */
+  mediaUrl: string;
+  /** 'video' or 'image' — selects the content-part type. */
+  kind: 'video' | 'image';
+  prompt: string;
+}
+
+export interface UnderstandResult {
+  text: string;
+  model: string;
+  costUsd?: number; // when the vendor reports usage cost (rare); else undefined → caller estimates
+}
+
+function errText(body: unknown, status: number): string {
+  const b = body as { error?: { message?: string } | string; message?: string; msg?: string } | undefined;
+  const e = b?.error;
+  const msg = (typeof e === 'string' ? e : e?.message) || b?.message || b?.msg;
+  return msg ? `Atlas understand error (${status}): ${msg}` : `Atlas understand error (${status})`;
+}
+
+export async function understandMedia(req: UnderstandRequest): Promise<UnderstandResult> {
+  const base = (req.baseUrl?.trim() || 'https://api.atlascloud.ai').replace(/\/+$/, '');
+  const model = req.model?.trim() || DEFAULT_VIDEO_UNDERSTAND_MODEL;
+  const partType = req.kind === 'video' ? 'video_url' : 'image_url';
+  const content = [
+    { type: 'text', text: req.prompt },
+    { type: partType, [partType]: { url: req.mediaUrl } },
+  ];
+  const res = await fetch(`${base}/v1/chat/completions`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${req.atlasKey}`, 'content-type': 'application/json' },
+    body: JSON.stringify({ model, messages: [{ role: 'user', content }] }),
+  });
+  const d = (await res.json().catch(() => ({}))) as {
+    choices?: { message?: { content?: string } }[];
+    usage?: { cost?: number };
+  } & Record<string, unknown>;
+  if (!res.ok) throw new Error(errText(d, res.status));
+  const text = d.choices?.[0]?.message?.content?.trim();
+  if (!text) throw new Error('the model returned no text');
+  return { text, model, costUsd: typeof d.usage?.cost === 'number' ? d.usage.cost : undefined };
+}

src/memory/memory-mcp.ts

@@ -87,6 +87,8 @@ const DISCORD_EGRESS = process.env.DISCORD_EGRESS === '1';
 const IMAGE_GEN = process.env.IMAGE_GEN === '1';
 // VIDEO_GEN: '1' when a video backend (fal/Atlas) is configured — exposes video_generate.
 const VIDEO_GEN = process.env.VIDEO_GEN === '1';
+// VIDEO_UNDERSTAND: '1' when Atlas is configured (its multimodal LLMs do video→text) — exposes video_understand.
+const VIDEO_UNDERSTAND = process.env.VIDEO_UNDERSTAND === '1';
 
 const SLACK_SEND_TOOL = {
   name: 'slack_send',
@@ -216,6 +218,28 @@ const VIDEO_GENERATE_TOOL = {
   },
 };
 
+const VIDEO_UNDERSTAND_TOOL = {
+  name: 'video_understand',
+  description:
+    'WATCH a video and answer a question about it (video → text). Use this whenever you need to know what is ' +
+    "IN a video — you cannot see video natively. It delegates to a video-capable model and returns a TEXT " +
+    'description/answer directly (no artifact). Pass `video` (a Library artifact id, e.g. one video_generate ' +
+    'just made; a file path in your working folder, written OR uploaded into the session; or an http(s) video ' +
+    'URL) and an optional `prompt` for what to find out ("summarise", "is there a person?", "transcribe any ' +
+    'on-screen text", "what happens at the end?"). Omit `prompt` for a general description. Also works on an ' +
+    'image if you set `kind:"image"` (but you can usually read images directly). Governed (cost-metered + audited).',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      video: { type: 'string', description: 'The video to watch: a Library artifact id, a file path in your working folder (written or uploaded), or an http(s) video URL.' },
+      prompt: { type: 'string', description: 'What to find out about the video. Omit for a general detailed description.' },
+      kind: { type: 'string', enum: ['video', 'image'], description: 'Media type (default "video"). Set "image" to analyse a still image instead.' },
+      model: { type: 'string', description: 'Optional Atlas multimodal model id override (must accept video input). Omit for the default.' },
+    },
+    required: ['video'],
+  },
+};
+
 const TOOLS = [
   {
     name: 'recall',
@@ -1163,6 +1187,24 @@ async function videoGenerate(args: Record<string, unknown>): Promise<string> {
   return `Video is rendering with ${d.model ?? 'the default model'}${cost} (job ${d.jobId ?? '?'}). It'll appear in the Library and post an inbox card when done — no need to wait; let the user know it's on the way.`;
 }
 
+async function videoUnderstand(args: Record<string, unknown>): Promise<string> {
+  const video = String(args.video ?? '').trim();
+  if (!video) return 'A `video` is required (a Library artifact id, a working-folder file path, or a video URL).';
+  const body: Record<string, unknown> = { session: SESSION, video };
+  if (typeof args.prompt === 'string' && args.prompt.trim()) body.prompt = args.prompt.trim();
+  if (args.kind === 'image') body.kind = 'image';
+  if (typeof args.model === 'string' && args.model.trim()) body.model = args.model.trim();
+  const res = await fetch(AOS_URL + '/api/agent/video/understand', {
+    method: 'POST',
+    headers: H({ 'content-type': 'application/json' }),
+    body: JSON.stringify(body),
+  });
+  const d = (await res.json()) as { ok?: boolean; error?: string; text?: string; model?: string; costUsd?: number };
+  if (!d.ok) return `Could not analyse the video: ${d.error ?? 'unknown error'}`;
+  const cost = typeof d.costUsd === 'number' ? ` · ~$${d.costUsd.toFixed(3)}` : '';
+  return `${d.text ?? '(no answer)'}\n\n— via ${d.model ?? 'a multimodal model'}${cost}`;
+}
+
 async function slackSend(args: Record<string, unknown>): Promise<string> {
   const channel = String(args.channel ?? '').trim();
   const text = String(args.text ?? '').trim();
@@ -2048,6 +2090,7 @@ async function handle(req: JsonRpc): Promise<void> {
       ...(DISCORD_EGRESS ? [DISCORD_SEND_TOOL, DISCORD_DM_TOOL] : []),
       ...(IMAGE_GEN ? [IMAGE_GENERATE_TOOL, IMAGE_EDIT_TOOL] : []),
       ...(VIDEO_GEN ? [VIDEO_GENERATE_TOOL] : []),
+      ...(VIDEO_UNDERSTAND ? [VIDEO_UNDERSTAND_TOOL] : []),
     ] } });
     return;
   }
@@ -2084,6 +2127,7 @@ async function handle(req: JsonRpc): Promise<void> {
         : name === 'image_generate' ? await imageGenerate(args)
         : name === 'image_edit' ? await imageEdit(args)
         : name === 'video_generate' ? await videoGenerate(args)
+        : name === 'video_understand' ? await videoUnderstand(args)
         : name === 'list_capabilities' ? await listCapabilities()
         : name === 'policy_check' ? await policyCheck(args)
         : name === 'directory_lookup' ? await directoryLookup(args)

src/server.ts

@@ -904,6 +904,21 @@ async function handle(os: AgentOS, tm: TerminalManager, autos: Automations, req:
     });
     return sendJson(res, out.ok ? 200 : 400, out);
   }
+  // OS-owned video/image UNDERSTANDING (`video_understand` MCP tool): delegate to an Atlas multimodal LLM
+  // and return the text answer. Same loopback/gate posture; TerminalManager.understandVideo owns the path.
+  if (method === 'POST' && p === '/api/agent/video/understand') {
+    const b = await readBody(req);
+    const session = String(b.session || '');
+    if (!tm.hasSession(session)) return sendJson(res, 404, { error: 'unknown session' });
+    if (!sessionSecretOk(session)) return sendJson(res, 403, { error: 'bad session secret' });
+    const out = await tm.understandVideo(session, {
+      video: String(b.video || b.image || ''),
+      prompt: b.prompt ? String(b.prompt) : undefined,
+      model: b.model ? String(b.model) : undefined,
+      kind: b.kind === 'image' ? 'image' : 'video',
+    });
+    return sendJson(res, out.ok ? 200 : 400, out);
+  }
   // native Slack egress (proactive): DM a person by Slack user id or email. Audited as `slack.dm`.
   if (method === 'POST' && p === '/api/agent/slack/dm') {
     const b = await readBody(req);

src/terminal.ts

@@ -24,6 +24,7 @@ import { browseRepo, RemoteCatalog } from './governance/skill-registry';
 import { claudeSupportsReloadSkills } from './edge/claude-cli';
 import { DEFAULT_IMAGE_COST_USD, resolveImageBackend } from './edge/image-gen';
 import { DEFAULT_VIDEO_COST_PER_SEC_USD, DEFAULT_VIDEO_DURATION_SEC, resolveVideoBackend, videoBackend, VideoBackend } from './edge/video-gen';
+import { understandMedia } from './edge/media-understand';
 
 // Video render tuning: a submitted job renders async. The in-call path polls briefly for the fast case;
 // the tick poller finishes the rest, bounded by a TTL + a poll ceiling so a stuck render can't linger.
@@ -1521,6 +1522,8 @@ export class TerminalManager {
         ...(this.os.settings.imageGenConfigured() ? { IMAGE_GEN: '1' } : {}),
         // VIDEO_GEN: '1' exposes `video_generate` when a video backend key (fal/Atlas) is configured.
         ...(this.os.settings.videoGenConfigured() ? { VIDEO_GEN: '1' } : {}),
+        // VIDEO_UNDERSTAND: '1' exposes `video_understand` (video→text) — needs Atlas (its multimodal LLMs).
+        ...(this.os.settings.atlasKey() ? { VIDEO_UNDERSTAND: '1' } : {}),
       },
     };
     return JSON.stringify(config, null, 2);
@@ -2464,6 +2467,47 @@ export class TerminalManager {
     return { ok: true, artifacts: out, model: result.model, costUsd, ...(warning ? { warning } : {}) };
   }
 
+  /**
+   * Understand a VIDEO (or image) — the `video_understand` MCP path. Claude can't natively watch a video,
+   * so this delegates to an Atlas video-capable multimodal LLM (chat endpoint, a `video_url` content part)
+   * and returns the model's TEXT answer directly to the agent — no artifact. The source is any ref
+   * `resolveImageRef` accepts (Library id, working-folder file, or URL), resolved to a base64 data URL so
+   * a clip the agent just generated or was handed can be analysed with no public hosting. Governed like the
+   * other media calls: classified `video.understand` with a cost estimate (money-cap applies), audited.
+   */
+  async understandVideo(sessionId: string, input: { video: string; prompt?: string; model?: string; kind?: 'video' | 'image' }): Promise<{ ok: boolean; text?: string; model?: string; costUsd?: number; error?: string }> {
+    const agent = this.sessionAgent(sessionId);
+    if (!agent) return { ok: false, error: 'unknown session' };
+    const atlasKey = this.os.settings.atlasKey();
+    if (!atlasKey) return { ok: false, error: 'video understanding needs an Atlas Cloud key — set one in Settings → Integrations' };
+    const kind = input.kind === 'image' ? 'image' : 'video';
+    const ref = (input.video || '').trim();
+    if (!ref) return { ok: false, error: `a ${kind} is required — a Library artifact id, a working-folder path, or a URL` };
+    const resolved = this.resolveImageRef(agent, ref, kind);
+    if ('error' in resolved) return { ok: false, error: resolved.error };
+    const prompt = (input.prompt || '').trim() || (kind === 'video' ? 'Describe this video in detail — what happens, who/what is in it, notable actions and setting.' : 'Describe this image in detail.');
+
+    // Token-priced LLM call; the exact cost isn't known ahead of time. Estimate for the money-cap gate.
+    const estimateUsd = DEFAULT_IMAGE_COST_USD;
+    const model = input.model?.trim() || 'qwen/qwen3.5-27b';
+    const gate = this.gate(sessionId, agent, 'video.understand', { prompt, model, kind, amountUsd: estimateUsd }, `understand a ${kind} with ${model}`);
+    if (gate.decision === 'deny') return { ok: false, error: 'blocked by policy' };
+    if (gate.decision === 'pending') return { ok: false, error: 'this needs human approval — an approval request was filed; retry once it is approved' };
+
+    let result;
+    try {
+      result = await understandMedia({ atlasKey, model: input.model, mediaUrl: resolved.url, kind, prompt });
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e);
+      this.audit(sessionId, agent, 'video.understand.failed', { model, kind, error: msg });
+      return { ok: false, error: msg };
+    }
+
+    const costUsd = result.costUsd ?? estimateUsd;
+    this.audit(sessionId, agent, 'video.understood', { model: result.model, kind, costUsd, costSource: result.costUsd != null ? 'actual' : 'estimate', chars: result.text.length });
+    return { ok: true, text: result.text, model: result.model, costUsd };
+  }
+
   /**
    * Resolve an agent-supplied image reference for image-to-video into something a vendor accepts inline
    * (a `data:` URL or a passthrough http URL). Supports every place a session's image can live:
@@ -2475,10 +2519,11 @@ export class TerminalManager {
    * through to the Library. Non-image inputs and unresolvable refs are rejected. Inlining as base64 means
    * the agent needs no public hosting step for something it just made or was handed.
    */
-  private resolveImageRef(agent: string, ref: string): { url: string } | { error: string } {
+  private resolveImageRef(agent: string, ref: string, kind: 'image' | 'video' = 'image'): { url: string } | { error: string } {
     if (/^https?:\/\//i.test(ref) || ref.startsWith('data:')) return { url: ref };
+    const wantMime = new RegExp(`^${kind}/`);
     const toDataUrl = (absPath: string, mime: string): { url: string } | { error: string } => {
-      if (!/^image\//.test(mime)) return { error: `"${ref}" is ${mime}, not an image` };
+      if (!wantMime.test(mime)) return { error: `"${ref}" is ${mime}, not ${kind === 'image' ? 'an image' : 'a video'}` };
       try {
         return { url: `data:${mime};base64,${fs.readFileSync(absPath).toString('base64')}` };
       } catch (e) {
@@ -2502,7 +2547,7 @@ export class TerminalManager {
       const rp = this.os.artifacts.readPath(ref);
       if (rp) return toDataUrl(rp.absPath, a.mime || rp.mime);
     }
-    return { error: `couldn't resolve image "${ref}" — pass an http(s) URL, a file path in your working folder, or a Library artifact id` };
+    return { error: `couldn't resolve ${kind} "${ref}" — pass an http(s) URL, a file path in your working folder, or a Library artifact id` };
   }
 
   /**