From e411107a409f851be0e474d0cb4e6fbba8e066b1 Mon Sep 17 00:00:00 2001 From: Vikas Singhal Date: Mon, 13 Jul 2026 14:27:29 +0530 Subject: [PATCH] =?UTF-8?q?feat(media):=20video=5Funderstand=20=E2=80=94?= =?UTF-8?q?=20watch=20a=20video=20(video=E2=86=92text)=20(v0.140.0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New governed MCP tool: delegates to an Atlas multimodal LLM (chat endpoint, video_url content part) and returns a text answer about a video — Claude can't see video natively. Source is a Library id, working-folder file, or URL (inlined as base64). New backend media-understand.ts, route /api/agent/video/understand, capability video.understand (money-capped). Verified live. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 15 +++++++-- package-lock.json | 4 +-- package.json | 2 +- src/edge/media-understand.ts | 59 ++++++++++++++++++++++++++++++++++++ src/memory/memory-mcp.ts | 44 +++++++++++++++++++++++++++ src/server.ts | 15 +++++++++ src/terminal.ts | 51 +++++++++++++++++++++++++++++-- 7 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 src/edge/media-understand.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f8b8b8d..f33c790 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,19 @@ new version heading in the same commit. ## [Unreleased] -## [0.139.0] — 2026-07-13 -### Added +## [0.140.0] — 2026-07-13 +### Added +- **`video_understand`: agents can now "watch" a video (video → text).** Claude can't see video natively; + this new governed MCP tool delegates to an Atlas **multimodal LLM** (the ~10 catalog models with video + input — qwen3.5, glm-5v, kimi-k2…) via the OpenAI-compatible chat endpoint and returns a **text** + answer directly (no artifact). Pass `video` (a Library artifact id — e.g. one `video_generate` just made + — a working-folder file written *or* terminal-uploaded, or an http(s) URL) and an optional `prompt` + ("summarise", "transcribe on-screen text", "what happens at the end?"); omit it for a general + description. Local files are inlined as base64 (no hosting needed). Also handles stills with + `kind:"image"`. Governed like the other media calls: classified `video.understand` with a cost estimate + (money-cap applies), audited `video.understood`. New backend `src/edge/media-understand.ts`, route + `POST /api/agent/video/understand`, default model `qwen/qwen3.5-27b`; exposed whenever Atlas is + configured (`VIDEO_UNDERSTAND`). Verified live (correctly described a test clip). - **`image_edit` gains a `remove-background` preset.** Alongside prompt-guided edit and upscale, agents can now cut out an image's subject with `image_edit({ image, operation: "remove-background" })` — no prompt needed. It returns a **transparent PNG** saved as a new Library image (source untouched), via Atlas's diff --git a/package-lock.json b/package-lock.json index a4de3ef..078142d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.139.0", + "version": "0.140.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.139.0", + "version": "0.140.0", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index a7200ca..04cd523 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.139.0", + "version": "0.140.0", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", diff --git a/src/edge/media-understand.ts b/src/edge/media-understand.ts new file mode 100644 index 0000000..8c96178 --- /dev/null +++ b/src/edge/media-understand.ts @@ -0,0 +1,59 @@ +/** + * Media UNDERSTANDING (video/image → text) — the inverse of the generation tools. Claude can't natively + * watch a video, so this delegates to one of Atlas's video-capable multimodal LLMs (input modalities + * text+image+video) via the OpenAI-compatible chat endpoint: a message carrying a `video_url` content + * part (a URL or a base64 `data:` URL) + a text question returns a text answer the agent reads directly. + * + * Verified live: `{type:'video_url', video_url:{url}}` is the content shape Atlas's video LLMs accept + * (qwen3.5, glm-5v, kimi-k2…); a `{type:'video'}` shape is silently ignored by the model. + */ + +/** A solid default: mid-size, reliably describes motion/content. Overridable per call. */ +export const DEFAULT_VIDEO_UNDERSTAND_MODEL = 'qwen/qwen3.5-27b'; + +export interface UnderstandRequest { + atlasKey: string; + baseUrl?: string; + model?: string; + /** A URL or base64 data: URL for the media. */ + mediaUrl: string; + /** 'video' or 'image' — selects the content-part type. */ + kind: 'video' | 'image'; + prompt: string; +} + +export interface UnderstandResult { + text: string; + model: string; + costUsd?: number; // when the vendor reports usage cost (rare); else undefined → caller estimates +} + +function errText(body: unknown, status: number): string { + const b = body as { error?: { message?: string } | string; message?: string; msg?: string } | undefined; + const e = b?.error; + const msg = (typeof e === 'string' ? e : e?.message) || b?.message || b?.msg; + return msg ? `Atlas understand error (${status}): ${msg}` : `Atlas understand error (${status})`; +} + +export async function understandMedia(req: UnderstandRequest): Promise { + const base = (req.baseUrl?.trim() || 'https://api.atlascloud.ai').replace(/\/+$/, ''); + const model = req.model?.trim() || DEFAULT_VIDEO_UNDERSTAND_MODEL; + const partType = req.kind === 'video' ? 'video_url' : 'image_url'; + const content = [ + { type: 'text', text: req.prompt }, + { type: partType, [partType]: { url: req.mediaUrl } }, + ]; + const res = await fetch(`${base}/v1/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${req.atlasKey}`, 'content-type': 'application/json' }, + body: JSON.stringify({ model, messages: [{ role: 'user', content }] }), + }); + const d = (await res.json().catch(() => ({}))) as { + choices?: { message?: { content?: string } }[]; + usage?: { cost?: number }; + } & Record; + if (!res.ok) throw new Error(errText(d, res.status)); + const text = d.choices?.[0]?.message?.content?.trim(); + if (!text) throw new Error('the model returned no text'); + return { text, model, costUsd: typeof d.usage?.cost === 'number' ? d.usage.cost : undefined }; +} diff --git a/src/memory/memory-mcp.ts b/src/memory/memory-mcp.ts index bc03d70..fc80e60 100644 --- a/src/memory/memory-mcp.ts +++ b/src/memory/memory-mcp.ts @@ -87,6 +87,8 @@ const DISCORD_EGRESS = process.env.DISCORD_EGRESS === '1'; const IMAGE_GEN = process.env.IMAGE_GEN === '1'; // VIDEO_GEN: '1' when a video backend (fal/Atlas) is configured — exposes video_generate. const VIDEO_GEN = process.env.VIDEO_GEN === '1'; +// VIDEO_UNDERSTAND: '1' when Atlas is configured (its multimodal LLMs do video→text) — exposes video_understand. +const VIDEO_UNDERSTAND = process.env.VIDEO_UNDERSTAND === '1'; const SLACK_SEND_TOOL = { name: 'slack_send', @@ -216,6 +218,28 @@ const VIDEO_GENERATE_TOOL = { }, }; +const VIDEO_UNDERSTAND_TOOL = { + name: 'video_understand', + description: + 'WATCH a video and answer a question about it (video → text). Use this whenever you need to know what is ' + + "IN a video — you cannot see video natively. It delegates to a video-capable model and returns a TEXT " + + 'description/answer directly (no artifact). Pass `video` (a Library artifact id, e.g. one video_generate ' + + 'just made; a file path in your working folder, written OR uploaded into the session; or an http(s) video ' + + 'URL) and an optional `prompt` for what to find out ("summarise", "is there a person?", "transcribe any ' + + 'on-screen text", "what happens at the end?"). Omit `prompt` for a general description. Also works on an ' + + 'image if you set `kind:"image"` (but you can usually read images directly). Governed (cost-metered + audited).', + inputSchema: { + type: 'object', + properties: { + video: { type: 'string', description: 'The video to watch: a Library artifact id, a file path in your working folder (written or uploaded), or an http(s) video URL.' }, + prompt: { type: 'string', description: 'What to find out about the video. Omit for a general detailed description.' }, + kind: { type: 'string', enum: ['video', 'image'], description: 'Media type (default "video"). Set "image" to analyse a still image instead.' }, + model: { type: 'string', description: 'Optional Atlas multimodal model id override (must accept video input). Omit for the default.' }, + }, + required: ['video'], + }, +}; + const TOOLS = [ { name: 'recall', @@ -1163,6 +1187,24 @@ async function videoGenerate(args: Record): Promise { return `Video is rendering with ${d.model ?? 'the default model'}${cost} (job ${d.jobId ?? '?'}). It'll appear in the Library and post an inbox card when done — no need to wait; let the user know it's on the way.`; } +async function videoUnderstand(args: Record): Promise { + const video = String(args.video ?? '').trim(); + if (!video) return 'A `video` is required (a Library artifact id, a working-folder file path, or a video URL).'; + const body: Record = { session: SESSION, video }; + if (typeof args.prompt === 'string' && args.prompt.trim()) body.prompt = args.prompt.trim(); + if (args.kind === 'image') body.kind = 'image'; + if (typeof args.model === 'string' && args.model.trim()) body.model = args.model.trim(); + const res = await fetch(AOS_URL + '/api/agent/video/understand', { + method: 'POST', + headers: H({ 'content-type': 'application/json' }), + body: JSON.stringify(body), + }); + const d = (await res.json()) as { ok?: boolean; error?: string; text?: string; model?: string; costUsd?: number }; + if (!d.ok) return `Could not analyse the video: ${d.error ?? 'unknown error'}`; + const cost = typeof d.costUsd === 'number' ? ` · ~$${d.costUsd.toFixed(3)}` : ''; + return `${d.text ?? '(no answer)'}\n\n— via ${d.model ?? 'a multimodal model'}${cost}`; +} + async function slackSend(args: Record): Promise { const channel = String(args.channel ?? '').trim(); const text = String(args.text ?? '').trim(); @@ -2048,6 +2090,7 @@ async function handle(req: JsonRpc): Promise { ...(DISCORD_EGRESS ? [DISCORD_SEND_TOOL, DISCORD_DM_TOOL] : []), ...(IMAGE_GEN ? [IMAGE_GENERATE_TOOL, IMAGE_EDIT_TOOL] : []), ...(VIDEO_GEN ? [VIDEO_GENERATE_TOOL] : []), + ...(VIDEO_UNDERSTAND ? [VIDEO_UNDERSTAND_TOOL] : []), ] } }); return; } @@ -2084,6 +2127,7 @@ async function handle(req: JsonRpc): Promise { : name === 'image_generate' ? await imageGenerate(args) : name === 'image_edit' ? await imageEdit(args) : name === 'video_generate' ? await videoGenerate(args) + : name === 'video_understand' ? await videoUnderstand(args) : name === 'list_capabilities' ? await listCapabilities() : name === 'policy_check' ? await policyCheck(args) : name === 'directory_lookup' ? await directoryLookup(args) diff --git a/src/server.ts b/src/server.ts index 92f8937..02eca2e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -904,6 +904,21 @@ async function handle(os: AgentOS, tm: TerminalManager, autos: Automations, req: }); return sendJson(res, out.ok ? 200 : 400, out); } + // OS-owned video/image UNDERSTANDING (`video_understand` MCP tool): delegate to an Atlas multimodal LLM + // and return the text answer. Same loopback/gate posture; TerminalManager.understandVideo owns the path. + if (method === 'POST' && p === '/api/agent/video/understand') { + const b = await readBody(req); + const session = String(b.session || ''); + if (!tm.hasSession(session)) return sendJson(res, 404, { error: 'unknown session' }); + if (!sessionSecretOk(session)) return sendJson(res, 403, { error: 'bad session secret' }); + const out = await tm.understandVideo(session, { + video: String(b.video || b.image || ''), + prompt: b.prompt ? String(b.prompt) : undefined, + model: b.model ? String(b.model) : undefined, + kind: b.kind === 'image' ? 'image' : 'video', + }); + return sendJson(res, out.ok ? 200 : 400, out); + } // native Slack egress (proactive): DM a person by Slack user id or email. Audited as `slack.dm`. if (method === 'POST' && p === '/api/agent/slack/dm') { const b = await readBody(req); diff --git a/src/terminal.ts b/src/terminal.ts index b4c1006..db8fa33 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -24,6 +24,7 @@ import { browseRepo, RemoteCatalog } from './governance/skill-registry'; import { claudeSupportsReloadSkills } from './edge/claude-cli'; import { DEFAULT_IMAGE_COST_USD, resolveImageBackend } from './edge/image-gen'; import { DEFAULT_VIDEO_COST_PER_SEC_USD, DEFAULT_VIDEO_DURATION_SEC, resolveVideoBackend, videoBackend, VideoBackend } from './edge/video-gen'; +import { understandMedia } from './edge/media-understand'; // Video render tuning: a submitted job renders async. The in-call path polls briefly for the fast case; // the tick poller finishes the rest, bounded by a TTL + a poll ceiling so a stuck render can't linger. @@ -1521,6 +1522,8 @@ export class TerminalManager { ...(this.os.settings.imageGenConfigured() ? { IMAGE_GEN: '1' } : {}), // VIDEO_GEN: '1' exposes `video_generate` when a video backend key (fal/Atlas) is configured. ...(this.os.settings.videoGenConfigured() ? { VIDEO_GEN: '1' } : {}), + // VIDEO_UNDERSTAND: '1' exposes `video_understand` (video→text) — needs Atlas (its multimodal LLMs). + ...(this.os.settings.atlasKey() ? { VIDEO_UNDERSTAND: '1' } : {}), }, }; return JSON.stringify(config, null, 2); @@ -2464,6 +2467,47 @@ export class TerminalManager { return { ok: true, artifacts: out, model: result.model, costUsd, ...(warning ? { warning } : {}) }; } + /** + * Understand a VIDEO (or image) — the `video_understand` MCP path. Claude can't natively watch a video, + * so this delegates to an Atlas video-capable multimodal LLM (chat endpoint, a `video_url` content part) + * and returns the model's TEXT answer directly to the agent — no artifact. The source is any ref + * `resolveImageRef` accepts (Library id, working-folder file, or URL), resolved to a base64 data URL so + * a clip the agent just generated or was handed can be analysed with no public hosting. Governed like the + * other media calls: classified `video.understand` with a cost estimate (money-cap applies), audited. + */ + async understandVideo(sessionId: string, input: { video: string; prompt?: string; model?: string; kind?: 'video' | 'image' }): Promise<{ ok: boolean; text?: string; model?: string; costUsd?: number; error?: string }> { + const agent = this.sessionAgent(sessionId); + if (!agent) return { ok: false, error: 'unknown session' }; + const atlasKey = this.os.settings.atlasKey(); + if (!atlasKey) return { ok: false, error: 'video understanding needs an Atlas Cloud key — set one in Settings → Integrations' }; + const kind = input.kind === 'image' ? 'image' : 'video'; + const ref = (input.video || '').trim(); + if (!ref) return { ok: false, error: `a ${kind} is required — a Library artifact id, a working-folder path, or a URL` }; + const resolved = this.resolveImageRef(agent, ref, kind); + if ('error' in resolved) return { ok: false, error: resolved.error }; + const prompt = (input.prompt || '').trim() || (kind === 'video' ? 'Describe this video in detail — what happens, who/what is in it, notable actions and setting.' : 'Describe this image in detail.'); + + // Token-priced LLM call; the exact cost isn't known ahead of time. Estimate for the money-cap gate. + const estimateUsd = DEFAULT_IMAGE_COST_USD; + const model = input.model?.trim() || 'qwen/qwen3.5-27b'; + const gate = this.gate(sessionId, agent, 'video.understand', { prompt, model, kind, amountUsd: estimateUsd }, `understand a ${kind} with ${model}`); + if (gate.decision === 'deny') return { ok: false, error: 'blocked by policy' }; + if (gate.decision === 'pending') return { ok: false, error: 'this needs human approval — an approval request was filed; retry once it is approved' }; + + let result; + try { + result = await understandMedia({ atlasKey, model: input.model, mediaUrl: resolved.url, kind, prompt }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + this.audit(sessionId, agent, 'video.understand.failed', { model, kind, error: msg }); + return { ok: false, error: msg }; + } + + const costUsd = result.costUsd ?? estimateUsd; + this.audit(sessionId, agent, 'video.understood', { model: result.model, kind, costUsd, costSource: result.costUsd != null ? 'actual' : 'estimate', chars: result.text.length }); + return { ok: true, text: result.text, model: result.model, costUsd }; + } + /** * Resolve an agent-supplied image reference for image-to-video into something a vendor accepts inline * (a `data:` URL or a passthrough http URL). Supports every place a session's image can live: @@ -2475,10 +2519,11 @@ export class TerminalManager { * through to the Library. Non-image inputs and unresolvable refs are rejected. Inlining as base64 means * the agent needs no public hosting step for something it just made or was handed. */ - private resolveImageRef(agent: string, ref: string): { url: string } | { error: string } { + private resolveImageRef(agent: string, ref: string, kind: 'image' | 'video' = 'image'): { url: string } | { error: string } { if (/^https?:\/\//i.test(ref) || ref.startsWith('data:')) return { url: ref }; + const wantMime = new RegExp(`^${kind}/`); const toDataUrl = (absPath: string, mime: string): { url: string } | { error: string } => { - if (!/^image\//.test(mime)) return { error: `"${ref}" is ${mime}, not an image` }; + if (!wantMime.test(mime)) return { error: `"${ref}" is ${mime}, not ${kind === 'image' ? 'an image' : 'a video'}` }; try { return { url: `data:${mime};base64,${fs.readFileSync(absPath).toString('base64')}` }; } catch (e) { @@ -2502,7 +2547,7 @@ export class TerminalManager { const rp = this.os.artifacts.readPath(ref); if (rp) return toDataUrl(rp.absPath, a.mime || rp.mime); } - return { error: `couldn't resolve image "${ref}" — pass an http(s) URL, a file path in your working folder, or a Library artifact id` }; + return { error: `couldn't resolve ${kind} "${ref}" — pass an http(s) URL, a file path in your working folder, or a Library artifact id` }; } /**