diff --git a/CHANGELOG.md b/CHANGELOG.md index 134403e..ae78d8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,18 @@ new version heading in the same commit. ## [Unreleased] -## [0.137.0] — 2026-07-13 -### Added +## [0.138.0] — 2026-07-13 +### Added +- **`image_edit`: agents can now edit or upscale an existing image, not just generate from scratch.** A new + governed MCP tool takes a source **`image`** (a Library artifact id, a working-folder file path — written + *or* terminal-uploaded — or an http(s) URL) and either a **`prompt`** (prompt-guided image-to-image edit, + e.g. "make it a watercolor", "remove the background") or a **`scale`** of 2/4 (upscale). The result is + saved as a **new** Library image (the source is never mutated) + an inbox card. Same governance as + `image_generate`: classified `image.edit` with a cost estimate (money-cap applies), audited `image.edited`. + Reuses the shared image-ref resolver (local files/artifacts sent inline as base64) and Atlas's + `generateImage` submit+poll. Defaults: edit → `google/nano-banana-2/edit`, upscale → `atlascloud/image-upscaler` + (override via `model`). Atlas-only (OpenRouter's image API is text-to-image). New route + `POST /api/agent/image/edit`. Verified against the live Atlas API (edit + upscale both return images). - **Per-member personal context + a self-service Profile page.** Each member can now add free-text **"My context"** that is injected into the system prompt of every session that runs **as them** (their working style, standing preferences, domain notes) — read at launch by `buildCompanyMd` and labelled as diff --git a/package-lock.json b/package-lock.json index 2844706..e7ce8f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.137.0", + "version": "0.138.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.137.0", + "version": "0.138.0", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index 3e49a6b..24b965b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.137.0", + "version": "0.138.0", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", diff --git a/src/edge/image-gen.ts b/src/edge/image-gen.ts index ecb0a91..a170ef5 100644 --- a/src/edge/image-gen.ts +++ b/src/edge/image-gen.ts @@ -40,10 +40,23 @@ export interface ImageGenRequest { n: number; } +/** Edit or upscale an EXISTING image. `images` are already-resolved inputs the vendor accepts inline + * (data: URLs or http URLs). `scale` set (>1) ⇒ upscale mode (uses the upscaler model, `prompt` ignored); + * otherwise it's a prompt-guided edit (image-to-image). */ +export interface ImageEditRequest { + images: string[]; + prompt?: string; + model?: string; + size?: string; + scale?: number; +} + export interface ImageBackend { readonly name: 'openrouter' | 'atlas'; readonly defaultModel: string; generate(req: ImageGenRequest): Promise; + /** Edit/upscale an existing image (image-to-image). Not every backend supports it — those throw. */ + editImage(req: ImageEditRequest): Promise; } /** What the factory needs from Settings — passed in (not the whole store) to keep this testable. */ @@ -145,6 +158,10 @@ class OpenRouterBackend implements ImageBackend { const images = await Promise.all(entries.map(toBytes)); return { images, model, costUsd: typeof d.usage?.cost === 'number' ? d.usage.cost : undefined }; } + + async editImage(_req: ImageEditRequest): Promise { + throw new Error('image editing requires an Atlas Cloud key (the OpenRouter image API is text-to-image only)'); + } } // ── Atlas Cloud ────────────────────────────────────────────────────────────── @@ -164,10 +181,13 @@ interface AtlasPrediction { class AtlasBackend implements ImageBackend { readonly name = 'atlas' as const; readonly defaultModel: string; + readonly editModel: string; // image-to-image default when an edit names no model + readonly upscaleModel = 'atlascloud/image-upscaler'; private readonly base: string; constructor(private readonly key: string, baseUrl?: string, defaultModel?: string) { this.base = (baseUrl?.trim() || 'https://api.atlascloud.ai').replace(/\/+$/, ''); this.defaultModel = defaultModel?.trim() || 'google/nano-banana-2/text-to-image'; + this.editModel = 'google/nano-banana-2/edit'; } private headers(): Record { @@ -178,13 +198,30 @@ class AtlasBackend implements ImageBackend { const model = req.model?.trim() || this.defaultModel; const body: Record = { model, prompt: req.prompt }; if (req.size) body.size = req.size; - // Submit + return this.submitAndPoll(body, model); + } + + async editImage(req: ImageEditRequest): Promise { + if (!req.images.length) throw new Error('an input image is required'); + const upscale = typeof req.scale === 'number' && req.scale > 1; + const model = req.model?.trim() || (upscale ? this.upscaleModel : this.editModel); + // Edit and upscale ride the SAME generateImage submit+poll — only the body shape differs: an upscaler + // takes a single `image` + `outscale`; an image-to-image edit takes `images[]` + a `prompt`. + const body: Record = upscale + ? { model, image: req.images[0], outscale: req.scale, output_format: 'jpeg' } + : { model, prompt: req.prompt ?? '', images: req.images }; + if (!upscale && req.size) body.size = req.size; + return this.submitAndPoll(body, model); + } + + /** Submit a generateImage body then poll the prediction to completion (image renders in seconds, so a + * bounded internal poll keeps the sync ImageBackend contract). Shared by generate + editImage. */ + private async submitAndPoll(body: Record, model: string): Promise { const sres = await fetch(`${this.base}/api/v1/model/generateImage`, { method: 'POST', headers: this.headers(), body: JSON.stringify(body) }); const sbody = (await sres.json().catch(() => ({}))) as { data?: AtlasPrediction; message?: string } & AtlasPrediction; if (!sres.ok) throw new Error(atlasErr(sbody, sres.status)); const id = sbody.data?.id ?? sbody.id; if (!id) throw new Error('Atlas did not return a prediction id'); - // Poll to completion (image is fast — bounded internal poll keeps the sync contract). const deadline = Date.now() + 90_000; for (;;) { await new Promise((r) => setTimeout(r, 2000)); diff --git a/src/memory/memory-mcp.ts b/src/memory/memory-mcp.ts index eeac474..bab928e 100644 --- a/src/memory/memory-mcp.ts +++ b/src/memory/memory-mcp.ts @@ -169,6 +169,28 @@ const IMAGE_GENERATE_TOOL = { }, }; +const IMAGE_EDIT_TOOL = { + name: 'image_edit', + description: + 'Edit or upscale an EXISTING image (image-to-image). Use this to transform an image you already have — ' + + 'change its style/content ("make the sky purple", "add a hat"), or upscale it — WITHOUT redrawing from ' + + 'scratch. The source `image` can be a Library artifact id (e.g. from a prior image_generate), a file path ' + + 'in your working folder (a file you wrote OR one uploaded into the session), or an http(s) image URL. ' + + 'The result is saved as a NEW image in the Library (the source is never changed) and the tool returns the ' + + 'new artifact id. Pass `prompt` to describe the edit, OR `scale` (2 or 4) to upscale. Governed ' + + '(cost-metered + audited). Requires an Atlas Cloud key.', + inputSchema: { + type: 'object', + properties: { + image: { type: 'string', description: 'The image to edit: a Library artifact id, a file path in your working folder (written or uploaded), or an http(s) image URL.' }, + prompt: { type: 'string', description: 'How to change the image (e.g. "turn it into a watercolor painting", "remove the background"). Required unless upscaling.' }, + scale: { type: 'number', description: 'Upscale factor (2 or 4). When set, upscales the image and `prompt` is ignored.' }, + model: { type: 'string', description: 'Optional model id override (an Atlas image-to-image / upscaler model). Omit for the default.' }, + }, + required: ['image'], + }, +}; + const VIDEO_GENERATE_TOOL = { name: 'video_generate', description: @@ -1091,6 +1113,29 @@ async function imageGenerate(args: Record): Promise { return `Generated ${d.artifacts?.length ?? 0} image(s) with ${d.model ?? 'the default model'}${cost}. Saved to the Library: ${list}.`; } +async function imageEdit(args: Record): Promise { + const image = String(args.image ?? '').trim(); + if (!image) return 'An input `image` is required (a Library artifact id, a working-folder file path, or an image URL).'; + const scale = args.scale !== undefined ? Number(args.scale) : undefined; + const prompt = typeof args.prompt === 'string' ? args.prompt.trim() : ''; + if (!(scale && scale > 1) && !prompt) return 'Describe the edit in `prompt`, or pass `scale` (2 or 4) to upscale.'; + const body: Record = { session: SESSION, image }; + if (prompt) body.prompt = prompt; + if (scale !== undefined) body.scale = scale; + if (typeof args.model === 'string' && args.model.trim()) body.model = args.model.trim(); + const res = await fetch(AOS_URL + '/api/agent/image/edit', { + method: 'POST', + headers: H({ 'content-type': 'application/json' }), + body: JSON.stringify(body), + }); + const d = (await res.json()) as { ok?: boolean; error?: string; artifacts?: { id: string; filename: string }[]; model?: string; costUsd?: number }; + if (!d.ok) return `Could not edit image: ${d.error ?? 'unknown error'}`; + const list = (d.artifacts ?? []).map((a) => `${a.id} (${a.filename})`).join(', '); + const cost = typeof d.costUsd === 'number' ? ` · ~$${d.costUsd.toFixed(3)}` : ''; + const what = scale && scale > 1 ? `Upscaled ${scale}×` : 'Edited'; + return `${what} with ${d.model ?? 'the default model'}${cost}. New image saved to the Library: ${list}.`; +} + async function videoGenerate(args: Record): Promise { const prompt = String(args.prompt ?? '').trim(); if (!prompt) return 'A prompt is required.'; @@ -1995,7 +2040,7 @@ async function handle(req: JsonRpc): Promise { ...(DISCORD_REPLY ? [DISCORD_REPLY_TOOL] : []), ...(SLACK_EGRESS ? [SLACK_SEND_TOOL, SLACK_DM_TOOL] : []), ...(DISCORD_EGRESS ? [DISCORD_SEND_TOOL, DISCORD_DM_TOOL] : []), - ...(IMAGE_GEN ? [IMAGE_GENERATE_TOOL] : []), + ...(IMAGE_GEN ? [IMAGE_GENERATE_TOOL, IMAGE_EDIT_TOOL] : []), ...(VIDEO_GEN ? [VIDEO_GENERATE_TOOL] : []), ] } }); return; @@ -2031,6 +2076,7 @@ async function handle(req: JsonRpc): Promise { : name === 'discord_send' ? await discordSend(args) : name === 'discord_dm' ? await discordDm(args) : name === 'image_generate' ? await imageGenerate(args) + : name === 'image_edit' ? await imageEdit(args) : name === 'video_generate' ? await videoGenerate(args) : name === 'list_capabilities' ? await listCapabilities() : name === 'policy_check' ? await policyCheck(args) diff --git a/src/server.ts b/src/server.ts index 9f4da4c..cfcdb90 100644 --- a/src/server.ts +++ b/src/server.ts @@ -873,6 +873,21 @@ async function handle(os: AgentOS, tm: TerminalManager, autos: Automations, req: }); return sendJson(res, out.ok ? 200 : 400, out); } + // OS-owned image EDIT/upscale (`image_edit` MCP tool): edit or upscale an existing image → a new + // artifact. Same loopback/gate posture as image/generate; TerminalManager.editImage owns the path. + if (method === 'POST' && p === '/api/agent/image/edit') { + const b = await readBody(req); + const session = String(b.session || ''); + if (!tm.hasSession(session)) return sendJson(res, 404, { error: 'unknown session' }); + if (!sessionSecretOk(session)) return sendJson(res, 403, { error: 'bad session secret' }); + const out = await tm.editImage(session, { + image: String(b.image || ''), + prompt: b.prompt ? String(b.prompt) : undefined, + scale: b.scale !== undefined ? Number(b.scale) : undefined, + model: b.model ? String(b.model) : undefined, + }); + return sendJson(res, out.ok ? 200 : 400, out); + } // OS-owned VIDEO generation (`video_generate` MCP tool). Async: submits + persists a job, briefly // polls, then the tick poller finishes it. Pre-auth loopback, session-secret gated like the others. if (method === 'POST' && p === '/api/agent/video/generate') { diff --git a/src/terminal.ts b/src/terminal.ts index 880410c..45baafc 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -2378,6 +2378,77 @@ export class TerminalManager { return { ok: true, artifacts: out, model: result.model, costUsd }; } + /** + * Edit or upscale an EXISTING image (the `image_edit` MCP path). Same governance + storage as + * `generateImage`: classified `image.edit` with an estimated `amountUsd` (money-cap applies), the + * result is `ingest`ed as a NEW `image` artifact (the source is never mutated) + an owner-scoped inbox + * card, audited `image.edited`. The source image is any ref `resolveImageRef` accepts — a Library + * artifact id, a working-folder file (written or terminal-uploaded), or a URL. `scale` (>1) upscales + * (prompt ignored); otherwise `prompt` drives an image-to-image edit. Atlas-only (OpenRouter throws). + */ + async editImage(sessionId: string, input: { image: string; prompt?: string; scale?: number; model?: string }): Promise<{ ok: boolean; artifacts?: { id: string; filename: string; mime: string }[]; model?: string; costUsd?: number; error?: string }> { + const agent = this.sessionAgent(sessionId); + if (!agent) return { ok: false, error: 'unknown session' }; + if (!this.os.artifacts.enabled) return { ok: false, error: 'artifacts store is disabled (no data home)' }; + const upscale = typeof input.scale === 'number' && input.scale > 1; + const prompt = (input.prompt || '').trim(); + if (!upscale && !prompt) return { ok: false, error: 'describe the edit in `prompt` (or pass `scale` to upscale)' }; + const imageRef = (input.image || '').trim(); + if (!imageRef) return { ok: false, error: 'an input `image` is required — a Library artifact id, a working-folder path, or an image URL' }; + const resolved = this.resolveImageRef(agent, imageRef); + if ('error' in resolved) return { ok: false, error: resolved.error }; + + const backend = resolveImageBackend({ + openRouterKey: this.os.settings.openRouterKey(), + atlasKey: this.os.settings.atlasKey(), + defaultModel: this.os.settings.imageDefaultModel() || undefined, + }); + if (!backend) return { ok: false, error: 'image editing is not configured — set an Atlas Cloud key in Settings → Integrations' }; + + const estimateUsd = DEFAULT_IMAGE_COST_USD; + const op = upscale ? 'upscale' : 'edit'; + const model = input.model?.trim() || (upscale ? 'atlascloud/image-upscaler' : op); + const gate = this.gate(sessionId, agent, 'image.edit', { prompt, model, upscale, amountUsd: estimateUsd }, `${op} an image with ${model}`); + if (gate.decision === 'deny') return { ok: false, error: 'blocked by policy' }; + if (gate.decision === 'pending') return { ok: false, error: 'this edit needs human approval — an approval request was filed; retry once it is approved' }; + + let result; + try { + result = await backend.editImage({ images: [resolved.url], prompt, model: input.model, scale: input.scale }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + this.audit(sessionId, agent, 'image.failed', { model, op, error: msg }); + return { ok: false, error: msg }; + } + + const srow = this.db.prepare('SELECT spawned_by, run_as FROM term_sessions WHERE id = ?').get<{ spawned_by: string | null; run_as: string | null }>(sessionId); + const source = srow?.run_as ?? srow?.spawned_by ?? undefined; + const title = upscale ? `Upscaled ${input.scale}×` : (prompt.length > 60 ? prompt.slice(0, 57) + '…' : prompt); + const stamp = Date.now(); + const costUsd = result.costUsd ?? estimateUsd; + const perImageUsd = +(costUsd / result.images.length).toFixed(6); + const out: { id: string; filename: string; mime: string }[] = []; + result.images.forEach((img, i) => { + const filename = `${op}-${stamp}${result.images.length > 1 ? `-${i + 1}` : ''}.${img.ext}`; + const r = this.os.artifacts.ingest({ + sessionId, agent, source, title, description: upscale ? `Upscaled ${input.scale}× from ${imageRef}` : prompt, + folder: 'edited-images', filename, bytes: img.bytes, kind: 'image', costUsd: perImageUsd, + }); + if (!r.ok) return; + const a = r.artifact; + out.push({ id: a.id, filename: a.filename, mime: a.mime }); + this.addMessage({ + type: 'artifact', sessionId, agent, title: `Image — ${agent}`, body: a.title, status: 'open', + source, args: { artifactId: a.id, filename: a.filename, mime: a.mime, kind: a.kind }, + audienceKind: 'sessionOwner', audienceId: sessionId, + }); + }); + if (!out.length) return { ok: false, error: 'edit succeeded but no image could be stored' }; + + this.audit(sessionId, agent, 'image.edited', { model: result.model, backend: backend.name, op, count: out.length, costUsd, costSource: result.costUsd != null ? 'actual' : 'estimate', artifactIds: out.map((o) => o.id) }); + return { ok: true, artifacts: out, model: result.model, costUsd }; + } + /** * Resolve an agent-supplied image reference for image-to-video into something a vendor accepts inline * (a `data:` URL or a passthrough http URL). Supports every place a session's image can live: