feat(media): image_edit — edit or upscale an existing image (v0.138.0)

CHANGELOG.md

@@ -8,8 +8,18 @@ new version heading in the same commit.
 
 ## [Unreleased]
 
-## [0.137.0] — 2026-07-13
-### Added
+## [0.138.0] — 2026-07-13
+### Added
+- **`image_edit`: agents can now edit or upscale an existing image, not just generate from scratch.** A new
+  governed MCP tool takes a source **`image`** (a Library artifact id, a working-folder file path — written
+  *or* terminal-uploaded — or an http(s) URL) and either a **`prompt`** (prompt-guided image-to-image edit,
+  e.g. "make it a watercolor", "remove the background") or a **`scale`** of 2/4 (upscale). The result is
+  saved as a **new** Library image (the source is never mutated) + an inbox card. Same governance as
+  `image_generate`: classified `image.edit` with a cost estimate (money-cap applies), audited `image.edited`.
+  Reuses the shared image-ref resolver (local files/artifacts sent inline as base64) and Atlas's
+  `generateImage` submit+poll. Defaults: edit → `google/nano-banana-2/edit`, upscale → `atlascloud/image-upscaler`
+  (override via `model`). Atlas-only (OpenRouter's image API is text-to-image). New route
+  `POST /api/agent/image/edit`. Verified against the live Atlas API (edit + upscale both return images).
 - **Per-member personal context + a self-service Profile page.** Each member can now add free-text
   **"My context"** that is injected into the system prompt of every session that runs **as them** (their
   working style, standing preferences, domain notes) — read at launch by `buildCompanyMd` and labelled as

package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-os",
-  "version": "0.137.0",
+  "version": "0.138.0",
   "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.",
   "license": "MIT",
   "type": "commonjs",

src/edge/image-gen.ts

@@ -40,10 +40,23 @@ export interface ImageGenRequest {
   n: number;
 }
 
+/** Edit or upscale an EXISTING image. `images` are already-resolved inputs the vendor accepts inline
+ *  (data: URLs or http URLs). `scale` set (>1) ⇒ upscale mode (uses the upscaler model, `prompt` ignored);
+ *  otherwise it's a prompt-guided edit (image-to-image). */
+export interface ImageEditRequest {
+  images: string[];
+  prompt?: string;
+  model?: string;
+  size?: string;
+  scale?: number;
+}
+
 export interface ImageBackend {
   readonly name: 'openrouter' | 'atlas';
   readonly defaultModel: string;
   generate(req: ImageGenRequest): Promise<ImageGenResult>;
+  /** Edit/upscale an existing image (image-to-image). Not every backend supports it — those throw. */
+  editImage(req: ImageEditRequest): Promise<ImageGenResult>;
 }
 
 /** What the factory needs from Settings — passed in (not the whole store) to keep this testable. */
@@ -145,6 +158,10 @@ class OpenRouterBackend implements ImageBackend {
     const images = await Promise.all(entries.map(toBytes));
     return { images, model, costUsd: typeof d.usage?.cost === 'number' ? d.usage.cost : undefined };
   }
+
+  async editImage(_req: ImageEditRequest): Promise<ImageGenResult> {
+    throw new Error('image editing requires an Atlas Cloud key (the OpenRouter image API is text-to-image only)');
+  }
 }
 
 // ── Atlas Cloud ──────────────────────────────────────────────────────────────
@@ -164,10 +181,13 @@ interface AtlasPrediction {
 class AtlasBackend implements ImageBackend {
   readonly name = 'atlas' as const;
   readonly defaultModel: string;
+  readonly editModel: string; // image-to-image default when an edit names no model
+  readonly upscaleModel = 'atlascloud/image-upscaler';
   private readonly base: string;
   constructor(private readonly key: string, baseUrl?: string, defaultModel?: string) {
     this.base = (baseUrl?.trim() || 'https://api.atlascloud.ai').replace(/\/+$/, '');
     this.defaultModel = defaultModel?.trim() || 'google/nano-banana-2/text-to-image';
+    this.editModel = 'google/nano-banana-2/edit';
   }
 
   private headers(): Record<string, string> {
@@ -178,13 +198,30 @@ class AtlasBackend implements ImageBackend {
     const model = req.model?.trim() || this.defaultModel;
     const body: Record<string, unknown> = { model, prompt: req.prompt };
     if (req.size) body.size = req.size;
-    // Submit
+    return this.submitAndPoll(body, model);
+  }
+
+  async editImage(req: ImageEditRequest): Promise<ImageGenResult> {
+    if (!req.images.length) throw new Error('an input image is required');
+    const upscale = typeof req.scale === 'number' && req.scale > 1;
+    const model = req.model?.trim() || (upscale ? this.upscaleModel : this.editModel);
+    // Edit and upscale ride the SAME generateImage submit+poll — only the body shape differs: an upscaler
+    // takes a single `image` + `outscale`; an image-to-image edit takes `images[]` + a `prompt`.
+    const body: Record<string, unknown> = upscale
+      ? { model, image: req.images[0], outscale: req.scale, output_format: 'jpeg' }
+      : { model, prompt: req.prompt ?? '', images: req.images };
+    if (!upscale && req.size) body.size = req.size;
+    return this.submitAndPoll(body, model);
+  }
+
+  /** Submit a generateImage body then poll the prediction to completion (image renders in seconds, so a
+   *  bounded internal poll keeps the sync ImageBackend contract). Shared by generate + editImage. */
+  private async submitAndPoll(body: Record<string, unknown>, model: string): Promise<ImageGenResult> {
     const sres = await fetch(`${this.base}/api/v1/model/generateImage`, { method: 'POST', headers: this.headers(), body: JSON.stringify(body) });
     const sbody = (await sres.json().catch(() => ({}))) as { data?: AtlasPrediction; message?: string } & AtlasPrediction;
     if (!sres.ok) throw new Error(atlasErr(sbody, sres.status));
     const id = sbody.data?.id ?? sbody.id;
     if (!id) throw new Error('Atlas did not return a prediction id');
-    // Poll to completion (image is fast — bounded internal poll keeps the sync contract).
     const deadline = Date.now() + 90_000;
     for (;;) {
       await new Promise((r) => setTimeout(r, 2000));

src/memory/memory-mcp.ts

@@ -169,6 +169,28 @@ const IMAGE_GENERATE_TOOL = {
   },
 };
 
+const IMAGE_EDIT_TOOL = {
+  name: 'image_edit',
+  description:
+    'Edit or upscale an EXISTING image (image-to-image). Use this to transform an image you already have — ' +
+    'change its style/content ("make the sky purple", "add a hat"), or upscale it — WITHOUT redrawing from ' +
+    'scratch. The source `image` can be a Library artifact id (e.g. from a prior image_generate), a file path ' +
+    'in your working folder (a file you wrote OR one uploaded into the session), or an http(s) image URL. ' +
+    'The result is saved as a NEW image in the Library (the source is never changed) and the tool returns the ' +
+    'new artifact id. Pass `prompt` to describe the edit, OR `scale` (2 or 4) to upscale. Governed ' +
+    '(cost-metered + audited). Requires an Atlas Cloud key.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      image: { type: 'string', description: 'The image to edit: a Library artifact id, a file path in your working folder (written or uploaded), or an http(s) image URL.' },
+      prompt: { type: 'string', description: 'How to change the image (e.g. "turn it into a watercolor painting", "remove the background"). Required unless upscaling.' },
+      scale: { type: 'number', description: 'Upscale factor (2 or 4). When set, upscales the image and `prompt` is ignored.' },
+      model: { type: 'string', description: 'Optional model id override (an Atlas image-to-image / upscaler model). Omit for the default.' },
+    },
+    required: ['image'],
+  },
+};
+
 const VIDEO_GENERATE_TOOL = {
   name: 'video_generate',
   description:
@@ -1091,6 +1113,29 @@ async function imageGenerate(args: Record<string, unknown>): Promise<string> {
   return `Generated ${d.artifacts?.length ?? 0} image(s) with ${d.model ?? 'the default model'}${cost}. Saved to the Library: ${list}.`;
 }
 
+async function imageEdit(args: Record<string, unknown>): Promise<string> {
+  const image = String(args.image ?? '').trim();
+  if (!image) return 'An input `image` is required (a Library artifact id, a working-folder file path, or an image URL).';
+  const scale = args.scale !== undefined ? Number(args.scale) : undefined;
+  const prompt = typeof args.prompt === 'string' ? args.prompt.trim() : '';
+  if (!(scale && scale > 1) && !prompt) return 'Describe the edit in `prompt`, or pass `scale` (2 or 4) to upscale.';
+  const body: Record<string, unknown> = { session: SESSION, image };
+  if (prompt) body.prompt = prompt;
+  if (scale !== undefined) body.scale = scale;
+  if (typeof args.model === 'string' && args.model.trim()) body.model = args.model.trim();
+  const res = await fetch(AOS_URL + '/api/agent/image/edit', {
+    method: 'POST',
+    headers: H({ 'content-type': 'application/json' }),
+    body: JSON.stringify(body),
+  });
+  const d = (await res.json()) as { ok?: boolean; error?: string; artifacts?: { id: string; filename: string }[]; model?: string; costUsd?: number };
+  if (!d.ok) return `Could not edit image: ${d.error ?? 'unknown error'}`;
+  const list = (d.artifacts ?? []).map((a) => `${a.id} (${a.filename})`).join(', ');
+  const cost = typeof d.costUsd === 'number' ? ` · ~$${d.costUsd.toFixed(3)}` : '';
+  const what = scale && scale > 1 ? `Upscaled ${scale}×` : 'Edited';
+  return `${what} with ${d.model ?? 'the default model'}${cost}. New image saved to the Library: ${list}.`;
+}
+
 async function videoGenerate(args: Record<string, unknown>): Promise<string> {
   const prompt = String(args.prompt ?? '').trim();
   if (!prompt) return 'A prompt is required.';
@@ -1995,7 +2040,7 @@ async function handle(req: JsonRpc): Promise<void> {
       ...(DISCORD_REPLY ? [DISCORD_REPLY_TOOL] : []),
       ...(SLACK_EGRESS ? [SLACK_SEND_TOOL, SLACK_DM_TOOL] : []),
       ...(DISCORD_EGRESS ? [DISCORD_SEND_TOOL, DISCORD_DM_TOOL] : []),
-      ...(IMAGE_GEN ? [IMAGE_GENERATE_TOOL] : []),
+      ...(IMAGE_GEN ? [IMAGE_GENERATE_TOOL, IMAGE_EDIT_TOOL] : []),
       ...(VIDEO_GEN ? [VIDEO_GENERATE_TOOL] : []),
     ] } });
     return;
@@ -2031,6 +2076,7 @@ async function handle(req: JsonRpc): Promise<void> {
         : name === 'discord_send' ? await discordSend(args)
         : name === 'discord_dm' ? await discordDm(args)
         : name === 'image_generate' ? await imageGenerate(args)
+        : name === 'image_edit' ? await imageEdit(args)
         : name === 'video_generate' ? await videoGenerate(args)
         : name === 'list_capabilities' ? await listCapabilities()
         : name === 'policy_check' ? await policyCheck(args)

src/server.ts

@@ -873,6 +873,21 @@ async function handle(os: AgentOS, tm: TerminalManager, autos: Automations, req:
     });
     return sendJson(res, out.ok ? 200 : 400, out);
   }
+  // OS-owned image EDIT/upscale (`image_edit` MCP tool): edit or upscale an existing image → a new
+  // artifact. Same loopback/gate posture as image/generate; TerminalManager.editImage owns the path.
+  if (method === 'POST' && p === '/api/agent/image/edit') {
+    const b = await readBody(req);
+    const session = String(b.session || '');
+    if (!tm.hasSession(session)) return sendJson(res, 404, { error: 'unknown session' });
+    if (!sessionSecretOk(session)) return sendJson(res, 403, { error: 'bad session secret' });
+    const out = await tm.editImage(session, {
+      image: String(b.image || ''),
+      prompt: b.prompt ? String(b.prompt) : undefined,
+      scale: b.scale !== undefined ? Number(b.scale) : undefined,
+      model: b.model ? String(b.model) : undefined,
+    });
+    return sendJson(res, out.ok ? 200 : 400, out);
+  }
   // OS-owned VIDEO generation (`video_generate` MCP tool). Async: submits + persists a job, briefly
   // polls, then the tick poller finishes it. Pre-auth loopback, session-secret gated like the others.
   if (method === 'POST' && p === '/api/agent/video/generate') {

src/terminal.ts

@@ -2378,6 +2378,77 @@ export class TerminalManager {
     return { ok: true, artifacts: out, model: result.model, costUsd };
   }
 
+  /**
+   * Edit or upscale an EXISTING image (the `image_edit` MCP path). Same governance + storage as
+   * `generateImage`: classified `image.edit` with an estimated `amountUsd` (money-cap applies), the
+   * result is `ingest`ed as a NEW `image` artifact (the source is never mutated) + an owner-scoped inbox
+   * card, audited `image.edited`. The source image is any ref `resolveImageRef` accepts — a Library
+   * artifact id, a working-folder file (written or terminal-uploaded), or a URL. `scale` (>1) upscales
+   * (prompt ignored); otherwise `prompt` drives an image-to-image edit. Atlas-only (OpenRouter throws).
+   */
+  async editImage(sessionId: string, input: { image: string; prompt?: string; scale?: number; model?: string }): Promise<{ ok: boolean; artifacts?: { id: string; filename: string; mime: string }[]; model?: string; costUsd?: number; error?: string }> {
+    const agent = this.sessionAgent(sessionId);
+    if (!agent) return { ok: false, error: 'unknown session' };
+    if (!this.os.artifacts.enabled) return { ok: false, error: 'artifacts store is disabled (no data home)' };
+    const upscale = typeof input.scale === 'number' && input.scale > 1;
+    const prompt = (input.prompt || '').trim();
+    if (!upscale && !prompt) return { ok: false, error: 'describe the edit in `prompt` (or pass `scale` to upscale)' };
+    const imageRef = (input.image || '').trim();
+    if (!imageRef) return { ok: false, error: 'an input `image` is required — a Library artifact id, a working-folder path, or an image URL' };
+    const resolved = this.resolveImageRef(agent, imageRef);
+    if ('error' in resolved) return { ok: false, error: resolved.error };
+
+    const backend = resolveImageBackend({
+      openRouterKey: this.os.settings.openRouterKey(),
+      atlasKey: this.os.settings.atlasKey(),
+      defaultModel: this.os.settings.imageDefaultModel() || undefined,
+    });
+    if (!backend) return { ok: false, error: 'image editing is not configured — set an Atlas Cloud key in Settings → Integrations' };
+
+    const estimateUsd = DEFAULT_IMAGE_COST_USD;
+    const op = upscale ? 'upscale' : 'edit';
+    const model = input.model?.trim() || (upscale ? 'atlascloud/image-upscaler' : op);
+    const gate = this.gate(sessionId, agent, 'image.edit', { prompt, model, upscale, amountUsd: estimateUsd }, `${op} an image with ${model}`);
+    if (gate.decision === 'deny') return { ok: false, error: 'blocked by policy' };
+    if (gate.decision === 'pending') return { ok: false, error: 'this edit needs human approval — an approval request was filed; retry once it is approved' };
+
+    let result;
+    try {
+      result = await backend.editImage({ images: [resolved.url], prompt, model: input.model, scale: input.scale });
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e);
+      this.audit(sessionId, agent, 'image.failed', { model, op, error: msg });
+      return { ok: false, error: msg };
+    }
+
+    const srow = this.db.prepare('SELECT spawned_by, run_as FROM term_sessions WHERE id = ?').get<{ spawned_by: string | null; run_as: string | null }>(sessionId);
+    const source = srow?.run_as ?? srow?.spawned_by ?? undefined;
+    const title = upscale ? `Upscaled ${input.scale}×` : (prompt.length > 60 ? prompt.slice(0, 57) + '…' : prompt);
+    const stamp = Date.now();
+    const costUsd = result.costUsd ?? estimateUsd;
+    const perImageUsd = +(costUsd / result.images.length).toFixed(6);
+    const out: { id: string; filename: string; mime: string }[] = [];
+    result.images.forEach((img, i) => {
+      const filename = `${op}-${stamp}${result.images.length > 1 ? `-${i + 1}` : ''}.${img.ext}`;
+      const r = this.os.artifacts.ingest({
+        sessionId, agent, source, title, description: upscale ? `Upscaled ${input.scale}× from ${imageRef}` : prompt,
+        folder: 'edited-images', filename, bytes: img.bytes, kind: 'image', costUsd: perImageUsd,
+      });
+      if (!r.ok) return;
+      const a = r.artifact;
+      out.push({ id: a.id, filename: a.filename, mime: a.mime });
+      this.addMessage({
+        type: 'artifact', sessionId, agent, title: `Image — ${agent}`, body: a.title, status: 'open',
+        source, args: { artifactId: a.id, filename: a.filename, mime: a.mime, kind: a.kind },
+        audienceKind: 'sessionOwner', audienceId: sessionId,
+      });
+    });
+    if (!out.length) return { ok: false, error: 'edit succeeded but no image could be stored' };
+
+    this.audit(sessionId, agent, 'image.edited', { model: result.model, backend: backend.name, op, count: out.length, costUsd, costSource: result.costUsd != null ? 'actual' : 'estimate', artifactIds: out.map((o) => o.id) });
+    return { ok: true, artifacts: out, model: result.model, costUsd };
+  }
+
   /**
    * Resolve an agent-supplied image reference for image-to-video into something a vendor accepts inline
    * (a `data:` URL or a passthrough http URL). Supports every place a session's image can live: