feat: web search + fetch tools for agents (SearXNG-backed)

.claude/skills/kild-cli/SKILL.md

@@ -44,6 +44,8 @@ driven from the cockpit UI over the engine's WebSocket, not the CLI.)
 | `kild worktree ls --project <p>` | List the project's `kild/*` worktrees |
 | `kild worktree rm <name> --project <p>` | Remove a worktree (frees disk; the `kild/<name>` branch persists) |
 | `kild worktree prune --project <p>` | Remove **and `-d`-delete the branch of** each `kild/*` worktree merged into the default branch (clean trees only; dirty/in-use ones are kept) |
+| `kild web fetch <url> [--format markdown\|html]` | Fetch a URL as markdown (in-process; keyless) |
+| `kild web search "<query>"` | Query the web via the configured SearXNG (`KILD_SEARXNG_URL`) |
 
 Add `--json` to any command for machine-readable output on stdout.
 
@@ -139,6 +141,10 @@ kild agent ls --project ~/projects/myapp --json | jq -r '.[].name'
   `.pi/agents/`, or globally in `~/.config/kild/agents/` or `~/.claude/agents/`.
   The built-in `default` agent uses pi's own prompt. To add an agent, drop a file —
   kild only reads them.
+- **Web access.** A `kild run` agent automatically gets `webfetch` (read a URL as
+  markdown — keyless, in-process) and, when `KILD_SEARXNG_URL` points at a SearXNG
+  instance, `web_search`. Works for any model. `KILD_WEB=off` disables both. Use
+  `kild web fetch/search` to exercise them directly without spending agent tokens.
 - **One-shot only.** `kild run` blocks until the agent finishes. For a long task,
   expect it to take a while; tool progress streams to stderr so you can see it
   working. There is no way yet to attach to or steer a running agent from the CLI.

CLAUDE.md

@@ -136,6 +136,7 @@ kild/
 │   │   │   ├── agents.ts       #     agents read from .kild/.claude/.pi convention dirs
 │   │   │   ├── sessions.ts     #     SessionManager: coding-agent SDK sessions → UiEvent stream
 │   │   │   ├── worktree.ts     #     [kild-owned] git worktree CRUD + ensureWorktree + merge-prune (NO @flue)
+│   │   │   ├── web/            #     [kild-owned] web_search (SearXNG seam) + webfetch (in-process) tools
 │   │   │   ├── run.ts          #     [Flue layer] one-shot run via Flue
 │   │   │   ├── comms-bus.ts    #     [Flue layer] agent-to-agent peer comms bus (Flue-demo; NOT the Room primitive)
 │   │   │   ├── brain.ts        #     [Flue layer] operator-mirror agent (kild capabilities as tools)
@@ -173,6 +174,13 @@ kild/
   self-contained (no `kild/*` imports) so it can be lifted into Flue verbatim.
   Worktrees **persist**: a session closing never removes one; only an explicit
   `kild worktree rm` / UI action or the automatic merge-prune does.
+- **The web boundary.** kild owns the `web_search` / `webfetch` *tools*
+  (`engine/src/kild/web/`); the search *backend* is external — a self-hosted SearXNG
+  kild only points at via `KILD_SEARXNG_URL`. The engine never spawns or manages the
+  container. The backend sits behind a `SearchProvider` seam, so DDG/fastCRW/Tavily
+  are a new impl + a switch arm later — no tool or worker changes. `webfetch` is
+  in-process (turndown), keyless, and needs no backend. Works for any model (e.g.
+  MiniMax) since it's a plain pi tool, not a provider-native capability.
 
 ### Naming conventions
 

engine/README.md

@@ -15,12 +15,30 @@ bun run cli -- project ls --json   # the kild CLI (secondary interface)
 bun run cli -- run --model anthropic/claude-haiku-4-5 "what files are here?"
 bun run cli -- run --worktree fix-auth "…"   # run isolated in a kild/fix-auth worktree
 bun run cli -- worktree ls --project <p>     # list/rm/prune kild worktrees
+bun run cli -- web fetch https://example.com # read a url as markdown (in-process)
+bun run cli -- web search "claude opus"      # query the web (needs KILD_SEARXNG_URL)
 bun run typecheck && bun run lint  # tsc + biome
 ```
 
 `pi` must be on PATH and authenticated (`~/.pi/agent/auth.json` — Claude Max /
 ChatGPT OAuth work natively).
 
+## Web tools
+
+Agents get `webfetch` (URL → markdown, in-process, keyless) and — when a search
+backend is configured — `web_search`. Search is backed by a **self-hosted SearXNG**
+kild only points at; it never runs the container:
+
+```bash
+cd ../infra/searxng && docker compose up -d
+export KILD_SEARXNG_URL=http://localhost:8888
+```
+
+`webfetch` works regardless; without `KILD_SEARXNG_URL`, `web_search` is simply not
+offered (the worker logs a one-line notice). `KILD_WEB=off` disables both. The search
+backend sits behind a `SearchProvider` seam (`kild/web/search.ts`) — DDG/fastCRW/Tavily
+are a new impl + a switch arm later, no tool/worker changes.
+
 ## Layout
 
 ```
@@ -34,6 +52,7 @@ src/
     agents.ts      agents from .kild/.claude/.pi convention dirs
     sessions.ts    SessionManager: coding-agent SDK sessions → UiEvent stream
     worktree.ts    [kild-owned] git worktree CRUD + ensureWorktree + merge-prune (no @flue)
+    web/           web_search (SearXNG seam) + webfetch (in-process turndown) tools
     run/rooms/brain/observability/auth.ts   [Flue layer]
   flue/
     worktree-sandbox.ts   [Flue layer] worktree() SandboxFactory — the upstream contribution

engine/package.json

@@ -25,12 +25,14 @@
     "@flue/runtime": "^0.9.2",
     "hono": "^4.8.3",
     "just-bash": "^3.0.1",
+    "turndown": "^7.2.4",
     "valibot": "^1.1.0"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.0.0",
     "@flue/cli": "^0.9.2",
     "@types/node": "^22.0.0",
+    "@types/turndown": "^5.0.6",
     "typescript": "^5.6.0"
   }
 }

engine/src/cli.ts

@@ -12,6 +12,9 @@ import { AuthStorage, createAgentSession, ModelRegistry } from '@earendil-works/
 import { listAgents, resolveAgentInstructions } from './kild/agents.ts';
 import { resolveModel, withRole } from './kild/models.ts';
 import { addProject, findProject, loadProjects, removeProject } from './kild/projects.ts';
+import { fetchUrl } from './kild/web/fetch.ts';
+import { webSearchProvider } from './kild/web/search.ts';
+import { webTools } from './kild/web/tools.ts';
 import {
   ensureWorktree,
   listWorktrees,
@@ -30,6 +33,7 @@ const { values, positionals } = parseArgs({
     model: { type: 'string' },
     worktree: { type: 'string' },
     participants: { type: 'string' }, // `kild room` participants, e.g. orchestrator,worker,reviewer
+    format: { type: 'string' }, // `kild web fetch` output format: markdown (default) | html
   },
 });
 
@@ -53,12 +57,14 @@ async function dispatch(): Promise<void> {
       return agent(action, rest);
     case 'worktree':
       return worktree(action, rest);
+    case 'web':
+      return web(action, rest);
     case 'run':
       return run([action, ...rest].filter(Boolean).join(' '));
     case 'room':
       return room([action, ...rest].filter(Boolean).join(' '));
     default:
-      console.error('usage: kild <project|agent|worktree|run|room> …');
+      console.error('usage: kild <project|agent|worktree|web|run|room> …');
       process.exit(2);
   }
 }
@@ -168,6 +174,32 @@ async function worktree(action: string | undefined, args: string[]): Promise<voi
   }
 }
 
+// Debug surface for the web tools — exercise the search backend / fetch a page
+// without spending agent tokens (CLI-first). The agent itself uses the tools directly.
+async function web(action: string | undefined, args: string[]): Promise<void> {
+  if (action === 'search') {
+    const query = args.join(' ');
+    if (!query) throw new Error('usage: kild web search "<query>"');
+    const provider = webSearchProvider();
+    if (!provider) throw new Error('web search needs KILD_SEARXNG_URL (see infra/searxng)');
+    const hits = await provider.search(query, 8);
+    if (json) return void console.log(JSON.stringify(hits, null, 2));
+    if (hits.length === 0) return void console.error('no results');
+    for (const [i, h] of hits.entries()) {
+      console.log(`${i + 1}. ${h.title}\n   ${h.url}${h.snippet ? `\n   ${h.snippet}` : ''}`);
+    }
+  } else if (action === 'fetch') {
+    const [url] = args;
+    if (!url) throw new Error('usage: kild web fetch <url> [--format markdown|html]');
+    const format = values.format === 'html' ? 'html' : 'markdown';
+    const text = await fetchUrl(url, format);
+    if (json) console.log(JSON.stringify({ url, format, text }, null, 2));
+    else console.log(text);
+  } else {
+    throw new Error('usage: kild web <search|fetch>');
+  }
+}
+
 async function run(prompt: string): Promise<void> {
   if (!prompt) throw new Error('usage: kild run <prompt…>');
   // If the engine is up, run THROUGH it so the session shows up in the cockpit;
@@ -383,11 +415,13 @@ async function runInProcess(prompt: string): Promise<void> {
   const registry = ModelRegistry.create(authStorage);
   const model = resolveModel(registry, values.model);
 
+  const tools = webTools();
   const { session } = await createAgentSession({
     model,
     authStorage,
     modelRegistry: registry,
     cwd,
+    customTools: tools.length ? tools : undefined,
   });
 
   let text = '';

engine/src/kild/config.ts

@@ -11,3 +11,14 @@ export function kildHome(): string {
   const base = process.env.XDG_CONFIG_HOME ?? path.join(process.env.HOME ?? '', '.config');
   return path.join(base, 'kild');
 }
+
+/** Whether agents get the web tools. On by default; `KILD_WEB=off` is the kill-switch. */
+export function webEnabled(): boolean {
+  return (process.env.KILD_WEB ?? '').toLowerCase() !== 'off';
+}
+
+/** The self-hosted SearXNG base URL backing `web_search`, or undefined if unset.
+ *  kild only *points at* this instance — it never spawns or manages it. */
+export function searxngUrl(): string | undefined {
+  return process.env.KILD_SEARXNG_URL || undefined;
+}

engine/src/kild/web/fetch-tool.ts

@@ -0,0 +1,32 @@
+import type { ToolDefinition } from '@earendil-works/pi-coding-agent';
+import { Type } from 'typebox';
+
+import { fetchUrl } from './fetch.ts';
+
+/**
+ * The agent-facing `webfetch` tool. Plain HTTP + HTML→markdown (no headless browser,
+ * no service, no key). Always available when web is enabled — it needs no backend.
+ */
+export function createWebFetchTool(): ToolDefinition {
+  return {
+    name: 'webfetch',
+    label: 'Web Fetch',
+    description:
+      'Fetch a public http(s) URL and return its content as markdown (default) or raw ' +
+      'html. Use this to read a page found via web_search.',
+    promptSnippet: 'webfetch — read a url as markdown',
+    parameters: Type.Object({
+      url: Type.String({ description: 'The http(s) URL to fetch.' }),
+      format: Type.Optional(
+        Type.Union([Type.Literal('markdown'), Type.Literal('html')], {
+          description: 'Output format (default markdown).',
+        }),
+      ),
+    }),
+    async execute(_toolCallId, params) {
+      const { url, format } = params as { url: string; format?: 'markdown' | 'html' };
+      const text = await fetchUrl(url, format ?? 'markdown');
+      return { content: [{ type: 'text' as const, text }], details: { url } };
+    },
+  };
+}

engine/src/kild/web/fetch.test.ts

@@ -0,0 +1,44 @@
+import { afterEach, expect, test } from 'bun:test';
+
+import { fetchUrl } from './fetch.ts';
+
+const realFetch = globalThis.fetch;
+afterEach(() => {
+  globalThis.fetch = realFetch;
+});
+
+const serve = (body: string, contentType = 'text/html'): typeof fetch =>
+  (async () =>
+    new Response(body, { status: 200, headers: { 'content-type': contentType } })) as typeof fetch;
+
+test('converts HTML to markdown by default', async () => {
+  globalThis.fetch = serve('<html><body><h1>Hi</h1><p>Para</p></body></html>');
+  const md = await fetchUrl('https://example.com');
+  expect(md).toContain('# Hi');
+  expect(md).toContain('Para');
+});
+
+test('format=html returns the raw html', async () => {
+  globalThis.fetch = serve('<h1>Hi</h1>');
+  expect(await fetchUrl('https://example.com', 'html')).toContain('<h1>Hi</h1>');
+});
+
+test('non-html content is returned as-is (turndown would mangle it)', async () => {
+  globalThis.fetch = serve('{"a":1}', 'application/json');
+  expect(await fetchUrl('https://api.example.com/x')).toBe('{"a":1}');
+});
+
+test('truncates very large output', async () => {
+  globalThis.fetch = serve(`<html><body>${'x'.repeat(60 * 1024)}</body></html>`);
+  const out = await fetchUrl('https://example.com');
+  expect(out).toContain('[truncated at');
+  expect(out.length).toBeLessThan(55 * 1024);
+});
+
+test('rejects non-http(s) and private/loopback hosts before fetching', async () => {
+  await expect(fetchUrl('ftp://x/y')).rejects.toThrow(/http/);
+  await expect(fetchUrl('http://localhost/x')).rejects.toThrow(/private|loopback/);
+  await expect(fetchUrl('http://127.0.0.1/x')).rejects.toThrow(/private|loopback/);
+  await expect(fetchUrl('http://192.168.1.5/x')).rejects.toThrow(/private|loopback/);
+  await expect(fetchUrl('http://10.0.0.1/x')).rejects.toThrow(/private|loopback/);
+});

engine/src/kild/web/fetch.ts

@@ -0,0 +1,100 @@
+import TurndownService from 'turndown';
+
+export type FetchFormat = 'markdown' | 'html';
+
+const MAX_BYTES = 5 * 1024 * 1024; // 5 MB hard cap (mirrors opencode webfetch)
+const MAX_OUTPUT = 50 * 1024; // truncate returned text to ~50 KB
+const DEFAULT_TIMEOUT_MS = 30_000;
+const MAX_TIMEOUT_MS = 120_000;
+// A realistic desktop Chrome UA gets past most basic bot filters; some walls (e.g.
+// Cloudflare) instead trust a plain identifying agent, so we retry once with that.
+const BROWSER_UA =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36';
+const FALLBACK_UA = 'kild';
+
+const turndown = new TurndownService({
+  headingStyle: 'atx',
+  hr: '---',
+  bulletListMarker: '-',
+  codeBlockStyle: 'fenced',
+  emDelimiter: '*',
+});
+// Drop non-content elements entirely — otherwise their text (CSS, JS, <head> meta)
+// leaks into the markdown.
+turndown.remove(['script', 'style', 'head', 'noscript', 'iframe']);
+
+/** Block fetches to loopback / link-local / private ranges so the agent can't be
+ *  steered into the local network (basic SSRF guard). Literal-host check — does not
+ *  defend against DNS rebinding; acceptable for a single-user loopback tool. */
+function assertPublicHost(hostname: string): void {
+  const h = hostname.toLowerCase().replace(/^\[|\]$/g, ''); // strip IPv6 brackets
+  const blocked =
+    h === 'localhost' ||
+    h.endsWith('.localhost') ||
+    h === '0.0.0.0' ||
+    h === '::1' ||
+    h.startsWith('127.') ||
+    h.startsWith('10.') ||
+    h.startsWith('169.254.') ||
+    h.startsWith('192.168.') ||
+    /^172\.(1[6-9]|2\d|3[01])\./.test(h) ||
+    h.startsWith('fc') || // IPv6 unique-local
+    h.startsWith('fd') ||
+    h.startsWith('fe80:'); // IPv6 link-local
+  if (blocked) throw new Error(`refusing to fetch a private/loopback host: ${hostname}`);
+}
+
+async function doFetch(url: string, ua: string, timeoutMs: number): Promise<Response> {
+  return fetch(url, {
+    headers: { 'user-agent': ua, accept: 'text/html,application/xhtml+xml,*/*' },
+    redirect: 'follow',
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+}
+
+/** Fetch a public URL and return its content as markdown (default) or raw html/text.
+ *  Plain HTTP — no headless browser. Enforces http(s), a 5 MB cap, and a timeout. */
+export async function fetchUrl(
+  url: string,
+  format: FetchFormat = 'markdown',
+  timeoutSeconds?: number,
+): Promise<string> {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`invalid url: ${url}`);
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    throw new Error(`only http(s) urls are supported: ${url}`);
+  }
+  assertPublicHost(parsed.hostname);
+
+  const timeoutMs = Math.min(MAX_TIMEOUT_MS, (timeoutSeconds ?? DEFAULT_TIMEOUT_MS / 1000) * 1000);
+
+  let res = await doFetch(url, BROWSER_UA, timeoutMs);
+  if ((res.status === 403 || res.status === 429 || res.status === 503) && res.body)
+    res.body.cancel?.();
+  if (res.status === 403 || res.status === 429 || res.status === 503) {
+    res = await doFetch(url, FALLBACK_UA, timeoutMs); // some walls trust a plain UA
+  }
+  if (!res.ok) throw new Error(`fetch ${res.status} for ${url}`);
+
+  const len = Number(res.headers.get('content-length') ?? '0');
+  if (len > MAX_BYTES) throw new Error(`response too large (${len} bytes > ${MAX_BYTES})`);
+
+  const contentType = res.headers.get('content-type') ?? '';
+  const raw = await res.text();
+  if (raw.length > MAX_BYTES) throw new Error(`response too large (> ${MAX_BYTES} bytes)`);
+
+  // Non-HTML (json, plain text, etc.) is returned as-is — turndown would mangle it.
+  const isHtml = contentType.includes('html') || /<html[\s>]/i.test(raw);
+  let out: string;
+  if (format === 'html' || !isHtml) out = raw;
+  else out = turndown.turndown(raw);
+
+  if (out.length > MAX_OUTPUT) {
+    out = `${out.slice(0, MAX_OUTPUT)}\n\n…[truncated at ${MAX_OUTPUT} bytes]`;
+  }
+  return out;
+}