feat: measured web-data provider contract — Firecrawl/Exa/Context.dev/Aside/local, selectable per task

CHANGELOG.md

@@ -7,6 +7,48 @@
 > completion state and remaining P0 gates. No version bump or release claim is
 > made here while that status holds.
 
+## [1.65.0.0] - 2026-07-24
+
+**Scraping now has a menu. Measured, ranked,**
+**and it tells you what to install when the best tool is missing.**
+
+When a task needs web data at scale, gstack used to design a bespoke scraper without asking. A flightconnections planning session hand-rolled a cheerio pipeline for a thousand Wikipedia pages and never mentioned that better options exist. That gap is closed. Every skill now reads a WEB-DATA contract before bulk web-data work, checks for an official API first (Wikipedia has MediaWiki, no scraper needed), and otherwise offers a provider menu once: Firecrawl, Exa, Context.dev, the Aside browser, or the local browser. Your choice persists. Declining persists too, and you are never nagged again.
+
+The rankings are not vibes. `gstack-web-data recommend <task>` answers per task kind (scrape, crawl, search, batch, hostile, authenticated) using measured results, filtered by what is actually installed on your machine. When the best tool is missing, it says so with the exact setup step (signup URL, env var, or install) and recommends the best already-installed option for right now.
+
+### The numbers that matter
+
+Source: the reproducible provider bakeoff (`~/.gstack-dev/scrape-bench/bench.ts`, report in `~/.gstack/provider-bakeoff/2026-07-23/`), 7 live tasks per provider.
+
+| Task | Winner | Time | Runner-up |
+|---|---|---|---|
+| Amazon prices (anti-bot) | Firecrawl | 1.1s clean markdown | Aside 3.9s (real session) |
+| Find URLs (search) | Exa | 0.05-0.8s | Firecrawl 2.0s |
+| Crawl 10 pages | Context.dev | 1.1s | Aside 3.6s |
+| Batch 8 pages | Firecrawl | 5.6s | Exa (cache) |
+| Logged-in pages | Aside | only capable option | none |
+
+Firecrawl was the only API to pass all 7 tasks with live fetches. The plain local headless browser got bot-walled on Amazon (empty page). Aside took 22.6s on the 8-page batch, so it is ranked for logged-in and hostile work, never bulk.
+
+Hard rails: off-machine providers receive public URLs only, never authenticated pages, private addresses, project content, or credentials. Logged-in pages route to the user's own browser or nowhere.
+
+### What this means for you
+
+The next time a task needs web data, you get a one-time question with real options, honest availability, and a measured recommendation instead of a silently hand-rolled scraper. Pick once, or say none and never hear about it again. Run `gstack-web-data options` to see where your machine stands.
+
+### Itemized changes
+
+### Added
+
+- `references/WEB-DATA.md` in all five skills: the optional scraping-provider contract. Official API first, provider menu once via AskUserQuestion, best-available recommendation per task, exact setup steps for better-but-unconfigured providers, decline persisted, hand-rolled path always valid.
+- `gstack-web-data` CLI (`status`, `options`, `select`, `recommend`) with availability detection: Firecrawl/Exa env keys, Context.dev runtime consent, `aside` CLI on PATH, installed browse binary. Ships with the managed runtime.
+- Dispatch-protocol trigger in all five dispatchers: read WEB-DATA.md before designing or performing bulk web-data acquisition, whether the agent fetches directly or a plan designs the pipeline.
+
+### For contributors
+
+- `lib/web-data.ts`: provider registry, task rankings (bakeoff-sourced), selection store at `$GSTACK_HOME/web-data.json`, availability detection. 14 unit tests in `test/web-data.test.ts`.
+- `runtime/install.js`: `gstack-web-data` registered in `DEFAULT_RUNTIME_HELPERS`, `lib/web-data.ts` in the helper dependency closure.
+
 ## [1.64.10.0] - 2026-07-24
 
 **Marketing screenshots never required an API key.**

VERSION

@@ -1 +1 @@
-1.64.10.0
+1.65.0.0

bin/gstack-web-data

@@ -0,0 +1,128 @@
+#!/usr/bin/env bun
+/**
+ * gstack-web-data — pick a web-data (scraping/crawling) provider and get the
+ * best recommendation per task. OPTIONAL: with nothing selected, gstack works
+ * fine and callers hand-roll fetches or use the local browser.
+ *
+ * Usage:
+ *   gstack-web-data status [--json]        # selection + per-provider availability
+ *   gstack-web-data options                # list providers + availability + setup hints
+ *   gstack-web-data select <firecrawl|exa|context|aside|local-browser|none>
+ *   gstack-web-data recommend <task> [--json]
+ *     tasks: scrape | crawl | search | batch | hostile | authenticated
+ *
+ * Selecting an off-machine provider records the user's explicit consent to
+ * send PUBLIC target URLs to it — never authenticated pages, private
+ * addresses, project content, cookies, tokens, or credentials.
+ */
+
+import {
+  PROVIDERS,
+  TASK_IDS,
+  TASK_RANKINGS,
+  detectAvailability,
+  readSelection,
+  recommend,
+  setProvider,
+  type WebDataProviderId,
+  type WebDataTask,
+} from "../lib/web-data";
+
+function out(s: string): void {
+  process.stdout.write(`${s}\n`);
+}
+function fail(s: string): never {
+  process.stderr.write(`gstack-web-data: ${s}\n`);
+  process.exit(1);
+}
+
+function cmdStatus(json: boolean): void {
+  const sel = readSelection();
+  const avail = detectAvailability();
+  if (json) {
+    out(JSON.stringify({
+      selected: sel.provider,
+      declined: sel.declined,
+      providers: avail.map((a) => {
+        const p = PROVIDERS.find((x) => x.id === a.id)!;
+        return { id: p.id, label: p.label, offMachine: p.offMachine, available: a.available, detail: a.detail, note: p.note, setupHint: p.setupHint };
+      }),
+      tasks: TASK_IDS,
+    }));
+    return;
+  }
+  out(`selected: ${sel.provider ?? (sel.declined ? "none (declined — do not ask again)" : "none (never asked)")}`);
+  for (const a of avail) {
+    const p = PROVIDERS.find((x) => x.id === a.id)!;
+    out(`  ${p.label.padEnd(20)} [${a.available ? "available" : "not available"}] ${a.detail}`);
+  }
+}
+
+function cmdOptions(): void {
+  out("Web-data providers (all optional; public URLs only for off-machine providers):\n");
+  const avail = new Map(detectAvailability().map((a) => [a.id, a]));
+  for (const p of PROVIDERS) {
+    const a = avail.get(p.id)!;
+    out(`  ${p.id === "firecrawl" ? "*" : " "} ${p.label.padEnd(20)} [${a.available ? "available" : "not available"}] — ${p.note}`);
+    if (!a.available) out(`      setup: ${p.setupHint}`);
+  }
+  out("\nSelect one with: gstack-web-data select <provider>");
+  out("Per-task best: gstack-web-data recommend <scrape|crawl|search|batch|hostile|authenticated>");
+}
+
+function cmdSelect(arg: string | undefined): void {
+  if (arg === "none") {
+    setProvider(null);
+    out("web-data provider declined; gstack hand-rolls fetches / uses the local browser and will not ask again");
+    return;
+  }
+  const provider = PROVIDERS.find((p) => p.id === arg);
+  if (!provider) fail("Usage: select <firecrawl|exa|context|aside|local-browser|none>");
+  setProvider(provider.id as WebDataProviderId);
+  out(`selected ${provider.label}.`);
+  if (provider.offMachine) out(`${provider.label} receives public target URLs only — never authenticated pages, private addresses, project content, or credentials.`);
+}
+
+function cmdRecommend(rest: string[]): void {
+  const json = rest.includes("--json");
+  const task = rest.find((a) => !a.startsWith("--")) as WebDataTask | undefined;
+  if (!task || !TASK_IDS.includes(task)) {
+    fail(`Usage: recommend <${TASK_IDS.join("|")}> [--json]`);
+  }
+  const r = recommend(task);
+  if (json) {
+    out(JSON.stringify(r));
+    return;
+  }
+  out(`task: ${task} — ${r.why}`);
+  if (r.best) {
+    out(`recommended (best available): ${r.best.label} (${r.best.detail})`);
+  } else {
+    out("no capable provider is set up for this task.");
+  }
+  for (const u of r.betterUnavailable) {
+    out(`  better but not set up: ${u.label} — ${u.setupHint}`);
+  }
+  if (!r.best && task === "authenticated") {
+    out("  logged-in pages must stay in the user's own browser; never send them to an API provider.");
+  }
+  out(`full ranking: ${TASK_RANKINGS[task].order.join(" > ")}`);
+}
+
+function main(): void {
+  const [action, ...rest] = process.argv.slice(2);
+  switch (action) {
+    case "status":
+      return cmdStatus(rest.includes("--json"));
+    case "options":
+      return cmdOptions();
+    case "select":
+      return cmdSelect(rest[0]);
+    case "recommend":
+      return cmdRecommend(rest);
+    default:
+      fail("Usage: status [--json] | options | select <provider|none> | recommend <task> [--json]");
+  }
+}
+
+main();