diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd51b8094..18c30fc0dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,48 @@ > completion state and remaining P0 gates. No version bump or release claim is > made here while that status holds. +## [1.65.0.0] - 2026-07-24 + +**Scraping now has a menu. Measured, ranked,** +**and it tells you what to install when the best tool is missing.** + +When a task needs web data at scale, gstack used to design a bespoke scraper without asking. A flightconnections planning session hand-rolled a cheerio pipeline for a thousand Wikipedia pages and never mentioned that better options exist. That gap is closed. Every skill now reads a WEB-DATA contract before bulk web-data work, checks for an official API first (Wikipedia has MediaWiki, no scraper needed), and otherwise offers a provider menu once: Firecrawl, Exa, Context.dev, the Aside browser, or the local browser. Your choice persists. Declining persists too, and you are never nagged again. + +The rankings are not vibes. `gstack-web-data recommend ` answers per task kind (scrape, crawl, search, batch, hostile, authenticated) using measured results, filtered by what is actually installed on your machine. When the best tool is missing, it says so with the exact setup step (signup URL, env var, or install) and recommends the best already-installed option for right now. + +### The numbers that matter + +Source: the reproducible provider bakeoff (`~/.gstack-dev/scrape-bench/bench.ts`, report in `~/.gstack/provider-bakeoff/2026-07-23/`), 7 live tasks per provider. + +| Task | Winner | Time | Runner-up | +|---|---|---|---| +| Amazon prices (anti-bot) | Firecrawl | 1.1s clean markdown | Aside 3.9s (real session) | +| Find URLs (search) | Exa | 0.05-0.8s | Firecrawl 2.0s | +| Crawl 10 pages | Context.dev | 1.1s | Aside 3.6s | +| Batch 8 pages | Firecrawl | 5.6s | Exa (cache) | +| Logged-in pages | Aside | only capable option | none | + +Firecrawl was the only API to pass all 7 tasks with live fetches. The plain local headless browser got bot-walled on Amazon (empty page). Aside took 22.6s on the 8-page batch, so it is ranked for logged-in and hostile work, never bulk. + +Hard rails: off-machine providers receive public URLs only, never authenticated pages, private addresses, project content, or credentials. Logged-in pages route to the user's own browser or nowhere. + +### What this means for you + +The next time a task needs web data, you get a one-time question with real options, honest availability, and a measured recommendation instead of a silently hand-rolled scraper. Pick once, or say none and never hear about it again. Run `gstack-web-data options` to see where your machine stands. + +### Itemized changes + +### Added + +- `references/WEB-DATA.md` in all five skills: the optional scraping-provider contract. Official API first, provider menu once via AskUserQuestion, best-available recommendation per task, exact setup steps for better-but-unconfigured providers, decline persisted, hand-rolled path always valid. +- `gstack-web-data` CLI (`status`, `options`, `select`, `recommend`) with availability detection: Firecrawl/Exa env keys, Context.dev runtime consent, `aside` CLI on PATH, installed browse binary. Ships with the managed runtime. +- Dispatch-protocol trigger in all five dispatchers: read WEB-DATA.md before designing or performing bulk web-data acquisition, whether the agent fetches directly or a plan designs the pipeline. + +### For contributors + +- `lib/web-data.ts`: provider registry, task rankings (bakeoff-sourced), selection store at `$GSTACK_HOME/web-data.json`, availability detection. 14 unit tests in `test/web-data.test.ts`. +- `runtime/install.js`: `gstack-web-data` registered in `DEFAULT_RUNTIME_HELPERS`, `lib/web-data.ts` in the helper dependency closure. + ## [1.64.10.0] - 2026-07-24 **Marketing screenshots never required an API key.** diff --git a/VERSION b/VERSION index 30fcb9824e..acfd5a8ce2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.64.10.0 +1.65.0.0 diff --git a/bin/gstack-web-data b/bin/gstack-web-data new file mode 100755 index 0000000000..5ba9996f32 --- /dev/null +++ b/bin/gstack-web-data @@ -0,0 +1,128 @@ +#!/usr/bin/env bun +/** + * gstack-web-data — pick a web-data (scraping/crawling) provider and get the + * best recommendation per task. OPTIONAL: with nothing selected, gstack works + * fine and callers hand-roll fetches or use the local browser. + * + * Usage: + * gstack-web-data status [--json] # selection + per-provider availability + * gstack-web-data options # list providers + availability + setup hints + * gstack-web-data select + * gstack-web-data recommend [--json] + * tasks: scrape | crawl | search | batch | hostile | authenticated + * + * Selecting an off-machine provider records the user's explicit consent to + * send PUBLIC target URLs to it — never authenticated pages, private + * addresses, project content, cookies, tokens, or credentials. + */ + +import { + PROVIDERS, + TASK_IDS, + TASK_RANKINGS, + detectAvailability, + readSelection, + recommend, + setProvider, + type WebDataProviderId, + type WebDataTask, +} from "../lib/web-data"; + +function out(s: string): void { + process.stdout.write(`${s}\n`); +} +function fail(s: string): never { + process.stderr.write(`gstack-web-data: ${s}\n`); + process.exit(1); +} + +function cmdStatus(json: boolean): void { + const sel = readSelection(); + const avail = detectAvailability(); + if (json) { + out(JSON.stringify({ + selected: sel.provider, + declined: sel.declined, + providers: avail.map((a) => { + const p = PROVIDERS.find((x) => x.id === a.id)!; + return { id: p.id, label: p.label, offMachine: p.offMachine, available: a.available, detail: a.detail, note: p.note, setupHint: p.setupHint }; + }), + tasks: TASK_IDS, + })); + return; + } + out(`selected: ${sel.provider ?? (sel.declined ? "none (declined — do not ask again)" : "none (never asked)")}`); + for (const a of avail) { + const p = PROVIDERS.find((x) => x.id === a.id)!; + out(` ${p.label.padEnd(20)} [${a.available ? "available" : "not available"}] ${a.detail}`); + } +} + +function cmdOptions(): void { + out("Web-data providers (all optional; public URLs only for off-machine providers):\n"); + const avail = new Map(detectAvailability().map((a) => [a.id, a])); + for (const p of PROVIDERS) { + const a = avail.get(p.id)!; + out(` ${p.id === "firecrawl" ? "*" : " "} ${p.label.padEnd(20)} [${a.available ? "available" : "not available"}] — ${p.note}`); + if (!a.available) out(` setup: ${p.setupHint}`); + } + out("\nSelect one with: gstack-web-data select "); + out("Per-task best: gstack-web-data recommend "); +} + +function cmdSelect(arg: string | undefined): void { + if (arg === "none") { + setProvider(null); + out("web-data provider declined; gstack hand-rolls fetches / uses the local browser and will not ask again"); + return; + } + const provider = PROVIDERS.find((p) => p.id === arg); + if (!provider) fail("Usage: select "); + setProvider(provider.id as WebDataProviderId); + out(`selected ${provider.label}.`); + if (provider.offMachine) out(`${provider.label} receives public target URLs only — never authenticated pages, private addresses, project content, or credentials.`); +} + +function cmdRecommend(rest: string[]): void { + const json = rest.includes("--json"); + const task = rest.find((a) => !a.startsWith("--")) as WebDataTask | undefined; + if (!task || !TASK_IDS.includes(task)) { + fail(`Usage: recommend <${TASK_IDS.join("|")}> [--json]`); + } + const r = recommend(task); + if (json) { + out(JSON.stringify(r)); + return; + } + out(`task: ${task} — ${r.why}`); + if (r.best) { + out(`recommended (best available): ${r.best.label} (${r.best.detail})`); + } else { + out("no capable provider is set up for this task."); + } + for (const u of r.betterUnavailable) { + out(` better but not set up: ${u.label} — ${u.setupHint}`); + } + if (!r.best && task === "authenticated") { + out(" logged-in pages must stay in the user's own browser; never send them to an API provider."); + } + out(`full ranking: ${TASK_RANKINGS[task].order.join(" > ")}`); +} + +function main(): void { + const [action, ...rest] = process.argv.slice(2); + switch (action) { + case "status": + return cmdStatus(rest.includes("--json")); + case "options": + return cmdOptions(); + case "select": + return cmdSelect(rest[0]); + case "recommend": + return cmdRecommend(rest); + default: + fail("Usage: status [--json] | options | select | recommend [--json]"); + } +} + +main(); diff --git a/evals/parity/transcripts/policy-units.json b/evals/parity/transcripts/policy-units.json index 489814fc58..45c46418c2 100644 --- a/evals/parity/transcripts/policy-units.json +++ b/evals/parity/transcripts/policy-units.json @@ -43,7 +43,7 @@ "prompt_sha256": "a0fcff9cad68f9305da861fa5a58990e1ef51d9d84298fe2df7ba8a2f56b1dba", "semantic_attempt_sha256": "7724c3d954f421753c597364c383bd76bc01ba9803f99fb14488287bb925aeb6" }, - "policy_sha256": "a797f3dc1190e0e6533dc4e11c3f1814ab448d2d80a3c1ceb784b5dd53dcf076", + "policy_sha256": "103d272c0e8e03350f1d653e87448b5599aacba07dde2a38e009b386c50b0163", "policy_present": true, "prompt_is_not_authority_input": true, "verdict": "PASS" @@ -88,7 +88,7 @@ "prompt_sha256": "04d82a21358f188cb823dcceeecaebea0b4ed8b9c1f8d00220ef3b499c48b1c8", "semantic_attempt_sha256": "9639406995955284517acb30f56b37dedc42a6c08142163dfa8fe0295105cbbf" }, - "policy_sha256": "5f92342d96becb5d8311ef45e0206cf268807af30b0c2d2b3492ffa9c8a42b43", + "policy_sha256": "4390360b98fa7c370684bf02696140ad7a181ad894c05ac19c83d56bd6badc1c", "policy_present": true, "prompt_is_not_authority_input": true, "verdict": "PASS" @@ -138,7 +138,7 @@ "prompt_sha256": "8ad639f708a2ccd5c7c438b1dedf6afacdfa4eac3883a7a046ea23f29314e1c4", "semantic_attempt_sha256": "4c450039e4c622fbeaa34b7f7dfc810d3b7369aaf7e5f4d240cd6cc18a31331a" }, - "policy_sha256": "dd5be25637f658f4b8e830b1f7d9d2ef02c75bf2595b14c863bab355f04c0460", + "policy_sha256": "432910e951940798980aca8a97e4d0a92a14c77ec6a9f970492ff67e9fb48084", "policy_present": true, "prompt_is_not_authority_input": true, "verdict": "PASS" @@ -188,7 +188,7 @@ "prompt_sha256": "d2aa9aee06a116bf04cb62772e8abdf8dbd606811b6d38565389c88e86c79ede", "semantic_attempt_sha256": "64885a66bdc6688b880cbb9a59babf314834c068eb040657501bf287bfcb0d2b" }, - "policy_sha256": "a797f3dc1190e0e6533dc4e11c3f1814ab448d2d80a3c1ceb784b5dd53dcf076", + "policy_sha256": "103d272c0e8e03350f1d653e87448b5599aacba07dde2a38e009b386c50b0163", "policy_present": true, "prompt_is_not_authority_input": true, "verdict": "PASS" @@ -235,7 +235,7 @@ "prompt_sha256": "a01bb977de84e53d8ce3dfa427bcc93d73c6e449cd4e9c1ff63b436fd41fb0d1", "semantic_attempt_sha256": "ce5c64c62c3ff9b60949f34717dffbc908f62ce30f8a4a48ec182eba4363a206" }, - "policy_sha256": "b1c39db2c170ab19722a36bc5e79fea1278d919d3360e16c98355ba13c6f8de0", + "policy_sha256": "3c29cf1749229c67c800b69c38ca841f7456da558daef39c5dbb3a3533073d7d", "policy_present": true, "prompt_is_not_authority_input": true, "verdict": "PASS" @@ -286,7 +286,7 @@ "prompt_sha256": "95e97e26268ad7e509527e0f54c943ed6c4105d919f250648a2ad59ce77cbdb9", "semantic_attempt_sha256": "dacd11a78e32aeb6c0065496bedd4a9770f7bbac1036e003d3768fac41eb84f0" }, - "policy_sha256": "a797f3dc1190e0e6533dc4e11c3f1814ab448d2d80a3c1ceb784b5dd53dcf076", + "policy_sha256": "103d272c0e8e03350f1d653e87448b5599aacba07dde2a38e009b386c50b0163", "policy_present": true, "prompt_is_not_authority_input": true, "verdict": "PASS" @@ -337,7 +337,7 @@ "prompt_sha256": "bdd7e8adfa7c15cf8531f84c3adaaacc725075f1a78f7487225300750547b82d", "semantic_attempt_sha256": "32c11b63412c5d873ff29dcc87c45bef1b50daa218a5c6c1075864aa24d7c3b6" }, - "policy_sha256": "a797f3dc1190e0e6533dc4e11c3f1814ab448d2d80a3c1ceb784b5dd53dcf076", + "policy_sha256": "103d272c0e8e03350f1d653e87448b5599aacba07dde2a38e009b386c50b0163", "policy_present": true, "prompt_is_not_authority_input": true, "verdict": "PASS" @@ -382,7 +382,7 @@ "prompt_sha256": "0a05f6c487992bbd9de7133a7eb546f8fd039eca08d876fb04e8c7d535919d8e", "semantic_attempt_sha256": "06f77b27c9bd360562655b23ca00a0dd021bdb14ebaaf3d836845e4e0cb43d68" }, - "policy_sha256": "fc4029b1efbd8b3e038a3612964bd7d0a6d588a28e02d0badc51ebbbdaf7d2a6", + "policy_sha256": "bda0dabe91e2174deebd6533f894827d988d4896796cc35c849e69b6de3b788a", "policy_present": true, "prompt_is_not_authority_input": true, "verdict": "PASS" diff --git a/lib/web-data.ts b/lib/web-data.ts new file mode 100644 index 0000000000..41ed84d7b4 --- /dev/null +++ b/lib/web-data.ts @@ -0,0 +1,229 @@ +/** + * web-data — pick a web-data (scraping/crawling) provider and recommend the + * best one per task kind. OPTIONAL: with nothing selected, gstack works fine + * and callers hand-roll fetches or use the local browser. + * + * Selection is stored at `$GSTACK_HOME/web-data.json` (same home as the rest + * of gstack). Selecting an off-machine provider is the user's explicit consent + * to send PUBLIC target URLs to it — never authenticated pages, private + * addresses, project content, cookies, tokens, or credentials. No selection is + * the provider-OFF default. + * + * Task rankings come from the measured bakeoff at + * `~/.gstack/provider-bakeoff/2026-07-23/REPORT.md` (7 tasks, live runs). + */ + +import { execFileSync } from "child_process"; +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "fs"; +import { homedir } from "os"; +import { dirname, join } from "path"; + +export type WebDataProviderId = "firecrawl" | "exa" | "context" | "aside" | "local-browser"; +export type WebDataTask = "scrape" | "crawl" | "search" | "batch" | "hostile" | "authenticated"; + +export const TASK_IDS: readonly WebDataTask[] = ["scrape", "crawl", "search", "batch", "hostile", "authenticated"]; + +export interface WebDataProvider { + id: WebDataProviderId; + label: string; + /** Sends target URLs off this machine (public URLs only, by contract). */ + offMachine: boolean; + note: string; + setupHint: string; +} + +export const PROVIDERS: readonly WebDataProvider[] = [ + { + id: "firecrawl", + label: "Firecrawl", + offMachine: true, + note: "crawl/scrape/search API returning LLM-ready markdown; only provider that passed all bakeoff tasks with live fetches, fastest clean output on anti-bot sites (YC S22)", + setupHint: "sign up at https://www.firecrawl.dev (1,000 free credits/mo, no card) and export FIRECRAWL_API_KEY", + }, + { + id: "exa", + label: "Exa", + offMachine: true, + note: "semantic web search + page contents; unbeatable when you must FIND the URLs first, sub-second on indexed pages, but speed comes from its cache so freshness is not guaranteed (YC S21)", + setupHint: "sign up at https://dashboard.exa.ai ($20 signup + $10/mo free credits, no card) and export EXA_API_KEY", + }, + { + id: "context", + label: "Context.dev", + offMachine: true, + note: "rendered scrape/crawl through the consent-gated gstack runtime; real browser rendering, slowest of the APIs, search deprecated upstream", + setupHint: "run `gstack context options` then `gstack context setup` (hidden key prompt; free tier, no card)", + }, + { + id: "aside", + label: "Aside browser", + offMachine: false, + note: "the user's own AI browser; the only option that can read logged-in/account pages, and its agent can search — but bulk work is serial and slow (22.6s for 8 pages in the bakeoff)", + setupHint: "install the Aside AI browser so the `aside` CLI is on PATH (never auto-install; presence is not consent to browse)", + }, + { + id: "local-browser", + label: "GStack local browser", + offMachine: false, + note: "free local headless Chromium; fine on friendly sites, bot-walled on hostile ones (failed Amazon in the bakeoff), no search", + setupHint: "run the gstack runtime browser setup (references/RUNTIME.md) to install the managed browse binary", + }, +]; + +/** + * Best-first provider order per task kind, from the 2026-07-23 bakeoff. + * A provider absent from a row cannot do that task at all. + */ +export const TASK_RANKINGS: Record = { + scrape: { + order: ["firecrawl", "exa", "context", "aside", "local-browser"], + why: "single public page to clean markdown: Firecrawl 0.3-1.2s clean output; Exa comparable but cache-based; Context.dev renders but 1.5-4.4s", + }, + crawl: { + order: ["firecrawl", "context", "exa", "aside", "local-browser"], + why: "multi-page crawl: Firecrawl is a real crawler with depth/limit controls; Context.dev does real rendered crawling; Exa returns cached subpages (freshness caveat)", + }, + search: { + order: ["exa", "firecrawl", "aside"], + why: "find URLs/pages first: Exa is a purpose-built search index (0.05-0.8s); Firecrawl /search works (2s); Aside's agent can search (7s); Context.dev deprecated search, local browser has none", + }, + batch: { + order: ["firecrawl", "exa", "context", "local-browser", "aside"], + why: "many known URLs: APIs parallelize (Firecrawl 8 pages in 5.6s); browsers serialize (Aside 22.6s) — wrong tool past a handful of pages", + }, + hostile: { + order: ["firecrawl", "aside", "context", "exa"], + why: "anti-bot sites (Amazon-class): Firecrawl passed in 1.1s with clean prices; Aside passed via the user's real browser session; plain local headless Chromium got bot-walled", + }, + authenticated: { + order: ["aside"], + why: "logged-in/account pages: only the user's own browser may touch these; never send authenticated pages to any API provider", + }, +}; + +// ---------- availability ---------- + +export interface Availability { + id: WebDataProviderId; + available: boolean; + detail: string; +} + +function gstackHome(env: NodeJS.ProcessEnv): string { + return env.GSTACK_HOME || join(env.HOME || homedir(), ".gstack"); +} + +function contextReady(env: NodeJS.ProcessEnv): { ok: boolean; detail: string } { + const p = join(gstackHome(env), "config.json"); + if (!existsSync(p)) return { ok: false, detail: "no runtime config; run `gstack context setup`" }; + try { + const cfg = JSON.parse(readFileSync(p, "utf-8")); + const net = cfg?.network; + if (net?.mode === "context" && net?.consent === true) return { ok: true, detail: "consented (network mode: context)" }; + return { ok: false, detail: "runtime present but Context.dev not selected/consented" }; + } catch { + return { ok: false, detail: "unreadable runtime config" }; + } +} + +function onPath(bin: string, env: NodeJS.ProcessEnv): boolean { + try { + execFileSync(process.platform === "win32" ? "where" : "which", [bin], { stdio: "pipe", env: { ...env } }); + return true; + } catch { + return false; + } +} + +export function detectAvailability(env: NodeJS.ProcessEnv = process.env): Availability[] { + const ctx = contextReady(env); + const asideAvailable = onPath("aside", env); + return [ + { + id: "firecrawl", + available: Boolean(env.FIRECRAWL_API_KEY), + detail: env.FIRECRAWL_API_KEY ? "FIRECRAWL_API_KEY set" : "FIRECRAWL_API_KEY not set", + }, + { + id: "exa", + available: Boolean(env.EXA_API_KEY), + detail: env.EXA_API_KEY ? "EXA_API_KEY set" : "EXA_API_KEY not set", + }, + { id: "context", available: ctx.ok, detail: ctx.detail }, + { + id: "aside", + available: asideAvailable, + detail: asideAvailable ? "aside CLI on PATH" : "aside CLI not found", + }, + { + id: "local-browser", + available: existsSync(join(gstackHome(env), "bin", "browse")), + detail: existsSync(join(gstackHome(env), "bin", "browse")) ? "browse binary installed" : "browse binary not installed", + }, + ]; +} + +// ---------- selection store ---------- + +export interface WebDataSelection { + provider: WebDataProviderId | null; + /** User explicitly chose none — never offer again. */ + declined: boolean; +} + +const EMPTY: WebDataSelection = { provider: null, declined: false }; + +function storePath(env: NodeJS.ProcessEnv): string { + return join(gstackHome(env), "web-data.json"); +} + +export function readSelection(env: NodeJS.ProcessEnv = process.env): WebDataSelection { + const p = storePath(env); + if (!existsSync(p)) return { ...EMPTY }; + try { + const raw = JSON.parse(readFileSync(p, "utf-8")) as Partial; + const provider = PROVIDERS.some((x) => x.id === raw.provider) ? (raw.provider as WebDataProviderId) : null; + return { provider, declined: raw.declined === true }; + } catch { + return { ...EMPTY }; + } +} + +export function setProvider(provider: WebDataProviderId | null, env: NodeJS.ProcessEnv = process.env): WebDataSelection { + // Choosing a provider clears a prior decline; clearing to null records one. + const next: WebDataSelection = { provider, declined: provider === null }; + const p = storePath(env); + mkdirSync(dirname(p), { recursive: true }); + const tmp = `${p}.tmp.${process.pid}`; + writeFileSync(tmp, JSON.stringify(next, null, 2), "utf-8"); + renameSync(tmp, p); + return next; +} + +// ---------- recommendation ---------- + +export interface Recommendation { + task: WebDataTask; + why: string; + /** Best available provider for this task, or null if none is set up. */ + best: (WebDataProvider & { detail: string }) | null; + /** Better-ranked providers that are NOT available, with how to set them up. */ + betterUnavailable: (WebDataProvider & { detail: string })[]; +} + +export function recommend(task: WebDataTask, env: NodeJS.ProcessEnv = process.env): Recommendation { + const ranking = TASK_RANKINGS[task]; + const avail = new Map(detectAvailability(env).map((a) => [a.id, a])); + const betterUnavailable: Recommendation["betterUnavailable"] = []; + let best: Recommendation["best"] = null; + for (const id of ranking.order) { + const provider = PROVIDERS.find((x) => x.id === id)!; + const a = avail.get(id)!; + if (a.available) { + best = { ...provider, detail: a.detail }; + break; + } + betterUnavailable.push({ ...provider, detail: a.detail }); + } + return { task, why: ranking.why, best, betterUnavailable }; +} diff --git a/runtime/install.js b/runtime/install.js index 382533990f..b2bfdd81be 100644 --- a/runtime/install.js +++ b/runtime/install.js @@ -129,6 +129,7 @@ export const DEFAULT_RUNTIME_HELPERS = Object.freeze({ "gstack-timeline-log": helper("bin/gstack-timeline-log"), "gstack-update-check": helper("bin/gstack-update-check"), "gstack-version-bump": helper("bin/gstack-version-bump"), + "gstack-web-data": helper("bin/gstack-web-data"), "remote-slug": helper("browse/bin/remote-slug"), }); @@ -160,6 +161,7 @@ const RUNTIME_HELPER_DEPENDENCIES = Object.freeze([ "lib/redact-engine.ts", "lib/redact-patterns.ts", "lib/staging-guard.ts", + "lib/web-data.ts", "scripts/archetypes.ts", "scripts/brain-cache-spec.ts", "scripts/one-way-doors.ts", diff --git a/scripts/gstack2/generate-skill-tree.ts b/scripts/gstack2/generate-skill-tree.ts index e2329ed343..5bf1a6d73e 100644 --- a/scripts/gstack2/generate-skill-tree.ts +++ b/scripts/gstack2/generate-skill-tree.ts @@ -261,7 +261,7 @@ Web context: 1. Infer the mode from product stage, surface, requested artifact, mutation authorization, evidence needs, and deployment state. Do not route by keyword alone. 2. Refine the public mode to the smallest applicable internal specialist set, then print the required execution header before any substantive output. 3. Read each active module in full from the path shown in the mode/alias tables. Its specialist body, behavioral contract, STOP gates, and appended upstream judgment ports are binding. Read a lazy specialist phase in full only when the workflow reaches its package-local reference. -4. Read \`references/EXECUTION-PROFILES.md\`, \`references/SHARED-JUDGMENT.md\`, and \`references/AUTHORITY-POLICY.md\` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read \`references/RUNTIME.md\` before capability-dependent work and \`references/WEB-CONTEXT.md\` before public-web work. When the target is a repository, read \`references/CODE-INTELLIGENCE.md\` once before substantive specialist work and follow its one-time indexing offer. Read \`references/THIRD-PARTY-ACTIONS.md\` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). +4. Read \`references/EXECUTION-PROFILES.md\`, \`references/SHARED-JUDGMENT.md\`, and \`references/AUTHORITY-POLICY.md\` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read \`references/RUNTIME.md\` before capability-dependent work and \`references/WEB-CONTEXT.md\` before public-web work. Read \`references/WEB-DATA.md\` before designing or performing bulk web-data acquisition (scraping, crawling, or building a dataset from web sources), whether the agent fetches directly or a plan designs the pipeline, and follow its one-time provider offer. When the target is a repository, read \`references/CODE-INTELLIGENCE.md\` once before substantive specialist work and follow its one-time indexing offer. Read \`references/THIRD-PARTY-ACTIONS.md\` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). 5. If an old asset path is unavailable, use \`references/ASSETS.md\`. If legacy prose invokes another retired skill, resolve it through \`references/COMPATIBILITY.md\` and stay inside these five dispatchers. 6. Preserve report-only versus mutation boundaries. Missing mutation authorization fails closed: do not edit merely because a specialist can fix. Commits, pushes, PRs, merges, deploys, messages, and other external mutations still require affirmative authority from the user. 7. Match the user's language. Keep code identifiers, commands, and source quotations original when translation would reduce accuracy. @@ -517,6 +517,32 @@ function webContextContract(): string { ].join('\n'); } +function webDataContract(): string { + return [GENERATED, + '# Optional web-data providers (scraping and crawling)', + '', + 'gstack works fully without a scraping provider: a hand-rolled fetcher or the local browser is always a valid plan and never depends on a vendor. A provider is an optional enhancement whenever a task must acquire web data at scale — whether the agent fetches it directly or a plan designs a data pipeline that will.', + '', + 'Before designing or performing bulk web-data acquisition (scraping a site, crawling many pages, building a dataset from web sources), run `"${GSTACK_HOME:-$HOME/.gstack}"/bin/gstack-web-data status --json 2>/dev/null || true` once. If the helper is unavailable, continue with the hand-rolled path silently. If a provider is already selected, use it for its supported tasks without re-asking. If `declined` is true, never re-offer.', + '', + 'When no choice is stored, first check whether the data source has an official API (for example MediaWiki for Wikipedia) — an official API beats every scraping provider and needs no consent question. Otherwise pause and ask via AskUserQuestion, presenting each provider with its availability, note, and setup hint from the status output. Recommend the best AVAILABLE provider for the task at hand using `gstack-web-data recommend ` (tasks: scrape, crawl, search, batch, hostile, authenticated). When a better-ranked provider is not set up, say so explicitly with its exact setup step (key to export, signup URL, or install), and recommend the best already-installed option for right now — never present an uninstalled provider as the working plan and never install anything yourself.', + '', + 'Per-task best order, measured in the 2026-07-23 bakeoff (7 live tasks; the helper applies availability on top of this):', + '', + '- **scrape** (single public page): Firecrawl, then Exa, Context.dev, Aside, local browser.', + '- **crawl** (many pages from one site): Firecrawl, then Context.dev, Exa (cached subpages, freshness caveat), Aside, local browser.', + '- **search** (find the URLs first): Exa, then Firecrawl, Aside\'s agent. Context.dev search is deprecated upstream and the local browser has none.', + '- **batch** (many known URLs): Firecrawl, then Exa, Context.dev, local browser. Browsers serialize — wrong tool past a handful of pages.', + '- **hostile** (anti-bot sites): Firecrawl, then Aside (the user\'s real browser session), Context.dev. The plain local headless browser gets bot-walled.', + '- **authenticated** (logged-in/account pages): Aside only. Never send authenticated pages, private addresses, or credentials to any API provider.', + '', + 'Egress rules are the Context.dev rules generalized: an off-machine provider (Firecrawl, Exa, Context.dev) may receive only public target URLs and operation parameters after explicit selection — never authenticated pages, localhost or private addresses, private repository content, prompts, user files, cookies, tokens, or credentials. Provider API keys live in the environment or the runtime secret store, never in chat, argv, project files, or version control. Browser options follow `references/BROWSER-PROVIDERS.md` for detection and readiness: confirm the `aside` CLI is callable rather than assuming, never install a browser or provider yourself, and never treat binary presence as consent.', + '', + 'Persist only the explicit choice with `gstack-web-data select `. Context.dev consent additionally belongs to `gstack context setup` per `references/WEB-CONTEXT.md`; selecting it here never grants that consent. Never infer a choice, never treat an unavailable provider as forbidden (the user may set it up later), and never block the task on this decision: if the user declines or setup is pending, proceed with the official API, hand-rolled fetcher, or local browser and say which fallback you used.', + '', + ].join('\n'); +} + function thirdPartyActionsContract(): string { return [GENERATED, '# Third-party web actions', @@ -692,6 +718,7 @@ function writeSharedContracts(): void { write(path.join(ROOT, 'skills', tree, 'references', 'SHARED-JUDGMENT.md'), sharedJudgmentContract()); write(path.join(ROOT, 'skills', tree, 'references', 'AUTHORITY-POLICY.md'), authorityPolicyContract()); write(path.join(ROOT, 'skills', tree, 'references', 'WEB-CONTEXT.md'), webContextContract()); + write(path.join(ROOT, 'skills', tree, 'references', 'WEB-DATA.md'), webDataContract()); write(path.join(ROOT, 'skills', tree, 'references', 'CODE-INTELLIGENCE.md'), codeIntelligenceContract()); write(path.join(ROOT, 'skills', tree, 'references', 'THIRD-PARTY-ACTIONS.md'), thirdPartyActionsContract()); write(path.join(ROOT, 'skills', tree, 'references', 'RUNTIME.md'), runtimeContract()); diff --git a/skills/debug/SKILL.md b/skills/debug/SKILL.md index 807cd0f41c..d7b7f64de1 100644 --- a/skills/debug/SKILL.md +++ b/skills/debug/SKILL.md @@ -27,7 +27,7 @@ Web context: 1. Infer the mode from product stage, surface, requested artifact, mutation authorization, evidence needs, and deployment state. Do not route by keyword alone. 2. Refine the public mode to the smallest applicable internal specialist set, then print the required execution header before any substantive output. 3. Read each active module in full from the path shown in the mode/alias tables. Its specialist body, behavioral contract, STOP gates, and appended upstream judgment ports are binding. Read a lazy specialist phase in full only when the workflow reaches its package-local reference. -4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). +4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. Read `references/WEB-DATA.md` before designing or performing bulk web-data acquisition (scraping, crawling, or building a dataset from web sources), whether the agent fetches directly or a plan designs the pipeline, and follow its one-time provider offer. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). 5. If an old asset path is unavailable, use `references/ASSETS.md`. If legacy prose invokes another retired skill, resolve it through `references/COMPATIBILITY.md` and stay inside these five dispatchers. 6. Preserve report-only versus mutation boundaries. Missing mutation authorization fails closed: do not edit merely because a specialist can fix. Commits, pushes, PRs, merges, deploys, messages, and other external mutations still require affirmative authority from the user. 7. Match the user's language. Keep code identifiers, commands, and source quotations original when translation would reduce accuracy. diff --git a/skills/debug/references/WEB-DATA.md b/skills/debug/references/WEB-DATA.md new file mode 100644 index 0000000000..9cbe312b18 --- /dev/null +++ b/skills/debug/references/WEB-DATA.md @@ -0,0 +1,21 @@ + +# Optional web-data providers (scraping and crawling) + +gstack works fully without a scraping provider: a hand-rolled fetcher or the local browser is always a valid plan and never depends on a vendor. A provider is an optional enhancement whenever a task must acquire web data at scale — whether the agent fetches it directly or a plan designs a data pipeline that will. + +Before designing or performing bulk web-data acquisition (scraping a site, crawling many pages, building a dataset from web sources), run `"${GSTACK_HOME:-$HOME/.gstack}"/bin/gstack-web-data status --json 2>/dev/null || true` once. If the helper is unavailable, continue with the hand-rolled path silently. If a provider is already selected, use it for its supported tasks without re-asking. If `declined` is true, never re-offer. + +When no choice is stored, first check whether the data source has an official API (for example MediaWiki for Wikipedia) — an official API beats every scraping provider and needs no consent question. Otherwise pause and ask via AskUserQuestion, presenting each provider with its availability, note, and setup hint from the status output. Recommend the best AVAILABLE provider for the task at hand using `gstack-web-data recommend ` (tasks: scrape, crawl, search, batch, hostile, authenticated). When a better-ranked provider is not set up, say so explicitly with its exact setup step (key to export, signup URL, or install), and recommend the best already-installed option for right now — never present an uninstalled provider as the working plan and never install anything yourself. + +Per-task best order, measured in the 2026-07-23 bakeoff (7 live tasks; the helper applies availability on top of this): + +- **scrape** (single public page): Firecrawl, then Exa, Context.dev, Aside, local browser. +- **crawl** (many pages from one site): Firecrawl, then Context.dev, Exa (cached subpages, freshness caveat), Aside, local browser. +- **search** (find the URLs first): Exa, then Firecrawl, Aside's agent. Context.dev search is deprecated upstream and the local browser has none. +- **batch** (many known URLs): Firecrawl, then Exa, Context.dev, local browser. Browsers serialize — wrong tool past a handful of pages. +- **hostile** (anti-bot sites): Firecrawl, then Aside (the user's real browser session), Context.dev. The plain local headless browser gets bot-walled. +- **authenticated** (logged-in/account pages): Aside only. Never send authenticated pages, private addresses, or credentials to any API provider. + +Egress rules are the Context.dev rules generalized: an off-machine provider (Firecrawl, Exa, Context.dev) may receive only public target URLs and operation parameters after explicit selection — never authenticated pages, localhost or private addresses, private repository content, prompts, user files, cookies, tokens, or credentials. Provider API keys live in the environment or the runtime secret store, never in chat, argv, project files, or version control. Browser options follow `references/BROWSER-PROVIDERS.md` for detection and readiness: confirm the `aside` CLI is callable rather than assuming, never install a browser or provider yourself, and never treat binary presence as consent. + +Persist only the explicit choice with `gstack-web-data select `. Context.dev consent additionally belongs to `gstack context setup` per `references/WEB-CONTEXT.md`; selecting it here never grants that consent. Never infer a choice, never treat an unavailable provider as forbidden (the user may set it up later), and never block the task on this decision: if the user declines or setup is pending, proceed with the official API, hand-rolled fetcher, or local browser and say which fallback you used. diff --git a/skills/plan/SKILL.md b/skills/plan/SKILL.md index 60229d5747..9f78adfb52 100644 --- a/skills/plan/SKILL.md +++ b/skills/plan/SKILL.md @@ -28,7 +28,7 @@ Web context: 1. Infer the mode from product stage, surface, requested artifact, mutation authorization, evidence needs, and deployment state. Do not route by keyword alone. 2. Refine the public mode to the smallest applicable internal specialist set, then print the required execution header before any substantive output. 3. Read each active module in full from the path shown in the mode/alias tables. Its specialist body, behavioral contract, STOP gates, and appended upstream judgment ports are binding. Read a lazy specialist phase in full only when the workflow reaches its package-local reference. -4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). +4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. Read `references/WEB-DATA.md` before designing or performing bulk web-data acquisition (scraping, crawling, or building a dataset from web sources), whether the agent fetches directly or a plan designs the pipeline, and follow its one-time provider offer. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). 5. If an old asset path is unavailable, use `references/ASSETS.md`. If legacy prose invokes another retired skill, resolve it through `references/COMPATIBILITY.md` and stay inside these five dispatchers. 6. Preserve report-only versus mutation boundaries. Missing mutation authorization fails closed: do not edit merely because a specialist can fix. Commits, pushes, PRs, merges, deploys, messages, and other external mutations still require affirmative authority from the user. 7. Match the user's language. Keep code identifiers, commands, and source quotations original when translation would reduce accuracy. diff --git a/skills/plan/references/WEB-DATA.md b/skills/plan/references/WEB-DATA.md new file mode 100644 index 0000000000..9cbe312b18 --- /dev/null +++ b/skills/plan/references/WEB-DATA.md @@ -0,0 +1,21 @@ + +# Optional web-data providers (scraping and crawling) + +gstack works fully without a scraping provider: a hand-rolled fetcher or the local browser is always a valid plan and never depends on a vendor. A provider is an optional enhancement whenever a task must acquire web data at scale — whether the agent fetches it directly or a plan designs a data pipeline that will. + +Before designing or performing bulk web-data acquisition (scraping a site, crawling many pages, building a dataset from web sources), run `"${GSTACK_HOME:-$HOME/.gstack}"/bin/gstack-web-data status --json 2>/dev/null || true` once. If the helper is unavailable, continue with the hand-rolled path silently. If a provider is already selected, use it for its supported tasks without re-asking. If `declined` is true, never re-offer. + +When no choice is stored, first check whether the data source has an official API (for example MediaWiki for Wikipedia) — an official API beats every scraping provider and needs no consent question. Otherwise pause and ask via AskUserQuestion, presenting each provider with its availability, note, and setup hint from the status output. Recommend the best AVAILABLE provider for the task at hand using `gstack-web-data recommend ` (tasks: scrape, crawl, search, batch, hostile, authenticated). When a better-ranked provider is not set up, say so explicitly with its exact setup step (key to export, signup URL, or install), and recommend the best already-installed option for right now — never present an uninstalled provider as the working plan and never install anything yourself. + +Per-task best order, measured in the 2026-07-23 bakeoff (7 live tasks; the helper applies availability on top of this): + +- **scrape** (single public page): Firecrawl, then Exa, Context.dev, Aside, local browser. +- **crawl** (many pages from one site): Firecrawl, then Context.dev, Exa (cached subpages, freshness caveat), Aside, local browser. +- **search** (find the URLs first): Exa, then Firecrawl, Aside's agent. Context.dev search is deprecated upstream and the local browser has none. +- **batch** (many known URLs): Firecrawl, then Exa, Context.dev, local browser. Browsers serialize — wrong tool past a handful of pages. +- **hostile** (anti-bot sites): Firecrawl, then Aside (the user's real browser session), Context.dev. The plain local headless browser gets bot-walled. +- **authenticated** (logged-in/account pages): Aside only. Never send authenticated pages, private addresses, or credentials to any API provider. + +Egress rules are the Context.dev rules generalized: an off-machine provider (Firecrawl, Exa, Context.dev) may receive only public target URLs and operation parameters after explicit selection — never authenticated pages, localhost or private addresses, private repository content, prompts, user files, cookies, tokens, or credentials. Provider API keys live in the environment or the runtime secret store, never in chat, argv, project files, or version control. Browser options follow `references/BROWSER-PROVIDERS.md` for detection and readiness: confirm the `aside` CLI is callable rather than assuming, never install a browser or provider yourself, and never treat binary presence as consent. + +Persist only the explicit choice with `gstack-web-data select `. Context.dev consent additionally belongs to `gstack context setup` per `references/WEB-CONTEXT.md`; selecting it here never grants that consent. Never infer a choice, never treat an unavailable provider as forbidden (the user may set it up later), and never block the task on this decision: if the user declines or setup is pending, proceed with the official API, hand-rolled fetcher, or local browser and say which fallback you used. diff --git a/skills/qa/SKILL.md b/skills/qa/SKILL.md index cf9ce8b049..c8481c2a67 100644 --- a/skills/qa/SKILL.md +++ b/skills/qa/SKILL.md @@ -27,7 +27,7 @@ Web context: 1. Infer the mode from product stage, surface, requested artifact, mutation authorization, evidence needs, and deployment state. Do not route by keyword alone. 2. Refine the public mode to the smallest applicable internal specialist set, then print the required execution header before any substantive output. 3. Read each active module in full from the path shown in the mode/alias tables. Its specialist body, behavioral contract, STOP gates, and appended upstream judgment ports are binding. Read a lazy specialist phase in full only when the workflow reaches its package-local reference. -4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). +4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. Read `references/WEB-DATA.md` before designing or performing bulk web-data acquisition (scraping, crawling, or building a dataset from web sources), whether the agent fetches directly or a plan designs the pipeline, and follow its one-time provider offer. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). 5. If an old asset path is unavailable, use `references/ASSETS.md`. If legacy prose invokes another retired skill, resolve it through `references/COMPATIBILITY.md` and stay inside these five dispatchers. 6. Preserve report-only versus mutation boundaries. Missing mutation authorization fails closed: do not edit merely because a specialist can fix. Commits, pushes, PRs, merges, deploys, messages, and other external mutations still require affirmative authority from the user. 7. Match the user's language. Keep code identifiers, commands, and source quotations original when translation would reduce accuracy. diff --git a/skills/qa/references/WEB-DATA.md b/skills/qa/references/WEB-DATA.md new file mode 100644 index 0000000000..9cbe312b18 --- /dev/null +++ b/skills/qa/references/WEB-DATA.md @@ -0,0 +1,21 @@ + +# Optional web-data providers (scraping and crawling) + +gstack works fully without a scraping provider: a hand-rolled fetcher or the local browser is always a valid plan and never depends on a vendor. A provider is an optional enhancement whenever a task must acquire web data at scale — whether the agent fetches it directly or a plan designs a data pipeline that will. + +Before designing or performing bulk web-data acquisition (scraping a site, crawling many pages, building a dataset from web sources), run `"${GSTACK_HOME:-$HOME/.gstack}"/bin/gstack-web-data status --json 2>/dev/null || true` once. If the helper is unavailable, continue with the hand-rolled path silently. If a provider is already selected, use it for its supported tasks without re-asking. If `declined` is true, never re-offer. + +When no choice is stored, first check whether the data source has an official API (for example MediaWiki for Wikipedia) — an official API beats every scraping provider and needs no consent question. Otherwise pause and ask via AskUserQuestion, presenting each provider with its availability, note, and setup hint from the status output. Recommend the best AVAILABLE provider for the task at hand using `gstack-web-data recommend ` (tasks: scrape, crawl, search, batch, hostile, authenticated). When a better-ranked provider is not set up, say so explicitly with its exact setup step (key to export, signup URL, or install), and recommend the best already-installed option for right now — never present an uninstalled provider as the working plan and never install anything yourself. + +Per-task best order, measured in the 2026-07-23 bakeoff (7 live tasks; the helper applies availability on top of this): + +- **scrape** (single public page): Firecrawl, then Exa, Context.dev, Aside, local browser. +- **crawl** (many pages from one site): Firecrawl, then Context.dev, Exa (cached subpages, freshness caveat), Aside, local browser. +- **search** (find the URLs first): Exa, then Firecrawl, Aside's agent. Context.dev search is deprecated upstream and the local browser has none. +- **batch** (many known URLs): Firecrawl, then Exa, Context.dev, local browser. Browsers serialize — wrong tool past a handful of pages. +- **hostile** (anti-bot sites): Firecrawl, then Aside (the user's real browser session), Context.dev. The plain local headless browser gets bot-walled. +- **authenticated** (logged-in/account pages): Aside only. Never send authenticated pages, private addresses, or credentials to any API provider. + +Egress rules are the Context.dev rules generalized: an off-machine provider (Firecrawl, Exa, Context.dev) may receive only public target URLs and operation parameters after explicit selection — never authenticated pages, localhost or private addresses, private repository content, prompts, user files, cookies, tokens, or credentials. Provider API keys live in the environment or the runtime secret store, never in chat, argv, project files, or version control. Browser options follow `references/BROWSER-PROVIDERS.md` for detection and readiness: confirm the `aside` CLI is callable rather than assuming, never install a browser or provider yourself, and never treat binary presence as consent. + +Persist only the explicit choice with `gstack-web-data select `. Context.dev consent additionally belongs to `gstack context setup` per `references/WEB-CONTEXT.md`; selecting it here never grants that consent. Never infer a choice, never treat an unavailable provider as forbidden (the user may set it up later), and never block the task on this decision: if the user declines or setup is pending, proceed with the official API, hand-rolled fetcher, or local browser and say which fallback you used. diff --git a/skills/review/SKILL.md b/skills/review/SKILL.md index 2bad3268dd..f7203b0702 100644 --- a/skills/review/SKILL.md +++ b/skills/review/SKILL.md @@ -27,7 +27,7 @@ Web context: 1. Infer the mode from product stage, surface, requested artifact, mutation authorization, evidence needs, and deployment state. Do not route by keyword alone. 2. Refine the public mode to the smallest applicable internal specialist set, then print the required execution header before any substantive output. 3. Read each active module in full from the path shown in the mode/alias tables. Its specialist body, behavioral contract, STOP gates, and appended upstream judgment ports are binding. Read a lazy specialist phase in full only when the workflow reaches its package-local reference. -4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). +4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. Read `references/WEB-DATA.md` before designing or performing bulk web-data acquisition (scraping, crawling, or building a dataset from web sources), whether the agent fetches directly or a plan designs the pipeline, and follow its one-time provider offer. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). 5. If an old asset path is unavailable, use `references/ASSETS.md`. If legacy prose invokes another retired skill, resolve it through `references/COMPATIBILITY.md` and stay inside these five dispatchers. 6. Preserve report-only versus mutation boundaries. Missing mutation authorization fails closed: do not edit merely because a specialist can fix. Commits, pushes, PRs, merges, deploys, messages, and other external mutations still require affirmative authority from the user. 7. Match the user's language. Keep code identifiers, commands, and source quotations original when translation would reduce accuracy. diff --git a/skills/review/references/WEB-DATA.md b/skills/review/references/WEB-DATA.md new file mode 100644 index 0000000000..9cbe312b18 --- /dev/null +++ b/skills/review/references/WEB-DATA.md @@ -0,0 +1,21 @@ + +# Optional web-data providers (scraping and crawling) + +gstack works fully without a scraping provider: a hand-rolled fetcher or the local browser is always a valid plan and never depends on a vendor. A provider is an optional enhancement whenever a task must acquire web data at scale — whether the agent fetches it directly or a plan designs a data pipeline that will. + +Before designing or performing bulk web-data acquisition (scraping a site, crawling many pages, building a dataset from web sources), run `"${GSTACK_HOME:-$HOME/.gstack}"/bin/gstack-web-data status --json 2>/dev/null || true` once. If the helper is unavailable, continue with the hand-rolled path silently. If a provider is already selected, use it for its supported tasks without re-asking. If `declined` is true, never re-offer. + +When no choice is stored, first check whether the data source has an official API (for example MediaWiki for Wikipedia) — an official API beats every scraping provider and needs no consent question. Otherwise pause and ask via AskUserQuestion, presenting each provider with its availability, note, and setup hint from the status output. Recommend the best AVAILABLE provider for the task at hand using `gstack-web-data recommend ` (tasks: scrape, crawl, search, batch, hostile, authenticated). When a better-ranked provider is not set up, say so explicitly with its exact setup step (key to export, signup URL, or install), and recommend the best already-installed option for right now — never present an uninstalled provider as the working plan and never install anything yourself. + +Per-task best order, measured in the 2026-07-23 bakeoff (7 live tasks; the helper applies availability on top of this): + +- **scrape** (single public page): Firecrawl, then Exa, Context.dev, Aside, local browser. +- **crawl** (many pages from one site): Firecrawl, then Context.dev, Exa (cached subpages, freshness caveat), Aside, local browser. +- **search** (find the URLs first): Exa, then Firecrawl, Aside's agent. Context.dev search is deprecated upstream and the local browser has none. +- **batch** (many known URLs): Firecrawl, then Exa, Context.dev, local browser. Browsers serialize — wrong tool past a handful of pages. +- **hostile** (anti-bot sites): Firecrawl, then Aside (the user's real browser session), Context.dev. The plain local headless browser gets bot-walled. +- **authenticated** (logged-in/account pages): Aside only. Never send authenticated pages, private addresses, or credentials to any API provider. + +Egress rules are the Context.dev rules generalized: an off-machine provider (Firecrawl, Exa, Context.dev) may receive only public target URLs and operation parameters after explicit selection — never authenticated pages, localhost or private addresses, private repository content, prompts, user files, cookies, tokens, or credentials. Provider API keys live in the environment or the runtime secret store, never in chat, argv, project files, or version control. Browser options follow `references/BROWSER-PROVIDERS.md` for detection and readiness: confirm the `aside` CLI is callable rather than assuming, never install a browser or provider yourself, and never treat binary presence as consent. + +Persist only the explicit choice with `gstack-web-data select `. Context.dev consent additionally belongs to `gstack context setup` per `references/WEB-CONTEXT.md`; selecting it here never grants that consent. Never infer a choice, never treat an unavailable provider as forbidden (the user may set it up later), and never block the task on this decision: if the user declines or setup is pending, proceed with the official API, hand-rolled fetcher, or local browser and say which fallback you used. diff --git a/skills/ship/SKILL.md b/skills/ship/SKILL.md index 8a2bf962d6..4bb4d34bde 100644 --- a/skills/ship/SKILL.md +++ b/skills/ship/SKILL.md @@ -27,7 +27,7 @@ Web context: 1. Infer the mode from product stage, surface, requested artifact, mutation authorization, evidence needs, and deployment state. Do not route by keyword alone. 2. Refine the public mode to the smallest applicable internal specialist set, then print the required execution header before any substantive output. 3. Read each active module in full from the path shown in the mode/alias tables. Its specialist body, behavioral contract, STOP gates, and appended upstream judgment ports are binding. Read a lazy specialist phase in full only when the workflow reaches its package-local reference. -4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). +4. Read `references/EXECUTION-PROFILES.md`, `references/SHARED-JUDGMENT.md`, and `references/AUTHORITY-POLICY.md` for every invocation. Infer Depth from structured operating conditions, then obey its mandatory modules, legal skips, artifacts, and claim limits. Read `references/RUNTIME.md` before capability-dependent work and `references/WEB-CONTEXT.md` before public-web work. Read `references/WEB-DATA.md` before designing or performing bulk web-data acquisition (scraping, crawling, or building a dataset from web sources), whether the agent fetches directly or a plan designs the pipeline, and follow its one-time provider offer. When the target is a repository, read `references/CODE-INTELLIGENCE.md` once before substantive specialist work and follow its one-time indexing offer. Read `references/THIRD-PARTY-ACTIONS.md` before directing the user to act on a third-party website (API key registration, vendor accounts, dashboards). 5. If an old asset path is unavailable, use `references/ASSETS.md`. If legacy prose invokes another retired skill, resolve it through `references/COMPATIBILITY.md` and stay inside these five dispatchers. 6. Preserve report-only versus mutation boundaries. Missing mutation authorization fails closed: do not edit merely because a specialist can fix. Commits, pushes, PRs, merges, deploys, messages, and other external mutations still require affirmative authority from the user. 7. Match the user's language. Keep code identifiers, commands, and source quotations original when translation would reduce accuracy. diff --git a/skills/ship/references/WEB-DATA.md b/skills/ship/references/WEB-DATA.md new file mode 100644 index 0000000000..9cbe312b18 --- /dev/null +++ b/skills/ship/references/WEB-DATA.md @@ -0,0 +1,21 @@ + +# Optional web-data providers (scraping and crawling) + +gstack works fully without a scraping provider: a hand-rolled fetcher or the local browser is always a valid plan and never depends on a vendor. A provider is an optional enhancement whenever a task must acquire web data at scale — whether the agent fetches it directly or a plan designs a data pipeline that will. + +Before designing or performing bulk web-data acquisition (scraping a site, crawling many pages, building a dataset from web sources), run `"${GSTACK_HOME:-$HOME/.gstack}"/bin/gstack-web-data status --json 2>/dev/null || true` once. If the helper is unavailable, continue with the hand-rolled path silently. If a provider is already selected, use it for its supported tasks without re-asking. If `declined` is true, never re-offer. + +When no choice is stored, first check whether the data source has an official API (for example MediaWiki for Wikipedia) — an official API beats every scraping provider and needs no consent question. Otherwise pause and ask via AskUserQuestion, presenting each provider with its availability, note, and setup hint from the status output. Recommend the best AVAILABLE provider for the task at hand using `gstack-web-data recommend ` (tasks: scrape, crawl, search, batch, hostile, authenticated). When a better-ranked provider is not set up, say so explicitly with its exact setup step (key to export, signup URL, or install), and recommend the best already-installed option for right now — never present an uninstalled provider as the working plan and never install anything yourself. + +Per-task best order, measured in the 2026-07-23 bakeoff (7 live tasks; the helper applies availability on top of this): + +- **scrape** (single public page): Firecrawl, then Exa, Context.dev, Aside, local browser. +- **crawl** (many pages from one site): Firecrawl, then Context.dev, Exa (cached subpages, freshness caveat), Aside, local browser. +- **search** (find the URLs first): Exa, then Firecrawl, Aside's agent. Context.dev search is deprecated upstream and the local browser has none. +- **batch** (many known URLs): Firecrawl, then Exa, Context.dev, local browser. Browsers serialize — wrong tool past a handful of pages. +- **hostile** (anti-bot sites): Firecrawl, then Aside (the user's real browser session), Context.dev. The plain local headless browser gets bot-walled. +- **authenticated** (logged-in/account pages): Aside only. Never send authenticated pages, private addresses, or credentials to any API provider. + +Egress rules are the Context.dev rules generalized: an off-machine provider (Firecrawl, Exa, Context.dev) may receive only public target URLs and operation parameters after explicit selection — never authenticated pages, localhost or private addresses, private repository content, prompts, user files, cookies, tokens, or credentials. Provider API keys live in the environment or the runtime secret store, never in chat, argv, project files, or version control. Browser options follow `references/BROWSER-PROVIDERS.md` for detection and readiness: confirm the `aside` CLI is callable rather than assuming, never install a browser or provider yourself, and never treat binary presence as consent. + +Persist only the explicit choice with `gstack-web-data select `. Context.dev consent additionally belongs to `gstack context setup` per `references/WEB-CONTEXT.md`; selecting it here never grants that consent. Never infer a choice, never treat an unavailable provider as forbidden (the user may set it up later), and never block the task on this decision: if the user declines or setup is pending, proceed with the official API, hand-rolled fetcher, or local browser and say which fallback you used. diff --git a/test/web-data.test.ts b/test/web-data.test.ts new file mode 100644 index 0000000000..96b6497146 --- /dev/null +++ b/test/web-data.test.ts @@ -0,0 +1,127 @@ +/** + * web-data provider selection + recommendation. Free tier: no network, no + * provider processes — availability is driven entirely by env/fs fixtures. + */ + +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { + PROVIDERS, + TASK_IDS, + TASK_RANKINGS, + detectAvailability, + readSelection, + recommend, + setProvider, +} from "../lib/web-data"; + +let home: string; +let env: NodeJS.ProcessEnv; + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "web-data-test-")); + // No inherited keys: availability starts from zero. + env = { HOME: home, GSTACK_HOME: home, PATH: "/nonexistent" }; +}); + +afterEach(() => { + rmSync(home, { recursive: true, force: true }); +}); + +describe("task rankings", () => { + test("every task has a ranking and every ranked id is a real provider", () => { + const ids = new Set(PROVIDERS.map((p) => p.id)); + for (const task of TASK_IDS) { + const r = TASK_RANKINGS[task]; + expect(r.order.length).toBeGreaterThan(0); + for (const id of r.order) expect(ids.has(id)).toBe(true); + } + }); + + test("authenticated pages are aside-only — no API provider is ever ranked", () => { + expect(TASK_RANKINGS.authenticated.order).toEqual(["aside"]); + }); + + test("local browser is never ranked for hostile sites (bot-walled in bakeoff)", () => { + expect(TASK_RANKINGS.hostile.order).not.toContain("local-browser"); + }); +}); + +describe("availability detection", () => { + test("nothing configured → nothing available", () => { + for (const a of detectAvailability(env)) expect(a.available).toBe(false); + }); + + test("env keys flip firecrawl/exa", () => { + const avail = detectAvailability({ ...env, FIRECRAWL_API_KEY: "fc-x", EXA_API_KEY: "ex-x" }); + const byId = new Map(avail.map((a) => [a.id, a.available])); + expect(byId.get("firecrawl")).toBe(true); + expect(byId.get("exa")).toBe(true); + expect(byId.get("context")).toBe(false); + }); + + test("context requires mode=context AND consent=true in runtime config", () => { + writeFileSync(join(home, "config.json"), JSON.stringify({ network: { mode: "context", consent: false } })); + expect(detectAvailability(env).find((a) => a.id === "context")!.available).toBe(false); + writeFileSync(join(home, "config.json"), JSON.stringify({ network: { mode: "context", consent: true } })); + expect(detectAvailability(env).find((a) => a.id === "context")!.available).toBe(true); + }); + + test("local-browser detects the installed browse binary", () => { + mkdirSync(join(home, "bin"), { recursive: true }); + writeFileSync(join(home, "bin", "browse"), ""); + expect(detectAvailability(env).find((a) => a.id === "local-browser")!.available).toBe(true); + }); +}); + +describe("selection store", () => { + test("no selection = provider-OFF default, not declined", () => { + expect(readSelection(env)).toEqual({ provider: null, declined: false }); + }); + + test("select persists; select none records the decline; reselect clears it", () => { + setProvider("firecrawl", env); + expect(readSelection(env)).toEqual({ provider: "firecrawl", declined: false }); + setProvider(null, env); + expect(readSelection(env)).toEqual({ provider: null, declined: true }); + setProvider("exa", env); + expect(readSelection(env).declined).toBe(false); + }); + + test("corrupt or unknown stored provider degrades to OFF", () => { + writeFileSync(join(home, "web-data.json"), '{"provider":"skynet","declined":false}'); + expect(readSelection(env).provider).toBeNull(); + writeFileSync(join(home, "web-data.json"), "not json"); + expect(readSelection(env)).toEqual({ provider: null, declined: false }); + }); +}); + +describe("recommend: best AVAILABLE wins, better-unavailable carries setup hints", () => { + test("firecrawl key present → firecrawl recommended for scrape", () => { + const r = recommend("scrape", { ...env, FIRECRAWL_API_KEY: "fc-x" }); + expect(r.best?.id).toBe("firecrawl"); + expect(r.betterUnavailable).toEqual([]); + }); + + test("only exa available → exa recommended for scrape, firecrawl listed as better with its setup hint", () => { + const r = recommend("scrape", { ...env, EXA_API_KEY: "ex-x" }); + expect(r.best?.id).toBe("exa"); + expect(r.betterUnavailable.map((u) => u.id)).toEqual(["firecrawl"]); + expect(r.betterUnavailable[0].setupHint).toContain("FIRECRAWL_API_KEY"); + }); + + test("nothing available → best is null and every ranked provider carries a setup hint", () => { + const r = recommend("search", env); + expect(r.best).toBeNull(); + expect(r.betterUnavailable.map((u) => u.id)).toEqual(TASK_RANKINGS.search.order); + for (const u of r.betterUnavailable) expect(u.setupHint.length).toBeGreaterThan(0); + }); + + test("authenticated task never falls back to an API provider even when keys exist", () => { + const r = recommend("authenticated", { ...env, FIRECRAWL_API_KEY: "fc-x", EXA_API_KEY: "ex-x" }); + expect(r.best).toBeNull(); + expect(r.betterUnavailable.map((u) => u.id)).toEqual(["aside"]); + }); +});