feat: migrate scrapegraph-js from github pin to npm v2.1.0

.github/workflows/ci.yml

@@ -14,10 +14,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v2
       - run: bun install
-      # scrapegraph-js is pinned to a GitHub commit (PR #13 head, not yet on npm)
-      # and ships without a prebuilt dist/ — build it in-place so module
-      # resolution works.
-      - run: cd node_modules/scrapegraph-js && bun install && bun run build
       - run: bun test
 
   lint:
@@ -27,5 +23,4 @@ jobs:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v2
       - run: bun install
-      - run: cd node_modules/scrapegraph-js && bun install && bun run build
       - run: bun run check

package.json

@@ -28,7 +28,7 @@
 		"chalk": "^5.4.1",
 		"citty": "^0.1.6",
 		"dotenv": "^17.2.4",
-		"scrapegraph-js": "github:ScrapeGraphAI/scrapegraph-js#096c110"
+		"scrapegraph-js": "^2.1.0"
 	},
 	"devDependencies": {
 		"@biomejs/biome": "^1.9.4",

src/commands/crawl.ts

@@ -1,27 +1,13 @@
 import { defineCommand } from "citty";
 import { crawl } from "scrapegraph-js";
-import type { ApiCrawlRequest, ApiScrapeFormatEntry } from "scrapegraph-js";
+import type { CrawlRequest, FetchConfig } from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
+import { BASE_FORMATS, type BaseFormat, buildBaseFormat } from "../lib/formats.js";
 import * as log from "../lib/log.js";
-
-const FORMATS = [
-	"markdown",
-	"html",
-	"screenshot",
-	"branding",
-	"links",
-	"images",
-	"summary",
-] as const;
-type Format = (typeof FORMATS)[number];
+import { parseIntArg, parseJsonArg } from "../lib/parse.js";
 
 const POLL_INTERVAL_MS = 3000;
 
-function buildFormat(f: Format): ApiScrapeFormatEntry {
-	if (f === "markdown" || f === "html") return { type: f, mode: "normal" };
-	return { type: f } as ApiScrapeFormatEntry;
-}
-
 export default defineCommand({
 	meta: {
 		name: "crawl",
@@ -36,7 +22,7 @@ export default defineCommand({
 		format: {
 			type: "string",
 			alias: "f",
-			description: `Per-page format(s), comma-separated: ${FORMATS.join(", ")} (default: markdown)`,
+			description: `Per-page format(s), comma-separated: ${BASE_FORMATS.join(", ")} (default: markdown)`,
 		},
 		"max-pages": { type: "string", description: "Maximum pages to crawl (default 50, max 1000)" },
 		"max-depth": { type: "string", description: "Crawl depth (default 2)" },
@@ -64,24 +50,40 @@ export default defineCommand({
 			.map((f) => f.trim())
 			.filter(Boolean);
 		for (const f of requested) {
-			if (!FORMATS.includes(f as Format))
-				out.error(`Unknown format: ${f}. Valid: ${FORMATS.join(", ")}`);
+			if (!BASE_FORMATS.includes(f as BaseFormat))
+				out.error(`Unknown format: ${f}. Valid: ${BASE_FORMATS.join(", ")}`);
 		}
-		const formats = requested.map((f) => buildFormat(f as Format));
-
-		const params: ApiCrawlRequest = { url: args.url, formats };
-		const mut = params as Record<string, unknown>;
-		if (args["max-pages"]) mut.maxPages = Number(args["max-pages"]);
-		if (args["max-depth"]) mut.maxDepth = Number(args["max-depth"]);
-		if (args["max-links-per-page"]) mut.maxLinksPerPage = Number(args["max-links-per-page"]);
-		if (args["allow-external"]) mut.allowExternal = true;
-		if (args["include-patterns"]) mut.includePatterns = JSON.parse(args["include-patterns"]);
-		if (args["exclude-patterns"]) mut.excludePatterns = JSON.parse(args["exclude-patterns"]);
+		const formats = requested.map((f) => buildBaseFormat(f as BaseFormat));
 
 		const fetchConfig: Record<string, unknown> = {};
 		if (args.mode) fetchConfig.mode = args.mode;
 		if (args.stealth) fetchConfig.stealth = true;
-		if (Object.keys(fetchConfig).length > 0) mut.fetchConfig = fetchConfig;
+
+		const params: CrawlRequest = {
+			url: args.url,
+			formats,
+			...(args["max-pages"] && { maxPages: parseIntArg(args["max-pages"], "max-pages", out) }),
+			...(args["max-depth"] && { maxDepth: parseIntArg(args["max-depth"], "max-depth", out) }),
+			...(args["max-links-per-page"] && {
+				maxLinksPerPage: parseIntArg(args["max-links-per-page"], "max-links-per-page", out),
+			}),
+			...(args["allow-external"] && { allowExternal: true }),
+			...(args["include-patterns"] && {
+				includePatterns: parseJsonArg(
+					args["include-patterns"],
+					"include-patterns",
+					out,
+				) as string[],
+			}),
+			...(args["exclude-patterns"] && {
+				excludePatterns: parseJsonArg(
+					args["exclude-patterns"],
+					"exclude-patterns",
+					out,
+				) as string[],
+			}),
+			...(Object.keys(fetchConfig).length > 0 && { fetchConfig: fetchConfig as FetchConfig }),
+		};
 
 		out.start("Starting crawl");
 		const job = await crawl.start(apiKey, params);

src/commands/extract.ts

@@ -1,8 +1,9 @@
 import { defineCommand } from "citty";
 import { extract } from "scrapegraph-js";
-import type { ApiExtractRequest } from "scrapegraph-js";
+import type { ExtractRequest, FetchConfig } from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
+import { parseIntArg, parseJsonArg } from "../lib/parse.js";
 
 export default defineCommand({
 	meta: {
@@ -42,16 +43,20 @@ export default defineCommand({
 		const fetchConfig: Record<string, unknown> = {};
 		if (args.mode) fetchConfig.mode = args.mode;
 		if (args.stealth) fetchConfig.stealth = true;
-		if (args.scrolls) fetchConfig.scrolls = Number(args.scrolls);
-		if (args.cookies) fetchConfig.cookies = JSON.parse(args.cookies);
-		if (args.headers) fetchConfig.headers = JSON.parse(args.headers);
+		if (args.scrolls) fetchConfig.scrolls = parseIntArg(args.scrolls, "scrolls", out);
+		if (args.cookies) fetchConfig.cookies = parseJsonArg(args.cookies, "cookies", out);
+		if (args.headers) fetchConfig.headers = parseJsonArg(args.headers, "headers", out);
 		if (args.country) fetchConfig.country = args.country;
 
-		const params: ApiExtractRequest = { url: args.url, prompt: args.prompt };
-		if (args.schema) (params as Record<string, unknown>).schema = JSON.parse(args.schema);
-		if (args["html-mode"]) (params as Record<string, unknown>).mode = args["html-mode"];
-		if (Object.keys(fetchConfig).length > 0)
-			(params as Record<string, unknown>).fetchConfig = fetchConfig;
+		const params: ExtractRequest = {
+			url: args.url,
+			prompt: args.prompt,
+			...(args.schema && {
+				schema: parseJsonArg(args.schema, "schema", out) as Record<string, unknown>,
+			}),
+			...(args["html-mode"] && { mode: args["html-mode"] as "normal" | "reader" | "prune" }),
+			...(Object.keys(fetchConfig).length > 0 && { fetchConfig: fetchConfig as FetchConfig }),
+		};
 
 		out.start("Extracting");
 		const result = await extract(apiKey, params);

src/commands/history.ts

@@ -2,20 +2,21 @@ import * as p from "@clack/prompts";
 import chalk from "chalk";
 import { defineCommand } from "citty";
 import { history } from "scrapegraph-js";
-import type { ApiHistoryEntry, ApiHistoryService } from "scrapegraph-js";
+import type { HistoryEntry, Service } from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
+import { parseIntArg } from "../lib/parse.js";
 
 const SERVICES = ["scrape", "extract", "search", "monitor", "crawl"] as const;
 const VALID = SERVICES.join(", ");
 const LOAD_MORE = "__load_more__";
 
-function entryUrl(e: ApiHistoryEntry): string {
+function entryUrl(e: HistoryEntry): string {
 	const params = e.params as Record<string, unknown>;
 	return String(params.url ?? params.query ?? "");
 }
 
-function entryLabel(e: ApiHistoryEntry): string {
+function entryLabel(e: HistoryEntry): string {
 	const short = e.id.length > 12 ? `${e.id.slice(0, 12)}…` : e.id;
 	const url = entryUrl(e);
 	const urlShort = url.length > 50 ? `${url.slice(0, 49)}…` : url;
@@ -24,7 +25,7 @@ function entryLabel(e: ApiHistoryEntry): string {
 	return `${chalk.dim(short)}  ${color(e.status)}  ${urlShort}`;
 }
 
-function entryHint(e: ApiHistoryEntry): string {
+function entryHint(e: HistoryEntry): string {
 	if (!e.createdAt) return "";
 	const d = new Date(e.createdAt);
 	return Number.isNaN(d.getTime()) ? e.createdAt : d.toLocaleString();
@@ -49,11 +50,14 @@ export default defineCommand({
 		const quiet = !!args.json;
 		const out = log.create(quiet);
 		const apiKey = await resolveApiKey(quiet);
-		const service = args.service as ApiHistoryService | undefined;
-		if (service && !SERVICES.includes(service)) out.error(`Invalid service. Valid: ${VALID}`);
+		const rawService = args.service;
+		if (rawService && !SERVICES.includes(rawService as Service)) {
+			out.error(`Invalid service. Valid: ${VALID}`);
+		}
+		const service = rawService as Service | undefined;
 		const requestId = (args as { _: string[] })._.at(1);
-		const limit = args["page-size"] ? Number(args["page-size"]) : 20;
-		let page = args.page ? Number(args.page) : 1;
+		const limit = args["page-size"] ? parseIntArg(args["page-size"], "page-size", out) : 20;
+		let page = args.page ? parseIntArg(args.page, "page", out) : 1;
 
 		const fetchPage = async (pg: number) => {
 			const r = await history.list(apiKey, {
@@ -62,7 +66,7 @@ export default defineCommand({
 				...(service ? { service } : {}),
 			});
 			if (!r.data) out.error(r.error);
-			const d = r.data as { data: ApiHistoryEntry[]; pagination: { total: number } };
+			const d = r.data as { data: HistoryEntry[]; pagination: { total: number } };
 			return {
 				rows: d.data ?? [],
 				hasMore: (d.pagination?.total ?? 0) > pg * limit,

src/commands/monitor.ts

@@ -3,12 +3,15 @@ import chalk from "chalk";
 import { defineCommand } from "citty";
 import { monitor } from "scrapegraph-js";
 import type {
-	ApiMonitorCreateInput,
-	ApiMonitorUpdateInput,
-	ApiScrapeFormatEntry,
+	FetchConfig,
+	FormatConfig,
+	MonitorCreateRequest,
+	MonitorUpdateRequest,
 } from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
+import { BASE_FORMATS, type BaseFormat, buildBaseFormat } from "../lib/formats.js";
 import * as log from "../lib/log.js";
+import { parseIntArg } from "../lib/parse.js";
 
 const ACTIONS = [
 	"create",
@@ -22,24 +25,14 @@ const ACTIONS = [
 ] as const;
 type Action = (typeof ACTIONS)[number];
 
-const FORMATS = [
-	"markdown",
-	"html",
-	"screenshot",
-	"branding",
-	"links",
-	"images",
-	"summary",
-] as const;
-
-function buildFormats(raw: string): ApiScrapeFormatEntry[] {
+function buildFormats(raw: string, onInvalid: (f: string) => never): FormatConfig[] {
 	return raw
 		.split(",")
 		.map((f) => f.trim())
 		.filter(Boolean)
 		.map((f) => {
-			if (f === "markdown" || f === "html") return { type: f, mode: "normal" as const };
-			return { type: f } as ApiScrapeFormatEntry;
+			if (!BASE_FORMATS.includes(f as BaseFormat)) onInvalid(f);
+			return buildBaseFormat(f as BaseFormat);
 		});
 }
 
@@ -64,7 +57,7 @@ export default defineCommand({
 		format: {
 			type: "string",
 			alias: "f",
-			description: `Formats to track, comma-separated: ${FORMATS.join(", ")} (default: markdown)`,
+			description: `Formats to track, comma-separated: ${BASE_FORMATS.join(", ")} (default: markdown)`,
 		},
 		"webhook-url": { type: "string", description: "Webhook URL for change notifications" },
 		mode: { type: "string", alias: "m", description: "Fetch mode: auto (default), fast, js" },
@@ -81,33 +74,33 @@ export default defineCommand({
 
 		if (!ACTIONS.includes(action)) {
 			out.error(`Unknown action: ${action}. Valid: ${ACTIONS.join(", ")}`);
-			return;
 		}
 
 		const needsId: Action[] = ["get", "update", "delete", "pause", "resume", "activity"];
 		if (needsId.includes(action) && !args.id) {
 			out.error(`--id is required for ${action}`);
-			return;
 		}
 
 		const fetchConfig: Record<string, unknown> = {};
 		if (args.mode) fetchConfig.mode = args.mode;
 		if (args.stealth) fetchConfig.stealth = true;
 
+		const onInvalidFormat = (f: string): never =>
+			out.error(`Unknown format: ${f}. Valid: ${BASE_FORMATS.join(", ")}`);
+
 		switch (action) {
 			case "create": {
-				if (!args.url) return out.error("--url is required for create");
-				if (!args.interval) return out.error("--interval is required for create");
+				if (!args.url) out.error("--url is required for create");
+				if (!args.interval) out.error("--interval is required for create");
 
-				const params: ApiMonitorCreateInput = {
+				const params: MonitorCreateRequest = {
 					url: args.url,
 					interval: args.interval,
-					formats: buildFormats(args.format ?? "markdown"),
+					formats: buildFormats(args.format ?? "markdown", onInvalidFormat),
+					...(args.name && { name: args.name }),
+					...(args["webhook-url"] && { webhookUrl: args["webhook-url"] }),
+					...(Object.keys(fetchConfig).length > 0 && { fetchConfig: fetchConfig as FetchConfig }),
 				};
-				const mut = params as Record<string, unknown>;
-				if (args.name) mut.name = args.name;
-				if (args["webhook-url"]) mut.webhookUrl = args["webhook-url"];
-				if (Object.keys(fetchConfig).length > 0) mut.fetchConfig = fetchConfig;
 
 				out.start("Creating monitor");
 				const result = await monitor.create(apiKey, params);
@@ -146,13 +139,13 @@ export default defineCommand({
 			}
 
 			case "update": {
-				const params: ApiMonitorUpdateInput = {};
-				const mut = params as Record<string, unknown>;
-				if (args.name) mut.name = args.name;
-				if (args.interval) mut.interval = args.interval;
-				if (args["webhook-url"]) mut.webhookUrl = args["webhook-url"];
-				if (args.format) mut.formats = buildFormats(args.format);
-				if (Object.keys(fetchConfig).length > 0) mut.fetchConfig = fetchConfig;
+				const params: MonitorUpdateRequest = {
+					...(args.name && { name: args.name }),
+					...(args.interval && { interval: args.interval }),
+					...(args["webhook-url"] && { webhookUrl: args["webhook-url"] }),
+					...(args.format && { formats: buildFormats(args.format, onInvalidFormat) }),
+					...(Object.keys(fetchConfig).length > 0 && { fetchConfig: fetchConfig as FetchConfig }),
+				};
 
 				out.start("Updating monitor");
 				const result = await monitor.update(apiKey, args.id as string, params);
@@ -190,9 +183,10 @@ export default defineCommand({
 			}
 
 			case "activity": {
-				const qp: { limit?: number; cursor?: string } = {};
-				if (args.limit) qp.limit = Number(args.limit);
-				if (args.cursor) qp.cursor = args.cursor;
+				const qp: { limit?: number; cursor?: string } = {
+					...(args.limit && { limit: parseIntArg(args.limit, "limit", out) }),
+					...(args.cursor && { cursor: args.cursor }),
+				};
 
 				out.start("Fetching monitor activity");
 				const result = await monitor.activity(apiKey, args.id as string, qp);