diff --git a/package.json b/package.json index 4ede9ce4..da590d2e 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "sync:discovered": "bun scripts/batch/sync-kv.ts", "validate-favicons": "bun scripts/validate-favicons.ts", "validate:batch": "bun scripts/batch/validate-results.ts", + "detect:stale-apis": "bun scripts/batch/stale-apis-guru.ts", "dev": "bun run normalize && astro dev", "build": "ASTRO_TELEMETRY_DISABLED=1 bun run normalize && ASTRO_TELEMETRY_DISABLED=1 astro build", "preview": "astro preview", diff --git a/scripts/batch/stale-apis-guru.test.ts b/scripts/batch/stale-apis-guru.test.ts new file mode 100644 index 00000000..a2c28484 --- /dev/null +++ b/scripts/batch/stale-apis-guru.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, test } from "bun:test"; +import { analyzeStaleEntry, detectPotentiallyStaleApiGuruEntries, type ApiGuruSpecLite } from "./stale-apis-guru.ts"; + +type MockFetch = (url: string) => Promise; + +function mockFetch(responses: Record): MockFetch { + return async (url: string) => { + const response = responses[url]; + if (!response) throw new Error(`unmocked URL: ${url}`); + return new Response(response.body, { + status: response.status, + headers: { "content-type": response.contentType ?? "text/plain" }, + }); + }; +} + +function entry(provider: string, updated = "2020-01-01T00:00:00.000Z", swaggerUrl?: string, link?: string, service?: string): ApiGuruSpecLite { + return { + provider, + service: service ?? null, + updated, + swaggerUrl, + link, + }; +} + +describe("stale API.guru detection", () => { + const nowMs = Date.parse("2026-07-24T00:00:00.000Z"); + test("flags old specs when declared spec endpoints stop returning payloads", async () => { + const specs = [entry("getsandbox.com", "2024-01-01T00:00:00.000Z", "https://getsandbox.com/api.json")]; + const fetchImpl = mockFetch({ + "https://getsandbox.com/api.json": { + status: 200, + contentType: "text/html", + body: "parked", + }, + }); + + const stale = await detectPotentiallyStaleApiGuruEntries(specs, fetchImpl, { + staleAfterDays: 365, + timeoutMs: 5000, + parallelism: 1, + nowMs, + }); + + expect(stale).toHaveLength(1); + expect(stale[0]?.provider).toBe("getsandbox.com"); + expect(stale[0]?.reasons[0]).toContain("no accessible OpenAPI/Swagger payload"); + }); + + test("does not flag recent specs even if endpoint is unavailable", async () => { + const specs = [entry("fastapi.com", "2026-06-20T00:00:00.000Z", "https://fastapi.com/api.json")]; + const fetchImpl = mockFetch({ + "https://fastapi.com/api.json": { + status: 500, + contentType: "text/plain", + body: "down", + }, + }); + + const stale = await detectPotentiallyStaleApiGuruEntries(specs, fetchImpl, { + staleAfterDays: 365, + timeoutMs: 5000, + parallelism: 1, + nowMs, + }); + + expect(stale).toHaveLength(0); + }); + + test("keeps old records with accessible OpenAPI markers", async () => { + const specs = [entry("modernapi.com", "2024-01-01T00:00:00.000Z", "https://modernapi.com/openapi.json")]; + const fetchImpl = mockFetch({ + "https://modernapi.com/openapi.json": { + status: 200, + contentType: "application/json", + body: '{"openapi":"3.0.3","info":{"title":"Modern","version":"1.0.0"}}', + }, + }); + + const stale = await detectPotentiallyStaleApiGuruEntries(specs, fetchImpl, { + staleAfterDays: 365, + timeoutMs: 5000, + parallelism: 1, + nowMs, + }); + + expect(stale).toHaveLength(0); + }); + + test("skips records without detectable spec endpoints", () => { + const probes = [] as const; + const spec = entry("nospec.com", "2024-01-01T00:00:00.000Z"); + const result = analyzeStaleEntry(spec, [...probes], 365, nowMs); + expect(result.stale).toBe(false); + }); +}); + diff --git a/scripts/batch/stale-apis-guru.ts b/scripts/batch/stale-apis-guru.ts new file mode 100644 index 00000000..ae333fce --- /dev/null +++ b/scripts/batch/stale-apis-guru.ts @@ -0,0 +1,229 @@ +import { getFlag, getNumberFlag, hasFlag, mapLimit, parseArgs, readJson, ROOT, usage } from "./shared.ts"; +import { join, dirname } from "node:path"; +import { mkdirSync, writeFileSync } from "node:fs"; + +export const DEFAULT_STALE_DAYS = 365; +export const DEFAULT_TIMEOUT_MS = 5000; +export const DEFAULT_PARALLELISM = 8; +export const DEFAULT_OUT_FILE = "reports/stale-api-guru.json"; + +type FetchLike = (url: string, init?: RequestInit) => Promise; + +export interface ApiGuruSpecLite { + provider: string; + service?: string | null; + updated?: string; + link?: string; + origin?: string; + swaggerUrl?: string; +} + +export interface ApiGuruSpecsFile { + specs: ApiGuruSpecLite[]; +} + +interface ProbeBase { + kind: "spec" | "origin"; + url: string; + status: "ok" | "error"; + httpStatus?: number; + contentType: string | null; + specLike: boolean; + error?: string; +} + +export interface StaleProbe extends ProbeBase {} + +export interface StaleApiGuruEntry { + provider: string; + service?: string; + domain: string; + updated?: string; + ageDays: number | null; + reasons: string[]; + probes: StaleProbe[]; +} + +interface AnalyzeOptions { + staleAfterDays: number; + timeoutMs: number; + parallelism: number; + nowMs: number; +} + +const HELP = ` +Usage: bun scripts/batch/stale-apis-guru.ts --source file [options] + +Scan apis.guru source rows for likely stale API definitions. + +Options: + --source path Path to api-guru-openapi.json (default: sources/api-guru-openapi.json) + --out path Write stale candidates JSON summary (default: reports/stale-api-guru.json) + --max-age-days number Only flag entries older than this value (default: ${DEFAULT_STALE_DAYS}) + --timeout-ms number Per-request timeout in milliseconds (default: ${DEFAULT_TIMEOUT_MS}) + --parallel number Concurrent request limit (default: ${DEFAULT_PARALLELISM}) + --json Print JSON report to stdout + --strict Exit non-zero if any stale entries are found + --help Show this help +`; + +function isHttpUrl(value: unknown): value is string { + if (typeof value !== "string") return false; + try { + const parsed = new URL(value); + return parsed.protocol === "http:" || parsed.protocol === "https:"; + } catch { + return false; + } +} + +function ageInDays(updated: string | undefined, nowMs: number): number | null { + if (!updated) return null; + const parsed = Date.parse(updated); + if (!Number.isFinite(parsed)) return null; + return Math.max(0, (nowMs - parsed) / (1000 * 60 * 60 * 24)); +} + +function isSpecLikely(head: string, contentType: string | null): boolean { + const text = head.toLowerCase(); + const ct = (contentType ?? "").toLowerCase(); + if (ct.includes("json") || ct.includes("yaml") || ct.includes("yml")) return true; + return /["']openapi["']\s*[:=]/i.test(text) || + /["']swagger["']\s*[:=]/i.test(text) || + /\bopenapi:\s*/i.test(text) || + /\bswagger:\s*/i.test(text); +} + +function readHeadChunk(response: Response, maxChars = 4000): Promise { + if (!response.body) return Promise.resolve(""); + const decoder = new TextDecoder(); + return response.body.getReader().read().then(({ value }) => { + const chunk = value ? decoder.decode(value, { stream: true }) : ""; + return chunk.slice(0, maxChars); + }); +} + +async function probeEndpoint(url: string, kind: StaleProbe["kind"], fetchImpl: FetchLike, timeoutMs: number): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(url, { + redirect: "follow", + signal: controller.signal, + }); + const head = await readHeadChunk(response); + const contentType = response.headers.get("content-type"); + const ok = response.status >= 200 && response.status < 400; + clearTimeout(timeout); + return { + kind, + url, + status: ok ? "ok" : "error", + httpStatus: response.status, + contentType, + specLike: kind === "spec" ? isSpecLikely(head, contentType) : false, + }; + } catch (error) { + clearTimeout(timeout); + const message = error instanceof Error ? error.message : String(error); + return { kind, url, status: "error", contentType: null, specLike: false, error: message }; + } finally { + clearTimeout(timeout); + } +} + +export function analyzeStaleEntry( + spec: ApiGuruSpecLite, + probes: StaleProbe[], + staleAfterDays: number, + nowMs: number, +): { stale: boolean; entry?: StaleApiGuruEntry } { + const reasons: string[] = []; + const ageDays = ageInDays(spec.updated, nowMs); + if (ageDays === null || ageDays < staleAfterDays) return { stale: false }; + + const specTargets = probes.filter((probe) => probe.kind === "spec"); + if (specTargets.length === 0) return { stale: false }; + + const specReachable = specTargets.some((probe) => probe.status === "ok" && probe.specLike); + if (!specReachable) { + reasons.push("no accessible OpenAPI/Swagger payload at declared spec endpoints"); + } + + if (reasons.length === 0) return { stale: false }; + return { + stale: true, + entry: { + provider: spec.provider, + service: spec.service ?? undefined, + domain: spec.provider, + updated: spec.updated, + ageDays, + reasons, + probes, + }, + }; +} + +export async function detectPotentiallyStaleApiGuruEntries( + specs: readonly ApiGuruSpecLite[], + fetchImpl: FetchLike = fetch, + options: AnalyzeOptions, +): Promise { + const rows = specs.filter((spec) => isHttpUrl(spec.provider) === false); + const results = await mapLimit(rows, options.parallelism, async (spec) => { + const targets = [spec.swaggerUrl, spec.link].filter(isHttpUrl); + const probes = await Promise.all( + targets.map((url) => probeEndpoint(url, "spec", fetchImpl, options.timeoutMs)), + ); + const outcome = analyzeStaleEntry(spec, probes, options.staleAfterDays, options.nowMs); + return outcome.entry; + }); + return results.filter((entry): entry is StaleApiGuruEntry => Boolean(entry)).sort((a, b) => b.ageDays - a.ageDays); +} + +async function main(): Promise { + const args = parseArgs(); + if (hasFlag(args, "help")) usage(HELP); + const source = getFlag(args, "source", join(ROOT, "sources", "api-guru-openapi.json"))!; + const out = getFlag(args, "out", DEFAULT_OUT_FILE); + const staleAfterDays = getNumberFlag(args, "max-age-days", DEFAULT_STALE_DAYS); + const timeoutMs = getNumberFlag(args, "timeout-ms", DEFAULT_TIMEOUT_MS); + const parallelism = getNumberFlag(args, "parallel", DEFAULT_PARALLELISM); + const strict = hasFlag(args, "strict"); + const outputJson = hasFlag(args, "json"); + + const payload = readJson(source); + const specs = payload.specs ?? []; + const nowMs = Date.now(); + const staleEntries = await detectPotentiallyStaleApiGuruEntries(specs, fetch, { + staleAfterDays, + timeoutMs, + parallelism, + nowMs, + }); + + if (outputJson) { + console.log(JSON.stringify({ staleEntries, scanned: specs.length, staleAfterDays }, null, 2)); + } else { + console.log(`scanned ${specs.length} API.guru entries; stale candidates: ${staleEntries.length}`); + for (const entry of staleEntries) { + const ageDays = entry.ageDays === null ? "unknown" : `${Math.round(entry.ageDays)}d`; + const service = entry.service ? ` (${entry.service})` : ""; + const summary = [entry.provider, service, `age:${ageDays}`, ...(entry.updated ? [`updated:${entry.updated}`] : [])].filter(Boolean); + console.log(`- ${summary.join(" ")} -> ${entry.reasons.join("; ")}`); + } + } + + if (out) { + mkdirSync(dirname(out), { recursive: true }); + writeFileSync(out, `${JSON.stringify(staleEntries, null, 2)}\n`); + console.log(`wrote stale analysis to ${out}`); + } + + if (strict && staleEntries.length > 0) { + process.exit(1); + } +} + +if (import.meta.main) await main();