diff --git a/README.md b/README.md index 9ba4c55..7f6161d 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Deployable **GitHub language chart generator** — embeddable SVGs for READMEs a - **Dynamic Layout:** The legend automatically shifts to a **two-column layout** when displaying 9 or more languages. - Automatically fetches all public GitHub repositories. Users or organization sources with a token include private repos exposed to that token. - Ignores forks and optionally specific repositories (`IGNORED_REPOS`). -- Uses **hourly caching** to reduce API calls and improve performance. +- Uses **hourly caching** to reduce API calls and improve performance. On fetch failure the last good chart is served and refresh retries after 5 minutes, or when GitHub rate-limits the real reset time. ## Customize Your Charts Prefer a visual workflow? Use the [@gh-top-languages/builder](https://github.com/gh-top-languages/builder) to preview themes, colours, and layout options interactively, then easily copy the generated embed URL to quickly deploy. @@ -132,6 +132,7 @@ Common error messages: - `GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration.` — malformed JSON array in env config - `GitHub API error: {status} {statusText}` — GitHub API request failed - `No language data available` — no public repositories found +- Rate limiting (`403`/`429` with exhausted quota) is detected separately and logged as `GitHub rate limit exceeded; resets at HH:MM:SS`; the cached chart is served until the reset. ## License MIT License - see [LICENSE](./LICENSE) for details. diff --git a/src/github.ts b/src/github/fetch.ts similarity index 57% rename from src/github.ts rename to src/github/fetch.ts index 723d74c..e3dbef8 100644 --- a/src/github.ts +++ b/src/github/fetch.ts @@ -1,75 +1,38 @@ -import type { Language } from "@gh-top-languages/lib/charts/types.js"; +import { parseNextLink, parseSources } from "./parse.js"; +import { FALLBACK_RETRY_MS, rateLimitReset } from "./rateLimit.js"; +import type { LanguageBytes } from "./types.js"; + const REFRESH_INTERVAL = 1000 * 60 * 60; +let cachedLanguageData: LanguageBytes | null = null; +let lastRefresh = 0; +let inFlightFetch: Promise | null = null; + type Repo = { name: string; fork: boolean; full_name: string; }; -type Source = { name: string; token?: string }; - -type LanguageBytes = Record; - -let cachedLanguageData: LanguageBytes | null = null; -let lastRefresh = 0; -let inFlightFetch: Promise | null = null; - -function parseSources(env: string | undefined): Source[] { - if (!env) return []; - - const trimmed = env.trim(); - if (trimmed.startsWith('[')) { - try { - const parsed = JSON.parse(env); - return (parsed as unknown[]).map(entry => { - if (typeof entry === "string" && entry.trim()) return { name: entry.trim() }; - if (entry && typeof entry === "object" && "name" in entry && typeof entry.name === "string" && entry.name.trim()) { - const source: Source = { name: entry.name.trim() }; - if ("token" in entry - && typeof entry.token === "string" - && entry.token.trim() - ) source.token = entry.token.trim(); - return source; - } - return null; - }).filter((s): s is Source => !!s); - } catch { - console.error("Failed to parse configuration JSON array."); - throw new Error("GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration."); - } +export async function fetchLanguageData(useTestData = false): Promise { + if (useTestData) { + const testData = await import ("../test-data.json", { with: { type: "json" } }); + return testData.default; } - return trimmed.split(',').map(s => ({ name: s.trim().replace(/^["']|["']$/g, "") })).filter(s => s.name); -} - -function makeOptions(token?: string): RequestInit { - return token ? { headers: { Authorization: `Bearer ${token}` } } : {}; -} + const now = Date.now(); + if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData; -function parseNextLink(linkHeader: string | null): string | null { - if (!linkHeader) return null; - const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/); - return match?.[1] ?? null; + if (inFlightFetch) return inFlightFetch; + inFlightFetch = fetchAndAggregate(now).finally(() => { inFlightFetch = null; }); + return inFlightFetch; } -async function fetchAllRepos(url: string, token?: string): Promise { - const options = makeOptions(token); - let nextUrl: string | null = url; - const repos: Repo[] = []; - - while (nextUrl) { - const response = await fetch(nextUrl, options); - if (!response.ok) throw new Error(`GitHub API error: ${response.status} ${response.statusText}`); - repos.push(...(await response.json() as Repo[])); - nextUrl = parseNextLink(response.headers.get("Link")); - if (nextUrl && !nextUrl.startsWith("https://api.github.com/")) throw new Error( - `Unexpected pagination URL: ${nextUrl}` - ); - } - - return repos; +export function resetCache(): void { + cachedLanguageData = null; + lastRefresh = 0; + inFlightFetch = null; } async function fetchAndAggregate(now: number): Promise { @@ -80,12 +43,16 @@ async function fetchAndAggregate(now: number): Promise { "At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set" ); + let rateLimitResetAt: number | null = null; + const noteReset = (ms: number) => { rateLimitResetAt = Math.max(rateLimitResetAt ?? 0, ms); }; + let hadFetchFailure = false; const repoGroups = await Promise.all([ ...usernames.map(u => fetchAllRepos( u.token ? `https://api.github.com/user/repos?per_page=100&visibility=all&affiliation=owner` : `https://api.github.com/users/${encodeURIComponent(u.name)}/repos?per_page=100`, + noteReset, u.token ) .then(repos => ({ token: u.token, repos })) @@ -96,13 +63,17 @@ async function fetchAndAggregate(now: number): Promise { }) ), ...orgs.map(o => - fetchAllRepos(`https://api.github.com/orgs/${encodeURIComponent(o.name)}/repos?per_page=100`, o.token) - .then(repos => ({ token: o.token, repos })) - .catch(() => { - hadFetchFailure = true; - console.error(`Skipping org "${o.name}": failed to fetch repositories.`); - return { token: o.token, repos: [] as Repo[] }; - }) + fetchAllRepos( + `https://api.github.com/orgs/${encodeURIComponent(o.name)}/repos?per_page=100`, + noteReset, + o.token + ) + .then(repos => ({ token: o.token, repos })) + .catch(() => { + hadFetchFailure = true; + console.error(`Skipping org "${o.name}": failed to fetch repositories.`); + return { token: o.token, repos: [] as Repo[] }; + }) ) ]); @@ -114,6 +85,8 @@ async function fetchAndAggregate(now: number): Promise { .then(r => { if (r.ok) return r.json() as Promise; hadFetchFailure = true; + const reset = rateLimitReset(r); + if (reset) noteReset(reset); return {} as LanguageBytes; }) .catch(() => { hadFetchFailure = true; return {} as LanguageBytes; }) @@ -130,8 +103,15 @@ async function fetchAndAggregate(now: number): Promise { }, {}); if (hadFetchFailure) { + if (rateLimitResetAt !== null) console.error( + `GitHub rate limit exceeded; resets at ${new Date(rateLimitResetAt).toLocaleTimeString()}` + ); + if (cachedLanguageData !== null) { - lastRefresh = now - REFRESH_INTERVAL + (5 * 60 * 1000); + const retryDelay = rateLimitResetAt + ? Math.min(Math.max(rateLimitResetAt - now, 60_000), REFRESH_INTERVAL) + : FALLBACK_RETRY_MS; + lastRefresh = now - REFRESH_INTERVAL + retryDelay; return cachedLanguageData; } return result; @@ -142,35 +122,32 @@ async function fetchAndAggregate(now: number): Promise { return result; } -export async function fetchLanguageData(useTestData = false): Promise { - if (useTestData) { - const testData = await import ("./test-data.json", { with: { type: "json" } }); - return testData.default; - } - - const now = Date.now(); - if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData; - - if (inFlightFetch) return inFlightFetch; - inFlightFetch = fetchAndAggregate(now).finally(() => { inFlightFetch = null; }); - return inFlightFetch; -} - -export function processLanguageData(languageBytes: LanguageBytes, count: number): Language[] { - if (Object.keys(languageBytes).length === 0) throw new Error("No language data available"); +async function fetchAllRepos( + url: string, + onRateLimit: (resetMs: number) => void, + token?: string +): Promise { + const options = makeOptions(token); + let nextUrl: string | null = url; + const repos: Repo[] = []; - const totalBytes = Object.values(languageBytes).reduce((a, b) => a + b, 0); - - const sortedLanguages = Object.entries(languageBytes) - .map(([lang, bytes]) => ({ lang, pct: (bytes / totalBytes) * 100 })) - .sort((a, b) => b.pct - a.pct); + while (nextUrl) { + const response = await fetch(nextUrl, options); + if (!response.ok) { + const reset = rateLimitReset(response); + if (reset) onRateLimit(reset); + throw new Error(`GitHub API error: ${response.status} ${response.statusText}`); + } + repos.push(...(await response.json() as Repo[])); + nextUrl = parseNextLink(response.headers.get("Link")); + if (nextUrl && !nextUrl.startsWith("https://api.github.com/")) throw new Error( + `Unexpected pagination URL: ${nextUrl}` + ); + } - return sortedLanguages.slice(0, count); + return repos; } -export function resetCache(): void { - cachedLanguageData = null; - lastRefresh = 0; - inFlightFetch = null; +function makeOptions(token?: string): RequestInit { + return token ? { headers: { Authorization: `Bearer ${token}` } } : {}; } - diff --git a/src/github/parse.ts b/src/github/parse.ts new file mode 100644 index 0000000..a09b110 --- /dev/null +++ b/src/github/parse.ts @@ -0,0 +1,35 @@ +import type { Source } from "./types.js"; + +export function parseSources(env: string | undefined): Source[] { + if (!env) return []; + + const trimmed = env.trim(); + if (trimmed.startsWith('[')) { + try { + const parsed = JSON.parse(env); + return (parsed as unknown[]).map(entry => { + if (typeof entry === "string" && entry.trim()) return { name: entry.trim() }; + if (entry && typeof entry === "object" && "name" in entry && typeof entry.name === "string" && entry.name.trim()) { + const source: Source = { name: entry.name.trim() }; + if ("token" in entry + && typeof entry.token === "string" + && entry.token.trim() + ) source.token = entry.token.trim(); + return source; + } + return null; + }).filter((s): s is Source => !!s); + } catch { + console.error("Failed to parse configuration JSON array."); + throw new Error("GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration."); + } + } + + return trimmed.split(',').map(s => ({ name: s.trim().replace(/^["']|["']$/g, "") })).filter(s => s.name); +} + +export function parseNextLink(linkHeader: string | null): string | null { + if (!linkHeader) return null; + const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/); + return match?.[1] ?? null; +} diff --git a/src/github/process.ts b/src/github/process.ts new file mode 100644 index 0000000..caf1dd2 --- /dev/null +++ b/src/github/process.ts @@ -0,0 +1,14 @@ +import type { Language } from "@gh-top-languages/lib/charts/types.js"; +import type { LanguageBytes } from "./types.js"; + +export function processLanguageData(languageBytes: LanguageBytes, count: number): Language[] { + if (Object.keys(languageBytes).length === 0) throw new Error("No language data available"); + + const totalBytes = Object.values(languageBytes).reduce((a, b) => a + b, 0); + + const sortedLanguages = Object.entries(languageBytes) + .map(([lang, bytes]) => ({ lang, pct: (bytes / totalBytes) * 100 })) + .sort((a, b) => b.pct - a.pct); + + return sortedLanguages.slice(0, count); +} diff --git a/src/github/rateLimit.ts b/src/github/rateLimit.ts new file mode 100644 index 0000000..394ea92 --- /dev/null +++ b/src/github/rateLimit.ts @@ -0,0 +1,13 @@ +export const FALLBACK_RETRY_MS = 5 * 60 * 1000; + +export function rateLimitReset(r: Response): number | null { + const retryAfter = r.headers.get("Retry-After"); + const primary = (r.status === 403 || r.status === 429) && r.headers.get("X-RateLimit-Remaining") === "0"; + if (!retryAfter && !primary) return null; + + const resetHeader = r.headers.get("X-RateLimit-Reset"); + const resetMs = retryAfter + ? Date.now() + Number(retryAfter) * 1000 + : resetHeader ? Number(resetHeader) * 1000 : null; + return resetMs !== null && Number.isFinite(resetMs) ? resetMs : null; +} diff --git a/src/github/types.ts b/src/github/types.ts new file mode 100644 index 0000000..d1d89e3 --- /dev/null +++ b/src/github/types.ts @@ -0,0 +1,3 @@ +export type Source = { name: string; token?: string }; + +export type LanguageBytes = Record; diff --git a/src/handler.ts b/src/handler.ts index 7625ccb..ed42960 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -1,9 +1,10 @@ -import { DEFAULT_CONFIG } from "@gh-top-languages/lib/constants/config.js"; -import { parseQueryParams, type QueryParams } from "@gh-top-languages/lib/utils/params.js"; -import { generateChartData } from "@gh-top-languages/lib/charts/generate.js"; -import { renderSvg } from "@gh-top-languages/lib/render/svg.js"; -import { renderError } from "@gh-top-languages/lib/render/error.js"; -import { fetchLanguageData, processLanguageData } from "./github.js"; +import { DEFAULT_CONFIG } from "@gh-top-languages/lib/constants/config.js"; +import { parseQueryParams, type QueryParams } from "@gh-top-languages/lib/utils/params.js"; +import { generateChartData } from "@gh-top-languages/lib/charts/generate.js"; +import { renderSvg } from "@gh-top-languages/lib/render/svg.js"; +import { renderError } from "@gh-top-languages/lib/render/error.js"; +import { fetchLanguageData } from "./github/fetch.js"; +import { processLanguageData } from "./github/process.js"; export type ChartResponse = { status: number; diff --git a/src/server.ts b/src/server.ts index f7db2ae..d587e2a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,5 +1,5 @@ -import { createServer, type Server } from "node:http"; -import { pathToFileURL } from "node:url"; +import { createServer, type Server } from "node:http"; +import { pathToFileURL } from "node:url"; import { handleLanguages, type RawQuery } from "./handler.js"; export function queryFromUrl(url: URL): RawQuery { diff --git a/tests/api/languages/index.test.ts b/tests/api/languages/index.test.ts index d1e2c3e..ba19aa2 100644 --- a/tests/api/languages/index.test.ts +++ b/tests/api/languages/index.test.ts @@ -6,13 +6,15 @@ import { generateChartData } from "@gh-top-languages/lib/ch import { renderSvg } from "@gh-top-languages/lib/render/svg.js"; import { renderError } from "@gh-top-languages/lib/render/error.js"; import handler from "../../../api/languages/index.js"; -import { fetchLanguageData, processLanguageData } from "../../../src/github.js"; +import { fetchLanguageData } from "../../../src/github/fetch.js"; +import { processLanguageData } from "../../../src/github/process.js"; vi.mock("@gh-top-languages/lib/utils/params.js"); vi.mock("@gh-top-languages/lib/charts/generate.js"); vi.mock("@gh-top-languages/lib/render/svg.js"); vi.mock("@gh-top-languages/lib/render/error.js"); -vi.mock("../../../src/github.js"); +vi.mock("../../../src/github/fetch.js"); +vi.mock("../../../src/github/process.js"); describe("handler", () => { let req: VercelRequest; diff --git a/tests/github.test.ts b/tests/github/fetch.test.ts similarity index 89% rename from tests/github.test.ts rename to tests/github/fetch.test.ts index 6a9add4..a44288f 100644 --- a/tests/github.test.ts +++ b/tests/github/fetch.test.ts @@ -1,5 +1,5 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { fetchLanguageData, processLanguageData, resetCache } from "../src/github.js"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { fetchLanguageData, resetCache } from "../../src/github/fetch.js"; const repos = [ { name: "repo1", fork: false, full_name: "user/repo1" }, @@ -21,9 +21,10 @@ const mockResponse = (data: unknown, link: string | null = null) => ({ headers: { get: (h: string) => h === "Link" ? link : null } }) as unknown as Response; -const mockErrorResponse = (status: number, statusText = "") => ( - { ok: false, status, statusText } -) as unknown as Response; +const mockErrorResponse = (status: number, statusText = "", headers: Record = {}) => ({ + ok: false, status, statusText, + headers: { get: (h: string) => headers[h] ?? null } +}) as unknown as Response; describe("fetchLanguageData", () => { beforeEach(() => { @@ -435,45 +436,34 @@ describe("fetchLanguageData", () => { vi.restoreAllMocks(); }); -}); - -describe("processLanguageData", () => { - it("calculates percentages correctly", () => { - const data = { JavaScript: 5000, Python: 3000, HTML: 2000 }; - const result = processLanguageData(data, 3); - expect(result).toHaveLength(3); - expect(result[0]).toEqual({ lang: "JavaScript", pct: 50 }); - expect(result[1]).toEqual({ lang: "Python", pct: 30 }); - expect(result[2]).toEqual({ lang: "HTML", pct: 20 }); - }); - - it("sorts by percentage descending", () => { - const data = { HTML: 1000, JavaScript: 5000, Python: 3000 }; - const result = processLanguageData(data, 3); - expect(result.map(l => l.lang)).toEqual(["JavaScript", "Python", "HTML"]); - }); - - it("limits to count", () => { - const data = { JavaScript: 5000, Python: 3000, HTML: 2000, CSS: 1000 }; - const result = processLanguageData(data, 2); - - expect(result).toHaveLength(2); - expect(result.map(l => l.lang)).toEqual(["JavaScript", "Python"]); - }); + it("backs off until the rate-limit reset time when limited", async () => { + mockFetch() + .mockResolvedValueOnce(mockResponse([repos[0]])) + .mockResolvedValueOnce(mockResponse(languages)); + await fetchLanguageData(); - it("does not renormalize percentages after slicing", () => { - const data = { JavaScript: 5000, Python: 3000, HTML: 2000 }; - const result = processLanguageData(data, 2); + const base = Date.now() + 1000 * 60 * 61; + const resetSec = Math.floor(base / 1000) + 30 * 60; + vi.spyOn(Date, 'now').mockReturnValue(base); - expect(result[0]).toEqual({ lang: "JavaScript", pct: 50 }); - expect(result[1]).toEqual({ lang: "Python", pct: 30 }); + mockFetch().mockResolvedValueOnce(mockErrorResponse(403, "Forbidden", { + "X-RateLimit-Remaining": "0", + "X-RateLimit-Reset": String(resetSec) + })); + const stale = await fetchLanguageData(); + expect(stale).toEqual(languages); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("rate limit exceeded")); - const totalPct = result.reduce((sum, l) => sum + l.pct, 0); - expect(totalPct).toBeCloseTo(80); - }); + vi.mocked(Date.now).mockReturnValue(base + 10 * 60 * 1000); + const calls = mockFetch().mock.calls.length; + await fetchLanguageData(); + expect(mockFetch().mock.calls.length).toBe(calls); - it("throws when no language data", () => { - expect(() => processLanguageData({}, 5)).toThrow("No language data available"); + vi.mocked(Date.now).mockReturnValue(resetSec * 1000 + 1000); + mockFetch() + .mockResolvedValueOnce(mockResponse([repos[0]])) + .mockResolvedValueOnce(mockResponse({ Go: 100 })); + expect(await fetchLanguageData()).toEqual({ Go: 100 }); }); }); diff --git a/tests/github/process.test.ts b/tests/github/process.test.ts new file mode 100644 index 0000000..c730151 --- /dev/null +++ b/tests/github/process.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect } from "vitest"; +import { processLanguageData } from "../../src/github/process.js"; + +describe("processLanguageData", () => { + it("calculates percentages correctly", () => { + const data = { JavaScript: 5000, Python: 3000, HTML: 2000 }; + const result = processLanguageData(data, 3); + expect(result).toHaveLength(3); + expect(result[0]).toEqual({ lang: "JavaScript", pct: 50 }); + expect(result[1]).toEqual({ lang: "Python", pct: 30 }); + expect(result[2]).toEqual({ lang: "HTML", pct: 20 }); + }); + + it("sorts by percentage descending", () => { + const data = { HTML: 1000, JavaScript: 5000, Python: 3000 }; + const result = processLanguageData(data, 3); + + expect(result.map(l => l.lang)).toEqual(["JavaScript", "Python", "HTML"]); + }); + + it("limits to count", () => { + const data = { JavaScript: 5000, Python: 3000, HTML: 2000, CSS: 1000 }; + const result = processLanguageData(data, 2); + + expect(result).toHaveLength(2); + expect(result.map(l => l.lang)).toEqual(["JavaScript", "Python"]); + }); + + it("does not renormalize percentages after slicing", () => { + const data = { JavaScript: 5000, Python: 3000, HTML: 2000 }; + const result = processLanguageData(data, 2); + + expect(result[0]).toEqual({ lang: "JavaScript", pct: 50 }); + expect(result[1]).toEqual({ lang: "Python", pct: 30 }); + + const totalPct = result.reduce((sum, l) => sum + l.pct, 0); + expect(totalPct).toBeCloseTo(80); + }); + + it("throws when no language data", () => { + expect(() => processLanguageData({}, 5)).toThrow("No language data available"); + }); +}); diff --git a/tests/github/rateLimit.test.ts b/tests/github/rateLimit.test.ts new file mode 100644 index 0000000..7870498 --- /dev/null +++ b/tests/github/rateLimit.test.ts @@ -0,0 +1,34 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { rateLimitReset } from "../../src/github/rateLimit.js"; + +const res = (status: number, headers: Record = {}) => ({ + status, headers: { get: (h: string) => headers[h] ?? null } +}) as unknown as Response; + +describe("rateLimitReset", () => { + afterEach(() => { vi.restoreAllMocks(); }); + + it("returns reset time for 403 with exhausted primary limit", () => { + expect(rateLimitReset(res(403, { "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "1800000000" }))) + .toBe(1800000000 * 1000); + }); + + it("detects 429 secondary limit the same way", () => { + expect(rateLimitReset(res(429, { "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "1800000000" }))) + .toBe(1800000000 * 1000); + }); + + it("computes reset from Retry-After relative to now", () => { + vi.spyOn(Date, "now").mockReturnValue(1_000_000); + expect(rateLimitReset(res(403, { "Retry-After": "60" }))).toBe(1_000_000 + 60_000); + }); + + it("returns null for a plain 403 with remaining quota", () => { + expect(rateLimitReset(res(403, { "X-RateLimit-Remaining": "42" }))).toBeNull(); + }); + + it("returns null when limited but reset header is missing or garbage", () => { + expect(rateLimitReset(res(403, { "X-RateLimit-Remaining": "0" }))).toBeNull(); + expect(rateLimitReset(res(403, { "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "soon" }))).toBeNull(); + }); +});