feat: rate-limit detection, back off until reset

README.md

@@ -46,7 +46,7 @@ Deployable **GitHub language chart generator** — embeddable SVGs for READMEs a
 - **Dynamic Layout:** The legend automatically shifts to a **two-column layout** when displaying 9 or more languages.
 - Automatically fetches all public GitHub repositories. Users or organization sources with a token include private repos exposed to that token.
 - Ignores forks and optionally specific repositories (`IGNORED_REPOS`).
-- Uses **hourly caching** to reduce API calls and improve performance.
+- Uses **hourly caching** to reduce API calls and improve performance. On fetch failure the last good chart is served and refresh retries after 5 minutes, or when GitHub rate-limits the real reset time.
 
 ## Customize Your Charts
 Prefer a visual workflow? Use the [@gh-top-languages/builder](https://github.com/gh-top-languages/builder) to preview themes, colours, and layout options interactively, then easily copy the generated embed URL to quickly deploy.
@@ -132,6 +132,7 @@ Common error messages:
 - `GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration.` — malformed JSON array in env config
 - `GitHub API error: {status} {statusText}` — GitHub API request failed
 - `No language data available` — no public repositories found
+- Rate limiting (`403`/`429` with exhausted quota) is detected separately and logged as `GitHub rate limit exceeded; resets at HH:MM:SS`; the cached chart is served until the reset.
 
 ## License
 MIT License - see [LICENSE](./LICENSE) for details.

src/github.ts → src/github/fetch.ts

@@ -1,75 +1,38 @@
-import type { Language } from "@gh-top-languages/lib/charts/types.js";
+import { parseNextLink, parseSources       } from "./parse.js";
+import { FALLBACK_RETRY_MS, rateLimitReset } from "./rateLimit.js";
+import type { LanguageBytes                } from "./types.js";
+
 
 const REFRESH_INTERVAL = 1000 * 60 * 60;
 
+let cachedLanguageData: LanguageBytes | null = null;
+let lastRefresh = 0;
+let inFlightFetch: Promise<LanguageBytes> | null = null;
+
 type Repo = {
   name:      string;
   fork:      boolean;
   full_name: string;
 };
 
-type Source = { name: string; token?: string };
-
-type LanguageBytes = Record<string, number>;
-
-let cachedLanguageData: LanguageBytes | null = null;
-let lastRefresh = 0;
-let inFlightFetch: Promise<LanguageBytes> | null = null;
-
-function parseSources(env: string | undefined): Source[] {
-  if (!env) return [];
-
-  const trimmed = env.trim();
-  if (trimmed.startsWith('[')) {
-    try {
-      const parsed = JSON.parse(env);
-      return (parsed as unknown[]).map(entry => {
-        if (typeof entry === "string" && entry.trim()) return { name: entry.trim() };
-        if (entry && typeof entry === "object" && "name" in entry && typeof entry.name === "string" && entry.name.trim()) {
-          const source: Source = { name: entry.name.trim() };
-          if ("token" in entry
-           && typeof entry.token === "string"
-           && entry.token.trim()
-          ) source.token = entry.token.trim();
-          return source;
-        }
-        return null;
-      }).filter((s): s is Source => !!s);
-    } catch {
-      console.error("Failed to parse configuration JSON array.");
-      throw new Error("GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration.");
-    }
+export async function fetchLanguageData(useTestData = false): Promise<LanguageBytes> {
+  if (useTestData) {
+    const testData = await import ("../test-data.json", { with: { type: "json" } });
+    return testData.default;
   }
 
-  return trimmed.split(',').map(s => ({ name: s.trim().replace(/^["']|["']$/g, "") })).filter(s => s.name);
-}
-
-function makeOptions(token?: string): RequestInit {
-  return token ? { headers: { Authorization: `Bearer ${token}` } } : {};
-}
+  const now = Date.now();
+  if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData;
 
-function parseNextLink(linkHeader: string | null): string | null {
-  if (!linkHeader) return null;
-  const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
-  return match?.[1] ?? null;
+  if (inFlightFetch) return inFlightFetch;
+  inFlightFetch = fetchAndAggregate(now).finally(() => { inFlightFetch = null; });
+  return inFlightFetch;
 }
 
-async function fetchAllRepos(url: string, token?: string): Promise<Repo[]> {
-  const options = makeOptions(token);
-  let nextUrl: string | null = url;
-  const repos: Repo[]        = [];
-
-  while (nextUrl) {
-    const response = await fetch(nextUrl, options);
-    if (!response.ok) throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
-    repos.push(...(await response.json() as Repo[]));
-    nextUrl = parseNextLink(response.headers.get("Link"));
-    if (nextUrl && !nextUrl.startsWith("https://api.github.com/")) throw new Error(
-      `Unexpected pagination URL: ${nextUrl}`
-    );
-  }
-
-  return repos;
+export function resetCache(): void {
+  cachedLanguageData = null;
+  lastRefresh = 0;
+  inFlightFetch = null;
 }
 
 async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
@@ -80,12 +43,16 @@ async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
     "At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set"
   );
 
+  let rateLimitResetAt: number | null = null;
+  const noteReset = (ms: number) => { rateLimitResetAt = Math.max(rateLimitResetAt ?? 0, ms); };
+
   let hadFetchFailure = false;
   const repoGroups = await Promise.all([
     ...usernames.map(u =>
       fetchAllRepos(
         u.token ? `https://api.github.com/user/repos?per_page=100&visibility=all&affiliation=owner`
                 : `https://api.github.com/users/${encodeURIComponent(u.name)}/repos?per_page=100`,
+        noteReset,
         u.token
       )
       .then(repos => ({ token: u.token, repos }))
@@ -96,13 +63,17 @@ async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
       })
     ),
     ...orgs.map(o =>
-      fetchAllRepos(`https://api.github.com/orgs/${encodeURIComponent(o.name)}/repos?per_page=100`, o.token)
-        .then(repos => ({ token: o.token, repos }))
-        .catch(() => {
-          hadFetchFailure = true;
-          console.error(`Skipping org "${o.name}": failed to fetch repositories.`);
-          return { token: o.token, repos: [] as Repo[] };
-        })
+      fetchAllRepos(
+        `https://api.github.com/orgs/${encodeURIComponent(o.name)}/repos?per_page=100`,
+        noteReset,
+        o.token
+      )
+      .then(repos => ({ token: o.token, repos }))
+      .catch(() => {
+        hadFetchFailure = true;
+        console.error(`Skipping org "${o.name}": failed to fetch repositories.`);
+        return { token: o.token, repos: [] as Repo[] };
+      })
     )
   ]);
 
@@ -114,6 +85,8 @@ async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
         .then(r => {
           if (r.ok) return r.json() as Promise<LanguageBytes>;
           hadFetchFailure = true;
+          const reset = rateLimitReset(r);
+          if (reset) noteReset(reset);
           return {} as LanguageBytes;
         })
         .catch(() => { hadFetchFailure = true; return {} as LanguageBytes; })
@@ -130,8 +103,15 @@ async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
   }, {});
 
   if (hadFetchFailure) {
+    if (rateLimitResetAt !== null) console.error(
+      `GitHub rate limit exceeded; resets at ${new Date(rateLimitResetAt).toLocaleTimeString()}`
+    );
+
     if (cachedLanguageData !== null) {
-      lastRefresh = now - REFRESH_INTERVAL + (5 * 60 * 1000);
+      const retryDelay = rateLimitResetAt
+        ? Math.min(Math.max(rateLimitResetAt - now, 60_000), REFRESH_INTERVAL)
+        : FALLBACK_RETRY_MS;
+      lastRefresh = now - REFRESH_INTERVAL + retryDelay;
       return cachedLanguageData;
     }
     return result;
@@ -142,35 +122,32 @@ async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
   return result;
 }
 
-export async function fetchLanguageData(useTestData = false): Promise<LanguageBytes> {
-  if (useTestData) {
-    const testData = await import ("./test-data.json", { with: { type: "json" } });
-    return testData.default;
-  }
-
-  const now = Date.now();
-  if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData;
-
-  if (inFlightFetch) return inFlightFetch;
-  inFlightFetch = fetchAndAggregate(now).finally(() => { inFlightFetch = null; });
-  return inFlightFetch;
-}
-
-export function processLanguageData(languageBytes: LanguageBytes, count: number): Language[] {
-  if (Object.keys(languageBytes).length === 0) throw new Error("No language data available");
+async function fetchAllRepos(
+  url: string,
+  onRateLimit: (resetMs: number) => void,
+  token?: string
+): Promise<Repo[]> {
+  const options = makeOptions(token);
+  let nextUrl: string | null = url;
+  const repos: Repo[]        = [];
 
-  const totalBytes = Object.values(languageBytes).reduce((a, b) => a + b, 0);
-
-  const sortedLanguages = Object.entries(languageBytes)
-    .map(([lang, bytes]) => ({ lang, pct: (bytes / totalBytes) * 100 }))
-    .sort((a, b) => b.pct - a.pct);
+  while (nextUrl) {
+    const response = await fetch(nextUrl, options);
+    if (!response.ok) {
+      const reset = rateLimitReset(response);
+      if (reset) onRateLimit(reset);
+      throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
+    }
+    repos.push(...(await response.json() as Repo[]));
+    nextUrl = parseNextLink(response.headers.get("Link"));
+    if (nextUrl && !nextUrl.startsWith("https://api.github.com/")) throw new Error(
+      `Unexpected pagination URL: ${nextUrl}`
+    );
+  }
 
-  return sortedLanguages.slice(0, count);
+  return repos;
 }
 
-export function resetCache(): void {
-  cachedLanguageData = null;
-  lastRefresh = 0;
-  inFlightFetch = null;
+function makeOptions(token?: string): RequestInit {
+  return token ? { headers: { Authorization: `Bearer ${token}` } } : {};
 }
-

src/github/parse.ts

@@ -0,0 +1,35 @@
+import type { Source } from "./types.js";
+
+export function parseSources(env: string | undefined): Source[] {
+  if (!env) return [];
+
+  const trimmed = env.trim();
+  if (trimmed.startsWith('[')) {
+    try {
+      const parsed = JSON.parse(env);
+      return (parsed as unknown[]).map(entry => {
+        if (typeof entry === "string" && entry.trim()) return { name: entry.trim() };
+        if (entry && typeof entry === "object" && "name" in entry && typeof entry.name === "string" && entry.name.trim()) {
+          const source: Source = { name: entry.name.trim() };
+          if ("token" in entry
+           && typeof entry.token === "string"
+           && entry.token.trim()
+          ) source.token = entry.token.trim();
+          return source;
+        }
+        return null;
+      }).filter((s): s is Source => !!s);
+    } catch {
+      console.error("Failed to parse configuration JSON array.");
+      throw new Error("GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration.");
+    }
+  }
+
+  return trimmed.split(',').map(s => ({ name: s.trim().replace(/^["']|["']$/g, "") })).filter(s => s.name);
+}
+
+export function parseNextLink(linkHeader: string | null): string | null {
+  if (!linkHeader) return null;
+  const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
+  return match?.[1] ?? null;
+}

src/github/process.ts

@@ -0,0 +1,14 @@
+import type { Language      } from "@gh-top-languages/lib/charts/types.js";
+import type { LanguageBytes } from "./types.js";
+
+export function processLanguageData(languageBytes: LanguageBytes, count: number): Language[] {
+  if (Object.keys(languageBytes).length === 0) throw new Error("No language data available");
+
+  const totalBytes = Object.values(languageBytes).reduce((a, b) => a + b, 0);
+
+  const sortedLanguages = Object.entries(languageBytes)
+    .map(([lang, bytes]) => ({ lang, pct: (bytes / totalBytes) * 100 }))
+    .sort((a, b) => b.pct - a.pct);
+
+  return sortedLanguages.slice(0, count);
+}

src/github/rateLimit.ts

@@ -0,0 +1,13 @@
+export const FALLBACK_RETRY_MS = 5 * 60 * 1000;
+
+export function rateLimitReset(r: Response): number | null {
+  const retryAfter = r.headers.get("Retry-After");
+  const primary = (r.status === 403 || r.status === 429) && r.headers.get("X-RateLimit-Remaining") === "0";
+  if (!retryAfter && !primary) return null;
+
+  const resetHeader = r.headers.get("X-RateLimit-Reset");
+  const resetMs = retryAfter
+    ? Date.now() + Number(retryAfter) * 1000
+    : resetHeader ? Number(resetHeader) * 1000 : null;
+  return resetMs !== null && Number.isFinite(resetMs) ? resetMs : null;
+}

src/github/types.ts

@@ -0,0 +1,3 @@
+export type Source = { name: string; token?: string };
+
+export type LanguageBytes = Record<string, number>;

src/handler.ts

@@ -1,9 +1,10 @@
-import { DEFAULT_CONFIG                         } from "@gh-top-languages/lib/constants/config.js";
-import { parseQueryParams, type QueryParams     } from "@gh-top-languages/lib/utils/params.js";
-import { generateChartData                      } from "@gh-top-languages/lib/charts/generate.js";
-import { renderSvg                              } from "@gh-top-languages/lib/render/svg.js";
-import { renderError                            } from "@gh-top-languages/lib/render/error.js";
-import { fetchLanguageData, processLanguageData } from "./github.js";
+import { DEFAULT_CONFIG                     } from "@gh-top-languages/lib/constants/config.js";
+import { parseQueryParams, type QueryParams } from "@gh-top-languages/lib/utils/params.js";
+import { generateChartData                  } from "@gh-top-languages/lib/charts/generate.js";
+import { renderSvg                          } from "@gh-top-languages/lib/render/svg.js";
+import { renderError                        } from "@gh-top-languages/lib/render/error.js";
+import { fetchLanguageData                  } from "./github/fetch.js";
+import { processLanguageData                } from "./github/process.js";
 
 export type ChartResponse = {
   status:  number;

src/server.ts

@@ -1,5 +1,5 @@
-import { createServer, type Server } from "node:http";
-import { pathToFileURL             } from "node:url";
+import { createServer, type Server      } from "node:http";
+import { pathToFileURL                  } from "node:url";
 import { handleLanguages, type RawQuery } from "./handler.js";
 
 export function queryFromUrl(url: URL): RawQuery {

tests/api/languages/index.test.ts

@@ -6,13 +6,15 @@ import { generateChartData                      } from "@gh-top-languages/lib/ch
 import { renderSvg                              } from "@gh-top-languages/lib/render/svg.js";
 import { renderError                            } from "@gh-top-languages/lib/render/error.js";
 import   handler                                  from "../../../api/languages/index.js";
-import { fetchLanguageData, processLanguageData } from "../../../src/github.js";
+import { fetchLanguageData                      } from "../../../src/github/fetch.js";
+import { processLanguageData                    } from "../../../src/github/process.js";
 
 vi.mock("@gh-top-languages/lib/utils/params.js");
 vi.mock("@gh-top-languages/lib/charts/generate.js");
 vi.mock("@gh-top-languages/lib/render/svg.js");
 vi.mock("@gh-top-languages/lib/render/error.js");
-vi.mock("../../../src/github.js");
+vi.mock("../../../src/github/fetch.js");
+vi.mock("../../../src/github/process.js");
 
 describe("handler", () => {
   let req: VercelRequest;