fix: GitHub fetch caching

README.md

@@ -44,7 +44,7 @@ Deployable **GitHub language chart generator** — embeddable SVGs for READMEs a
     - **Theming**: Supports `default`, `light`, and `dark` themes.
     - **Custom Colours**: Set background (`bg`), text (`text`), and individual language colours (`c1`-`c16`) via query parameters. 
 - **Dynamic Layout:** The legend automatically shifts to a **two-column layout** when displaying 9 or more languages.
-- Automatically fetches all public GitHub repositories, and private repositories with a token.
+- Automatically fetches all public GitHub repositories. Organization sources with a token also include private repos visible to that token.
 - Ignores forks and optionally specific repositories (`IGNORED_REPOS`).
 - Uses **hourly caching** to reduce API calls and improve performance.
 
@@ -95,7 +95,7 @@ npm install
 Copy `.env.example` to `.env`, and update the variables.
 - `GITHUB_USERNAMES`: GitHub usernames to fetch repositories from. Accepts a single value (`masonlet`), comma-separated (`masonlet,secondlet`), or a JSON array with optional per-user tokens (`["masonlet", {"name": "other", "token": "github_pat_..."}]`).
 - `GITHUB_ORGS`: GitHub organization names to fetch repositories from. Accepts a single value (`gh-top-languages`), comma-separated (`gh-top-languages,starweb-libs`), or a JSON array with optional per-org tokens (`["gh-top-languages", {"name": "starweb-libs", "token": "github_pat_..."}]`)
-- `IGNORED_REPOS`: Optional comma-separated repo names to exclude from the chart.
+- `IGNORED_REPOS`: Optional comma-separated repo names to exclude from the chart. Accepts a bare name (`repo`, excluded from all sources) or `owner/name` (`org/repo`, scoped to one source).
 
 ### Running Locally
 Your endpoint will be available at http://localhost:3000/api/languages (or your configured PORT)
@@ -121,16 +121,17 @@ npm start
 
 Or, deploy with Vercel:
 
-[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https://github.com/gh-top-languages/api&env=GITHUB_USERNAMES,GITHUB_ORGS,IGNORED_REPOS&envDescription=GITHUB_USERNAMES%20and%2For%20GITHUB_ORGS%3A%20GitHub%20users%2Forgs%20to%20fetch%20repos%20from.%20IGNORED_REPOS%3A%20optional%20comma-separated%20repo%20names%20to%20exclude.&envLink=https%3A%2F%2Fgithub.com%2Fgh-top-languages%2Fapi%23configuration)
+[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https://github.com/gh-top-languages/api&env=GITHUB_USERNAMES,GITHUB_ORGS,IGNORED_REPOS&envDescription=GITHUB_USERNAMES%20and%2For%20GITHUB_ORGS%3A%20GitHub%20users%2Forgs%20to%20fetch%20repos%20from.%20IGNORED_REPOS%3A%20optional%20comma-separated%20repo%20names%20to%20exclude%20%28bare%20name%20or%20owner%2Fname%29.&envLink=https%3A%2F%2Fgithub.com%2Fgh-top-languages%2Fapi%23configuration)
 
 ## Error Responses
 
 All errors return HTTP 200 with an error SVG so they render in GitHub README embeds.
 
 Common error messages:
+- `At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set` — missing environment configuration
+- `GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration.` — malformed JSON array in env config
 - `GitHub API error: {status} {statusText}` — GitHub API request failed
 - `No language data available` — no public repositories found
-- `At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set` — missing environment configuration
 
 ## License
 MIT License - see [LICENSE](./LICENSE) for details.

src/github.ts

@@ -14,6 +14,7 @@ type LanguageBytes = Record<string, number>;
 
 let cachedLanguageData: LanguageBytes | null = null;
 let lastRefresh = 0;
+let inFlightFetch: Promise<LanguageBytes> | null = null;
 
 function parseSources(env: string | undefined): Source[] {
   if (!env) return [];
@@ -71,15 +72,7 @@ async function fetchAllRepos(url: string, token?: string): Promise<Repo[]> {
   return repos;
 }
 
-export async function fetchLanguageData(useTestData = false): Promise<LanguageBytes> {
-  if (useTestData) {
-    const testData = await import ("./test-data.json", { with: { type: "json" } });
-    return testData.default;
-  }
-
-  const now = Date.now();
-  if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData;
-
+async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
   const usernames = parseSources(process.env["GITHUB_USERNAMES"]);
   const orgs      = parseSources(process.env["GITHUB_ORGS"]);
 
@@ -109,12 +102,16 @@ export async function fetchLanguageData(useTestData = false): Promise<LanguageBy
     )
   ]);
 
-  const ignored = process.env["IGNORED_REPOS"]?.split(',').map(name => name.trim()) || [];
+  const ignored = process.env["IGNORED_REPOS"]?.split(',').map(s => s.trim()) || [];
 
   const languageFetches = repoGroups.flatMap(({ token, repos }) =>
-    repos.filter(repo => !repo.fork && !ignored.includes(repo.name)).map(repo =>
+    repos.filter(repo => !repo.fork && !ignored.includes(repo.name) && !ignored.includes(repo.full_name)).map(repo =>
       fetch(`https://api.github.com/repos/${repo.full_name.split('/').map(encodeURIComponent).join('/')}/languages`, makeOptions(token))
-        .then(r => r.ok ? (r.json() as Promise<LanguageBytes>) : ({} as LanguageBytes))
+        .then(r => {
+          if (r.ok) return r.json() as Promise<LanguageBytes>;
+          hadFetchFailure = true;
+          return {} as LanguageBytes;
+        })
         .catch(() => { hadFetchFailure = true; return {} as LanguageBytes; })
     )
   );
@@ -128,16 +125,33 @@ export async function fetchLanguageData(useTestData = false): Promise<LanguageBy
     return acc;
   }, {});
 
-  if (Object.keys(result).length === 0 && hadFetchFailure && cachedLanguageData !== null) {
-    lastRefresh = now - REFRESH_INTERVAL + (5 * 60 * 1000);
-    return cachedLanguageData;
+  if (hadFetchFailure) {
+    if (cachedLanguageData !== null) {
+      lastRefresh = now - REFRESH_INTERVAL + (5 * 60 * 1000);
+      return cachedLanguageData;
+    }
+    return result;
   }
 
   cachedLanguageData = result;
   lastRefresh = now;
   return result;
 }
 
+export async function fetchLanguageData(useTestData = false): Promise<LanguageBytes> {
+  if (useTestData) {
+    const testData = await import ("./test-data.json", { with: { type: "json" } });
+    return testData.default;
+  }
+
+  const now = Date.now();
+  if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData;
+
+  if (inFlightFetch) return inFlightFetch;
+  inFlightFetch = fetchAndAggregate(now).finally(() => { inFlightFetch = null; });
+  return inFlightFetch;
+}
+
 export function processLanguageData(languageBytes: LanguageBytes, count: number): Language[] {
   if (Object.keys(languageBytes).length === 0) throw new Error("No language data available");
 
@@ -153,4 +167,6 @@ export function processLanguageData(languageBytes: LanguageBytes, count: number)
 export function resetCache(): void {
   cachedLanguageData = null;
   lastRefresh = 0;
+  inFlightFetch = null;
 }
+

tests/github.test.ts

@@ -51,6 +51,12 @@ describe("fetchLanguageData", () => {
     await expect(fetchLanguageData()).rejects.toThrow("At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set");
   });
 
+  it("propagates a rejection to all concurrent callers", async () => {
+    vi.unstubAllEnvs();
+    await expect(Promise.all([fetchLanguageData(), fetchLanguageData()]))
+      .rejects.toThrow("At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set");
+  });
+
   it("handles missing IGNORED_REPOS env variable", async () => {
     vi.unstubAllEnvs();
     vi.stubEnv("GITHUB_USERNAMES", `["testuser"]`);
@@ -87,6 +93,28 @@ describe("fetchLanguageData", () => {
     );
   });
 
+  it("ignores repos by owner/name to scope across sources", async () => {
+    vi.unstubAllEnvs();
+    vi.stubEnv("GITHUB_USERNAMES", `["testuser"]`);
+    vi.stubEnv("IGNORED_REPOS", "otheruser/blog");
+
+    const scopedRepos = [
+      { name: "blog", fork: false, full_name: "testuser/blog" },
+      { name: "notes", fork: false, full_name: "testuser/notes" }
+    ];
+
+    mockFetch()
+      .mockResolvedValueOnce(mockResponse(scopedRepos))
+      .mockResolvedValueOnce(mockResponse(languages))
+      .mockResolvedValueOnce(mockResponse(languages));
+
+    await fetchLanguageData();
+
+    expect(global.fetch).toHaveBeenCalledWith(
+      "https://api.github.com/repos/testuser/blog/languages", {}
+    );
+  });
+
   it("aggregates language bytes across repos", async () => {
     const repos = [
       { name: "repo1", fork: false, full_name: "user/repo1" },
@@ -129,6 +157,20 @@ describe("fetchLanguageData", () => {
     expect(global.fetch).toHaveBeenCalledTimes(2);
   });
 
+  it("coalesces concurrent calls into a single fetch", async () => {
+    mockFetch()
+      .mockResolvedValueOnce(mockResponse([repos[0]]))
+      .mockResolvedValueOnce(mockResponse(languages));
+
+    const [result1, result2] = await Promise.all([
+      fetchLanguageData(),
+      fetchLanguageData()
+    ]);
+
+    expect(global.fetch).toHaveBeenCalledTimes(2);
+    expect(result1).toBe(result2);
+  });
+
   it("handles failed language fetch gracefully", async () => {
     const repos = [
       { name: "repo1", fork: false, full_name: "user/repo1" },
@@ -137,13 +179,68 @@ describe("fetchLanguageData", () => {
 
     mockFetch()
       .mockResolvedValueOnce(mockResponse(repos))
-      .mockResolvedValueOnce(mockErrorResponse(403)) // Failed language fetch
+      .mockResolvedValueOnce(mockErrorResponse(403))
       .mockResolvedValueOnce(mockResponse({ Python: 500 }));
 
     const result = await fetchLanguageData();
     expect(result).toEqual({ Python: 500 });
   });
 
+  it("does not cache a cold-start total failure and retries on next call", async () => {
+    mockFetch().mockResolvedValueOnce(mockErrorResponse(500, "Internal Server Error"));
+    await fetchLanguageData();
+
+    mockFetch().mockResolvedValueOnce(mockResponse([repos[0]])).mockResolvedValueOnce(mockResponse(languages));
+    const result = await fetchLanguageData();
+
+    expect(global.fetch).toHaveBeenCalledTimes(3);
+    expect(result).toEqual(languages);
+  });
+
+  it("does not cache a partial failure as complete and retries on next call", async () => {
+    const twoRepos = [
+      { name: "repo1", fork: false, full_name: "user/repo1" },
+      { name: "repo2", fork: false, full_name: "user/repo2" }
+    ];
+
+    mockFetch()
+      .mockResolvedValueOnce(mockResponse(twoRepos))
+      .mockRejectedValueOnce(new Error("Network error"))
+      .mockResolvedValueOnce(mockResponse({ Python: 500 }));
+    await fetchLanguageData();
+
+    mockFetch()
+      .mockResolvedValueOnce(mockResponse(twoRepos))
+      .mockResolvedValueOnce(mockResponse({ JavaScript: 1000 }))
+      .mockResolvedValueOnce(mockResponse({ Python: 500 }));
+    const result = await fetchLanguageData();
+
+    expect(global.fetch).toHaveBeenCalledTimes(6);
+    expect(result).toEqual({ JavaScript: 1000, Python: 500 });
+  });
+
+  it("treats a non-ok language-fetch response as a failure, skipping the cache", async () => {
+    const twoRepos = [
+      { name: "repo1", fork: false, full_name: "user/repo1" },
+      { name: "repo2", fork: false, full_name: "user/repo2" }
+    ];
+
+    mockFetch()
+      .mockResolvedValueOnce(mockResponse(twoRepos))
+      .mockResolvedValueOnce(mockErrorResponse(500))
+      .mockResolvedValueOnce(mockResponse({ Python: 500 }));
+    await fetchLanguageData();
+
+    mockFetch()
+      .mockResolvedValueOnce(mockResponse(twoRepos))
+      .mockResolvedValueOnce(mockResponse({ JavaScript: 1000 }))
+      .mockResolvedValueOnce(mockResponse({ Python: 500 }));
+    const result = await fetchLanguageData();
+
+    expect(global.fetch).toHaveBeenCalledTimes(6);
+    expect(result).toEqual({ JavaScript: 1000, Python: 500 });
+  });
+
   it("fetches from organizations", async () => {
     vi.unstubAllEnvs();
     vi.stubEnv("GITHUB_ORGS", `["test-org"]`);