diff --git a/README.md b/README.md index 628ef93..a270ef9 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Deployable **GitHub language chart generator** — embeddable SVGs for READMEs a - **Theming**: Supports `default`, `light`, and `dark` themes. - **Custom Colours**: Set background (`bg`), text (`text`), and individual language colours (`c1`-`c16`) via query parameters. - **Dynamic Layout:** The legend automatically shifts to a **two-column layout** when displaying 9 or more languages. -- Automatically fetches all public GitHub repositories, and private repositories with a token. +- Automatically fetches all public GitHub repositories. Organization sources with a token also include private repos visible to that token. - Ignores forks and optionally specific repositories (`IGNORED_REPOS`). - Uses **hourly caching** to reduce API calls and improve performance. @@ -95,7 +95,7 @@ npm install Copy `.env.example` to `.env`, and update the variables. - `GITHUB_USERNAMES`: GitHub usernames to fetch repositories from. Accepts a single value (`masonlet`), comma-separated (`masonlet,secondlet`), or a JSON array with optional per-user tokens (`["masonlet", {"name": "other", "token": "github_pat_..."}]`). - `GITHUB_ORGS`: GitHub organization names to fetch repositories from. Accepts a single value (`gh-top-languages`), comma-separated (`gh-top-languages,starweb-libs`), or a JSON array with optional per-org tokens (`["gh-top-languages", {"name": "starweb-libs", "token": "github_pat_..."}]`) -- `IGNORED_REPOS`: Optional comma-separated repo names to exclude from the chart. +- `IGNORED_REPOS`: Optional comma-separated repo names to exclude from the chart. Accepts a bare name (`repo`, excluded from all sources) or `owner/name` (`org/repo`, scoped to one source). ### Running Locally Your endpoint will be available at http://localhost:3000/api/languages (or your configured PORT) @@ -121,16 +121,17 @@ npm start Or, deploy with Vercel: -[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https://github.com/gh-top-languages/api&env=GITHUB_USERNAMES,GITHUB_ORGS,IGNORED_REPOS&envDescription=GITHUB_USERNAMES%20and%2For%20GITHUB_ORGS%3A%20GitHub%20users%2Forgs%20to%20fetch%20repos%20from.%20IGNORED_REPOS%3A%20optional%20comma-separated%20repo%20names%20to%20exclude.&envLink=https%3A%2F%2Fgithub.com%2Fgh-top-languages%2Fapi%23configuration) +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https://github.com/gh-top-languages/api&env=GITHUB_USERNAMES,GITHUB_ORGS,IGNORED_REPOS&envDescription=GITHUB_USERNAMES%20and%2For%20GITHUB_ORGS%3A%20GitHub%20users%2Forgs%20to%20fetch%20repos%20from.%20IGNORED_REPOS%3A%20optional%20comma-separated%20repo%20names%20to%20exclude%20%28bare%20name%20or%20owner%2Fname%29.&envLink=https%3A%2F%2Fgithub.com%2Fgh-top-languages%2Fapi%23configuration) ## Error Responses All errors return HTTP 200 with an error SVG so they render in GitHub README embeds. Common error messages: +- `At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set` — missing environment configuration +- `GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration.` — malformed JSON array in env config - `GitHub API error: {status} {statusText}` — GitHub API request failed - `No language data available` — no public repositories found -- `At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set` — missing environment configuration ## License MIT License - see [LICENSE](./LICENSE) for details. diff --git a/src/github.ts b/src/github.ts index f86694b..4baca51 100644 --- a/src/github.ts +++ b/src/github.ts @@ -14,6 +14,7 @@ type LanguageBytes = Record; let cachedLanguageData: LanguageBytes | null = null; let lastRefresh = 0; +let inFlightFetch: Promise | null = null; function parseSources(env: string | undefined): Source[] { if (!env) return []; @@ -71,15 +72,7 @@ async function fetchAllRepos(url: string, token?: string): Promise { return repos; } -export async function fetchLanguageData(useTestData = false): Promise { - if (useTestData) { - const testData = await import ("./test-data.json", { with: { type: "json" } }); - return testData.default; - } - - const now = Date.now(); - if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData; - +async function fetchAndAggregate(now: number): Promise { const usernames = parseSources(process.env["GITHUB_USERNAMES"]); const orgs = parseSources(process.env["GITHUB_ORGS"]); @@ -109,12 +102,16 @@ export async function fetchLanguageData(useTestData = false): Promise name.trim()) || []; + const ignored = process.env["IGNORED_REPOS"]?.split(',').map(s => s.trim()) || []; const languageFetches = repoGroups.flatMap(({ token, repos }) => - repos.filter(repo => !repo.fork && !ignored.includes(repo.name)).map(repo => + repos.filter(repo => !repo.fork && !ignored.includes(repo.name) && !ignored.includes(repo.full_name)).map(repo => fetch(`https://api.github.com/repos/${repo.full_name.split('/').map(encodeURIComponent).join('/')}/languages`, makeOptions(token)) - .then(r => r.ok ? (r.json() as Promise) : ({} as LanguageBytes)) + .then(r => { + if (r.ok) return r.json() as Promise; + hadFetchFailure = true; + return {} as LanguageBytes; + }) .catch(() => { hadFetchFailure = true; return {} as LanguageBytes; }) ) ); @@ -128,9 +125,12 @@ export async function fetchLanguageData(useTestData = false): Promise { + if (useTestData) { + const testData = await import ("./test-data.json", { with: { type: "json" } }); + return testData.default; + } + + const now = Date.now(); + if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData; + + if (inFlightFetch) return inFlightFetch; + inFlightFetch = fetchAndAggregate(now).finally(() => { inFlightFetch = null; }); + return inFlightFetch; +} + export function processLanguageData(languageBytes: LanguageBytes, count: number): Language[] { if (Object.keys(languageBytes).length === 0) throw new Error("No language data available"); @@ -153,4 +167,6 @@ export function processLanguageData(languageBytes: LanguageBytes, count: number) export function resetCache(): void { cachedLanguageData = null; lastRefresh = 0; + inFlightFetch = null; } + diff --git a/tests/github.test.ts b/tests/github.test.ts index 77aba81..2e4b887 100644 --- a/tests/github.test.ts +++ b/tests/github.test.ts @@ -51,6 +51,12 @@ describe("fetchLanguageData", () => { await expect(fetchLanguageData()).rejects.toThrow("At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set"); }); + it("propagates a rejection to all concurrent callers", async () => { + vi.unstubAllEnvs(); + await expect(Promise.all([fetchLanguageData(), fetchLanguageData()])) + .rejects.toThrow("At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set"); + }); + it("handles missing IGNORED_REPOS env variable", async () => { vi.unstubAllEnvs(); vi.stubEnv("GITHUB_USERNAMES", `["testuser"]`); @@ -87,6 +93,28 @@ describe("fetchLanguageData", () => { ); }); + it("ignores repos by owner/name to scope across sources", async () => { + vi.unstubAllEnvs(); + vi.stubEnv("GITHUB_USERNAMES", `["testuser"]`); + vi.stubEnv("IGNORED_REPOS", "otheruser/blog"); + + const scopedRepos = [ + { name: "blog", fork: false, full_name: "testuser/blog" }, + { name: "notes", fork: false, full_name: "testuser/notes" } + ]; + + mockFetch() + .mockResolvedValueOnce(mockResponse(scopedRepos)) + .mockResolvedValueOnce(mockResponse(languages)) + .mockResolvedValueOnce(mockResponse(languages)); + + await fetchLanguageData(); + + expect(global.fetch).toHaveBeenCalledWith( + "https://api.github.com/repos/testuser/blog/languages", {} + ); + }); + it("aggregates language bytes across repos", async () => { const repos = [ { name: "repo1", fork: false, full_name: "user/repo1" }, @@ -129,6 +157,20 @@ describe("fetchLanguageData", () => { expect(global.fetch).toHaveBeenCalledTimes(2); }); + it("coalesces concurrent calls into a single fetch", async () => { + mockFetch() + .mockResolvedValueOnce(mockResponse([repos[0]])) + .mockResolvedValueOnce(mockResponse(languages)); + + const [result1, result2] = await Promise.all([ + fetchLanguageData(), + fetchLanguageData() + ]); + + expect(global.fetch).toHaveBeenCalledTimes(2); + expect(result1).toBe(result2); + }); + it("handles failed language fetch gracefully", async () => { const repos = [ { name: "repo1", fork: false, full_name: "user/repo1" }, @@ -137,13 +179,68 @@ describe("fetchLanguageData", () => { mockFetch() .mockResolvedValueOnce(mockResponse(repos)) - .mockResolvedValueOnce(mockErrorResponse(403)) // Failed language fetch + .mockResolvedValueOnce(mockErrorResponse(403)) .mockResolvedValueOnce(mockResponse({ Python: 500 })); const result = await fetchLanguageData(); expect(result).toEqual({ Python: 500 }); }); + it("does not cache a cold-start total failure and retries on next call", async () => { + mockFetch().mockResolvedValueOnce(mockErrorResponse(500, "Internal Server Error")); + await fetchLanguageData(); + + mockFetch().mockResolvedValueOnce(mockResponse([repos[0]])).mockResolvedValueOnce(mockResponse(languages)); + const result = await fetchLanguageData(); + + expect(global.fetch).toHaveBeenCalledTimes(3); + expect(result).toEqual(languages); + }); + + it("does not cache a partial failure as complete and retries on next call", async () => { + const twoRepos = [ + { name: "repo1", fork: false, full_name: "user/repo1" }, + { name: "repo2", fork: false, full_name: "user/repo2" } + ]; + + mockFetch() + .mockResolvedValueOnce(mockResponse(twoRepos)) + .mockRejectedValueOnce(new Error("Network error")) + .mockResolvedValueOnce(mockResponse({ Python: 500 })); + await fetchLanguageData(); + + mockFetch() + .mockResolvedValueOnce(mockResponse(twoRepos)) + .mockResolvedValueOnce(mockResponse({ JavaScript: 1000 })) + .mockResolvedValueOnce(mockResponse({ Python: 500 })); + const result = await fetchLanguageData(); + + expect(global.fetch).toHaveBeenCalledTimes(6); + expect(result).toEqual({ JavaScript: 1000, Python: 500 }); + }); + + it("treats a non-ok language-fetch response as a failure, skipping the cache", async () => { + const twoRepos = [ + { name: "repo1", fork: false, full_name: "user/repo1" }, + { name: "repo2", fork: false, full_name: "user/repo2" } + ]; + + mockFetch() + .mockResolvedValueOnce(mockResponse(twoRepos)) + .mockResolvedValueOnce(mockErrorResponse(500)) + .mockResolvedValueOnce(mockResponse({ Python: 500 })); + await fetchLanguageData(); + + mockFetch() + .mockResolvedValueOnce(mockResponse(twoRepos)) + .mockResolvedValueOnce(mockResponse({ JavaScript: 1000 })) + .mockResolvedValueOnce(mockResponse({ Python: 500 })); + const result = await fetchLanguageData(); + + expect(global.fetch).toHaveBeenCalledTimes(6); + expect(result).toEqual({ JavaScript: 1000, Python: 500 }); + }); + it("fetches from organizations", async () => { vi.unstubAllEnvs(); vi.stubEnv("GITHUB_ORGS", `["test-org"]`);