Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
a5c624f
refactor: split single global cache into per-source entries
masonlet Jul 13, 2026
cf5723e
feat: add mode detection and source resolution
masonlet Jul 13, 2026
88c7b16
docs: reorganize .env.example and add hosted settings
masonlet Jul 13, 2026
afa5e64
test: github select coverage
masonlet Jul 13, 2026
16be3bd
feat: negative-cache missing accounts for 10 minutes
masonlet Jul 13, 2026
0c9520c
feat: wire ?source= selection through the handler
masonlet Jul 13, 2026
4e4097f
test: add source selection testing
masonlet Jul 13, 2026
34d4315
test: stub env for handler unit tests
masonlet Jul 13, 2026
baacbc6
docs: document hosted instances and ?source= selection
masonlet Jul 13, 2026
13acc1f
fix: make in-flight fetches strictness-neutral and surface strict fet…
masonlet Jul 13, 2026
3c89cfd
test: cover strict/non-strict race, negative-cache short-circuit, sou…
masonlet Jul 14, 2026
1fb53ae
docs: clean up .env.example
masonlet Jul 14, 2026
9d14de3
fix: dedupe configured sources and log skipped missing accounts
masonlet Jul 14, 2026
d805ce8
test: cover source dedupe
masonlet Jul 14, 2026
b59beb4
refactor: constants file, extract parseNames to parse module
masonlet Jul 14, 2026
f913d84
fix: dedupe allowlist entries and pin hosted endpoint boundary
masonlet Jul 14, 2026
d2722b5
fix: log fetch-failure detail and comment out .env.example defaults
masonlet Jul 14, 2026
ca480be
feat: optional referer gate for preview-only deployments
masonlet Jul 14, 2026
e172a6f
docs: document ALLOWED_REFERERS
masonlet Jul 14, 2026
18f3296
docs: add ALLOWED_REFERS to deploy button
masonlet Jul 14, 2026
3bf366c
docs: split deploy buttons per mode
masonlet Jul 14, 2026
9b56d8a
refactor: surface rate-limit reset time in error SVG
masonlet Jul 14, 2026
44ed03f
fix: indent
masonlet Jul 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions .env.example
Original file line number Diff line number Diff line change
@@ -1,8 +1,23 @@
GITHUB_TOKEN=your_token
GITHUB_USERNAMES=your_username
GITHUB_ORGS=your_organization
IGNORED_REPOS=repo1,repo2
# ===== Global Settings =====
# GITHUB_TOKEN=your_token
# IGNORED_REPOS=repo1,repo2
# ALLOWED_REFERERS=your-builder-domain.com

# Personal and Hosted instances are mutually exclusive.

# ===== Personal Instance =====
# Render a chart of your own accounts.
# Advanced: JSON array with optional per-entry tokens

# GITHUB_USERNAMES=your_username
# GITHUB_ORGS=your_organization

# GITHUB_USERNAMES=["your_username", {"name": "other_username", "token": "github_pat_"}]
# GITHUB_ORGS=["your_org", {"name": "private_org", "token": "github_pat_"}]

# ===== Hosted instance =====
# Serves charts for others' public repos via ?source=<name>[,<name>...].
# Comma-separated account names to allow, or * to allow any source.
# WARNING: * spends your GITHUB_TOKEN's rate limit on everyone who uses your instance.

# GITHUB_ALLOWED_SOURCES=
42 changes: 31 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,13 @@ Deployable **GitHub language chart generator** — embeddable SVGs for READMEs a
- **Theming**: Supports `default`, `light`, and `dark` themes.
- **Custom Colours**: Set background (`bg`), text (`text`), and individual language colours (`c1`-`c16`) via query parameters.
- **Dynamic Layout:** The legend automatically shifts to a **two-column layout** when displaying 9 or more languages.
- **Automatically fetches GitHub repositories:** Public user and organization sources are automatically fetched; private repos can be fetched if tokens have access (personal mode only).
- **Hosted instances:** Serve charts for others via `?source=` — an enumerated allowlist, or `*` for any GitHub account. Hosted modes fetch public repos only.
- **Global Token:** Optional deployment-wide default token, applied to any source without its own `token`, boosting limits from 60 req/hr to 5,000/hr.
- **Automatically fetches GitHub repositories:** Public user and organization sources are automatically fetched; private repos can be fetched if tokens have access.
- **Automatically ignores forks**.
- **Ignored repositories**: Repos ignored via env (`IGNORED_REPOS`).
- **Hourly caching**: Used to reduce API calls and improve performance. On fetch failure the last good chart is served and refresh retries after either 5 minutes, or when GitHub rate-limit timer resets.
- **Automatically ignores forks**.
- **Ignored repositories**: Repos ignored via optional (`IGNORED_REPOS`) env var.
- **Preview-only deployments:** Optional `ALLOWED_REFERERS` restricts rendering to your own site — blocks README embeds and hotlinking.

## Customize Your Charts
Prefer a visual workflow? Use the [@gh-top-languages/builder](https://github.com/gh-top-languages/builder) to preview themes, colours, and layout options interactively, then easily copy the generated embed URL to quickly deploy.
Expand All @@ -74,6 +76,7 @@ Prefer a visual workflow? Use the [@gh-top-languages/builder](https://github.com
| :--- | :--- | :--- | :--- | :--- |
| `test` | Boolean | Uses sample data instead of fetching from GitHub API. | `false` | `?test=true` |
| `error` | String | Forces an error SVG with the given message. For testing only. | — | `?error=test` |
| `source` | String | Hosted instances only: comma-separated GitHub account names to chart (max 10 when `GITHUB_ALLOWED_SOURCES=*`). | — | `?source=torvalds,rails` |

#### Query Parameters
Full parameter reference lives in the [lib README](https://github.com/gh-top-languages/lib#query-parameters).
Expand All @@ -94,11 +97,19 @@ npm install
```

### Configuration
Copy `.env.example` to `.env`, and update the variables.
- `GITHUB_USERNAMES`: GitHub usernames to fetch repositories from. Accepts a single value (`masonlet`), comma-separated (`masonlet,secondlet`), or a JSON array with optional per-user tokens (`["masonlet", {"name": "other", "token": "github_pat_..."}]`).
- `GITHUB_ORGS`: GitHub organization names to fetch repositories from. Accepts a single value (`gh-top-languages`), comma-separated (`gh-top-languages,starweb-libs`), or a JSON array with optional per-org tokens (`["gh-top-languages", {"name": "starweb-libs", "token": "github_pat_..."}]`)
Copy `.env.example` to `.env`, and update the variables. Configure exactly one mode: **Personal** or **Hosted**.

#### Global
- `IGNORED_REPOS`: Optional comma-separated repo names to exclude from the chart. Accepts a bare name (`repo`, excluded from all sources) or `owner/name` (`org/repo`, scoped to one source).
- `GITHUB_TOKEN`: Optional deployment-wide default token, applied to any source without its own `token`. A per-source token always takes precedence; the global token only authenticates public repos (no private repos) while boosting limits (5,000 req/hr instead of 60/hr).
- `ALLOWED_REFERERS`: Optional comma-separated hostnames. When set, charts render only for requests whose `Origin`/`Referer` hostname matches: for preview-only deployments (e.g. serving a builder site). Blocks README embeds, hotlinks, and direct navigation.

#### Personal
- `GITHUB_USERNAMES`: GitHub usernames to fetch repositories from. Accepts a single value (`masonlet`), comma-separated (`masonlet,secondlet`), or a JSON array with optional per-user tokens (`["masonlet", {"name": "other", "token": "github_pat_..."}]`).
- `GITHUB_ORGS`: GitHub organization names to fetch repositories from. Accepts a single value (`gh-top-languages`), comma-separated (`gh-top-languages,starweb-libs`), or a JSON array with optional per-org tokens (`["gh-top-languages", {"name": "starweb-libs", "token": "github_pat_..."}]`)

#### Hosted
- `GITHUB_ALLOWED_SOURCES`: Comma-separated account names selectable via `?source=` (bare URL renders their combined chart), or `*` to allow any GitHub account (`?source=` required, max 10 names per request). Hosted instances chart **public repositories only**; per-source tokens don't exist in this mode. **Warning:** with `*`, anyone can use your instance, and every request they make spends your GITHUB_TOKEN's rate limit.

### Running Locally
Your endpoint will be available at http://localhost:3000/api/languages (or your configured PORT)
Expand All @@ -123,18 +134,27 @@ npm start
```

Or, deploy with Vercel:
**Personal** instance (your own chart):

[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https://github.com/gh-top-languages/api&env=GITHUB_USERNAMES&envDescription=Your%20GitHub%20username%28s%29%2C%20comma-separated.%20Optional%20vars%20%28GITHUB_ORGS%2C%20GITHUB_TOKEN%2C%20IGNORED_REPOS%29%20are%20documented%20at%20the%20link%20and%20can%20be%20added%20in%20Vercel%20project%20settings.&envLink=https%3A%2F%2Fgithub.com%2Fgh-top-languages%2Fapi%23configuration)

**Hosted** instance (serve charts for others via `?source=`):

[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https://github.com/gh-top-languages/api&env=GITHUB_USERNAMES,GITHUB_ORGS,GITHUB_TOKEN,IGNORED_REPOS&envDescription=GITHUB_USERNAMES%20and%2For%20GITHUB_ORGS%3A%20GitHub%20users%2Forgs%20to%20fetch%20repos%20from.%20GITHUB_TOKEN%3A%20optional%20global%20fallback%20token%20for%20sources%20without%20their%20own.%20IGNORED_REPOS%3A%20optional%20comma-separated%20repo%20names%20to%20exclude%20%28bare%20name%20or%20owner%2Fname%29.&envLink=https%3A%2F%2Fgithub.com%2Fgh-top-languages%2Fapi%23configuration)
[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https://github.com/gh-top-languages/api&env=GITHUB_ALLOWED_SOURCES&envDescription=Account%20names%20to%20allow%2C%20comma-separated%2C%20or%20*%20for%20any%20account.%20Strongly%20recommended%3A%20add%20GITHUB_TOKEN%20in%20Vercel%20settings%20%28*%20mode%20is%20nearly%20unusable%20at%2060%20req%2Fhr%20without%20one%29.%20See%20link%20for%20all%20options.&envLink=https%3A%2F%2Fgithub.com%2Fgh-top-languages%2Fapi%23configuration)

## Error Responses

All errors return HTTP 200 with an error SVG so they render in GitHub README embeds.

Common error messages:
- `At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set` — missing environment configuration
- `GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration.` — malformed JSON array in env config
- `GitHub API error: {status} {statusText}` — GitHub API request failed
- `No language data available` — no public repositories found
- `Set GITHUB_USERNAMES/GITHUB_ORGS, or GITHUB_ALLOWED_SOURCES for a hosted instance`: missing environment configuration
- `GITHUB_ALLOWED_SOURCES cannot be combined with GITHUB_USERNAMES/GITHUB_ORGS`: both modes configured at once
- `Source selection is not enabled on this instance` / `Unknown or disallowed source` / `Invalid source name` / `Too many sources`: bad `?source=` request; these are CDN-cached for 5 minutes
- `Unknown GitHub account`: `?source=` named an account that doesn't exist (cached 10 minutes)
- `This instance only serves its own site - deploy your own from github.com/gh-top-languages/api`: request blocked by `ALLOWED_REFERERS` (cached 5 minutes)
- `GITHUB_USERNAMES/GITHUB_ORGS must be a valid JSON array. Check your configuration.`: malformed JSON array in env config
- `GitHub API error: {status} {statusText}`: GitHub API request failed
- `No language data available`: no public repositories found
- Rate limiting (`403`/`429` with exhausted quota) is detected separately and logged as `GitHub rate limit exceeded; resets at HH:MM:SS`; the cached chart is served until the reset.

## License
Expand Down
2 changes: 1 addition & 1 deletion api/languages/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import type { VercelRequest, VercelResponse } from "@vercel/node";
import { handleLanguages } from "../../src/handler.js";

export default async function handler(req: VercelRequest, res: VercelResponse): Promise<void> {
const { status, headers, body } = await handleLanguages(req.query);
const { status, headers, body } = await handleLanguages(req.query, req.headers);
for (const [key, value] of Object.entries(headers)) res.setHeader(key, value);
res.status(status).send(body);
}
5 changes: 5 additions & 0 deletions src/github/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
export const MAX_OPEN_SOURCES = 10;
export const VALID_LOGIN = /^[a-zA-Z0-9](?:-?[a-zA-Z0-9]){0,38}$/;

export const REFRESH_INTERVAL = 1000 * 60 * 60;
export const NEGATIVE_TTL = 1000 * 60 * 10;
182 changes: 103 additions & 79 deletions src/github/fetch.ts
Original file line number Diff line number Diff line change
@@ -1,91 +1,114 @@
import type {
CacheEntry,
FetchOutcome,
LanguageBytes,
Repo,
Source,
SourceKind
} from "./types.js";
import { NEGATIVE_TTL, REFRESH_INTERVAL } from "./constants.js";
import { parseNextLink, parseSources } from "./parse.js";
import { FALLBACK_RETRY_MS, rateLimitReset } from "./rateLimit.js";
import type { LanguageBytes } from "./types.js";

export class SourceNotFoundError extends Error {
constructor() { super("Unknown GitHub account"); }
}

const REFRESH_INTERVAL = 1000 * 60 * 60;

let cachedLanguageData: LanguageBytes | null = null;
let lastRefresh = 0;
let inFlightFetch: Promise<LanguageBytes> | null = null;
const cache = new Map<string, CacheEntry>();

type Repo = {
name: string;
fork: boolean;
full_name: string;
};
export async function fetchSelectedSources(names: string[], strict: boolean): Promise<LanguageBytes> {
const results = await Promise.all(names.map(n => fetchSource("user", { name: n }, strict)));
return mergeLanguages(results);
}

export async function fetchLanguageData(useTestData = false): Promise<LanguageBytes> {
if (useTestData) {
const testData = await import ("../test-data.json", { with: { type: "json" } });
return testData.default;
}

const now = Date.now();
if (cachedLanguageData && now - lastRefresh < REFRESH_INTERVAL) return cachedLanguageData;

if (inFlightFetch) return inFlightFetch;
inFlightFetch = fetchAndAggregate(now).finally(() => { inFlightFetch = null; });
return inFlightFetch;
}

export function resetCache(): void {
cachedLanguageData = null;
lastRefresh = 0;
inFlightFetch = null;
}

async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
const globalToken = process.env["GITHUB_TOKEN"]?.trim() || undefined;
const usernames = parseSources(process.env["GITHUB_USERNAMES"]);
const orgs = parseSources(process.env["GITHUB_ORGS"]);

if (usernames.length === 0 && orgs.length === 0) throw new Error(
"At least one of GITHUB_USERNAMES or GITHUB_ORGS must be set"
);

const dedupe = (sources: Source[]) => {
const seen = new Set<string>();
return sources.filter(s => {
const k = s.name.toLowerCase();
return seen.has(k) ? false : (seen.add(k), true);
});
};

const results = await Promise.all([
...dedupe(usernames).map(u => fetchSource("user", u)),
...dedupe(orgs).map (o => fetchSource("org", o))
]);

return mergeLanguages(results);
}

export function resetCache(): void { cache.clear(); }

async function fetchSource(kind: SourceKind, source: Source, strict = false): Promise<LanguageBytes> {
const key = `${kind}:${source.name.toLowerCase()}`;
let entry = cache.get(key);
if (!entry) { entry = { data: null, lastRefresh: 0, inFlight: null }; cache.set(key, entry); }

const now = Date.now();
if (entry.missingUntil && now < entry.missingUntil) {
if (strict) throw new SourceNotFoundError();
return {};
}
if (entry.data && now - entry.lastRefresh < REFRESH_INTERVAL) return entry.data;

const outcome = await (entry.inFlight ??= fetchOne(kind, source, entry, now).finally(() => { entry.inFlight = null; }));
if (outcome.kind === "missing") {
if (strict) throw new SourceNotFoundError();
console.error(`Skipping ${kind} "${source.name}": account not found.`);
return {};
}
if (outcome.kind === "failed" && strict) {
const msg = outcome.retryAt
? `GitHub rate limit exceeded; try again at ${new Date(outcome.retryAt).toLocaleTimeString()}`
: "GitHub API error: source fetch failed";
throw new Error(msg);
}
return outcome.data;
}

async function fetchOne(kind: SourceKind, source: Source, entry: CacheEntry, now: number): Promise<FetchOutcome> {
const token = source.token ?? (process.env["GITHUB_TOKEN"]?.trim() || undefined);

let rateLimitResetAt: number | null = null;
const noteReset = (ms: number) => { rateLimitResetAt = Math.max(rateLimitResetAt ?? 0, ms); };

let hadFetchFailure = false;
const repoGroups = await Promise.all([
...usernames.map(async u => {
const token = u.token ?? globalToken;
try {
const repos = await fetchAllRepos(
u.token ? `https://api.github.com/user/repos?per_page=100&visibility=all&affiliation=owner`
: `https://api.github.com/users/${encodeURIComponent(u.name)}/repos?per_page=100`,
noteReset,
token
);
return { token, repos };
} catch {
hadFetchFailure = true;
console.error(`Skipping user "${u.name}": failed to fetch repositories.`);
return { token, repos: [] as Repo[] };
}
}),
...orgs.map(async o => {
const token = o.token ?? globalToken;
try {
const repos = await fetchAllRepos(
`https://api.github.com/orgs/${encodeURIComponent(o.name)}/repos?per_page=100`,
noteReset,
token
);
return { token, repos };
} catch {
hadFetchFailure = true;
console.error(`Skipping org "${o.name}": failed to fetch repositories.`);
return { token, repos: [] as Repo[] };
}
})
]);

let repos: Repo[] = [];
try {
repos = await fetchAllRepos(
kind === "org" ? `https://api.github.com/orgs/${encodeURIComponent(source.name)}/repos?per_page=100`
: source.token ? `https://api.github.com/user/repos?per_page=100&visibility=all&affiliation=owner`
: `https://api.github.com/users/${encodeURIComponent(source.name)}/repos?per_page=100`,
noteReset,
token
);
} catch (e) {
if (e instanceof SourceNotFoundError) {
entry.missingUntil = now + NEGATIVE_TTL;
entry.data = null;
return { kind: "missing" };
}
hadFetchFailure = true;
console.error(`Skipping ${kind} "${source.name}": ${(e as Error).message}`);
}

const ignored = process.env["IGNORED_REPOS"]?.split(',').map(s => s.trim()) || [];

const languageFetches = repoGroups.flatMap(({ token, repos }) =>
repos.filter(repo => !repo.fork && !ignored.includes(repo.name) && !ignored.includes(repo.full_name)).map(repo =>
const langResults = await Promise.all(
repos.filter(r => !r.fork && !ignored.includes(r.name) && !ignored.includes(r.full_name)).map(repo =>
fetch(`https://api.github.com/repos/${repo.full_name.split('/').map(encodeURIComponent).join('/')}/languages`, makeOptions(token))
.then(r => {
if (r.ok) return r.json() as Promise<LanguageBytes>;
Expand All @@ -98,33 +121,26 @@ async function fetchAndAggregate(now: number): Promise<LanguageBytes> {
)
);

const langResults: LanguageBytes[] = await Promise.all(languageFetches);

const result = langResults.reduce<LanguageBytes>((acc, languages) => {
for (const [lang, bytes] of Object.entries(languages)) {
acc[lang] = (acc[lang] || 0) + bytes;
}
return acc;
}, {});
const result = mergeLanguages(langResults);

if (hadFetchFailure) {
if (rateLimitResetAt !== null) console.error(
`GitHub rate limit exceeded; resets at ${new Date(rateLimitResetAt).toLocaleTimeString()}`
);

if (cachedLanguageData !== null) {
if (entry.data !== null) {
const retryDelay = rateLimitResetAt
? Math.min(Math.max(rateLimitResetAt - now, 60_000), REFRESH_INTERVAL)
: FALLBACK_RETRY_MS;
lastRefresh = now - REFRESH_INTERVAL + retryDelay;
return cachedLanguageData;
entry.lastRefresh = now - REFRESH_INTERVAL + retryDelay;
return { kind: "ok", data: entry.data };
}
return result;
return { kind: "failed", data: result, retryAt: rateLimitResetAt };
}

cachedLanguageData = result;
lastRefresh = now;
return result;
entry.data = result;
entry.lastRefresh = now;
delete entry.missingUntil;
return { kind: "ok", data: result };
}

async function fetchAllRepos(
Expand All @@ -139,6 +155,7 @@ async function fetchAllRepos(
while (nextUrl) {
const response = await fetch(nextUrl, options);
if (!response.ok) {
if (response.status === 404 && repos.length === 0) throw new SourceNotFoundError();
const reset = rateLimitReset(response);
if (reset) onRateLimit(reset);
throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
Expand All @@ -153,6 +170,13 @@ async function fetchAllRepos(
return repos;
}

function mergeLanguages(all: LanguageBytes[]): LanguageBytes {
return all.reduce<LanguageBytes>((acc, languages) => {
for (const [lang, bytes] of Object.entries(languages)) acc[lang] = (acc[lang] || 0) + bytes;
return acc;
}, {});
}

function makeOptions(token?: string): RequestInit {
return token ? { headers: { Authorization: `Bearer ${token}` } } : {};
}
12 changes: 12 additions & 0 deletions src/github/parse.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { VALID_LOGIN } from "./constants.js";
import type { Source } from "./types.js";

export function parseSources(env: string | undefined): Source[] {
Expand Down Expand Up @@ -33,3 +34,14 @@ export function parseNextLink(linkHeader: string | null): string | null {
const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
return match?.[1] ?? null;
}

export function parseNames(raw: string, context: string): string[] {
const names: string[] = [];
const seen = new Set<string>();
for (const name of raw.split(',').map(s => s.trim()).filter(Boolean)) {
if (!VALID_LOGIN.test(name)) throw new Error(`${context}: "${name}" is not a valid GitHub account name`);
if (!seen.has(name.toLowerCase())) { seen.add(name.toLowerCase()); names.push(name); }
}
if (names.length === 0) throw new Error(`${context} contains no valid entries`);
return names;
}
Loading
Loading