From 6ced277c8f7561559edc44c9fe565eff34b6378a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSebastian?= <64795732+slegarraga@users.noreply.github.com> Date: Fri, 5 Jun 2026 00:30:07 -0400 Subject: [PATCH] Add provider error fixture corpus --- CHANGELOG.md | 14 ++++ README.md | 11 ++++ fixtures/README.md | 25 +++++++ .../cases/anthropic/fetch-context-length.json | 17 +++++ .../cases/anthropic/fetch-rate-limit.json | 18 +++++ .../cases/anthropic/sdk-overloaded-529.json | 17 +++++ fixtures/cases/gemini/fetch-unavailable.json | 19 ++++++ .../cases/gemini/rpc-permission-denied.json | 11 ++++ .../cases/gemini/rpc-resource-exhausted.json | 17 +++++ fixtures/cases/network/abort-error.json | 8 +++ fixtures/cases/network/node-timeout.json | 9 +++ .../cases/openai/fetch-context-length.json | 18 +++++ .../cases/openai/sdk-insufficient-quota.json | 13 ++++ fixtures/cases/openai/sdk-rate-limit.json | 18 +++++ .../anthropic/fetch-context-length.json | 8 +++ .../expected/anthropic/fetch-rate-limit.json | 9 +++ .../anthropic/sdk-overloaded-529.json | 8 +++ .../expected/gemini/fetch-unavailable.json | 9 +++ .../gemini/rpc-permission-denied.json | 8 +++ .../gemini/rpc-resource-exhausted.json | 9 +++ fixtures/expected/network/abort-error.json | 7 ++ fixtures/expected/network/node-timeout.json | 7 ++ .../expected/openai/fetch-context-length.json | 8 +++ .../openai/sdk-insufficient-quota.json | 8 +++ fixtures/expected/openai/sdk-rate-limit.json | 9 +++ package-lock.json | 4 +- package.json | 3 +- test/fixtures.test.ts | 66 +++++++++++++++++++ 28 files changed, 375 insertions(+), 3 deletions(-) create mode 100644 fixtures/README.md create mode 100644 fixtures/cases/anthropic/fetch-context-length.json create mode 100644 fixtures/cases/anthropic/fetch-rate-limit.json create mode 100644 fixtures/cases/anthropic/sdk-overloaded-529.json create mode 100644 fixtures/cases/gemini/fetch-unavailable.json create mode 100644 fixtures/cases/gemini/rpc-permission-denied.json create mode 100644 fixtures/cases/gemini/rpc-resource-exhausted.json create mode 100644 fixtures/cases/network/abort-error.json create mode 100644 fixtures/cases/network/node-timeout.json create mode 100644 fixtures/cases/openai/fetch-context-length.json create mode 100644 fixtures/cases/openai/sdk-insufficient-quota.json create mode 100644 fixtures/cases/openai/sdk-rate-limit.json create mode 100644 fixtures/expected/anthropic/fetch-context-length.json create mode 100644 fixtures/expected/anthropic/fetch-rate-limit.json create mode 100644 fixtures/expected/anthropic/sdk-overloaded-529.json create mode 100644 fixtures/expected/gemini/fetch-unavailable.json create mode 100644 fixtures/expected/gemini/rpc-permission-denied.json create mode 100644 fixtures/expected/gemini/rpc-resource-exhausted.json create mode 100644 fixtures/expected/network/abort-error.json create mode 100644 fixtures/expected/network/node-timeout.json create mode 100644 fixtures/expected/openai/fetch-context-length.json create mode 100644 fixtures/expected/openai/sdk-insufficient-quota.json create mode 100644 fixtures/expected/openai/sdk-rate-limit.json create mode 100644 test/fixtures.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d6baa5..472434d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.4] - 2026-06-05 + +### Added + +- Added a public provider error fixture corpus for OpenAI, Anthropic, Gemini and + transport-level failures, covering SDK-like objects, parsed fetch responses + and expected normalized outputs. +- Added fixture-driven regression tests and published `fixtures/` in the npm + package. + ## [0.1.3] - 2026-06-04 ### Changed @@ -44,4 +54,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). pair arrays. - Zero runtime dependencies; ESM + CJS builds with type declarations. +[0.1.4]: https://github.com/slegarraga/llm-errors/releases/tag/v0.1.4 +[0.1.3]: https://github.com/slegarraga/llm-errors/releases/tag/v0.1.3 +[0.1.2]: https://github.com/slegarraga/llm-errors/releases/tag/v0.1.2 +[0.1.1]: https://github.com/slegarraga/llm-errors/releases/tag/v0.1.1 [0.1.0]: https://github.com/slegarraga/llm-errors/releases/tag/v0.1.0 diff --git a/README.md b/README.md index ff47c94..da0ca42 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,17 @@ try { npm install llm-errors ``` +## Fixture corpus + +The npm package includes a public fixture corpus under +[`fixtures/`](./fixtures/README.md). It pairs raw SDK-like, fetch-like and +transport-level provider errors with the normalized output expected from +`normalizeError`. + +These fixtures are useful for downstream regression tests when you want to +verify provider-portable retry and error handling without importing OpenAI, +Anthropic or Gemini SDKs. + ## API ### `normalizeError(error, options?) => NormalizedError` diff --git a/fixtures/README.md b/fixtures/README.md new file mode 100644 index 0000000..a974b8f --- /dev/null +++ b/fixtures/README.md @@ -0,0 +1,25 @@ +# Provider Error Fixtures + +This corpus contains redacted, synthetic examples of provider error shapes that +`llm-errors` supports. The files are intentionally small JSON fixtures so they +can be reused by downstream test suites without importing any provider SDK. + +## Layout + +- `cases/` contains raw SDK-like, fetch-like and transport-level inputs. +- `expected/` contains the normalized output for the matching case path. + +For example, `cases/openai/sdk-rate-limit.json` is paired with +`expected/openai/sdk-rate-limit.json`. + +## Scope + +The corpus covers: + +- OpenAI SDK `APIError`-style objects and parsed fetch responses. +- Anthropic SDK envelopes and parsed fetch responses. +- Gemini / Google RPC error envelopes. +- Transport failures such as Node timeout codes and browser abort errors. + +The fixtures are not recordings of private traffic and do not contain API keys, +request IDs, account IDs or user content. diff --git a/fixtures/cases/anthropic/fetch-context-length.json b/fixtures/cases/anthropic/fetch-context-length.json new file mode 100644 index 0000000..d0b635d --- /dev/null +++ b/fixtures/cases/anthropic/fetch-context-length.json @@ -0,0 +1,17 @@ +{ + "name": "Anthropic parsed fetch response prompt too long", + "shape": "anthropic-fetch-parsed-body", + "error": { + "status": 400, + "headers": { + "anthropic-version": "2023-06-01" + }, + "body": { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "prompt is too long: 250000 tokens > 200000 maximum" + } + } + } +} diff --git a/fixtures/cases/anthropic/fetch-rate-limit.json b/fixtures/cases/anthropic/fetch-rate-limit.json new file mode 100644 index 0000000..0fcc37a --- /dev/null +++ b/fixtures/cases/anthropic/fetch-rate-limit.json @@ -0,0 +1,18 @@ +{ + "name": "Anthropic parsed fetch response rate limit", + "shape": "anthropic-fetch-parsed-body", + "error": { + "status": 429, + "headers": { + "anthropic-ratelimit-requests-limit": "50", + "retry-after": "30" + }, + "body": { + "type": "error", + "error": { + "type": "rate_limit_error", + "message": "rate limit exceeded" + } + } + } +} diff --git a/fixtures/cases/anthropic/sdk-overloaded-529.json b/fixtures/cases/anthropic/sdk-overloaded-529.json new file mode 100644 index 0000000..09032f9 --- /dev/null +++ b/fixtures/cases/anthropic/sdk-overloaded-529.json @@ -0,0 +1,17 @@ +{ + "name": "Anthropic SDK overloaded response", + "shape": "anthropic-sdk-apierror", + "error": { + "status": 529, + "headers": { + "anthropic-version": "2023-06-01" + }, + "error": { + "type": "error", + "error": { + "type": "overloaded_error", + "message": "Overloaded" + } + } + } +} diff --git a/fixtures/cases/gemini/fetch-unavailable.json b/fixtures/cases/gemini/fetch-unavailable.json new file mode 100644 index 0000000..d0107f7 --- /dev/null +++ b/fixtures/cases/gemini/fetch-unavailable.json @@ -0,0 +1,19 @@ +{ + "name": "Gemini parsed fetch response unavailable", + "shape": "google-rpc-fetch-parsed-body", + "error": { + "response": { + "status": 503, + "headers": { + "retry-after": "4" + }, + "data": { + "error": { + "code": 503, + "message": "The model is overloaded. Please try again later.", + "status": "UNAVAILABLE" + } + } + } + } +} diff --git a/fixtures/cases/gemini/rpc-permission-denied.json b/fixtures/cases/gemini/rpc-permission-denied.json new file mode 100644 index 0000000..b4a70cf --- /dev/null +++ b/fixtures/cases/gemini/rpc-permission-denied.json @@ -0,0 +1,11 @@ +{ + "name": "Gemini Google RPC permission denied", + "shape": "google-rpc-error-envelope", + "error": { + "error": { + "code": 403, + "message": "The caller does not have permission.", + "status": "PERMISSION_DENIED" + } + } +} diff --git a/fixtures/cases/gemini/rpc-resource-exhausted.json b/fixtures/cases/gemini/rpc-resource-exhausted.json new file mode 100644 index 0000000..918263a --- /dev/null +++ b/fixtures/cases/gemini/rpc-resource-exhausted.json @@ -0,0 +1,17 @@ +{ + "name": "Gemini Google RPC resource exhausted", + "shape": "google-rpc-error-envelope", + "error": { + "error": { + "code": 429, + "message": "Resource has been exhausted.", + "status": "RESOURCE_EXHAUSTED", + "details": [ + { + "@type": "type.googleapis.com/google.rpc.RetryInfo", + "retryDelay": "17s" + } + ] + } + } +} diff --git a/fixtures/cases/network/abort-error.json b/fixtures/cases/network/abort-error.json new file mode 100644 index 0000000..fb8ca36 --- /dev/null +++ b/fixtures/cases/network/abort-error.json @@ -0,0 +1,8 @@ +{ + "name": "Browser fetch abort", + "shape": "dom-abort-error", + "error": { + "name": "AbortError", + "message": "The operation was aborted." + } +} diff --git a/fixtures/cases/network/node-timeout.json b/fixtures/cases/network/node-timeout.json new file mode 100644 index 0000000..81c2585 --- /dev/null +++ b/fixtures/cases/network/node-timeout.json @@ -0,0 +1,9 @@ +{ + "name": "Node transport timeout", + "shape": "node-error-code", + "error": { + "name": "Error", + "message": "connect ETIMEDOUT", + "code": "ETIMEDOUT" + } +} diff --git a/fixtures/cases/openai/fetch-context-length.json b/fixtures/cases/openai/fetch-context-length.json new file mode 100644 index 0000000..d06dc85 --- /dev/null +++ b/fixtures/cases/openai/fetch-context-length.json @@ -0,0 +1,18 @@ +{ + "name": "OpenAI parsed fetch response context length", + "shape": "openai-fetch-parsed-body", + "error": { + "status": 400, + "headers": { + "openai-processing-ms": "42" + }, + "body": { + "error": { + "message": "This model's maximum context length is 128000 tokens.", + "type": "invalid_request_error", + "code": "context_length_exceeded", + "param": "messages" + } + } + } +} diff --git a/fixtures/cases/openai/sdk-insufficient-quota.json b/fixtures/cases/openai/sdk-insufficient-quota.json new file mode 100644 index 0000000..ef7815e --- /dev/null +++ b/fixtures/cases/openai/sdk-insufficient-quota.json @@ -0,0 +1,13 @@ +{ + "name": "OpenAI SDK APIError insufficient quota", + "shape": "openai-sdk-apierror", + "error": { + "status": 429, + "error": { + "message": "You exceeded your current quota.", + "type": "insufficient_quota", + "code": "insufficient_quota", + "param": null + } + } +} diff --git a/fixtures/cases/openai/sdk-rate-limit.json b/fixtures/cases/openai/sdk-rate-limit.json new file mode 100644 index 0000000..ad5a54a --- /dev/null +++ b/fixtures/cases/openai/sdk-rate-limit.json @@ -0,0 +1,18 @@ +{ + "name": "OpenAI SDK APIError rate limit", + "shape": "openai-sdk-apierror", + "error": { + "status": 429, + "headers": { + "openai-version": "2020-10-01", + "retry-after-ms": "1250", + "retry-after": "2" + }, + "error": { + "message": "Rate limit reached for requests", + "type": "rate_limit_error", + "code": "rate_limit_exceeded", + "param": null + } + } +} diff --git a/fixtures/expected/anthropic/fetch-context-length.json b/fixtures/expected/anthropic/fetch-context-length.json new file mode 100644 index 0000000..9efbd88 --- /dev/null +++ b/fixtures/expected/anthropic/fetch-context-length.json @@ -0,0 +1,8 @@ +{ + "provider": "anthropic", + "category": "context_length_exceeded", + "message": "prompt is too long: 250000 tokens > 200000 maximum", + "status": 400, + "code": "invalid_request_error", + "retryable": false +} diff --git a/fixtures/expected/anthropic/fetch-rate-limit.json b/fixtures/expected/anthropic/fetch-rate-limit.json new file mode 100644 index 0000000..bab6519 --- /dev/null +++ b/fixtures/expected/anthropic/fetch-rate-limit.json @@ -0,0 +1,9 @@ +{ + "provider": "anthropic", + "category": "rate_limit", + "message": "rate limit exceeded", + "status": 429, + "code": "rate_limit_error", + "retryable": true, + "retryAfterMs": 30000 +} diff --git a/fixtures/expected/anthropic/sdk-overloaded-529.json b/fixtures/expected/anthropic/sdk-overloaded-529.json new file mode 100644 index 0000000..3672914 --- /dev/null +++ b/fixtures/expected/anthropic/sdk-overloaded-529.json @@ -0,0 +1,8 @@ +{ + "provider": "anthropic", + "category": "overloaded", + "message": "Overloaded", + "status": 529, + "code": "overloaded_error", + "retryable": true +} diff --git a/fixtures/expected/gemini/fetch-unavailable.json b/fixtures/expected/gemini/fetch-unavailable.json new file mode 100644 index 0000000..84b591e --- /dev/null +++ b/fixtures/expected/gemini/fetch-unavailable.json @@ -0,0 +1,9 @@ +{ + "provider": "gemini", + "category": "overloaded", + "message": "The model is overloaded. Please try again later.", + "status": 503, + "code": "UNAVAILABLE", + "retryable": true, + "retryAfterMs": 4000 +} diff --git a/fixtures/expected/gemini/rpc-permission-denied.json b/fixtures/expected/gemini/rpc-permission-denied.json new file mode 100644 index 0000000..e231899 --- /dev/null +++ b/fixtures/expected/gemini/rpc-permission-denied.json @@ -0,0 +1,8 @@ +{ + "provider": "gemini", + "category": "permission", + "message": "The caller does not have permission.", + "status": 403, + "code": "PERMISSION_DENIED", + "retryable": false +} diff --git a/fixtures/expected/gemini/rpc-resource-exhausted.json b/fixtures/expected/gemini/rpc-resource-exhausted.json new file mode 100644 index 0000000..359cb9b --- /dev/null +++ b/fixtures/expected/gemini/rpc-resource-exhausted.json @@ -0,0 +1,9 @@ +{ + "provider": "gemini", + "category": "rate_limit", + "message": "Resource has been exhausted.", + "status": 429, + "code": "RESOURCE_EXHAUSTED", + "retryable": true, + "retryAfterMs": 17000 +} diff --git a/fixtures/expected/network/abort-error.json b/fixtures/expected/network/abort-error.json new file mode 100644 index 0000000..2ebed21 --- /dev/null +++ b/fixtures/expected/network/abort-error.json @@ -0,0 +1,7 @@ +{ + "provider": "unknown", + "category": "timeout", + "message": "The operation was aborted.", + "code": "AbortError", + "retryable": true +} diff --git a/fixtures/expected/network/node-timeout.json b/fixtures/expected/network/node-timeout.json new file mode 100644 index 0000000..3a24d1e --- /dev/null +++ b/fixtures/expected/network/node-timeout.json @@ -0,0 +1,7 @@ +{ + "provider": "unknown", + "category": "timeout", + "message": "connect ETIMEDOUT", + "code": "ETIMEDOUT", + "retryable": true +} diff --git a/fixtures/expected/openai/fetch-context-length.json b/fixtures/expected/openai/fetch-context-length.json new file mode 100644 index 0000000..7f6718d --- /dev/null +++ b/fixtures/expected/openai/fetch-context-length.json @@ -0,0 +1,8 @@ +{ + "provider": "openai", + "category": "context_length_exceeded", + "message": "This model's maximum context length is 128000 tokens.", + "status": 400, + "code": "context_length_exceeded", + "retryable": false +} diff --git a/fixtures/expected/openai/sdk-insufficient-quota.json b/fixtures/expected/openai/sdk-insufficient-quota.json new file mode 100644 index 0000000..f0af1cf --- /dev/null +++ b/fixtures/expected/openai/sdk-insufficient-quota.json @@ -0,0 +1,8 @@ +{ + "provider": "openai", + "category": "insufficient_quota", + "message": "You exceeded your current quota.", + "status": 429, + "code": "insufficient_quota", + "retryable": false +} diff --git a/fixtures/expected/openai/sdk-rate-limit.json b/fixtures/expected/openai/sdk-rate-limit.json new file mode 100644 index 0000000..cf6ccd8 --- /dev/null +++ b/fixtures/expected/openai/sdk-rate-limit.json @@ -0,0 +1,9 @@ +{ + "provider": "openai", + "category": "rate_limit", + "message": "Rate limit reached for requests", + "status": 429, + "code": "rate_limit_exceeded", + "retryable": true, + "retryAfterMs": 1250 +} diff --git a/package-lock.json b/package-lock.json index 22fe2b8..520b74d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "llm-errors", - "version": "0.1.3", + "version": "0.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "llm-errors", - "version": "0.1.3", + "version": "0.1.4", "license": "MIT", "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/package.json b/package.json index 757b459..f56f2a5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llm-errors", - "version": "0.1.3", + "version": "0.1.4", "description": "Normalize OpenAI, Anthropic and Gemini API errors into one shape: category, retryable flag and Retry-After delay. Zero dependencies.", "keywords": [ "openai", @@ -46,6 +46,7 @@ }, "files": [ "dist", + "fixtures", "README.md", "LICENSE", "CHANGELOG.md" diff --git a/test/fixtures.test.ts b/test/fixtures.test.ts new file mode 100644 index 0000000..4d020fd --- /dev/null +++ b/test/fixtures.test.ts @@ -0,0 +1,66 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { normalizeError, type NormalizeOptions } from '../src/index.ts'; + +interface FixtureCase { + name: string; + shape: string; + error: unknown; + options?: NormalizeOptions; +} + +const casesRoot = fileURLToPath(new URL('../fixtures/cases/', import.meta.url)); +const expectedRoot = fileURLToPath( + new URL('../fixtures/expected/', import.meta.url), +); + +function jsonFiles(dir: string): string[] { + return readdirSync(dir, { withFileTypes: true }) + .flatMap((entry) => { + const path = join(dir, entry.name); + if (entry.isDirectory()) { + return jsonFiles(path); + } + return entry.isFile() && entry.name.endsWith('.json') ? [path] : []; + }) + .sort(); +} + +function readJson(path: string): T { + return JSON.parse(readFileSync(path, 'utf8')) as T; +} + +function withoutUndefined( + value: Record, +): Record { + return Object.fromEntries( + Object.entries(value).filter(([, entry]) => entry !== undefined), + ); +} + +describe('provider fixture corpus', () => { + for (const casePath of jsonFiles(casesRoot)) { + const fixture = readJson(casePath); + const fixtureName = relative(casesRoot, casePath); + const expectedPath = join(expectedRoot, fixtureName); + + it(`${fixtureName} normalizes ${fixture.shape}`, () => { + const normalized = normalizeError(fixture.error, fixture.options); + const actual = withoutUndefined({ + provider: normalized.provider, + category: normalized.category, + message: normalized.message, + status: normalized.status, + code: normalized.code, + retryable: normalized.retryable, + retryAfterMs: normalized.retryAfterMs, + }); + + expect(actual).toEqual(readJson(expectedPath)); + expect(normalized.raw).toBe(fixture.error); + expect(basename(casePath)).toBe(basename(expectedPath)); + }); + } +});