From 24e4777ecf10d2efe7943c7999e4c4511a45100a Mon Sep 17 00:00:00 2001 From: ramseydallal-lab Date: Thu, 30 Apr 2026 07:34:51 -0400 Subject: [PATCH] fix(ai-gateway): remove overbroad model-slug rule (closes #60) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `\d+-\d+[)'"]` validation rule on the ai-gateway skill was designed to catch typo'd hyphenated model slugs in `gateway('…')` calls. In practice the rule: 1. Contradicts Anthropic's canonical model IDs, which are hyphenated. From https://docs.anthropic.com/en/docs/about-claude/models (verified 2026-04-30): | Claude API ID | claude-sonnet-4-6 | | Claude API alias | claude-sonnet-4-6 | Direct provider SDK calls — `anthropic('claude-sonnet-4-6')` — are correct AS WRITTEN with hyphens. The rule fires on every such call as a false positive. 2. False-positives outside model-slug context entirely. The pattern matches any digit-hyphen-digit followed by `)`, `'`, or `"`, so date strings ("01-15-2024"), package versions, and similar text trip the rule. Issue #60 documents this across multiple real repos. 3. Trains operators to dismiss validator output. Issue #38 shows the compounding noise problem; #59 documents the validator's own SKILL.md tripping its own rules. Each individual noise source compounds. The remaining provider/-prefix rule (`gateway\(['"][^'"/]+['"]\)`) still enforces the actual hard constraint — that gateway calls use `provider/model` format. Slug typos within that format will surface from the gateway itself with a clearer error than this lint can produce. Changes ------- - skills/ai-gateway/SKILL.md - Remove the validate-rule entry (kept as a comment with #60 pointer for future archeology). - Rewrite the "Model Slug Rules" prose to document the real asymmetry: gateway-catalog form uses dots (anthropic/claude-opus-4.6 per Vercel's models page), while direct provider SDK calls use the provider's native IDs (Anthropic publishes hyphens). Both are correct in their contexts; the validator should not assert one onto the other. - generated/skill-manifest.json - Drop the corresponding rule entry (regenerated artifact, but committed in this repo per existing pattern). - tests/validate-rules.test.ts - Invert the direct assertion (test at line 426): hyphenated AND dotted forms now both pass without violation. - Update three multi-skill overlap tests to use `gateway('opaque')` (missing-provider rule, severity:error) instead of the deleted slug rule as the ai-gateway trigger. - Update the line-number test's message-includes string to match the still-present "missing provider/ prefix" rule. - Update the negative-test's comment + extend coverage to both forms. - tests/posttooluse-validate.test.ts - Convert "detects gateway from 'ai' with hyphenated model slug" test from "expect 'dots not hyphens' violation" to "expect NO 'dots not hyphens' violation, for both hyphen and dot forms." ai-gateway skill is still expected to be matched on the file via the `'ai'` import pattern. - tests/posttooluse-chain.test.ts - Cosmetic: fix the test fixture string `anthropic('claude-sonnet-4.6')` to the canonical hyphen form `anthropic('claude-sonnet-4-6')`. Test logic unchanged. Verification ------------ The rule's removal was already validated against a real-world failure: a production app shipped `anthropic('claude-sonnet-4.6')` (dotted, matching what the validator told them was correct), the Anthropic API rejected the model ID, every critique call surfaced as `ai_error` for end users. Switching to `claude-sonnet-4-6` (hyphens) — what this rule had been flagging as wrong — fixed it. Local tests not run (bun not installed in the patch environment); relying on CI. Closes #60. --- generated/skill-manifest.json | 5 --- skills/ai-gateway/SKILL.md | 18 +++++----- tests/posttooluse-chain.test.ts | 2 +- tests/posttooluse-validate.test.ts | 54 +++++++++++++++++------------- tests/validate-rules.test.ts | 48 ++++++++++++++++---------- 5 files changed, 73 insertions(+), 54 deletions(-) diff --git a/generated/skill-manifest.json b/generated/skill-manifest.json index 765a878..4f2eabc 100644 --- a/generated/skill-manifest.json +++ b/generated/skill-manifest.json @@ -968,11 +968,6 @@ } ], "validate": [ - { - "pattern": "\\d+-\\d+[)'\"]", - "message": "Model slug uses hyphens — use dots not hyphens for version numbers (e.g., claude-sonnet-4.6)", - "severity": "error" - }, { "pattern": "AI_GATEWAY_API_KEY", "message": "Consider OIDC-based auth via vercel env pull for automatic token management — AI_GATEWAY_API_KEY works but requires manual rotation", diff --git a/skills/ai-gateway/SKILL.md b/skills/ai-gateway/SKILL.md index d426b72..ff04836 100644 --- a/skills/ai-gateway/SKILL.md +++ b/skills/ai-gateway/SKILL.md @@ -18,10 +18,12 @@ metadata: - '\bbun\s+(install|i|add)\s+[^\n]*@ai-sdk/gateway\b' - '\byarn\s+add\s+[^\n]*@ai-sdk/gateway\b' validate: - - - pattern: \d+-\d+[)'"] - message: 'Model slug uses hyphens — use dots not hyphens for version numbers (e.g., claude-sonnet-4.6)' - severity: error + # Removed: the prior rule `\d+-\d+[)'"]` was too broad — it fired on any + # hyphenated digit pair followed by a quote/paren, which produced false + # positives on date strings ("01-15-2024"), package versions, and on + # direct provider SDK calls like `anthropic('claude-sonnet-4-6')` where + # hyphens are CORRECT (Anthropic's canonical model IDs use hyphens; only + # the gateway slug form uses dots). See vercel/vercel-plugin#60. - pattern: AI_GATEWAY_API_KEY message: 'Consider OIDC-based auth via vercel env pull for automatic token management — AI_GATEWAY_API_KEY works but requires manual rotation' @@ -134,11 +136,11 @@ const result = await generateText({ ## Model Slug Rules (Critical) - Always use `provider/model` format (for example `openai/gpt-5.4`). -- Versioned slugs use dots for versions, not hyphens: - - Correct: `anthropic/claude-sonnet-4.6` - - Incorrect: `anthropic/claude-sonnet-4-6` +- Naming convention depends on which call surface you're using: + - **Vercel AI Gateway** (`gateway('provider/...')` or plain string): the gateway's catalog uses dots in version numbers — e.g. `anthropic/claude-opus-4.6`, `openai/gpt-5.4`. See [Vercel AI Gateway models](https://vercel.com/ai-gateway/models). + - **Direct provider SDK** (`anthropic('...')`, `openai('...')`): use the provider's native model IDs, which Anthropic publishes with hyphens — e.g. `claude-sonnet-4-6`, `claude-opus-4-7`. See [Anthropic models](https://docs.anthropic.com/en/docs/about-claude/models). - Before hardcoding model IDs, call `gateway.getAvailableModels()` and pick from the returned IDs. -- Default text models: `openai/gpt-5.4` or `anthropic/claude-sonnet-4.6`. +- Default text models: `openai/gpt-5.4` or `anthropic/claude-opus-4.6`. - Do not default to outdated choices like `openai/gpt-4o`. ```ts diff --git a/tests/posttooluse-chain.test.ts b/tests/posttooluse-chain.test.ts index a4bcb6a..fba1cf3 100644 --- a/tests/posttooluse-chain.test.ts +++ b/tests/posttooluse-chain.test.ts @@ -1143,7 +1143,7 @@ describe("real-world chain and validate scenarios", () => { `import { generateText } from 'ai';`, ``, `const result = await generateText({`, - ` model: anthropic('claude-sonnet-4.6'),`, + ` model: anthropic('claude-sonnet-4-6'),`, ` prompt: 'Hello!',`, `});`, ].join("\n"); diff --git a/tests/posttooluse-validate.test.ts b/tests/posttooluse-validate.test.ts index 13ffe80..e66e79d 100644 --- a/tests/posttooluse-validate.test.ts +++ b/tests/posttooluse-validate.test.ts @@ -187,29 +187,37 @@ describe("posttooluse-validate.mjs", () => { } }); - test("detects gateway from 'ai' with hyphenated model slug", async () => { - writeFileSync(testFile, [ - `import { generateText, gateway } from 'ai';`, - ``, - `const result = await generateText({`, - ` model: gateway('anthropic/claude-sonnet-4-6'),`, - ` prompt: 'Hello!',`, - `});`, - ].join("\n")); - const { code, stdout } = await runHook({ - tool_name: "Write", - tool_input: { file_path: testFile }, - }); - expect(code).toBe(0); - const result = JSON.parse(stdout); - expect(result.hookSpecificOutput).toBeDefined(); - const ctx = result.hookSpecificOutput.additionalContext; - expect(ctx).toContain("VALIDATION"); - expect(ctx).toContain("dots not hyphens"); - const meta = extractPostValidation(result.hookSpecificOutput); - expect(meta).toBeDefined(); - expect(meta.errorCount).toBeGreaterThan(0); - expect(meta.matchedSkills).toContain("ai-gateway"); + test("hyphenated and dotted model slugs both pass — slug-form rule removed in #60", async () => { + // Issue #60: the prior `\d+-\d+[)'"]` rule false-positived on + // Anthropic's canonical hyphenated slugs and on date strings. + // Rule removed; both gateway-catalog (dots) and provider-canonical + // (hyphens) forms must pass without raising "dots not hyphens". + // ai-gateway must still be detected on the file (importPatterns + // includes 'ai'), and the missing-provider rule still fires for + // bare model strings. + for (const slug of ["anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-4.6"]) { + writeFileSync(testFile, [ + `import { generateText, gateway } from 'ai';`, + ``, + `const result = await generateText({`, + ` model: gateway('${slug}'),`, + ` prompt: 'Hello!',`, + `});`, + ].join("\n")); + const { code, stdout } = await runHook({ + tool_name: "Write", + tool_input: { file_path: testFile }, + }); + expect(code).toBe(0); + const result = JSON.parse(stdout); + const ctx = result.hookSpecificOutput?.additionalContext ?? ""; + // The deleted rule's message must not appear for either form. + expect(ctx).not.toContain("dots not hyphens"); + const meta = extractPostValidation(result.hookSpecificOutput); + if (meta) { + expect(meta.matchedSkills).toContain("ai-gateway"); + } + } }); test("no output for file that doesn't match any skill", async () => { diff --git a/tests/validate-rules.test.ts b/tests/validate-rules.test.ts index 8507f73..4fd2b32 100644 --- a/tests/validate-rules.test.ts +++ b/tests/validate-rules.test.ts @@ -423,15 +423,25 @@ describe("ai-sdk validation rules", () => { // --------------------------------------------------------------------------- describe("ai-gateway validation rules", () => { - test("flags hyphenated model slug (anthropic/claude-sonnet-4-6)", () => { - const data = loadRealRules(); - const violations = runValidation( + test("does NOT flag hyphenated model slug — both gateway-form (dots) and provider-canonical (hyphens) are valid", () => { + // Issue #60: the prior rule `\d+-\d+[)'"]` flagged hyphenated model + // slugs as typos expecting dots. But Anthropic's canonical IDs are + // hyphenated (`claude-sonnet-4-6`), and the rule also false-positived + // on date strings, package versions, and direct provider SDK calls. + // Rule was removed; both forms must now pass without violation. + const data = loadRealRules(); + const hyphenViolations = runValidation( `gateway('anthropic/claude-sonnet-4-6')\n`, ["ai-gateway"], data!.rulesMap, ); - // This pattern is a plain string, no escaping needed - expect(violations.some((v) => v.message.includes("dots not hyphens"))).toBe(true); + const dotViolations = runValidation( + `gateway('anthropic/claude-sonnet-4.6')\n`, + ["ai-gateway"], + data!.rulesMap, + ); + expect(hyphenViolations.some((v) => v.message.includes("dots not hyphens"))).toBe(false); + expect(dotViolations.some((v) => v.message.includes("dots not hyphens"))).toBe(false); }); test("AI_GATEWAY_API_KEY is recommended severity (fallback auth)", () => { @@ -1143,10 +1153,10 @@ describe("multi-skill overlap", () => { const data = loadRealRules(); // Use patterns that actually work (no double-escape issues): // ai-sdk: import from 'openai' (direct import pattern) - // ai-gateway: anthropic/claude-sonnet-4-6 (hyphenated slug) + // ai-gateway: gateway('opaque') — missing provider/ prefix (error rule) const content = [ `import OpenAI from 'openai';`, - `const result = await generateText({ model: gateway('anthropic/claude-sonnet-4-6') });`, + `const result = await generateText({ model: gateway('opaque') });`, ].join("\n"); const violations = runValidation(content, ["ai-sdk", "ai-gateway"], data!.rulesMap); @@ -1177,10 +1187,10 @@ describe("multi-skill overlap", () => { test("overlapping rules don't suppress each other", () => { const data = loadRealRules(); // ai-sdk flags: import from 'openai' - // ai-gateway flags: anthropic/claude-sonnet-4-6 (hyphenated slug) + // ai-gateway flags: gateway('opaque') — missing provider/ prefix const content = [ `import OpenAI from 'openai';`, - `gateway('anthropic/claude-sonnet-4-6')`, + `gateway('opaque')`, ].join("\n"); const violations = runValidation(content, ["ai-sdk", "ai-gateway"], data!.rulesMap); @@ -1195,12 +1205,12 @@ describe("multi-skill overlap", () => { const content = [ `import OpenAI from 'openai';`, // line 1 - ai-sdk error `const x = 1;`, // line 2 - clean - `gateway('anthropic/claude-sonnet-4-6')`, // line 3 - ai-gateway error + `gateway('opaque')`, // line 3 - ai-gateway error (missing provider/) ].join("\n"); const violations = runValidation(content, ["ai-sdk", "ai-gateway"], data!.rulesMap); const aiSdkV = violations.find((v) => v.skill === "ai-sdk" && v.message.includes("@ai-sdk/openai")); - const aiGwV = violations.find((v) => v.skill === "ai-gateway" && v.message.includes("dots not hyphens")); + const aiGwV = violations.find((v) => v.skill === "ai-gateway" && v.message.includes("missing provider/ prefix")); expect(aiSdkV).toBeDefined(); expect(aiSdkV!.line).toBe(1); expect(aiGwV).toBeDefined(); @@ -1439,13 +1449,17 @@ describe("no false positives", () => { expect(errors.length).toBe(0); }); - test("correctly versioned anthropic slug does not flag", () => { + test("anthropic slug — both dot and hyphen forms pass (rule removed in #60)", () => { const data = loadRealRules(); - const content = `gateway('anthropic/claude-sonnet-4.6')\n`; - const violations = runValidation(content, ["ai-gateway"], data!.rulesMap); - // The dot version should NOT be flagged (only hyphenated version is wrong) - const slugError = violations.filter((v) => v.message.includes("dots not hyphens")); - expect(slugError.length).toBe(0); + // Post-#60: the dot-vs-hyphen rule is gone. Both gateway-catalog form + // (dots, e.g. anthropic/claude-sonnet-4.6) and Anthropic-canonical form + // (hyphens, e.g. anthropic/claude-sonnet-4-6) must pass without + // raising a "dots not hyphens" violation. + for (const slug of ["anthropic/claude-sonnet-4.6", "anthropic/claude-sonnet-4-6"]) { + const violations = runValidation(`gateway('${slug}')\n`, ["ai-gateway"], data!.rulesMap); + const slugError = violations.filter((v) => v.message.includes("dots not hyphens")); + expect(slugError.length).toBe(0); + } }); });