diff --git a/.claude/skills/mcp-sdk-tier-audit/README.md b/.claude/skills/mcp-sdk-tier-audit/README.md index 77fc8cf8..1f33c9bd 100644 --- a/.claude/skills/mcp-sdk-tier-audit/README.md +++ b/.claude/skills/mcp-sdk-tier-audit/README.md @@ -38,7 +38,10 @@ For public repos, any authenticated token works (no special scopes needed — au ``` --repo GitHub repository (required) --branch Branch to check ---skip-conformance Skip conformance tests +--skip-conformance Skip conformance tests (server and client) +--skip-server-conformance Skip only the server conformance test suite +--skip-client-conformance Skip only the client conformance test suite +--skip-repo-health Skip all GitHub-backed repo-health checks --conformance-server-url URL of the already-running conformance server --client-cmd Command to run the SDK conformance client (for client conformance tests) --days Limit triage analysis to last N days @@ -46,18 +49,27 @@ For public repos, any authenticated token works (no special scopes needed — au --token GitHub token (defaults to GITHUB_TOKEN or gh auth token) ``` +When any check is skipped — either via `--skip-*` flags or because `--conformance-server-url` / `--client-cmd` were omitted — the run is **partial**: the scorecard's `partial_run` field is `true` and the tier classification is suppressed (shown as `N/A (partial run)`). Skipped checks appear as `status: "skipped"` in JSON output and as `○ skipped` in the human-readable formats. + ### What the CLI Checks -| Check | What it measures | -| ------------------ | ------------------------------------------------------------------------------ | -| Server Conformance | Pass rate of server implementation against the conformance test suite | -| Client Conformance | Pass rate of client implementation against the conformance test suite | -| Labels | Whether SEP-1730 label taxonomy is set up (supports GitHub native issue types) | -| Triage | How quickly issues get labeled after creation | -| P0 Resolution | Whether critical bugs are resolved within SLA | -| Stable Release | Whether a stable release >= 1.0.0 exists | -| Policy Signals | Presence of CHANGELOG, SECURITY, CONTRIBUTING, dependabot, ROADMAP | -| Spec Tracking | Gap between latest spec release and SDK release | +#### Conformance Tests + +| Check | What it measures | +| ------------------ | --------------------------------------------------------------------- | +| Server Conformance | Pass rate of server implementation against the conformance test suite | +| Client Conformance | Pass rate of client implementation against the conformance test suite | + +#### Repository Health + +| Check | What it measures | +| -------------- | ------------------------------------------------------------------------------ | +| Labels | Whether SEP-1730 label taxonomy is set up (supports GitHub native issue types) | +| Triage | How quickly issues get labeled after creation | +| P0 Resolution | Whether critical bugs are resolved within SLA | +| Stable Release | Whether a stable release >= 1.0.0 exists | +| Policy Signals | Presence of CHANGELOG, SECURITY, CONTRIBUTING, dependabot, ROADMAP | +| Spec Tracking | Gap between latest spec release and SDK release | ### Example Output @@ -144,7 +156,7 @@ dotnet run --project tests/ModelContextProtocol.ConformanceServer --framework ne /mcp-sdk-tier-audit ~/src/mcp/csharp-sdk http://localhost:3003 "dotnet run --project ~/src/mcp/csharp-sdk/tests/ModelContextProtocol.ConformanceClient" ``` -The skill derives `owner/repo` from git remote, runs the CLI, launches parallel evaluations for docs and policy, and writes detailed reports to `results/`. +The skill derives `owner/repo` from git remote, runs the CLI, launches parallel evaluations for docs and policy, and writes detailed reports to `results/`. For scoped runs, `--skip-repo-health` also suppresses the AI policy review so repo-health behavior stays aligned end-to-end; `--skip-docs-eval` only suppresses the documentation evaluation. ### Any Other AI Coding Agent diff --git a/.claude/skills/mcp-sdk-tier-audit/SKILL.md b/.claude/skills/mcp-sdk-tier-audit/SKILL.md index 5da0fbde..6e700481 100644 --- a/.claude/skills/mcp-sdk-tier-audit/SKILL.md +++ b/.claude/skills/mcp-sdk-tier-audit/SKILL.md @@ -5,7 +5,7 @@ description: >- Produces tier classification (1/2/3) with evidence table, gap list, and remediation guide. Works for any official MCP SDK (TypeScript, Python, Go, C#, Java, Kotlin, PHP, Swift, Rust, Ruby). -argument-hint: ' [client-cmd] [--branch ]' +argument-hint: ' [client-cmd] [--branch ] [--scope ] [--skip-*]' --- # MCP SDK Tier Audit @@ -44,6 +44,18 @@ Extract from the user's input: - **conformance-server-url**: URL where the SDK's everything server is already running (e.g. `http://localhost:3000/mcp`) - **client-cmd** (optional): command to run the SDK's conformance client (e.g. `npx tsx test/conformance/src/everythingClient.ts`). If not provided, client conformance tests are skipped and noted as a gap in the report. - **branch** (optional): Git branch to check on GitHub (e.g. `--branch fweinberger/v1x-governance-docs`). If not provided, derive from the local checkout's current branch: `cd && git rev-parse --abbrev-ref HEAD`. This is passed to the tier-check CLI so that policy signal file checks use the correct branch instead of the repo's default branch. +- **scope** (optional): one of `server`, `client`, `conformance`, `repo-health`, or `full` (default). A partial scope runs only a subset of the audit and **suppresses tier classification** — the report will show "Tier: N/A (partial run)". Scope presets expand to skip flags: + - `--scope server` → `--skip-client-conformance --skip-repo-health --skip-docs-eval` + - `--scope client` → `--skip-server-conformance --skip-repo-health --skip-docs-eval` + - `--scope conformance` → `--skip-repo-health --skip-docs-eval` + - `--scope repo-health` → `--skip-conformance --skip-docs-eval` + - `--scope full` → default, no skips +- **individual skip flags** (optional; override/augment `--scope` if both supplied): + - `--skip-server-conformance`, `--skip-client-conformance`, `--skip-conformance` + - `--skip-repo-health` (skips all repo-health work: the CLI's GitHub-backed checks plus the AI policy evaluation in Step 3) + - `--skip-docs-eval` (skips Evaluation 1 in Step 3) + +Note: repo health is treated as a single concern across the deterministic CLI checks and the AI-assisted policy review. `--skip-repo-health` skips both of those surfaces. `--skip-docs-eval` remains the only separate evaluation skip. The first two arguments are required. If either is missing, ask the user to provide it. @@ -63,10 +75,15 @@ npm run --silent tier-check -- \ --branch \ --conformance-server-url \ --client-cmd '' \ - --output json + --output json \ + [--skip-server-conformance] \ + [--skip-client-conformance] \ + [--skip-repo-health] ``` -If no client-cmd was detected, omit the `--client-cmd` flag (client conformance will be skipped). The `--branch` flag should always be included (derived from the local checkout if not explicitly provided). +Forward whichever `--skip-*` flags the user passed (or that the `--scope` preset resolved to). If no scope/skip flags were supplied, run the CLI without them for a full assessment. + +If no client-cmd was detected, omit the `--client-cmd` flag. Client conformance will be skipped, and because the run is no longer complete, the final result should be treated as a partial run (`Tier: N/A (partial run)`). The `--branch` flag should always be included (derived from the local checkout if not explicitly provided). The CLI output includes server conformance pass rate, client conformance pass rate (with per-spec-version breakdown), issue triage compliance, P0 resolution times, label taxonomy, stable release status, policy signal files, and spec tracking gap. Parse the JSON output to feed into Step 4. @@ -84,12 +101,14 @@ If found, read the file. It lists known/expected conformance failures. This cont ## Step 3: Launch Parallel Evaluations -Launch 2 evaluations in parallel. Each reads the SDK from the local checkout path. +Launch up to 2 evaluations in parallel. Each reads the SDK from the local checkout path. -**IMPORTANT**: Launch both evaluations at the same time (in the same response) so they run in parallel. +**IMPORTANT**: Launch both evaluations at the same time (in the same response) so they run in parallel. If documentation coverage is skipped via `--skip-docs-eval`, or repo-health work is skipped via `--skip-repo-health` (or via a `--scope` preset), omit the relevant evaluation and treat its section in the report as "Not run — excluded by scope". ### Evaluation 1: Documentation Coverage +**Skip if** `--skip-docs-eval` is set. + Use the prompt from `references/docs-coverage-prompt.md`. Pass the local path. This evaluation checks: @@ -100,6 +119,8 @@ This evaluation checks: ### Evaluation 2: Policy Evaluation +**Skip if** `--skip-repo-health` is set. Repo health is intentionally aligned here: if the repo-health checks are excluded from the CLI run, the AI policy evaluation is excluded too. + Use the prompt from `references/policy-evaluation-prompt.md`. Pass the local path, the derived `owner/repo`, and the `policy_signals` section from the CLI JSON output. The CLI has already checked which policy files exist (ROADMAP.md, DEPENDENCY_POLICY.md, dependabot.yml, VERSIONING.md, etc.). The AI evaluation reads only the files the CLI found to judge whether the content is substantive — it does NOT search for files in other locations. @@ -113,7 +134,9 @@ This evaluation checks: ## Step 4: Compute Final Tier -Combine the deterministic scorecard (from the CLI) with the evaluation results (docs, policies). Apply the tier logic: +**If the run is partial** (any `--scope` other than `full`, or any individual `--skip-*` flag is set, or the CLI JSON has `partial_run: true`): **do not classify a tier**. Report the result as "Tier: N/A (partial run)" and list which checks ran vs. which were skipped. Do not output Tier 1/2 blockers for skipped checks — the skipped list already conveys that information. Still present the full table for the sections that did run so the user gets signal on those. + +Otherwise (full run): combine the deterministic scorecard (from the CLI) with the evaluation results (docs, policies). Apply the tier logic: ### Tier 1 requires ALL of: @@ -145,7 +168,7 @@ If any Tier 2 requirement is not met, the SDK is Tier 3. **Important edge cases:** - If GitHub issue labels are not set up per SEP-1730, triage metrics cannot be computed. Note this as a gap. However, repos may use GitHub's native issue types instead of type labels — the CLI checks for both. -- If client conformance was skipped (no client command found), note this as a gap but do not block tier advancement based on it alone. +- If client conformance was skipped (no client command found), note this as a gap and keep the overall result as a partial run rather than assigning a definitive tier. **Conformance Breakdown:** @@ -182,7 +205,7 @@ When evaluating P0 metrics, flag potentially mislabeled P0 issues: ## Step 5: Generate Output -Write detailed reports to files using subagents, then show a concise summary to the user. +Write detailed reports to files using subagents, then show a concise summary to the user. **Always write both files**, even for partial runs — skipped sections should render as `○ skipped` / "Not run — excluded by scope (`--skip-…` / `--scope `)" so the file shape stays stable. Include a `**Scope**` line in the header of each report (e.g., `Scope: partial run (server conformance only)` or `Scope: full`). ### Output files (write via subagents) @@ -208,7 +231,7 @@ Pass all the gathered data to a subagent and instruct it to write the remediatio ### Console output (shown to the user) -After the subagents finish, output a short executive summary directly to the user: +After the subagents finish, output a short executive summary directly to the user. For **partial runs**, replace the `## — Tier ` header with `## — Tier N/A (partial run)`, list the skipped sections directly below, and render rows for skipped checks as `○ skipped` in the tables (no ✓/✗ for T2/T1 columns on those rows). If you include a legend, state that `○` means "skipped". Omit the "For Tier 2" / "For Tier 1" sections entirely on partial runs — a partial run cannot definitively identify tier gaps. ``` ## — Tier @@ -304,4 +327,10 @@ Read these reference files when you need the detailed content for evaluation pro # Any SDK — server conformance only (no client) /mcp-sdk-tier-audit ~/src/mcp/some-sdk http://localhost:3004 + +# Scope presets — run only a subset of the audit +/mcp-sdk-tier-audit ~/src/mcp/typescript-sdk http://localhost:3000/mcp --scope server +/mcp-sdk-tier-audit ~/src/mcp/typescript-sdk http://localhost:3000/mcp "" --scope client +/mcp-sdk-tier-audit ~/src/mcp/typescript-sdk http://localhost:3000/mcp "" --scope conformance +/mcp-sdk-tier-audit ~/src/mcp/typescript-sdk --scope repo-health ``` diff --git a/.claude/skills/mcp-sdk-tier-audit/references/report-template.md b/.claude/skills/mcp-sdk-tier-audit/references/report-template.md index d77e1993..278e331a 100644 --- a/.claude/skills/mcp-sdk-tier-audit/references/report-template.md +++ b/.claude/skills/mcp-sdk-tier-audit/references/report-template.md @@ -12,14 +12,17 @@ Write two files to `results/` in the conformance repo: **Date**: {date} **Branch**: {branch} +**Scope**: {full | partial run — skipped: } **Auditor**: mcp-sdk-tier-audit skill (automated + subagent evaluation) -## Tier Assessment: Tier {X} +## Tier Assessment: {Tier X | N/A (partial run)} -{Brief 1-2 sentence summary of the overall assessment and key factors.} +{Brief 1-2 sentence summary of the overall assessment and key factors. For partial runs: state which scopes ran and note that no definitive tier is assigned.} ### Requirements Summary +For partial runs, rows corresponding to skipped checks should render with `Current Value` = `○ skipped (excluded by scope)` and both `T1?` / `T2?` = `N/A`. Do not mark skipped rows as PASS or FAIL. + | # | Requirement | Tier 1 Standard | Tier 2 Standard | Current Value | T1? | T2? | Gap | | --- | ----------------------- | --------------------------------- | ---------------------------- | --------------------------------- | ----------- | ----------- | ------------------ | | 1a | Server Conformance | 100% pass rate | >= 80% pass rate | {X}% ({passed}/{total}) | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | @@ -108,7 +111,10 @@ Labels: {present/missing list} # Remediation Guide: {repo} **Date**: {date} -**Current Tier**: {X} +**Scope**: {full | partial run — skipped: } +**Current Tier**: {X | N/A (partial run)} + +> For **partial runs**, omit the "Path to Tier 2" and "Path to Tier 1" sections (a partial run cannot identify all tier gaps). Instead, include a single section titled **Findings from scoped run** that lists remediation items only for the sections that were actually run, and a note that a full audit is required to enumerate remaining tier gaps. ## Path to Tier 2 diff --git a/README.md b/README.md index dc84f102..edd045db 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,17 @@ npm run --silent tier-check -- --repo modelcontextprotocol/typescript-sdk --skip npm run --silent tier-check -- \ --repo modelcontextprotocol/typescript-sdk \ --conformance-server-url http://localhost:3000/mcp + +# Partial runs (tier classification is suppressed — reported as "N/A (partial run)"): +# --skip-server-conformance / --skip-client-conformance target one conformance suite +# --skip-conformance skip both conformance suites +# --skip-repo-health skip all GitHub-backed checks +# Example: run only client conformance +npm run --silent tier-check -- \ + --repo modelcontextprotocol/typescript-sdk \ + --conformance-server-url http://localhost:3000/mcp \ + --client-cmd '' \ + --skip-server-conformance --skip-repo-health ``` For a full AI-assisted assessment with remediation guide, use Claude Code: diff --git a/src/tier-check/checks/skipped.ts b/src/tier-check/checks/skipped.ts new file mode 100644 index 00000000..2bd5afa2 --- /dev/null +++ b/src/tier-check/checks/skipped.ts @@ -0,0 +1,69 @@ +import { + ConformanceResult, + LabelsResult, + TriageResult, + P0Result, + ReleaseResult, + PolicySignalsResult, + SpecTrackingResult +} from '../types'; + +export const skippedConformance = (): ConformanceResult => ({ + status: 'skipped', + pass_rate: 0, + passed: 0, + failed: 0, + total: 0, + details: [] +}); + +export const skippedLabels = (): LabelsResult => ({ + status: 'skipped', + present: 0, + required: 0, + missing: [], + found: [], + uses_issue_types: false +}); + +export const skippedTriage = (): TriageResult => ({ + status: 'skipped', + compliance_rate: 0, + total_issues: 0, + triaged_within_sla: 0, + exceeding_sla: 0, + median_hours: 0, + p95_hours: 0, + days_analyzed: undefined +}); + +export const skippedP0 = (): P0Result => ({ + status: 'skipped', + open_p0s: 0, + open_p0_details: [], + closed_within_7d: 0, + closed_within_14d: 0, + closed_total: 0, + all_p0s_resolved_within_7d: false, + all_p0s_resolved_within_14d: false +}); + +export const skippedRelease = (): ReleaseResult => ({ + status: 'skipped', + version: null, + is_stable: false, + is_prerelease: false +}); + +export const skippedPolicySignals = (): PolicySignalsResult => ({ + status: 'skipped', + files: {} +}); + +export const skippedSpecTracking = (): SpecTrackingResult => ({ + status: 'skipped', + latest_spec_release: null, + latest_sdk_release: null, + sdk_release_within_30d: null, + days_gap: null +}); diff --git a/src/tier-check/index.test.ts b/src/tier-check/index.test.ts new file mode 100644 index 00000000..3cdb612a --- /dev/null +++ b/src/tier-check/index.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { resolveTierCheckPlan } from './index'; + +describe('resolveTierCheckPlan', () => { + it('treats omitted conformance inputs as skipped', () => { + const plan = resolveTierCheckPlan({}); + + expect(plan.runServer).toBe(false); + expect(plan.runClient).toBe(false); + expect(plan.serverSkipReason).toBe('no --conformance-server-url'); + expect(plan.clientSkipReason).toBe('no --client-cmd'); + expect(plan.nothingToRun).toBe(false); + }); + + it('reports nothing to run when repo health is also skipped', () => { + const plan = resolveTierCheckPlan({ skipRepoHealth: true }); + + expect(plan.runServer).toBe(false); + expect(plan.runClient).toBe(false); + expect(plan.nothingToRun).toBe(true); + }); + + it('respects explicit scope exclusions even when inputs are present', () => { + const plan = resolveTierCheckPlan({ + conformanceServerUrl: 'http://localhost:3000/mcp', + clientCmd: 'npm run conformance:client', + skipClientConformance: true + }); + + expect(plan.runServer).toBe(true); + expect(plan.runClient).toBe(false); + expect(plan.clientSkipReason).toBe('excluded by scope'); + expect(plan.nothingToRun).toBe(false); + }); +}); diff --git a/src/tier-check/index.ts b/src/tier-check/index.ts index 1cd32c19..0aa38871 100644 --- a/src/tier-check/index.ts +++ b/src/tier-check/index.ts @@ -10,6 +10,15 @@ import { checkP0Resolution } from './checks/p0'; import { checkStableRelease } from './checks/release'; import { checkPolicySignals } from './checks/files'; import { checkSpecTracking } from './checks/spec-tracking'; +import { + skippedConformance, + skippedLabels, + skippedTriage, + skippedP0, + skippedRelease, + skippedPolicySignals, + skippedSpecTracking +} from './checks/skipped'; import { computeTier } from './tier-logic'; import { formatJson, formatMarkdown, formatTerminal } from './output'; import { TierScorecard } from './types'; @@ -22,6 +31,41 @@ function parseRepo(repo: string): { owner: string; repo: string } { return { owner: parts[0], repo: parts[1] }; } +export function resolveTierCheckPlan(options: { + skipServerConformance?: boolean; + skipClientConformance?: boolean; + skipConformance?: boolean; + skipRepoHealth?: boolean; + conformanceServerUrl?: string; + clientCmd?: string; +}) { + const skipServerExplicit = + !!options.skipServerConformance || !!options.skipConformance; + const skipClientExplicit = + !!options.skipClientConformance || !!options.skipConformance; + const skipRepoHealth = !!options.skipRepoHealth; + + const runServer = !skipServerExplicit && !!options.conformanceServerUrl; + const runClient = !skipClientExplicit && !!options.clientCmd; + + return { + runServer, + runClient, + skipRepoHealth, + nothingToRun: !runServer && !runClient && skipRepoHealth, + serverSkipReason: runServer + ? null + : skipServerExplicit + ? 'excluded by scope' + : 'no --conformance-server-url', + clientSkipReason: runClient + ? null + : skipClientExplicit + ? 'excluded by scope' + : 'no --client-cmd' + }; +} + export function createTierCheckCommand(): Command { const tierCheck = new Command('tier-check') .description('Run SDK tier assessment checks against a GitHub repository') @@ -38,7 +82,19 @@ export function createTierCheckCommand(): Command { '--client-cmd ', 'Command to run the SDK conformance client (for client conformance tests)' ) - .option('--skip-conformance', 'Skip conformance tests') + .option('--skip-conformance', 'Skip conformance tests (server and client)') + .option( + '--skip-server-conformance', + 'Skip only the server conformance test suite' + ) + .option( + '--skip-client-conformance', + 'Skip only the client conformance test suite' + ) + .option( + '--skip-repo-health', + 'Skip all GitHub-backed repo-health checks (labels, triage, P0, release, policy signals, spec tracking)' + ) .option('--days ', 'Limit triage check to issues created in last N days') .option( '--output ', @@ -55,13 +111,27 @@ export function createTierCheckCommand(): Command { ) .action(async (options) => { const { owner, repo } = parseRepo(options.repo); + + const runPlan = resolveTierCheckPlan(options); + const { runServer, runClient, skipRepoHealth } = runPlan; + + if (runPlan.nothingToRun) { + console.error( + 'All checks are skipped — nothing to run. Provide --conformance-server-url and/or --client-cmd, or remove a --skip-* flag.' + ); + process.exit(1); + } + let token = options.token || process.env.GITHUB_TOKEN; const specVersion = options.specVersion ? resolveSpecVersion(options.specVersion) : undefined; - if (!token) { + // Token is only required if at least one GitHub-backed check will run. + const needsGitHub = !skipRepoHealth; + + if (!token && needsGitHub) { // Try to get token from GitHub CLI try { const { execSync } = await import('child_process'); @@ -71,7 +141,7 @@ export function createTierCheckCommand(): Command { } } - if (!token) { + if (!token && needsGitHub) { console.error( 'GitHub token required. Either:\n' + ' gh auth login\n' + @@ -86,7 +156,12 @@ export function createTierCheckCommand(): Command { console.error('Running tier assessment checks...\n'); - // Run all checks + const note = (label: string, didRun: boolean, reason?: string | null) => + ` ${didRun ? '\u2713' : '\u25cb'} ${label}${didRun ? '' : ` (skipped${reason ? `: ${reason}` : ''})`}`; + + // Run all non-skipped checks in parallel. Skipped checks resolve to + // a canned {status: 'skipped'} payload so downstream formatting and + // the tier scorecard schema remain stable. const [ conformance, clientConformance, @@ -97,46 +172,92 @@ export function createTierCheckCommand(): Command { files, specTracking ] = await Promise.all([ - checkConformance({ - serverUrl: options.conformanceServerUrl, - skip: options.skipConformance, - specVersion - }).then((r) => { - console.error(' ✓ Server Conformance'); - return r; - }), - checkClientConformance({ - clientCmd: options.clientCmd, - skip: options.skipConformance || !options.clientCmd, - specVersion - }).then((r) => { - console.error(' ✓ Client Conformance'); - return r; - }), - checkLabels(octokit, owner, repo).then((r) => { - console.error(' ✓ Labels'); - return r; - }), - checkTriage(octokit, owner, repo, days).then((r) => { - console.error(' \u2713 Triage'); - return r; - }), - checkP0Resolution(octokit, owner, repo).then((r) => { - console.error(' \u2713 P0 Resolution'); - return r; - }), - checkStableRelease(octokit, owner, repo).then((r) => { - console.error(' \u2713 Stable Release'); - return r; - }), - checkPolicySignals(octokit, owner, repo, options.branch).then((r) => { - console.error(' \u2713 Policy Signals'); - return r; - }), - checkSpecTracking(octokit, owner, repo).then((r) => { - console.error(' \u2713 Spec Tracking'); - return r; - }) + runServer + ? checkConformance({ + serverUrl: options.conformanceServerUrl, + skip: false, + specVersion + }).then((r) => { + console.error(note('Server Conformance', true)); + return r; + }) + : Promise.resolve(skippedConformance()).then((r) => { + console.error( + note('Server Conformance', false, runPlan.serverSkipReason) + ); + return r; + }), + runClient + ? checkClientConformance({ + clientCmd: options.clientCmd, + skip: false, + specVersion + }).then((r) => { + console.error(note('Client Conformance', true)); + return r; + }) + : Promise.resolve(skippedConformance()).then((r) => { + console.error( + note('Client Conformance', false, runPlan.clientSkipReason) + ); + return r; + }), + skipRepoHealth + ? Promise.resolve(skippedLabels()).then((r) => { + console.error(note('Labels', false)); + return r; + }) + : checkLabels(octokit, owner, repo).then((r) => { + console.error(note('Labels', true)); + return r; + }), + skipRepoHealth + ? Promise.resolve(skippedTriage()).then((r) => { + console.error(note('Triage', false)); + return r; + }) + : checkTriage(octokit, owner, repo, days).then((r) => { + console.error(note('Triage', true)); + return r; + }), + skipRepoHealth + ? Promise.resolve(skippedP0()).then((r) => { + console.error(note('P0 Resolution', false)); + return r; + }) + : checkP0Resolution(octokit, owner, repo).then((r) => { + console.error(note('P0 Resolution', true)); + return r; + }), + skipRepoHealth + ? Promise.resolve(skippedRelease()).then((r) => { + console.error(note('Stable Release', false)); + return r; + }) + : checkStableRelease(octokit, owner, repo).then((r) => { + console.error(note('Stable Release', true)); + return r; + }), + skipRepoHealth + ? Promise.resolve(skippedPolicySignals()).then((r) => { + console.error(note('Policy Signals', false)); + return r; + }) + : checkPolicySignals(octokit, owner, repo, options.branch).then( + (r) => { + console.error(note('Policy Signals', true)); + return r; + } + ), + skipRepoHealth + ? Promise.resolve(skippedSpecTracking()).then((r) => { + console.error(note('Spec Tracking', false)); + return r; + }) + : checkSpecTracking(octokit, owner, repo).then((r) => { + console.error(note('Spec Tracking', true)); + return r; + }) ]); const checks = { @@ -149,6 +270,9 @@ export function createTierCheckCommand(): Command { policy_signals: files, spec_tracking: specTracking }; + const partialRun = Object.values(checks).some( + (check) => check.status === 'skipped' + ); const implied_tier = computeTier(checks); @@ -157,6 +281,7 @@ export function createTierCheckCommand(): Command { branch: options.branch || null, timestamp: new Date().toISOString(), version: release.version, + partial_run: partialRun, checks, implied_tier }; diff --git a/src/tier-check/output.ts b/src/tier-check/output.ts index 9e7aa571..5b4094dd 100644 --- a/src/tier-check/output.ts +++ b/src/tier-check/output.ts @@ -19,7 +19,7 @@ function statusIcon(status: CheckStatus): string { case 'partial': return `${COLORS.YELLOW}~${COLORS.RESET}`; case 'skipped': - return `${COLORS.DIM}-${COLORS.RESET}`; + return `${COLORS.DIM}\u25cb${COLORS.RESET}`; } } @@ -99,11 +99,27 @@ function formatCell(cell: Cell | undefined): string { return `${cell.passed}/${cell.total}`; } +function formatConformanceCell( + cell: Cell | undefined, + status: CheckStatus | null +): string { + return status === 'skipped' ? '\u25cb' : formatCell(cell); +} + function formatRate(cell: Cell): string { if (cell.total === 0) return '0/0'; return `${cell.passed}/${cell.total} (${Math.round((cell.passed / cell.total) * 100)}%)`; } +function formatConformanceTotal( + cell: Cell, + status: CheckStatus | null, + verbose = false +): string { + if (status === 'skipped') return verbose ? '\u25cb skipped' : '\u25cb'; + return formatRate(cell); +} + export function formatJson(scorecard: TierScorecard): string { return JSON.stringify(scorecard, null, 2); } @@ -112,12 +128,24 @@ export function formatMarkdown(scorecard: TierScorecard): string { const lines: string[] = []; const c = scorecard.checks; - lines.push(`# Tier Assessment: Tier ${scorecard.implied_tier.tier}`); + const tierLabel = scorecard.partial_run + ? 'N/A (partial run)' + : `Tier ${scorecard.implied_tier.tier}`; + lines.push(`# Tier Assessment: ${tierLabel}`); lines.push(''); lines.push(`**Repo**: ${scorecard.repo}`); if (scorecard.branch) lines.push(`**Branch**: ${scorecard.branch}`); if (scorecard.version) lines.push(`**Version**: ${scorecard.version}`); lines.push(`**Timestamp**: ${scorecard.timestamp}`); + if (scorecard.partial_run) { + const skipped = Object.entries(c) + .filter(([, v]) => (v as { status: CheckStatus }).status === 'skipped') + .map(([k]) => k); + lines.push( + `**Scope**: partial run — skipped: ${skipped.join(', ') || '(none)'}` + ); + } + lines.push('_Legend: ✓ pass, ✗ fail, ~ partial, ○ skipped_'); lines.push(''); lines.push('## Check Results'); lines.push(''); @@ -128,21 +156,32 @@ export function formatMarkdown(scorecard: TierScorecard): string { c.conformance as ConformanceResult, c.client_conformance as ConformanceResult ); + const skippedSuites = [ + c.conformance.status === 'skipped' ? 'server' : null, + c.client_conformance.status === 'skipped' ? 'client' : null + ].filter((suite): suite is string => suite !== null); + + if (skippedSuites.length > 0) { + lines.push( + `> ○ Skipped conformance suites: ${skippedSuites.join(', ')}. Provide the missing inputs to include them in a full assessment.` + ); + lines.push(''); + } // Tier-scoring matrix lines.push(''); lines.push(`| | ${TIER_SPEC_VERSIONS.join(' | ')} | All* |`); lines.push(`|---|${TIER_SPEC_VERSIONS.map(() => '---|').join('')}---|`); - const mdRows: [string, MatrixRow][] = [ - ['Server', matrix.server], - ['Client: Core', matrix.clientCore], - ['Client: Auth', matrix.clientAuth] + const mdRows: [string, MatrixRow, CheckStatus | null, boolean][] = [ + ['Server', matrix.server, c.conformance.status, true], + ['Client: Core', matrix.clientCore, c.client_conformance.status, false], + ['Client: Auth', matrix.clientAuth, c.client_conformance.status, false] ]; - for (const [label, row] of mdRows) { + for (const [label, row, status, verboseSkipped] of mdRows) { lines.push( - `| ${label} | ${TIER_SPEC_VERSIONS.map((v) => formatCell(row.cells.get(v))).join(' | ')} | ${formatRate(row.tierUnique)} |` + `| ${label} | ${TIER_SPEC_VERSIONS.map((v) => formatConformanceCell(row.cells.get(v), status)).join(' | ')} | ${formatConformanceTotal(row.tierUnique, status, verboseSkipped)} |` ); } @@ -164,43 +203,59 @@ export function formatMarkdown(scorecard: TierScorecard): string { lines.push(''); lines.push(`| | ${INFO_SPEC_VERSIONS.join(' | ')} |`); lines.push(`|---|${INFO_SPEC_VERSIONS.map(() => '---|').join('')}`); - for (const [label, row] of mdRows) { + for (const [label, row, status] of mdRows) { const hasData = INFO_SPEC_VERSIONS.some((v) => { const cell = row.cells.get(v); return cell && cell.total > 0; }); - if (!hasData) continue; + if (!hasData && status !== 'skipped') continue; lines.push( - `| ${label} | ${INFO_SPEC_VERSIONS.map((v) => formatCell(row.cells.get(v))).join(' | ')} |` + `| ${label} | ${INFO_SPEC_VERSIONS.map((v) => formatConformanceCell(row.cells.get(v), status)).join(' | ')} |` ); } } lines.push(''); + const skippedDetail = '○ skipped'; lines.push( - `| Labels | ${c.labels.status} | ${c.labels.present}/${c.labels.required} required labels${c.labels.missing.length > 0 ? ` (missing: ${c.labels.missing.join(', ')})` : ''} |` + c.labels.status === 'skipped' + ? `| Labels | ${skippedDetail} | ${skippedDetail} |` + : `| Labels | ${c.labels.status} | ${c.labels.present}/${c.labels.required} required labels${c.labels.missing.length > 0 ? ` (missing: ${c.labels.missing.join(', ')})` : ''} |` ); lines.push( - `| Triage | ${c.triage.status} | ${Math.round(c.triage.compliance_rate * 100)}% within 2BD, median ${c.triage.median_hours}h, p95 ${c.triage.p95_hours}h |` + c.triage.status === 'skipped' + ? `| Triage | ${skippedDetail} | ${skippedDetail} |` + : `| Triage | ${c.triage.status} | ${Math.round(c.triage.compliance_rate * 100)}% within 2BD, median ${c.triage.median_hours}h, p95 ${c.triage.p95_hours}h |` ); lines.push( - `| P0 Resolution | ${c.p0_resolution.status} | ${c.p0_resolution.open_p0s} open, ${c.p0_resolution.closed_within_7d}/${c.p0_resolution.closed_total} closed within 7d |` + c.p0_resolution.status === 'skipped' + ? `| P0 Resolution | ${skippedDetail} | ${skippedDetail} |` + : `| P0 Resolution | ${c.p0_resolution.status} | ${c.p0_resolution.open_p0s} open, ${c.p0_resolution.closed_within_7d}/${c.p0_resolution.closed_total} closed within 7d |` ); lines.push( - `| Stable Release | ${c.stable_release.status} | ${c.stable_release.version || 'none'} (stable: ${c.stable_release.is_stable}) |` + c.stable_release.status === 'skipped' + ? `| Stable Release | ${skippedDetail} | ${skippedDetail} |` + : `| Stable Release | ${c.stable_release.status} | ${c.stable_release.version || 'none'} (stable: ${c.stable_release.is_stable}) |` ); lines.push( - `| Policy Signals | ${c.policy_signals.status} | ${Object.entries( - c.policy_signals.files - ) - .map(([f, e]) => `${f}: ${e ? '\u2713' : '\u2717'}`) - .join(', ')} |` + c.policy_signals.status === 'skipped' + ? `| Policy Signals | ${skippedDetail} | ${skippedDetail} |` + : `| Policy Signals | ${c.policy_signals.status} | ${Object.entries( + c.policy_signals.files + ) + .map(([f, e]) => `${f}: ${e ? '\u2713' : '\u2717'}`) + .join(', ')} |` ); lines.push( - `| Spec Tracking | ${c.spec_tracking.status} | ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : 'N/A'} |` + c.spec_tracking.status === 'skipped' + ? `| Spec Tracking | ${skippedDetail} | ${skippedDetail} |` + : `| Spec Tracking | ${c.spec_tracking.status} | ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : 'N/A'} |` ); lines.push(''); - if (scorecard.implied_tier.tier1_blockers.length > 0) { + if ( + !scorecard.partial_run && + scorecard.implied_tier.tier1_blockers.length > 0 + ) { lines.push('## Tier 1 Blockers'); lines.push(''); for (const blocker of scorecard.implied_tier.tier1_blockers) { @@ -209,7 +264,10 @@ export function formatMarkdown(scorecard: TierScorecard): string { lines.push(''); } - lines.push(`> ${scorecard.implied_tier.note}`); + const closingNote = scorecard.partial_run + ? 'Partial run — tier classification suppressed. Re-run without --skip-* flags for a full assessment.' + : scorecard.implied_tier.note; + lines.push(`> ${closingNote}`); return lines.join('\n'); } @@ -217,16 +275,35 @@ export function formatMarkdown(scorecard: TierScorecard): string { export function formatTerminal(scorecard: TierScorecard): void { const c = scorecard.checks; const tier = scorecard.implied_tier.tier; - const tierColor = - tier === 1 ? COLORS.GREEN : tier === 2 ? COLORS.YELLOW : COLORS.RED; + const partial = !!scorecard.partial_run; + const tierColor = partial + ? COLORS.DIM + : tier === 1 + ? COLORS.GREEN + : tier === 2 + ? COLORS.YELLOW + : COLORS.RED; + const tierText = partial ? 'N/A (partial run)' : `Tier ${tier}`; console.log( - `\n${COLORS.BOLD}Tier Assessment: ${tierColor}Tier ${tier}${COLORS.RESET}\n` + `\n${COLORS.BOLD}Tier Assessment: ${tierColor}${tierText}${COLORS.RESET}\n` ); console.log(`Repo: ${scorecard.repo}`); if (scorecard.branch) console.log(`Branch: ${scorecard.branch}`); if (scorecard.version) console.log(`Version: ${scorecard.version}`); - console.log(`Timestamp: ${scorecard.timestamp}\n`); + console.log(`Timestamp: ${scorecard.timestamp}`); + if (partial) { + const skipped = Object.entries(c) + .filter(([, v]) => (v as { status: CheckStatus }).status === 'skipped') + .map(([k]) => k); + console.log( + `${COLORS.DIM}Scope: partial — skipped: ${skipped.join(', ') || '(none)'}${COLORS.RESET}` + ); + } + console.log( + `${COLORS.DIM}Legend: ${statusIcon('pass')} pass ${statusIcon('fail')} fail ${statusIcon('partial')} partial ${statusIcon('skipped')} skipped${COLORS.RESET}` + ); + console.log(''); console.log(`${COLORS.BOLD}Conformance:${COLORS.RESET}\n`); @@ -235,6 +312,10 @@ export function formatTerminal(scorecard: TierScorecard): void { c.conformance as ConformanceResult, c.client_conformance as ConformanceResult ); + const skippedConformanceSuites = [ + c.conformance.status === 'skipped' ? 'server' : null, + c.client_conformance.status === 'skipped' ? 'client' : null + ].filter((suite): suite is string => suite !== null); const vw = 10; // column width for version cells const lw = 14; // label column width @@ -242,6 +323,15 @@ export function formatTerminal(scorecard: TierScorecard): void { const rp = (s: string, w: number) => s.padStart(w); const lp = (s: string, w: number) => s.padEnd(w); + if (skippedConformanceSuites.length > 0) { + console.log( + ` ${statusIcon('skipped')} Skipped suites: ${skippedConformanceSuites.join(', ')}` + ); + console.log( + ` ${COLORS.DIM}Provide the missing conformance inputs to include them in a full assessment.${COLORS.RESET}\n` + ); + } + // Tier-scoring matrix (date-versioned specs only) console.log( ` ${COLORS.DIM}${lp('', lw + 2)} ${TIER_SPEC_VERSIONS.map((v) => rp(v, vw)).join(' ')} ${rp('All*', tw)}${COLORS.RESET}` @@ -249,8 +339,8 @@ export function formatTerminal(scorecard: TierScorecard): void { const rows: [string, MatrixRow, CheckStatus | null, boolean][] = [ ['Server', matrix.server, c.conformance.status, true], - ['Client: Core', matrix.clientCore, null, false], - ['Client: Auth', matrix.clientAuth, null, false] + ['Client: Core', matrix.clientCore, c.client_conformance.status, false], + ['Client: Auth', matrix.clientAuth, c.client_conformance.status, false] ]; for (const [label, row, status, bold] of rows) { @@ -258,7 +348,7 @@ export function formatTerminal(scorecard: TierScorecard): void { const b = bold ? COLORS.BOLD : ''; const r = bold ? COLORS.RESET : ''; console.log( - ` ${icon}${b}${lp(label, lw)}${r} ${TIER_SPEC_VERSIONS.map((v) => rp(formatCell(row.cells.get(v)), vw)).join(' ')} ${b}${rp(formatRate(row.tierUnique), tw)}${r}` + ` ${icon}${b}${lp(label, lw)}${r} ${TIER_SPEC_VERSIONS.map((v) => rp(formatConformanceCell(row.cells.get(v), status), vw)).join(' ')} ${b}${rp(formatConformanceTotal(row.tierUnique, status, bold), tw)}${r}` ); } @@ -270,7 +360,7 @@ export function formatTerminal(scorecard: TierScorecard): void { matrix.clientCore.tierUnique.total + matrix.clientAuth.tierUnique.total }; console.log( - ` ${statusIcon(c.client_conformance.status)} ${COLORS.BOLD}${lp('Client Total', lw)}${COLORS.RESET} ${' '.repeat(TIER_SPEC_VERSIONS.length * (vw + 1) - 1)} ${COLORS.BOLD}${rp(formatRate(clientTierTotal), tw)}${COLORS.RESET}` + ` ${statusIcon(c.client_conformance.status)} ${COLORS.BOLD}${lp('Client Total', lw)}${COLORS.RESET} ${' '.repeat(TIER_SPEC_VERSIONS.length * (vw + 1) - 1)} ${COLORS.BOLD}${rp(formatConformanceTotal(clientTierTotal, c.client_conformance.status, true), tw)}${COLORS.RESET}` ); console.log( `\n ${COLORS.DIM}* unique scenarios — a scenario may apply to multiple spec versions${COLORS.RESET}` @@ -288,60 +378,100 @@ export function formatTerminal(scorecard: TierScorecard): void { console.log( ` ${COLORS.DIM}${lp('', lw + 2)} ${INFO_SPEC_VERSIONS.map((v) => rp(v, vw)).join(' ')}${COLORS.RESET}` ); - for (const [label, row, , bold] of rows) { + for (const [label, row, status, bold] of rows) { const hasData = INFO_SPEC_VERSIONS.some((v) => { const cell = row.cells.get(v); return cell && cell.total > 0; }); - if (!hasData) continue; + if (!hasData && status !== 'skipped') continue; const b = bold ? COLORS.BOLD : ''; const r = bold ? COLORS.RESET : ''; console.log( - ` ${b}${lp(label, lw)}${r} ${INFO_SPEC_VERSIONS.map((v) => rp(formatCell(row.cells.get(v)), vw)).join(' ')}` + ` ${b}${lp(label, lw)}${r} ${INFO_SPEC_VERSIONS.map((v) => rp(formatConformanceCell(row.cells.get(v), status), vw)).join(' ')}` ); } } console.log(`\n${COLORS.BOLD}Repository Health:${COLORS.RESET}\n`); - console.log( - ` ${statusIcon(c.labels.status)} Labels ${c.labels.present}/${c.labels.required} required labels` - ); - if (c.labels.missing.length > 0) + const rhLabel = (s: string) => s.padEnd(14); + if (c.labels.status === 'skipped') { console.log( - ` ${COLORS.DIM}Missing: ${c.labels.missing.join(', ')}${COLORS.RESET}` + ` ${statusIcon('skipped')} ${rhLabel('Labels')} ${COLORS.DIM}skipped${COLORS.RESET}` ); - console.log( - ` ${statusIcon(c.triage.status)} Triage ${Math.round(c.triage.compliance_rate * 100)}% within 2BD (${c.triage.total_issues} issues, median ${c.triage.median_hours}h)` - ); - console.log( - ` ${statusIcon(c.p0_resolution.status)} P0 Resolution ${c.p0_resolution.open_p0s} open, ${c.p0_resolution.closed_within_7d}/${c.p0_resolution.closed_total} closed within 7d` - ); - if (c.p0_resolution.open_p0_details.length > 0) { - for (const p0 of c.p0_resolution.open_p0_details) { + } else { + console.log( + ` ${statusIcon(c.labels.status)} ${rhLabel('Labels')} ${c.labels.present}/${c.labels.required} required labels` + ); + if (c.labels.missing.length > 0) console.log( - ` ${COLORS.RED}#${p0.number} (${p0.age_days}d old): ${p0.title}${COLORS.RESET}` + ` ${COLORS.DIM}Missing: ${c.labels.missing.join(', ')}${COLORS.RESET}` ); + } + if (c.triage.status === 'skipped') { + console.log( + ` ${statusIcon('skipped')} ${rhLabel('Triage')} ${COLORS.DIM}skipped${COLORS.RESET}` + ); + } else { + console.log( + ` ${statusIcon(c.triage.status)} ${rhLabel('Triage')} ${Math.round(c.triage.compliance_rate * 100)}% within 2BD (${c.triage.total_issues} issues, median ${c.triage.median_hours}h)` + ); + } + if (c.p0_resolution.status === 'skipped') { + console.log( + ` ${statusIcon('skipped')} ${rhLabel('P0 Resolution')} ${COLORS.DIM}skipped${COLORS.RESET}` + ); + } else { + console.log( + ` ${statusIcon(c.p0_resolution.status)} ${rhLabel('P0 Resolution')} ${c.p0_resolution.open_p0s} open, ${c.p0_resolution.closed_within_7d}/${c.p0_resolution.closed_total} closed within 7d` + ); + if (c.p0_resolution.open_p0_details.length > 0) { + for (const p0 of c.p0_resolution.open_p0_details) { + console.log( + ` ${COLORS.RED}#${p0.number} (${p0.age_days}d old): ${p0.title}${COLORS.RESET}` + ); + } } } - console.log( - ` ${statusIcon(c.stable_release.status)} Stable Release ${c.stable_release.version || 'none'}` - ); - console.log( - ` ${statusIcon(c.policy_signals.status)} Policy Signals ${Object.entries( - c.policy_signals.files - ) - .map(([f, e]) => `${e ? '\u2713' : '\u2717'} ${f}`) - .join(', ')}` - ); - console.log( - ` ${statusIcon(c.spec_tracking.status)} Spec Tracking ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : 'N/A'}` - ); + if (c.stable_release.status === 'skipped') { + console.log( + ` ${statusIcon('skipped')} ${rhLabel('Stable Release')} ${COLORS.DIM}skipped${COLORS.RESET}` + ); + } else { + console.log( + ` ${statusIcon(c.stable_release.status)} ${rhLabel('Stable Release')} ${c.stable_release.version || 'none'}` + ); + } + if (c.policy_signals.status === 'skipped') { + console.log( + ` ${statusIcon('skipped')} ${rhLabel('Policy Signals')} ${COLORS.DIM}skipped${COLORS.RESET}` + ); + } else { + console.log( + ` ${statusIcon(c.policy_signals.status)} ${rhLabel('Policy Signals')} ${Object.entries( + c.policy_signals.files + ) + .map(([f, e]) => `${e ? '\u2713' : '\u2717'} ${f}`) + .join(', ')}` + ); + } + if (c.spec_tracking.status === 'skipped') { + console.log( + ` ${statusIcon('skipped')} ${rhLabel('Spec Tracking')} ${COLORS.DIM}skipped${COLORS.RESET}` + ); + } else { + console.log( + ` ${statusIcon(c.spec_tracking.status)} ${rhLabel('Spec Tracking')} ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : 'N/A'}` + ); + } - if (scorecard.implied_tier.tier1_blockers.length > 0) { + if (!partial && scorecard.implied_tier.tier1_blockers.length > 0) { console.log(`\n${COLORS.BOLD}Tier 1 Blockers:${COLORS.RESET}`); for (const blocker of scorecard.implied_tier.tier1_blockers) { console.log(` ${COLORS.RED}\u2022${COLORS.RESET} ${blocker}`); } } - console.log(`\n${COLORS.DIM}${scorecard.implied_tier.note}${COLORS.RESET}\n`); + const closingNote = partial + ? 'Partial run — tier classification suppressed. Re-run without --skip-* flags for a full assessment.' + : scorecard.implied_tier.note; + console.log(`\n${COLORS.DIM}${closingNote}${COLORS.RESET}\n`); } diff --git a/src/tier-check/tier-logic.ts b/src/tier-check/tier-logic.ts index cf096c97..0d07efcc 100644 --- a/src/tier-check/tier-logic.ts +++ b/src/tier-check/tier-logic.ts @@ -19,15 +19,21 @@ export function computeTier( tier1Blockers.push('client_conformance'); } - if (checks.triage.compliance_rate < 0.9) { + if (checks.triage.status === 'skipped') { + tier1Blockers.push('triage (skipped)'); + } else if (checks.triage.compliance_rate < 0.9) { tier1Blockers.push('triage'); } - if (!checks.p0_resolution.all_p0s_resolved_within_7d) { + if (checks.p0_resolution.status === 'skipped') { + tier1Blockers.push('p0_resolution (skipped)'); + } else if (!checks.p0_resolution.all_p0s_resolved_within_7d) { tier1Blockers.push('p0_resolution'); } - if (!checks.stable_release.is_stable) { + if (checks.stable_release.status === 'skipped') { + tier1Blockers.push('stable_release (skipped)'); + } else if (!checks.stable_release.is_stable) { tier1Blockers.push('stable_release'); } @@ -35,11 +41,15 @@ export function computeTier( // they feed into the skill's judgment-based evaluation but don't independently // block tier advancement since SEP-1730 doesn't list specific files. - if (checks.spec_tracking.status === 'fail') { + if (checks.spec_tracking.status === 'skipped') { + tier1Blockers.push('spec_tracking (skipped)'); + } else if (checks.spec_tracking.status === 'fail') { tier1Blockers.push('spec_tracking'); } - if (checks.labels.missing.length > 0) { + if (checks.labels.status === 'skipped') { + tier1Blockers.push('labels (skipped)'); + } else if (checks.labels.missing.length > 0) { tier1Blockers.push('labels'); } @@ -49,8 +59,10 @@ export function computeTier( checks.conformance.pass_rate >= 0.8) && (checks.client_conformance.status === 'skipped' || checks.client_conformance.pass_rate >= 0.8) && - checks.p0_resolution.all_p0s_resolved_within_14d && - checks.stable_release.is_stable; + (checks.p0_resolution.status === 'skipped' || + checks.p0_resolution.all_p0s_resolved_within_14d) && + (checks.stable_release.status === 'skipped' || + checks.stable_release.is_stable); const tier = tier1Blockers.length === 0 ? 1 : tier2Met ? 2 : 3; diff --git a/src/tier-check/types.ts b/src/tier-check/types.ts index a9830f4c..33d9ce5f 100644 --- a/src/tier-check/types.ts +++ b/src/tier-check/types.ts @@ -71,6 +71,12 @@ export interface TierScorecard { branch: string | null; timestamp: string; version: string | null; + /** + * True if at least one check was skipped via --skip-* flags. + * When true, the implied_tier value should be treated as informational + * only — a partial run cannot definitively establish a tier. + */ + partial_run?: boolean; checks: { conformance: ConformanceResult; client_conformance: ConformanceResult;