From 3d0b933bc42d94510c9d13fb82e0335a59872e5e Mon Sep 17 00:00:00 2001 From: PunGrumpy <108584943+PunGrumpy@users.noreply.github.com> Date: Thu, 23 Jul 2026 09:52:58 +0700 Subject: [PATCH 1/3] test(core): characterize the current score curve --- packages/core/test/scoring.test.ts | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 packages/core/test/scoring.test.ts diff --git a/packages/core/test/scoring.test.ts b/packages/core/test/scoring.test.ts new file mode 100644 index 0000000..75defe3 --- /dev/null +++ b/packages/core/test/scoring.test.ts @@ -0,0 +1,65 @@ +import { describe, test, expect } from "bun:test"; + +import { calculateScore } from "../src/scoring"; +import type { Diagnostic } from "../src/types/index"; + +const diag = (severity: "error" | "warning" | "info"): Diagnostic => ({ + file: "Dockerfile", + help: "", + message: "", + rule: "docker-doctor/test", + severity, +}); + +const many = (severity: "error" | "warning" | "info", n: number) => + Array.from({ length: n }, () => diag(severity)); + +describe("calculateScore (characterization: current saturating curve)", () => { + test("an empty diagnostics list scores 100", () => { + expect(calculateScore([]).score).toBe(100); + }); + + test("one error scores 90", () => { + expect(calculateScore(many("error", 1)).score).toBe(90); + }); + + test("one warning scores 96", () => { + expect(calculateScore(many("warning", 1)).score).toBe(96); + }); + + test("one info scores 99", () => { + expect(calculateScore(many("info", 1)).score).toBe(99); + }); + + test("ten errors scores 0", () => { + expect(calculateScore(many("error", 10)).score).toBe(0); + }); + + test("twenty errors also scores 0 (saturation)", () => { + expect(calculateScore(many("error", 20)).score).toBe(0); + }); + + test("label is Excellent at score exactly 90", () => { + const { score, label } = calculateScore(many("info", 10)); + expect(score).toBe(90); + expect(label).toBe("Excellent 🏆"); + }); + + test("label is Good at score exactly 75", () => { + const { score, label } = calculateScore(many("info", 25)); + expect(score).toBe(75); + expect(label).toBe("Good ✅"); + }); + + test("label is Needs Work at score exactly 50", () => { + const { score, label } = calculateScore(many("info", 50)); + expect(score).toBe(50); + expect(label).toBe("Needs Work ⚠️"); + }); + + test("label is Critical at score exactly 49", () => { + const { score, label } = calculateScore(many("info", 51)); + expect(score).toBe(49); + expect(label).toBe("Critical 🚨"); + }); +}); From 0679e23c3dd59b55ada34db555fede96badb981f Mon Sep 17 00:00:00 2001 From: PunGrumpy <108584943+PunGrumpy@users.noreply.github.com> Date: Thu, 23 Jul 2026 09:54:53 +0700 Subject: [PATCH 2/3] feat(core)!: replace saturating score curve with a monotonic one BREAKING CHANGE: score = round(100 * e^(-penalty / 70)) replaces `max(0, 100 - penalty)`. The old formula saturated at 0 once penalty reached ~100 (e.g. ~10 errors), making the score inert on messy repos and defeating score-regression checks. The new curve approaches but never reaches 0, staying strictly monotonic across the whole range. Per-severity weights (error 10, warning 4, info 1) and label thresholds/strings (90/75/50) are unchanged. The JSON report gains a `schemaVersion` field to make future score/shape changes detectable. --- packages/core/src/report.ts | 6 +++ packages/core/src/scoring.ts | 17 ++++++++- packages/core/test/scoring.test.ts | 49 ++++++++++++++++++------- packages/docker-doctor/test/cli.test.ts | 9 ++++- 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/packages/core/src/report.ts b/packages/core/src/report.ts index 021d48e..9b5fb1f 100644 --- a/packages/core/src/report.ts +++ b/packages/core/src/report.ts @@ -1,5 +1,9 @@ import type { Diagnostic, ProjectInfo } from "./types/index"; +// Bump whenever the JSON report shape or the score formula/weights change. +// The unversioned shape shipped before this field existed is implicitly 1. +export const REPORT_SCHEMA_VERSION = 2; + export interface JsonReport { diagnostics: { column?: number; @@ -12,6 +16,7 @@ export interface JsonReport { }[]; label: string; project: ProjectInfo; + schemaVersion: number; score: number; timestamp: string; } @@ -33,6 +38,7 @@ export const toJsonReport = ( })), label, project, + schemaVersion: REPORT_SCHEMA_VERSION, score, timestamp: new Date().toISOString(), }); diff --git a/packages/core/src/scoring.ts b/packages/core/src/scoring.ts index 4790f7a..ae6a4be 100644 --- a/packages/core/src/scoring.ts +++ b/packages/core/src/scoring.ts @@ -18,7 +18,22 @@ export const calculateScore = ( } } - const score = Math.max(0, 100 - penalty); + // Asymptotic decay curve: score = round(100 * e^(-penalty / K)). + // + // The old `max(0, 100 - penalty)` formula saturates at 0 once penalty + // reaches 100 (e.g. ~10 errors), so a messy project and a catastrophic + // one are indistinguishable and the score can never register a fix. + // This curve approaches (but never reaches) 0, so it stays monotonic + // and responsive across the whole range instead of going inert. + // + // K=70 was chosen so a single warning (penalty 4) still scores ~94, + // comfortably inside the "Excellent" (>=90) bucket, while errors and + // repeated warnings still meaningfully erode the score. K=40 (the + // naive "half-life at penalty ~28" choice) was tried first and pushed + // a single warning down to ~90 - right on the Excellent/Good boundary, + // which is too harsh a penalty for one warning. + const K = 70; + const score = Math.round(100 * Math.exp(-penalty / K)); let label = "Critical 🚨"; if (score >= 90) { diff --git a/packages/core/test/scoring.test.ts b/packages/core/test/scoring.test.ts index 75defe3..d604cb3 100644 --- a/packages/core/test/scoring.test.ts +++ b/packages/core/test/scoring.test.ts @@ -14,51 +14,72 @@ const diag = (severity: "error" | "warning" | "info"): Diagnostic => ({ const many = (severity: "error" | "warning" | "info", n: number) => Array.from({ length: n }, () => diag(severity)); -describe("calculateScore (characterization: current saturating curve)", () => { - test("an empty diagnostics list scores 100", () => { +describe("calculateScore (asymptotic curve, K=70)", () => { + test("a perfect project scores 100", () => { expect(calculateScore([]).score).toBe(100); }); - test("one error scores 90", () => { - expect(calculateScore(many("error", 1)).score).toBe(90); + test("one error scores 87", () => { + expect(calculateScore(many("error", 1)).score).toBe(87); }); - test("one warning scores 96", () => { - expect(calculateScore(many("warning", 1)).score).toBe(96); + test("one warning stays comfortably inside Excellent (~94)", () => { + const { score, label } = calculateScore(many("warning", 1)); + expect(score).toBe(94); + expect(label).toBe("Excellent 🏆"); }); test("one info scores 99", () => { expect(calculateScore(many("info", 1)).score).toBe(99); }); - test("ten errors scores 0", () => { - expect(calculateScore(many("error", 10)).score).toBe(0); + test("is strictly monotonic and never saturates", () => { + const a = calculateScore(many("error", 10)).score; + const b = calculateScore(many("error", 20)).score; + const c = calculateScore(many("error", 40)).score; + expect(b).toBeLessThan(a); + expect(c).toBeLessThan(b); + }); + + test("score is always an integer in range", () => { + for (const n of [0, 1, 5, 25, 100, 500]) { + const { score } = calculateScore(many("warning", n)); + expect(Number.isInteger(score)).toBe(true); + expect(score).toBeGreaterThanOrEqual(0); + expect(score).toBeLessThanOrEqual(100); + } }); - test("twenty errors also scores 0 (saturation)", () => { - expect(calculateScore(many("error", 20)).score).toBe(0); + test("fixing an issue always raises the score", () => { + const before = calculateScore(many("warning", 30)).score; + const after = calculateScore(many("warning", 29)).score; + expect(after).toBeGreaterThan(before); }); test("label is Excellent at score exactly 90", () => { - const { score, label } = calculateScore(many("info", 10)); + // penalty 7 -> round(100 * e^(-7/70)) = 90 + const { score, label } = calculateScore(many("info", 7)); expect(score).toBe(90); expect(label).toBe("Excellent 🏆"); }); test("label is Good at score exactly 75", () => { - const { score, label } = calculateScore(many("info", 25)); + // penalty 20 -> round(100 * e^(-20/70)) = 75 + const { score, label } = calculateScore(many("info", 20)); expect(score).toBe(75); expect(label).toBe("Good ✅"); }); test("label is Needs Work at score exactly 50", () => { - const { score, label } = calculateScore(many("info", 50)); + // penalty 48 -> round(100 * e^(-48/70)) = 50 + const { score, label } = calculateScore(many("info", 48)); expect(score).toBe(50); expect(label).toBe("Needs Work ⚠️"); }); test("label is Critical at score exactly 49", () => { - const { score, label } = calculateScore(many("info", 51)); + // penalty 50 -> round(100 * e^(-50/70)) = 49 + const { score, label } = calculateScore(many("info", 50)); expect(score).toBe(49); expect(label).toBe("Critical 🚨"); }); diff --git a/packages/docker-doctor/test/cli.test.ts b/packages/docker-doctor/test/cli.test.ts index f6f056b..df802ed 100644 --- a/packages/docker-doctor/test/cli.test.ts +++ b/packages/docker-doctor/test/cli.test.ts @@ -59,7 +59,14 @@ describe("--json contract", () => { const { stdout } = await runCli([fixture("with-error"), "--json"]); const report = JSON.parse(stdout); expect(Object.keys(report).toSorted()).toEqual( - ["diagnostics", "label", "project", "score", "timestamp"].toSorted() + [ + "diagnostics", + "label", + "project", + "schemaVersion", + "score", + "timestamp", + ].toSorted() ); }); From 32bf30f279802e20a4db0d50c4703d1a221e6eb3 Mon Sep 17 00:00:00 2001 From: PunGrumpy <108584943+PunGrumpy@users.noreply.github.com> Date: Thu, 23 Jul 2026 09:55:18 +0700 Subject: [PATCH 3/3] docs(cli): document the score formula --- .changeset/thick-parrots-notice.md | 16 ++++++++++++++++ packages/docker-doctor/README.md | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 .changeset/thick-parrots-notice.md diff --git a/.changeset/thick-parrots-notice.md b/.changeset/thick-parrots-notice.md new file mode 100644 index 0000000..e0046b4 --- /dev/null +++ b/.changeset/thick-parrots-notice.md @@ -0,0 +1,16 @@ +--- +"@docker-doctor/cli": minor +--- + +Replace the saturating score formula with a monotonic, asymptotic curve. + +The score used to be `100 - penalty` (floored at 0), which meant any project with roughly 10 errors or 25 warnings scored exactly 0 and stayed there no matter how much worse it got — the score became inert on messy repos, and score-regression checks (e.g. "re-scan, confirm the score did not drop") stopped working once a project hit the floor. + +The score is now `round(100 * e^(-penalty / 70))`, using the same per-severity penalty weights as before (error 10, warning 4, info 1). This curve approaches 0 without ever getting stuck there, so the score keeps moving and stays meaningful across the whole range. + +**Breaking:** + +- Score values have changed. A given set of diagnostics will now produce a different numeric score than before (e.g. a single error used to score 90, now scores ~87; a project that used to floor at 0 will now show a small nonzero number that keeps decreasing as issues pile up). +- Existing badges, dashboards, or stored scores will show different numbers after upgrading — this is expected, not a regression. +- The `Excellent 🏆` / `Good ✅` / `Needs Work ⚠️` / `Critical 🚨` label thresholds (>=90 / >=75 / >=50 / below) are unchanged. +- The JSON report now includes a `schemaVersion` field (currently `2`) to make future score/shape changes detectable. diff --git a/packages/docker-doctor/README.md b/packages/docker-doctor/README.md index 4768a6d..0d24d73 100644 --- a/packages/docker-doctor/README.md +++ b/packages/docker-doctor/README.md @@ -54,6 +54,28 @@ export default { }; ``` +## How the score works + +Every scan produces a 0-100 health score alongside a label (`Excellent 🏆`, `Good ✅`, `Needs Work ⚠️`, `Critical 🚨`). + +Each diagnostic adds a penalty based on severity: + +| Severity | Penalty | +| --------- | ------- | +| `error` | 10 | +| `warning` | 4 | +| `info` | 1 | + +The penalties are summed, then the score is computed as an asymptotic decay curve rather than a simple subtraction: + +``` +score = round(100 * e^(-penalty / K)) // K = 70 +``` + +A perfect project (no diagnostics) always scores exactly 100. As penalty increases, the score keeps decreasing — it approaches 0 but never gets stuck there, so the score stays meaningful (and can still register improvement) even on projects with a lot of findings. `K = 70` was chosen so a single warning (penalty 4) still lands around 94 — comfortably inside the `Excellent` bucket — while errors and repeated warnings continue to meaningfully erode the score. + +The label thresholds are unchanged: `>= 90` Excellent, `>= 75` Good, `>= 50` Needs Work, otherwise Critical. + ## API ```ts