From 038bcd4e1a1cb55dc2f79f571cd735694ecc1d9b Mon Sep 17 00:00:00 2001 From: Asterios Raptis Date: Wed, 22 Jul 2026 11:14:41 +0200 Subject: [PATCH] feat(validate): W-INVISIBLE-CHAR author lint for invisible characters (Closes #75) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Text pasted from a PDF or a web page carries characters that render as nothing: zero-width spaces, byte-order marks, directional marks, soft hyphens. They are legal JSON string content, so every structural check passes them, and nobody spots them by reading the file. This is not a theoretical path: the reference app's book-text wizard asks the author to paste a textbook section. The rule that prompted this was our own. The blog style guide audited invisible characters with grep -oP '[^\x00-\x7F]' | sort -u # review every non-ASCII char which lists every umlaut and accent and asks a human to notice an invisible character among them. An invisible character is invisible in that output too. A check that names the codepoint is strictly better, and lesson content is where the paste actually lands. Detection lives in its own module. It walks every string in the lesson rather than a field list, which is what makes ext_payload covered without the engine knowing its shape, and keeps future schema fields covered for free. The warning names each distinct codepoint with its Unicode name, the occurrence count and the paths, aggregated once per lesson: pasted prose can carry dozens, and per-occurrence emission is the alert fatigue W-CARD-UNUSED was aggregated away from (#49). Deliberately NOT flagged: U+00A0 NO-BREAK SPACE and U+202F NARROW NO-BREAK SPACE. Both render as whitespace and are legitimate typography, notably in the French content this ecosystem carries ("Comment ca va ?" sets one before the question mark). Flagging them would make the lint usually-wrong, which is the over-sensitive behaviour it exists to avoid; two boundary tests pin the exclusion. TDD RED-first, 15 tests: one per codepoint class, ext_payload reach, per-lesson aggregation, and the silence cases (clean content, umlauts and accents, both no-break spaces, and that the lesson stays valid). Measured against real content rather than assumed: 528 lessons across eight content repositories produced 4 findings and 0 false positives. All four are genuine soft hyphens sitting mid-word in pasted prose ("Irrealität", "kulturspezifische"), invisible to a reader and untouched until now. Those are content fixes in other repos, reported rather than silently changed. Proven to reach content repos without their cooperation: with this build overlaid as the engine dependency, the UNMODIFIED adaptive-learner-content-test gate emitted the warning through its existing `make lint-warnings`, and `make lint` still exited 0. Warning tier, additive, no schema change and no schema_version bump. Release 0.13.2 (patch), verified free on npm. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 23 +++++ docs/lesson-format.md | 1 + package-lock.json | 4 +- package.json | 2 +- src/invisible-chars.test.ts | 182 ++++++++++++++++++++++++++++++++++++ src/invisible-chars.ts | 113 ++++++++++++++++++++++ src/validate.ts | 13 +++ 7 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 src/invisible-chars.test.ts create mode 100644 src/invisible-chars.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ae10a2d..9341ff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,29 @@ All notable changes to `learn-content-engine`. The format is inspired by [SemVer](https://semver.org/) (schema evolution is additive, see [docs/concepts.md](docs/concepts.md#schema-version-policy-additive)). +## [0.13.2] - 2026-07-22 + +New author lint `W-INVISIBLE-CHAR` (#75): warns when a lesson's text carries +characters that render as nothing - zero-width spaces, byte-order marks, +directional marks, soft hyphens. They are legal JSON and survive every +structural check, and no one spots them by reading the file; they arrive by +pasting from a PDF or a web page, which is exactly what the reference app's +book-text wizard asks the author to do. The warning names each codepoint +(`U+200B ZERO WIDTH SPACE`), its Unicode name, the occurrence count and the +paths, aggregated once per lesson (the `W-CARD-UNUSED` precedent from #49). +Every string is walked, including `ext_payload`, so extension text is covered +without the engine knowing its shape. + +Deliberately NOT flagged: `U+00A0` NO-BREAK SPACE and `U+202F` NARROW NO-BREAK +SPACE. Both render as whitespace and are legitimate typography, notably in the +French content this ecosystem carries. Measured on 528 real lessons across +eight content repositories: 4 findings, all genuine soft hyphens sitting +mid-word in pasted prose, 0 false positives. + +Warning tier, never blocks. Additive: no schema change, no `schema_version` +bump. Content repos pick it up through their existing `make lint` / +`make lint-warnings` without changing anything. + ## [0.13.1] - 2026-07-20 Docs/examples: sixth reference extension `ext:ref-dictation` (#68) - an audio diff --git a/docs/lesson-format.md b/docs/lesson-format.md index 55d3ae2..c4d9cc7 100644 --- a/docs/lesson-format.md +++ b/docs/lesson-format.md @@ -565,6 +565,7 @@ drifting. | `W-PIC-DUP-LABEL` | A `picture_choice` distractor shares its `label` with the correct image. | | `W-PIC-DATA-URI` | A `picture_choice` image `src` is an inline `data:` URI (schema v1.8 allows it for consumer-local content, e.g. uploaded images). Repo content should prefer a relative `assets/` path - inline data URIs bloat the lesson JSON and the git history. Advisory only, never blocks. | | `W-HINT-LENGTH` | A hint reveals the answer length (e.g. "four letters"). Consumers that display an answer-length indicator make such a hint redundant; on other consumers it gives part of the answer away. | +| `W-INVISIBLE-CHAR` | The lesson's text carries characters that render as nothing: zero-width spaces, byte-order marks, directional marks, soft hyphens. They are legal JSON and survive every structural check, and no one spots them by reading the file; they usually arrive by pasting from a PDF or a web page. The warning names each codepoint (`U+200B ZERO WIDTH SPACE`), its Unicode name, and where it sits, aggregated once per lesson. Every string is scanned, including `ext_payload`, so extension text is covered without the engine knowing its shape. Deliberately NOT flagged: `U+00A0` NO-BREAK SPACE and `U+202F` NARROW NO-BREAK SPACE, which render as whitespace and are legitimate typography (French sets one before `?` and `!`). | ## Linting diff --git a/package-lock.json b/package-lock.json index 7e89156..7424498 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "learn-content-engine", - "version": "0.13.1", + "version": "0.13.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "learn-content-engine", - "version": "0.13.1", + "version": "0.13.2", "license": "MIT", "dependencies": { "@rgrove/parse-xml": "^4.2.2", diff --git a/package.json b/package.json index 80d4db0..88acb5d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "learn-content-engine", - "version": "0.13.1", + "version": "0.13.2", "description": "Framework-agnostic TypeScript engine that parses lesson content from pluggable sources into a canonical lesson object.", "type": "module", "license": "MIT", diff --git a/src/invisible-chars.test.ts b/src/invisible-chars.test.ts new file mode 100644 index 0000000..06108d3 --- /dev/null +++ b/src/invisible-chars.test.ts @@ -0,0 +1,182 @@ +import { describe, it, expect } from "vitest"; + +import { validateLesson } from "./validate.js"; + +/** + * ``W-INVISIBLE-CHAR`` (#75): text pasted from a PDF or a web page carries + * characters that are invisible by definition (zero-width spaces, BOMs, + * directional marks, soft hyphens). They are legal JSON string content, so no + * structural check catches them, and no one spots them by reading the file. + * The app's book-text wizard invites exactly that paste, so this is a real + * content path rather than a theoretical one. + * + * The bar these tests set is TWO-sided: every invisible codepoint must be + * named, and clean content, INCLUDING legitimate typography that merely looks + * exotic, must stay silent. An over-sensitive lint is worse than none, because + * warnings that are usually wrong stop being read. + */ + +const lessonWith = (overrides: Record) => ({ + id: "l1", + title: "Lesson", + cards: [{ id: "c1", front: "coffee", back: "der Kaffee" }], + steps: [ + { + id: "s1", + type: "exercise", + exercise: { id: "e1", type: "free_text", prompt: "Translate.", accept: ["der Kaffee"] }, + }, + ], + ...overrides, +}); + +const warningsFor = (lesson: unknown) => + validateLesson(lesson).warnings.filter((issue) => issue.id === "W-INVISIBLE-CHAR"); + +const messageFor = (lesson: unknown) => warningsFor(lesson)[0]?.message ?? ""; + +describe("W-INVISIBLE-CHAR", () => { + it("flags a zero-width space in a card field and names the codepoint", () => { + const lesson = lessonWith({ + cards: [{ id: "c1", front: "cof​fee", back: "der Kaffee" }], + }); + expect(warningsFor(lesson)).toHaveLength(1); + expect(messageFor(lesson)).toContain("U+200B"); + expect(messageFor(lesson)).toContain("ZERO WIDTH SPACE"); + }); + + it("names the field path so the author can find it", () => { + const lesson = lessonWith({ + cards: [{ id: "c1", front: "cof​fee", back: "der Kaffee" }], + }); + expect(messageFor(lesson)).toContain("/cards/0/front"); + }); + + it("flags a byte-order mark", () => { + const lesson = lessonWith({ title: "Lesson" }); + expect(messageFor(lesson)).toContain("U+FEFF"); + expect(messageFor(lesson)).toContain("BYTE ORDER MARK"); + }); + + it("flags a left-to-right mark", () => { + const lesson = lessonWith({ title: "Lesson‎" }); + expect(messageFor(lesson)).toContain("U+200E"); + expect(messageFor(lesson)).toContain("LEFT-TO-RIGHT MARK"); + }); + + it("flags a right-to-left mark", () => { + const lesson = lessonWith({ title: "Lesson‏" }); + expect(messageFor(lesson)).toContain("U+200F"); + }); + + it("flags a soft hyphen, which a PDF copy routinely injects", () => { + const lesson = lessonWith({ title: "Kaf­fee" }); + expect(messageFor(lesson)).toContain("U+00AD"); + expect(messageFor(lesson)).toContain("SOFT HYPHEN"); + }); + + it("flags text inside an exercise prompt", () => { + const lesson = lessonWith({ + steps: [ + { + id: "s1", + type: "exercise", + exercise: { + id: "e1", + type: "free_text", + prompt: "Trans​late.", + accept: ["der Kaffee"], + }, + }, + ], + }); + expect(messageFor(lesson)).toContain("/steps/0/exercise/prompt"); + }); + + it("reaches into ext_payload without enumerating its fields", () => { + const lesson = lessonWith({ + requires_extensions: ["ext:al-dictation@1"], + steps: [ + { + id: "s1", + type: "exercise", + exercise: { + id: "e1", + type: "ext:al-dictation", + prompt: "Listen.", + ext_payload: { audio: "assets/a.mp3", accept: ["a cof​fee"] }, + }, + }, + ], + }); + const found = validateLesson(lesson, { + extensions: [{ type: "ext:al-dictation", major: 1, validate: () => [] }], + }).warnings.filter((issue) => issue.id === "W-INVISIBLE-CHAR"); + expect(found).toHaveLength(1); + expect(found[0]!.message).toContain("ext_payload"); + }); + + it("aggregates to ONE warning per lesson, not one per occurrence", () => { + // A pasted chapter can carry dozens; emitting one warning each is the + // alert fatigue W-CARD-UNUSED was aggregated to avoid (#49). + const lesson = lessonWith({ + title: "Les​son", + cards: [ + { id: "c1", front: "cof​fee", back: "der Kaf​fee" }, + { id: "c2", front: "milk​", back: "die Milch" }, + ], + }); + expect(warningsFor(lesson)).toHaveLength(1); + expect(messageFor(lesson)).toMatch(/4 occurrence|4 /); + }); + + it("reports every distinct codepoint it found, not just the first", () => { + const lesson = lessonWith({ title: "A​B­C" }); + expect(messageFor(lesson)).toContain("U+200B"); + expect(messageFor(lesson)).toContain("U+00AD"); + }); + + it("stays silent on clean content", () => { + expect(warningsFor(lessonWith({}))).toEqual([]); + }); + + it("stays silent on umlauts, accents and other legitimate non-ASCII text", () => { + const lesson = lessonWith({ + title: "Über die Prüfung", + cards: [{ id: "c1", front: "café", back: "das Café" }], + steps: [ + { + id: "s1", + type: "exercise", + exercise: { + id: "e1", + type: "free_text", + prompt: "Wie heißt „Kaffee“ auf Französisch?", + accept: ["café"], + }, + }, + ], + }); + expect(warningsFor(lesson)).toEqual([]); + }); + + it("boundary: does NOT flag a no-break space, which is legitimate French typography", () => { + // "Comment ça va ?" sets U+00A0 before the question mark by convention. + // Flagging it would reproduce the over-sensitive behaviour this lint + // exists to avoid, and this ecosystem ships French content. + const lesson = lessonWith({ title: "Comment ça va ?" }); + expect(warningsFor(lesson)).toEqual([]); + }); + + it("boundary: does NOT flag a narrow no-break space either", () => { + const lesson = lessonWith({ title: "1 000 Wörter" }); + expect(warningsFor(lesson)).toEqual([]); + }); + + it("never blocks: the lesson stays valid", () => { + const lesson = lessonWith({ title: "Les​son" }); + const result = validateLesson(lesson); + expect(result.valid).toBe(true); + expect(result.errors).toEqual([]); + }); +}); diff --git a/src/invisible-chars.ts b/src/invisible-chars.ts new file mode 100644 index 0000000..86c6475 --- /dev/null +++ b/src/invisible-chars.ts @@ -0,0 +1,113 @@ +/** + * Detection of invisible Unicode characters in lesson content (#75). + * + * Text pasted out of a PDF or a web page carries characters that cannot be + * seen: zero-width spaces, byte-order marks, directional marks, soft hyphens. + * They are legal JSON string content, so no structural check rejects them, and + * no one notices them by reading the file. The app's book-text wizard asks the + * author to paste a textbook section, which is precisely how they arrive. + * + * The rule is deliberately NARROW. Only characters that render as nothing at + * all are listed. A no-break space (``U+00A0``) and a narrow no-break space + * (``U+202F``) are excluded on purpose: they render as whitespace and are + * legitimate typography, notably in the French content this ecosystem carries + * ("Comment ca va ?" sets one before the question mark by convention). + * Flagging them would make the lint usually-wrong, and a lint that is usually + * wrong stops being read. + */ + +/** Invisible codepoints, with the Unicode name reported to the author. */ +const INVISIBLE_CHARACTERS = new Map([ + ["­", "SOFT HYPHEN"], + ["​", "ZERO WIDTH SPACE"], + ["‌", "ZERO WIDTH NON-JOINER"], + ["‍", "ZERO WIDTH JOINER"], + ["‎", "LEFT-TO-RIGHT MARK"], + ["‏", "RIGHT-TO-LEFT MARK"], + ["
", "LINE SEPARATOR"], + ["
", "PARAGRAPH SEPARATOR"], + ["‪", "LEFT-TO-RIGHT EMBEDDING"], + ["‫", "RIGHT-TO-LEFT EMBEDDING"], + ["‬", "POP DIRECTIONAL FORMATTING"], + ["‭", "LEFT-TO-RIGHT OVERRIDE"], + ["‮", "RIGHT-TO-LEFT OVERRIDE"], + ["⁠", "WORD JOINER"], + ["", "BYTE ORDER MARK"], +]); + +/** One invisible character found at one place in the lesson. */ +export interface InvisibleCharFinding { + /** JSON-pointer-ish location, e.g. ``/cards/0/front``. */ + path: string; + /** The offending character itself. */ + character: string; + /** ``U+200B`` style label. */ + codepoint: string; + /** The Unicode name, e.g. ``ZERO WIDTH SPACE``. */ + name: string; +} + +/** ``U+200B``-style label for a single character. */ +function toCodepointLabel(character: string): string { + return `U+${character.codePointAt(0)!.toString(16).toUpperCase().padStart(4, "0")}`; +} + +/** Collect every invisible character in one string value. */ +function findingsInString(value: string, path: string): InvisibleCharFinding[] { + const findings: InvisibleCharFinding[] = []; + for (const character of value) { + const name = INVISIBLE_CHARACTERS.get(character); + if (name) { + findings.push({ path, character, codepoint: toCodepointLabel(character), name }); + } + } + return findings; +} + +/** + * Walk every string in an arbitrary value, reporting invisible characters with + * the path they sit at. Walking the whole object rather than a field list is + * what lets this cover ``ext_payload`` (whose shape the engine does not know) + * and any field a later schema version adds. + */ +export function findInvisibleChars(value: unknown, path = ""): InvisibleCharFinding[] { + if (typeof value === "string") return findingsInString(value, path); + if (Array.isArray(value)) { + return value.flatMap((entry, index) => findInvisibleChars(entry, `${path}/${index}`)); + } + if (typeof value === "object" && value !== null) { + return Object.entries(value).flatMap(([key, entry]) => + findInvisibleChars(entry, `${path}/${key}`), + ); + } + return []; +} + +/** How many distinct paths to name before trailing off. Enough to start + * fixing, short enough that a heavily-affected pasted chapter stays readable. */ +const MAX_PATHS_LISTED = 5; + +/** + * One human-readable sentence describing every finding, or null when the + * content is clean. Aggregated per lesson rather than emitted per occurrence: + * a pasted chapter can carry dozens, and per-occurrence emission is the alert + * fatigue W-CARD-UNUSED was aggregated away from (#49). + */ +export function describeInvisibleChars(findings: readonly InvisibleCharFinding[]): string | null { + if (findings.length === 0) return null; + const byCodepoint = new Map(); + for (const finding of findings) byCodepoint.set(finding.codepoint, finding.name); + const kinds = [...byCodepoint] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([codepoint, name]) => `${codepoint} ${name}`) + .join(", "); + const paths = [...new Set(findings.map((finding) => finding.path))]; + const shown = paths.slice(0, MAX_PATHS_LISTED).join(", "); + const rest = paths.length > MAX_PATHS_LISTED ? `, and ${paths.length - MAX_PATHS_LISTED} more` : ""; + const plural = findings.length === 1 ? "occurrence" : "occurrences"; + return ( + `lesson text contains invisible characters (${kinds}): ` + + `${findings.length} ${plural} at ${shown}${rest}. ` + + "These usually arrive by pasting from a PDF or a web page; they are invisible when reading the file." + ); +} diff --git a/src/validate.ts b/src/validate.ts index 8d536bd..dae5ed3 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -29,6 +29,7 @@ import { Ajv2020 } from "ajv/dist/2020.js"; import type { ErrorObject, ValidateFunction } from "ajv"; import type { ExtensionRegistry } from "./extensions.js"; +import { describeInvisibleChars, findInvisibleChars } from "./invisible-chars.js"; import type { Exercise, Lesson, LessonStep } from "./types/lesson-schema.generated.js"; /** Whether an issue blocks (``error``) or merely advises (``warning``). */ @@ -451,6 +452,17 @@ function checkUnusedCards(lesson: Lesson, issues: ValidationIssue[]): void { ); } +/** Warn about invisible Unicode characters anywhere in the lesson's text + * (#75). Aggregated to ONE warning per lesson listing every distinct + * codepoint and where it sits: pasted text can carry dozens, and a warning + * per occurrence is the alert fatigue W-CARD-UNUSED was aggregated away from + * (#49). Never an error - the content is structurally valid, it just carries + * characters the author cannot see. */ +function checkInvisibleChars(lesson: Lesson, issues: ValidationIssue[]): void { + const description = describeInvisibleChars(findInvisibleChars(lesson)); + if (description) issues.push(warn("W-INVISIBLE-CHAR", "", description, "author-lints")); +} + /** Semantic + lint pass. Assumes the input is already structurally valid (so the * schema-typed shape is trustworthy). Returns a mixed error/warning list. */ function semanticIssues(lesson: Lesson, registry: ExtensionRegistry): ValidationIssue[] { @@ -461,6 +473,7 @@ function semanticIssues(lesson: Lesson, registry: ExtensionRegistry): Validation checkStep(step, `/steps/${index}`, knownCardIds, ext, issues); }); checkUnusedCards(lesson, issues); + checkInvisibleChars(lesson, issues); return issues; }