diff --git a/CHANGELOG.md b/CHANGELOG.md index ae10a2d..9341ff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,29 @@ All notable changes to `learn-content-engine`. The format is inspired by [SemVer](https://semver.org/) (schema evolution is additive, see [docs/concepts.md](docs/concepts.md#schema-version-policy-additive)). +## [0.13.2] - 2026-07-22 + +New author lint `W-INVISIBLE-CHAR` (#75): warns when a lesson's text carries +characters that render as nothing - zero-width spaces, byte-order marks, +directional marks, soft hyphens. They are legal JSON and survive every +structural check, and no one spots them by reading the file; they arrive by +pasting from a PDF or a web page, which is exactly what the reference app's +book-text wizard asks the author to do. The warning names each codepoint +(`U+200B ZERO WIDTH SPACE`), its Unicode name, the occurrence count and the +paths, aggregated once per lesson (the `W-CARD-UNUSED` precedent from #49). +Every string is walked, including `ext_payload`, so extension text is covered +without the engine knowing its shape. + +Deliberately NOT flagged: `U+00A0` NO-BREAK SPACE and `U+202F` NARROW NO-BREAK +SPACE. Both render as whitespace and are legitimate typography, notably in the +French content this ecosystem carries. Measured on 528 real lessons across +eight content repositories: 4 findings, all genuine soft hyphens sitting +mid-word in pasted prose, 0 false positives. + +Warning tier, never blocks. Additive: no schema change, no `schema_version` +bump. Content repos pick it up through their existing `make lint` / +`make lint-warnings` without changing anything. + ## [0.13.1] - 2026-07-20 Docs/examples: sixth reference extension `ext:ref-dictation` (#68) - an audio diff --git a/docs/lesson-format.md b/docs/lesson-format.md index 55d3ae2..c4d9cc7 100644 --- a/docs/lesson-format.md +++ b/docs/lesson-format.md @@ -565,6 +565,7 @@ drifting. | `W-PIC-DUP-LABEL` | A `picture_choice` distractor shares its `label` with the correct image. | | `W-PIC-DATA-URI` | A `picture_choice` image `src` is an inline `data:` URI (schema v1.8 allows it for consumer-local content, e.g. uploaded images). Repo content should prefer a relative `assets/` path - inline data URIs bloat the lesson JSON and the git history. Advisory only, never blocks. | | `W-HINT-LENGTH` | A hint reveals the answer length (e.g. "four letters"). Consumers that display an answer-length indicator make such a hint redundant; on other consumers it gives part of the answer away. | +| `W-INVISIBLE-CHAR` | The lesson's text carries characters that render as nothing: zero-width spaces, byte-order marks, directional marks, soft hyphens. They are legal JSON and survive every structural check, and no one spots them by reading the file; they usually arrive by pasting from a PDF or a web page. The warning names each codepoint (`U+200B ZERO WIDTH SPACE`), its Unicode name, and where it sits, aggregated once per lesson. Every string is scanned, including `ext_payload`, so extension text is covered without the engine knowing its shape. Deliberately NOT flagged: `U+00A0` NO-BREAK SPACE and `U+202F` NARROW NO-BREAK SPACE, which render as whitespace and are legitimate typography (French sets one before `?` and `!`). | ## Linting diff --git a/package-lock.json b/package-lock.json index 7e89156..7424498 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "learn-content-engine", - "version": "0.13.1", + "version": "0.13.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "learn-content-engine", - "version": "0.13.1", + "version": "0.13.2", "license": "MIT", "dependencies": { "@rgrove/parse-xml": "^4.2.2", diff --git a/package.json b/package.json index 80d4db0..88acb5d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "learn-content-engine", - "version": "0.13.1", + "version": "0.13.2", "description": "Framework-agnostic TypeScript engine that parses lesson content from pluggable sources into a canonical lesson object.", "type": "module", "license": "MIT", diff --git a/src/invisible-chars.test.ts b/src/invisible-chars.test.ts new file mode 100644 index 0000000..06108d3 --- /dev/null +++ b/src/invisible-chars.test.ts @@ -0,0 +1,182 @@ +import { describe, it, expect } from "vitest"; + +import { validateLesson } from "./validate.js"; + +/** + * ``W-INVISIBLE-CHAR`` (#75): text pasted from a PDF or a web page carries + * characters that are invisible by definition (zero-width spaces, BOMs, + * directional marks, soft hyphens). They are legal JSON string content, so no + * structural check catches them, and no one spots them by reading the file. + * The app's book-text wizard invites exactly that paste, so this is a real + * content path rather than a theoretical one. + * + * The bar these tests set is TWO-sided: every invisible codepoint must be + * named, and clean content, INCLUDING legitimate typography that merely looks + * exotic, must stay silent. An over-sensitive lint is worse than none, because + * warnings that are usually wrong stop being read. + */ + +const lessonWith = (overrides: Record) => ({ + id: "l1", + title: "Lesson", + cards: [{ id: "c1", front: "coffee", back: "der Kaffee" }], + steps: [ + { + id: "s1", + type: "exercise", + exercise: { id: "e1", type: "free_text", prompt: "Translate.", accept: ["der Kaffee"] }, + }, + ], + ...overrides, +}); + +const warningsFor = (lesson: unknown) => + validateLesson(lesson).warnings.filter((issue) => issue.id === "W-INVISIBLE-CHAR"); + +const messageFor = (lesson: unknown) => warningsFor(lesson)[0]?.message ?? ""; + +describe("W-INVISIBLE-CHAR", () => { + it("flags a zero-width space in a card field and names the codepoint", () => { + const lesson = lessonWith({ + cards: [{ id: "c1", front: "cof​fee", back: "der Kaffee" }], + }); + expect(warningsFor(lesson)).toHaveLength(1); + expect(messageFor(lesson)).toContain("U+200B"); + expect(messageFor(lesson)).toContain("ZERO WIDTH SPACE"); + }); + + it("names the field path so the author can find it", () => { + const lesson = lessonWith({ + cards: [{ id: "c1", front: "cof​fee", back: "der Kaffee" }], + }); + expect(messageFor(lesson)).toContain("/cards/0/front"); + }); + + it("flags a byte-order mark", () => { + const lesson = lessonWith({ title: "Lesson" }); + expect(messageFor(lesson)).toContain("U+FEFF"); + expect(messageFor(lesson)).toContain("BYTE ORDER MARK"); + }); + + it("flags a left-to-right mark", () => { + const lesson = lessonWith({ title: "Lesson‎" }); + expect(messageFor(lesson)).toContain("U+200E"); + expect(messageFor(lesson)).toContain("LEFT-TO-RIGHT MARK"); + }); + + it("flags a right-to-left mark", () => { + const lesson = lessonWith({ title: "Lesson‏" }); + expect(messageFor(lesson)).toContain("U+200F"); + }); + + it("flags a soft hyphen, which a PDF copy routinely injects", () => { + const lesson = lessonWith({ title: "Kaf­fee" }); + expect(messageFor(lesson)).toContain("U+00AD"); + expect(messageFor(lesson)).toContain("SOFT HYPHEN"); + }); + + it("flags text inside an exercise prompt", () => { + const lesson = lessonWith({ + steps: [ + { + id: "s1", + type: "exercise", + exercise: { + id: "e1", + type: "free_text", + prompt: "Trans​late.", + accept: ["der Kaffee"], + }, + }, + ], + }); + expect(messageFor(lesson)).toContain("/steps/0/exercise/prompt"); + }); + + it("reaches into ext_payload without enumerating its fields", () => { + const lesson = lessonWith({ + requires_extensions: ["ext:al-dictation@1"], + steps: [ + { + id: "s1", + type: "exercise", + exercise: { + id: "e1", + type: "ext:al-dictation", + prompt: "Listen.", + ext_payload: { audio: "assets/a.mp3", accept: ["a cof​fee"] }, + }, + }, + ], + }); + const found = validateLesson(lesson, { + extensions: [{ type: "ext:al-dictation", major: 1, validate: () => [] }], + }).warnings.filter((issue) => issue.id === "W-INVISIBLE-CHAR"); + expect(found).toHaveLength(1); + expect(found[0]!.message).toContain("ext_payload"); + }); + + it("aggregates to ONE warning per lesson, not one per occurrence", () => { + // A pasted chapter can carry dozens; emitting one warning each is the + // alert fatigue W-CARD-UNUSED was aggregated to avoid (#49). + const lesson = lessonWith({ + title: "Les​son", + cards: [ + { id: "c1", front: "cof​fee", back: "der Kaf​fee" }, + { id: "c2", front: "milk​", back: "die Milch" }, + ], + }); + expect(warningsFor(lesson)).toHaveLength(1); + expect(messageFor(lesson)).toMatch(/4 occurrence|4 /); + }); + + it("reports every distinct codepoint it found, not just the first", () => { + const lesson = lessonWith({ title: "A​B­C" }); + expect(messageFor(lesson)).toContain("U+200B"); + expect(messageFor(lesson)).toContain("U+00AD"); + }); + + it("stays silent on clean content", () => { + expect(warningsFor(lessonWith({}))).toEqual([]); + }); + + it("stays silent on umlauts, accents and other legitimate non-ASCII text", () => { + const lesson = lessonWith({ + title: "Über die Prüfung", + cards: [{ id: "c1", front: "café", back: "das Café" }], + steps: [ + { + id: "s1", + type: "exercise", + exercise: { + id: "e1", + type: "free_text", + prompt: "Wie heißt „Kaffee“ auf Französisch?", + accept: ["café"], + }, + }, + ], + }); + expect(warningsFor(lesson)).toEqual([]); + }); + + it("boundary: does NOT flag a no-break space, which is legitimate French typography", () => { + // "Comment ça va ?" sets U+00A0 before the question mark by convention. + // Flagging it would reproduce the over-sensitive behaviour this lint + // exists to avoid, and this ecosystem ships French content. + const lesson = lessonWith({ title: "Comment ça va ?" }); + expect(warningsFor(lesson)).toEqual([]); + }); + + it("boundary: does NOT flag a narrow no-break space either", () => { + const lesson = lessonWith({ title: "1 000 Wörter" }); + expect(warningsFor(lesson)).toEqual([]); + }); + + it("never blocks: the lesson stays valid", () => { + const lesson = lessonWith({ title: "Les​son" }); + const result = validateLesson(lesson); + expect(result.valid).toBe(true); + expect(result.errors).toEqual([]); + }); +}); diff --git a/src/invisible-chars.ts b/src/invisible-chars.ts new file mode 100644 index 0000000..86c6475 --- /dev/null +++ b/src/invisible-chars.ts @@ -0,0 +1,113 @@ +/** + * Detection of invisible Unicode characters in lesson content (#75). + * + * Text pasted out of a PDF or a web page carries characters that cannot be + * seen: zero-width spaces, byte-order marks, directional marks, soft hyphens. + * They are legal JSON string content, so no structural check rejects them, and + * no one notices them by reading the file. The app's book-text wizard asks the + * author to paste a textbook section, which is precisely how they arrive. + * + * The rule is deliberately NARROW. Only characters that render as nothing at + * all are listed. A no-break space (``U+00A0``) and a narrow no-break space + * (``U+202F``) are excluded on purpose: they render as whitespace and are + * legitimate typography, notably in the French content this ecosystem carries + * ("Comment ca va ?" sets one before the question mark by convention). + * Flagging them would make the lint usually-wrong, and a lint that is usually + * wrong stops being read. + */ + +/** Invisible codepoints, with the Unicode name reported to the author. */ +const INVISIBLE_CHARACTERS = new Map([ + ["­", "SOFT HYPHEN"], + ["​", "ZERO WIDTH SPACE"], + ["‌", "ZERO WIDTH NON-JOINER"], + ["‍", "ZERO WIDTH JOINER"], + ["‎", "LEFT-TO-RIGHT MARK"], + ["‏", "RIGHT-TO-LEFT MARK"], + ["
", "LINE SEPARATOR"], + ["
", "PARAGRAPH SEPARATOR"], + ["‪", "LEFT-TO-RIGHT EMBEDDING"], + ["‫", "RIGHT-TO-LEFT EMBEDDING"], + ["‬", "POP DIRECTIONAL FORMATTING"], + ["‭", "LEFT-TO-RIGHT OVERRIDE"], + ["‮", "RIGHT-TO-LEFT OVERRIDE"], + ["⁠", "WORD JOINER"], + ["", "BYTE ORDER MARK"], +]); + +/** One invisible character found at one place in the lesson. */ +export interface InvisibleCharFinding { + /** JSON-pointer-ish location, e.g. ``/cards/0/front``. */ + path: string; + /** The offending character itself. */ + character: string; + /** ``U+200B`` style label. */ + codepoint: string; + /** The Unicode name, e.g. ``ZERO WIDTH SPACE``. */ + name: string; +} + +/** ``U+200B``-style label for a single character. */ +function toCodepointLabel(character: string): string { + return `U+${character.codePointAt(0)!.toString(16).toUpperCase().padStart(4, "0")}`; +} + +/** Collect every invisible character in one string value. */ +function findingsInString(value: string, path: string): InvisibleCharFinding[] { + const findings: InvisibleCharFinding[] = []; + for (const character of value) { + const name = INVISIBLE_CHARACTERS.get(character); + if (name) { + findings.push({ path, character, codepoint: toCodepointLabel(character), name }); + } + } + return findings; +} + +/** + * Walk every string in an arbitrary value, reporting invisible characters with + * the path they sit at. Walking the whole object rather than a field list is + * what lets this cover ``ext_payload`` (whose shape the engine does not know) + * and any field a later schema version adds. + */ +export function findInvisibleChars(value: unknown, path = ""): InvisibleCharFinding[] { + if (typeof value === "string") return findingsInString(value, path); + if (Array.isArray(value)) { + return value.flatMap((entry, index) => findInvisibleChars(entry, `${path}/${index}`)); + } + if (typeof value === "object" && value !== null) { + return Object.entries(value).flatMap(([key, entry]) => + findInvisibleChars(entry, `${path}/${key}`), + ); + } + return []; +} + +/** How many distinct paths to name before trailing off. Enough to start + * fixing, short enough that a heavily-affected pasted chapter stays readable. */ +const MAX_PATHS_LISTED = 5; + +/** + * One human-readable sentence describing every finding, or null when the + * content is clean. Aggregated per lesson rather than emitted per occurrence: + * a pasted chapter can carry dozens, and per-occurrence emission is the alert + * fatigue W-CARD-UNUSED was aggregated away from (#49). + */ +export function describeInvisibleChars(findings: readonly InvisibleCharFinding[]): string | null { + if (findings.length === 0) return null; + const byCodepoint = new Map(); + for (const finding of findings) byCodepoint.set(finding.codepoint, finding.name); + const kinds = [...byCodepoint] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([codepoint, name]) => `${codepoint} ${name}`) + .join(", "); + const paths = [...new Set(findings.map((finding) => finding.path))]; + const shown = paths.slice(0, MAX_PATHS_LISTED).join(", "); + const rest = paths.length > MAX_PATHS_LISTED ? `, and ${paths.length - MAX_PATHS_LISTED} more` : ""; + const plural = findings.length === 1 ? "occurrence" : "occurrences"; + return ( + `lesson text contains invisible characters (${kinds}): ` + + `${findings.length} ${plural} at ${shown}${rest}. ` + + "These usually arrive by pasting from a PDF or a web page; they are invisible when reading the file." + ); +} diff --git a/src/validate.ts b/src/validate.ts index 8d536bd..dae5ed3 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -29,6 +29,7 @@ import { Ajv2020 } from "ajv/dist/2020.js"; import type { ErrorObject, ValidateFunction } from "ajv"; import type { ExtensionRegistry } from "./extensions.js"; +import { describeInvisibleChars, findInvisibleChars } from "./invisible-chars.js"; import type { Exercise, Lesson, LessonStep } from "./types/lesson-schema.generated.js"; /** Whether an issue blocks (``error``) or merely advises (``warning``). */ @@ -451,6 +452,17 @@ function checkUnusedCards(lesson: Lesson, issues: ValidationIssue[]): void { ); } +/** Warn about invisible Unicode characters anywhere in the lesson's text + * (#75). Aggregated to ONE warning per lesson listing every distinct + * codepoint and where it sits: pasted text can carry dozens, and a warning + * per occurrence is the alert fatigue W-CARD-UNUSED was aggregated away from + * (#49). Never an error - the content is structurally valid, it just carries + * characters the author cannot see. */ +function checkInvisibleChars(lesson: Lesson, issues: ValidationIssue[]): void { + const description = describeInvisibleChars(findInvisibleChars(lesson)); + if (description) issues.push(warn("W-INVISIBLE-CHAR", "", description, "author-lints")); +} + /** Semantic + lint pass. Assumes the input is already structurally valid (so the * schema-typed shape is trustworthy). Returns a mixed error/warning list. */ function semanticIssues(lesson: Lesson, registry: ExtensionRegistry): ValidationIssue[] { @@ -461,6 +473,7 @@ function semanticIssues(lesson: Lesson, registry: ExtensionRegistry): Validation checkStep(step, `/steps/${index}`, knownCardIds, ext, issues); }); checkUnusedCards(lesson, issues); + checkInvisibleChars(lesson, issues); return issues; }