feat(validate): W-INVISIBLE-CHAR author lint for invisible characters (Closes #75)

CHANGELOG.md

@@ -5,6 +5,29 @@ All notable changes to `learn-content-engine`. The format is inspired by
 [SemVer](https://semver.org/) (schema evolution is additive, see
 [docs/concepts.md](docs/concepts.md#schema-version-policy-additive)).
 
+## [0.13.2] - 2026-07-22
+
+New author lint `W-INVISIBLE-CHAR` (#75): warns when a lesson's text carries
+characters that render as nothing - zero-width spaces, byte-order marks,
+directional marks, soft hyphens. They are legal JSON and survive every
+structural check, and no one spots them by reading the file; they arrive by
+pasting from a PDF or a web page, which is exactly what the reference app's
+book-text wizard asks the author to do. The warning names each codepoint
+(`U+200B ZERO WIDTH SPACE`), its Unicode name, the occurrence count and the
+paths, aggregated once per lesson (the `W-CARD-UNUSED` precedent from #49).
+Every string is walked, including `ext_payload`, so extension text is covered
+without the engine knowing its shape.
+
+Deliberately NOT flagged: `U+00A0` NO-BREAK SPACE and `U+202F` NARROW NO-BREAK
+SPACE. Both render as whitespace and are legitimate typography, notably in the
+French content this ecosystem carries. Measured on 528 real lessons across
+eight content repositories: 4 findings, all genuine soft hyphens sitting
+mid-word in pasted prose, 0 false positives.
+
+Warning tier, never blocks. Additive: no schema change, no `schema_version`
+bump. Content repos pick it up through their existing `make lint` /
+`make lint-warnings` without changing anything.
+
 ## [0.13.1] - 2026-07-20
 
 Docs/examples: sixth reference extension `ext:ref-dictation` (#68) - an audio

docs/lesson-format.md

@@ -565,6 +565,7 @@ drifting.
 | `W-PIC-DUP-LABEL` | A `picture_choice` distractor shares its `label` with the correct image. |
 | `W-PIC-DATA-URI` | A `picture_choice` image `src` is an inline `data:` URI (schema v1.8 allows it for consumer-local content, e.g. uploaded images). Repo content should prefer a relative `assets/` path - inline data URIs bloat the lesson JSON and the git history. Advisory only, never blocks. |
 | `W-HINT-LENGTH` | A hint reveals the answer length (e.g. "four letters"). Consumers that display an answer-length indicator make such a hint redundant; on other consumers it gives part of the answer away. |
+| `W-INVISIBLE-CHAR` | The lesson's text carries characters that render as nothing: zero-width spaces, byte-order marks, directional marks, soft hyphens. They are legal JSON and survive every structural check, and no one spots them by reading the file; they usually arrive by pasting from a PDF or a web page. The warning names each codepoint (`U+200B ZERO WIDTH SPACE`), its Unicode name, and where it sits, aggregated once per lesson. Every string is scanned, including `ext_payload`, so extension text is covered without the engine knowing its shape. Deliberately NOT flagged: `U+00A0` NO-BREAK SPACE and `U+202F` NARROW NO-BREAK SPACE, which render as whitespace and are legitimate typography (French sets one before `?` and `!`). |
 
 ## Linting
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "learn-content-engine",
-  "version": "0.13.1",
+  "version": "0.13.2",
   "description": "Framework-agnostic TypeScript engine that parses lesson content from pluggable sources into a canonical lesson object.",
   "type": "module",
   "license": "MIT",

src/invisible-chars.test.ts

@@ -0,0 +1,182 @@
+import { describe, it, expect } from "vitest";
+
+import { validateLesson } from "./validate.js";
+
+/**
+ * ``W-INVISIBLE-CHAR`` (#75): text pasted from a PDF or a web page carries
+ * characters that are invisible by definition (zero-width spaces, BOMs,
+ * directional marks, soft hyphens). They are legal JSON string content, so no
+ * structural check catches them, and no one spots them by reading the file.
+ * The app's book-text wizard invites exactly that paste, so this is a real
+ * content path rather than a theoretical one.
+ *
+ * The bar these tests set is TWO-sided: every invisible codepoint must be
+ * named, and clean content, INCLUDING legitimate typography that merely looks
+ * exotic, must stay silent. An over-sensitive lint is worse than none, because
+ * warnings that are usually wrong stop being read.
+ */
+
+const lessonWith = (overrides: Record<string, unknown>) => ({
+  id: "l1",
+  title: "Lesson",
+  cards: [{ id: "c1", front: "coffee", back: "der Kaffee" }],
+  steps: [
+    {
+      id: "s1",
+      type: "exercise",
+      exercise: { id: "e1", type: "free_text", prompt: "Translate.", accept: ["der Kaffee"] },
+    },
+  ],
+  ...overrides,
+});
+
+const warningsFor = (lesson: unknown) =>
+  validateLesson(lesson).warnings.filter((issue) => issue.id === "W-INVISIBLE-CHAR");
+
+const messageFor = (lesson: unknown) => warningsFor(lesson)[0]?.message ?? "";
+
+describe("W-INVISIBLE-CHAR", () => {
+  it("flags a zero-width space in a card field and names the codepoint", () => {
+    const lesson = lessonWith({
+      cards: [{ id: "c1", front: "cof​fee", back: "der Kaffee" }],
+    });
+    expect(warningsFor(lesson)).toHaveLength(1);
+    expect(messageFor(lesson)).toContain("U+200B");
+    expect(messageFor(lesson)).toContain("ZERO WIDTH SPACE");
+  });
+
+  it("names the field path so the author can find it", () => {
+    const lesson = lessonWith({
+      cards: [{ id: "c1", front: "cof​fee", back: "der Kaffee" }],
+    });
+    expect(messageFor(lesson)).toContain("/cards/0/front");
+  });
+
+  it("flags a byte-order mark", () => {
+    const lesson = lessonWith({ title: "Lesson" });
+    expect(messageFor(lesson)).toContain("U+FEFF");
+    expect(messageFor(lesson)).toContain("BYTE ORDER MARK");
+  });
+
+  it("flags a left-to-right mark", () => {
+    const lesson = lessonWith({ title: "Lesson‎" });
+    expect(messageFor(lesson)).toContain("U+200E");
+    expect(messageFor(lesson)).toContain("LEFT-TO-RIGHT MARK");
+  });
+
+  it("flags a right-to-left mark", () => {
+    const lesson = lessonWith({ title: "Lesson‏" });
+    expect(messageFor(lesson)).toContain("U+200F");
+  });
+
+  it("flags a soft hyphen, which a PDF copy routinely injects", () => {
+    const lesson = lessonWith({ title: "Kaf­fee" });
+    expect(messageFor(lesson)).toContain("U+00AD");
+    expect(messageFor(lesson)).toContain("SOFT HYPHEN");
+  });
+
+  it("flags text inside an exercise prompt", () => {
+    const lesson = lessonWith({
+      steps: [
+        {
+          id: "s1",
+          type: "exercise",
+          exercise: {
+            id: "e1",
+            type: "free_text",
+            prompt: "Trans​late.",
+            accept: ["der Kaffee"],
+          },
+        },
+      ],
+    });
+    expect(messageFor(lesson)).toContain("/steps/0/exercise/prompt");
+  });
+
+  it("reaches into ext_payload without enumerating its fields", () => {
+    const lesson = lessonWith({
+      requires_extensions: ["ext:al-dictation@1"],
+      steps: [
+        {
+          id: "s1",
+          type: "exercise",
+          exercise: {
+            id: "e1",
+            type: "ext:al-dictation",
+            prompt: "Listen.",
+            ext_payload: { audio: "assets/a.mp3", accept: ["a cof​fee"] },
+          },
+        },
+      ],
+    });
+    const found = validateLesson(lesson, {
+      extensions: [{ type: "ext:al-dictation", major: 1, validate: () => [] }],
+    }).warnings.filter((issue) => issue.id === "W-INVISIBLE-CHAR");
+    expect(found).toHaveLength(1);
+    expect(found[0]!.message).toContain("ext_payload");
+  });
+
+  it("aggregates to ONE warning per lesson, not one per occurrence", () => {
+    // A pasted chapter can carry dozens; emitting one warning each is the
+    // alert fatigue W-CARD-UNUSED was aggregated to avoid (#49).
+    const lesson = lessonWith({
+      title: "Les​son",
+      cards: [
+        { id: "c1", front: "cof​fee", back: "der Kaf​fee" },
+        { id: "c2", front: "milk​", back: "die Milch" },
+      ],
+    });
+    expect(warningsFor(lesson)).toHaveLength(1);
+    expect(messageFor(lesson)).toMatch(/4 occurrence|4 /);
+  });
+
+  it("reports every distinct codepoint it found, not just the first", () => {
+    const lesson = lessonWith({ title: "A​B­C" });
+    expect(messageFor(lesson)).toContain("U+200B");
+    expect(messageFor(lesson)).toContain("U+00AD");
+  });
+
+  it("stays silent on clean content", () => {
+    expect(warningsFor(lessonWith({}))).toEqual([]);
+  });
+
+  it("stays silent on umlauts, accents and other legitimate non-ASCII text", () => {
+    const lesson = lessonWith({
+      title: "Über die Prüfung",
+      cards: [{ id: "c1", front: "café", back: "das Café" }],
+      steps: [
+        {
+          id: "s1",
+          type: "exercise",
+          exercise: {
+            id: "e1",
+            type: "free_text",
+            prompt: "Wie heißt „Kaffee“ auf Französisch?",
+            accept: ["café"],
+          },
+        },
+      ],
+    });
+    expect(warningsFor(lesson)).toEqual([]);
+  });
+
+  it("boundary: does NOT flag a no-break space, which is legitimate French typography", () => {
+    // "Comment ça va ?" sets U+00A0 before the question mark by convention.
+    // Flagging it would reproduce the over-sensitive behaviour this lint
+    // exists to avoid, and this ecosystem ships French content.
+    const lesson = lessonWith({ title: "Comment ça va ?" });
+    expect(warningsFor(lesson)).toEqual([]);
+  });
+
+  it("boundary: does NOT flag a narrow no-break space either", () => {
+    const lesson = lessonWith({ title: "1 000 Wörter" });
+    expect(warningsFor(lesson)).toEqual([]);
+  });
+
+  it("never blocks: the lesson stays valid", () => {
+    const lesson = lessonWith({ title: "Les​son" });
+    const result = validateLesson(lesson);
+    expect(result.valid).toBe(true);
+    expect(result.errors).toEqual([]);
+  });
+});

src/invisible-chars.ts

@@ -0,0 +1,113 @@
+/**
+ * Detection of invisible Unicode characters in lesson content (#75).
+ *
+ * Text pasted out of a PDF or a web page carries characters that cannot be
+ * seen: zero-width spaces, byte-order marks, directional marks, soft hyphens.
+ * They are legal JSON string content, so no structural check rejects them, and
+ * no one notices them by reading the file. The app's book-text wizard asks the
+ * author to paste a textbook section, which is precisely how they arrive.
+ *
+ * The rule is deliberately NARROW. Only characters that render as nothing at
+ * all are listed. A no-break space (``U+00A0``) and a narrow no-break space
+ * (``U+202F``) are excluded on purpose: they render as whitespace and are
+ * legitimate typography, notably in the French content this ecosystem carries
+ * ("Comment ca va ?" sets one before the question mark by convention).
+ * Flagging them would make the lint usually-wrong, and a lint that is usually
+ * wrong stops being read.
+ */
+
+/** Invisible codepoints, with the Unicode name reported to the author. */
+const INVISIBLE_CHARACTERS = new Map<string, string>([
+  ["­", "SOFT HYPHEN"],
+  ["​", "ZERO WIDTH SPACE"],
+  ["‌", "ZERO WIDTH NON-JOINER"],
+  ["‍", "ZERO WIDTH JOINER"],
+  ["‎", "LEFT-TO-RIGHT MARK"],
+  ["‏", "RIGHT-TO-LEFT MARK"],
+  ["
", "LINE SEPARATOR"],
+  ["
", "PARAGRAPH SEPARATOR"],
+  ["‪", "LEFT-TO-RIGHT EMBEDDING"],
+  ["‫", "RIGHT-TO-LEFT EMBEDDING"],
+  ["‬", "POP DIRECTIONAL FORMATTING"],
+  ["‭", "LEFT-TO-RIGHT OVERRIDE"],
+  ["‮", "RIGHT-TO-LEFT OVERRIDE"],
+  ["⁠", "WORD JOINER"],
+  ["", "BYTE ORDER MARK"],
+]);
+
+/** One invisible character found at one place in the lesson. */
+export interface InvisibleCharFinding {
+  /** JSON-pointer-ish location, e.g. ``/cards/0/front``. */
+  path: string;
+  /** The offending character itself. */
+  character: string;
+  /** ``U+200B`` style label. */
+  codepoint: string;
+  /** The Unicode name, e.g. ``ZERO WIDTH SPACE``. */
+  name: string;
+}
+
+/** ``U+200B``-style label for a single character. */
+function toCodepointLabel(character: string): string {
+  return `U+${character.codePointAt(0)!.toString(16).toUpperCase().padStart(4, "0")}`;
+}
+
+/** Collect every invisible character in one string value. */
+function findingsInString(value: string, path: string): InvisibleCharFinding[] {
+  const findings: InvisibleCharFinding[] = [];
+  for (const character of value) {
+    const name = INVISIBLE_CHARACTERS.get(character);
+    if (name) {
+      findings.push({ path, character, codepoint: toCodepointLabel(character), name });
+    }
+  }
+  return findings;
+}
+
+/**
+ * Walk every string in an arbitrary value, reporting invisible characters with
+ * the path they sit at. Walking the whole object rather than a field list is
+ * what lets this cover ``ext_payload`` (whose shape the engine does not know)
+ * and any field a later schema version adds.
+ */
+export function findInvisibleChars(value: unknown, path = ""): InvisibleCharFinding[] {
+  if (typeof value === "string") return findingsInString(value, path);
+  if (Array.isArray(value)) {
+    return value.flatMap((entry, index) => findInvisibleChars(entry, `${path}/${index}`));
+  }
+  if (typeof value === "object" && value !== null) {
+    return Object.entries(value).flatMap(([key, entry]) =>
+      findInvisibleChars(entry, `${path}/${key}`),
+    );
+  }
+  return [];
+}
+
+/** How many distinct paths to name before trailing off. Enough to start
+ *  fixing, short enough that a heavily-affected pasted chapter stays readable. */
+const MAX_PATHS_LISTED = 5;
+
+/**
+ * One human-readable sentence describing every finding, or null when the
+ * content is clean. Aggregated per lesson rather than emitted per occurrence:
+ * a pasted chapter can carry dozens, and per-occurrence emission is the alert
+ * fatigue W-CARD-UNUSED was aggregated away from (#49).
+ */
+export function describeInvisibleChars(findings: readonly InvisibleCharFinding[]): string | null {
+  if (findings.length === 0) return null;
+  const byCodepoint = new Map<string, string>();
+  for (const finding of findings) byCodepoint.set(finding.codepoint, finding.name);
+  const kinds = [...byCodepoint]
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([codepoint, name]) => `${codepoint} ${name}`)
+    .join(", ");
+  const paths = [...new Set(findings.map((finding) => finding.path))];
+  const shown = paths.slice(0, MAX_PATHS_LISTED).join(", ");
+  const rest = paths.length > MAX_PATHS_LISTED ? `, and ${paths.length - MAX_PATHS_LISTED} more` : "";
+  const plural = findings.length === 1 ? "occurrence" : "occurrences";
+  return (
+    `lesson text contains invisible characters (${kinds}): ` +
+    `${findings.length} ${plural} at ${shown}${rest}. ` +
+    "These usually arrive by pasting from a PDF or a web page; they are invisible when reading the file."
+  );
+}

src/validate.ts

@@ -29,6 +29,7 @@ import { Ajv2020 } from "ajv/dist/2020.js";
 import type { ErrorObject, ValidateFunction } from "ajv";
 
 import type { ExtensionRegistry } from "./extensions.js";
+import { describeInvisibleChars, findInvisibleChars } from "./invisible-chars.js";
 import type { Exercise, Lesson, LessonStep } from "./types/lesson-schema.generated.js";
 
 /** Whether an issue blocks (``error``) or merely advises (``warning``). */
@@ -451,6 +452,17 @@ function checkUnusedCards(lesson: Lesson, issues: ValidationIssue[]): void {
   );
 }
 
+/** Warn about invisible Unicode characters anywhere in the lesson's text
+ *  (#75). Aggregated to ONE warning per lesson listing every distinct
+ *  codepoint and where it sits: pasted text can carry dozens, and a warning
+ *  per occurrence is the alert fatigue W-CARD-UNUSED was aggregated away from
+ *  (#49). Never an error - the content is structurally valid, it just carries
+ *  characters the author cannot see. */
+function checkInvisibleChars(lesson: Lesson, issues: ValidationIssue[]): void {
+  const description = describeInvisibleChars(findInvisibleChars(lesson));
+  if (description) issues.push(warn("W-INVISIBLE-CHAR", "", description, "author-lints"));
+}
+
 /** Semantic + lint pass. Assumes the input is already structurally valid (so the
  *  schema-typed shape is trustworthy). Returns a mixed error/warning list. */
 function semanticIssues(lesson: Lesson, registry: ExtensionRegistry): ValidationIssue[] {
@@ -461,6 +473,7 @@ function semanticIssues(lesson: Lesson, registry: ExtensionRegistry): Validation
     checkStep(step, `/steps/${index}`, knownCardIds, ext, issues);
   });
   checkUnusedCards(lesson, issues);
+  checkInvisibleChars(lesson, issues);
   return issues;
 }