Improve Word parsing and harden NAS publishing

.gitignore

@@ -14,3 +14,9 @@ Files/*.pdf
 
 # Test artifacts and generated data
 output/
+
+# Fetched pdfium native binaries (downloaded by `npm run fetch:pdfium`,
+# shipped as Tauri resources at build time — not committed to the repo)
+src-tauri/lib/pdfium-windows/
+src-tauri/lib/pdfium-macos/
+src-tauri/lib/pdfium-linux/

package.json

@@ -3,15 +3,19 @@
   "version": "0.1.0",
   "private": true,
   "type": "module",
-  "description": "Epic 8 Tauri authoring studio for IELTS reading import, review, preview, and pack export.",
+  "description": "Epic 8 Tauri authoring studio for IELTS reading import, review, preview, and NAS publishing.",
   "scripts": {
     "dev": "vite",
     "build": "tsc --noEmit && vite build",
     "preview": "vite preview",
     "tauri": "tauri",
     "check": "tsc --noEmit",
+    "clean": "node scripts/clean-generated.mjs",
+    "clean:deep": "node scripts/clean-generated.mjs --deep",
+    "fetch:pdfium": "node scripts/fetch-pdfium.mjs",
     "e2e:ui-flow": "node sidecars/ui-flow-e2e/ui-flow-e2e.mjs",
     "test:pdf-regression": "node scripts/pdf-regression-sample.mjs",
+    "test:pdf-regression:smoke": "node scripts/pdf-regression-sample.mjs --smoke",
     "test:pdf-regression:sample30": "node scripts/pdf-regression-sample.mjs --sample 30",
     "test:live-pdf-regression": "node scripts/live-pdf-pipeline-regression.mjs",
     "audit:package": "node scripts/package-audit.mjs --platform macos",
@@ -20,9 +24,9 @@
     "audit:windows:signatures": "pwsh -NoProfile -ExecutionPolicy Bypass -File scripts/audit-windows-signatures.ps1",
     "repack:macos-dmg": "node scripts/repack-macos-dmg.mjs",
     "verify:release": "npm run verify:release:macos",
-    "verify:release:macos": "npm run tauri -- build --bundles app,dmg && npm run repack:macos-dmg && npm run audit:package:macos",
-    "verify:release:windows": "npm run tauri -- build --bundles nsis && npm run audit:package:windows",
-    "verify:release:windows:offline": "npm run tauri -- build --bundles nsis --config src-tauri/tauri.windows.offline.conf.json && npm run audit:package:windows -- --config src-tauri/tauri.windows.offline.conf.json"
+    "verify:release:macos": "npm run fetch:pdfium && npm run tauri -- build --bundles app,dmg --config src-tauri/tauri.macos.conf.json && npm run repack:macos-dmg && npm run audit:package:macos",
+    "verify:release:windows": "npm run fetch:pdfium && npm run tauri -- build --bundles nsis --config src-tauri/tauri.windows.conf.json && npm run audit:package:windows -- src-tauri/tauri.windows.conf.json",
+    "verify:release:windows:offline": "npm run fetch:pdfium && npm run tauri -- build --bundles nsis --config src-tauri/tauri.windows.offline.conf.json && npm run audit:package:windows -- src-tauri/tauri.windows.offline.conf.json"
   },
   "dependencies": {
     "@tauri-apps/api": "^2.5.0",

scripts/clean-generated.mjs

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+/**
+ * Removes generated build outputs, temp directories, and local logs that can
+ * be recreated from source. Use --deep to also remove node_modules.
+ *
+ * Usage:
+ *   node scripts/clean-generated.mjs
+ *   node scripts/clean-generated.mjs --deep
+ *   node scripts/clean-generated.mjs --dry-run
+ */
+import { existsSync, lstatSync, readdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+
+const DEEP = process.argv.includes("--deep");
+const DRY_RUN = process.argv.includes("--dry-run");
+
+const targets = [
+  "dist",
+  "output",
+  "tmp",
+  ".playwright-mcp",
+  "output-tauri-dev.log",
+  "output-tauri-dev.err.log",
+  join("src-tauri", "target"),
+];
+
+if (DEEP) {
+  targets.push("node_modules");
+}
+
+function sizeOf(path) {
+  const stat = lstatSync(path);
+  if (!stat.isDirectory()) {
+    return stat.size;
+  }
+
+  let total = 0;
+  for (const entry of readdirSync(path, { withFileTypes: true })) {
+    total += sizeOf(join(path, entry.name));
+  }
+  return total;
+}
+
+function formatBytes(bytes) {
+  if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(2)} GB`;
+  if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(2)} MB`;
+  if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(2)} KB`;
+  return `${bytes} B`;
+}
+
+let reclaimed = 0;
+let removed = 0;
+
+for (const target of targets) {
+  if (!existsSync(target)) {
+    console.log(`[clean] skip ${target} (not found)`);
+    continue;
+  }
+
+  const bytes = sizeOf(target);
+  reclaimed += bytes;
+  removed += 1;
+  console.log(`[clean] ${DRY_RUN ? "would remove" : "remove"} ${target} (${formatBytes(bytes)})`);
+
+  if (!DRY_RUN) {
+    rmSync(target, { recursive: true, force: true });
+  }
+}
+
+console.log(
+  `[clean] ${DRY_RUN ? "would reclaim" : "reclaimed"} ${formatBytes(reclaimed)} across ${removed} path(s).`,
+);

scripts/fetch-pdfium.mjs

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+/**
+ * Fetches the pdfium native library for the current platform and extracts it
+ * into src-tauri/lib/pdfium-<platform>/ so it is bundled as a Tauri resource.
+ *
+ * The pdfium-render Rust crate binds to this library at runtime (see
+ * src-tauri/src/pdf_geometry.rs -> pdfium_library_path), giving the app a
+ * real-coordinate PDF backend that works on machines with NO Python.
+ *
+ * Sources binaries from bblanchon/pdfium-binaries (official chromium builds).
+ * Idempotent: skips download when the library is already present and the
+ * stamped version matches.
+ *
+ * Usage: node scripts/fetch-pdfium.mjs [--force] [--platform <win|mac|linux>]
+ */
+import { createWriteStream, existsSync, readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+import https from "node:https";
+
+// Pin a known-good chromium release. Update deliberately — each bump should
+// be validated against the sample PDFs in fixtures/parser.
+const PDFIUM_RELEASE = "chromium/7920";
+
+const PLATFORM_OVERRIDE = process.argv.includes("--platform")
+  ? process.argv[process.argv.indexOf("--platform") + 1]
+  : null;
+const FORCE = process.argv.includes("--force");
+
+function detectPlatform() {
+  if (PLATFORM_OVERRIDE) return PLATFORM_OVERRIDE;
+  if (process.platform === "win32") return "win";
+  if (process.platform === "darwin") return "mac";
+  return "linux";
+}
+
+function platformAsset(platform) {
+  // bblanchon/pdfium-binaries asset names (x64). arm64 variants exist but are
+  // not fetched here; add them if you ship arm64 builds.
+  return `pdfium-${platform}-x64.tgz`;
+}
+
+function libraryFileName(platform) {
+  if (platform === "win") return "pdfium.dll";
+  if (platform === "mac") return "libpdfium.dylib";
+  return "libpdfium.so";
+}
+
+function platformFolder(platform) {
+  // MUST match the folder names checked by pdf_geometry.rs::platform_pdfium_folder.
+  return `pdfium-${platform === "win" ? "windows" : platform === "mac" ? "macos" : "linux"}`;
+}
+
+function download(url, dest) {
+  return new Promise((resolve, reject) => {
+    const file = createWriteStream(dest);
+    const req = https.get(url, (response) => {
+      // Follow redirects (GitHub releases 302 to a CDN).
+      if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
+        file.close();
+        return resolve(download(response.headers.location, dest));
+      }
+      if (response.statusCode !== 200) {
+        file.close();
+        return reject(new Error(`download failed: HTTP ${response.statusCode} for ${url}`));
+      }
+      response.pipe(file);
+      file.on("finish", () => file.close(() => resolve(dest)));
+    });
+    req.on("error", (error) => {
+      file.close();
+      reject(error);
+    });
+  });
+}
+
+function extractTgz(tgzPath, destDir) {
+  mkdirSync(destDir, { recursive: true });
+  // Use the system tar (available in git-bash on Windows, and on mac/linux).
+  const result = spawnSync("tar", ["-xzf", tgzPath, "-C", destDir], {
+    stdio: "inherit",
+  });
+  if (result.status !== 0) {
+    throw new Error(`tar extraction failed (exit ${result.status}). Ensure 'tar' is on PATH.`);
+  }
+}
+
+async function main() {
+  const platform = detectPlatform();
+  const asset = platformAsset(platform);
+  const url = `https://github.com/bblanchon/pdfium-binaries/releases/download/${PDFIUM_RELEASE}/${asset}`;
+  const libDir = join("src-tauri", "lib", platformFolder(platform));
+  const libFile = join(libDir, libraryFileName(platform));
+  const versionStamp = join(libDir, ".version");
+
+  if (existsSync(libFile) && existsSync(versionStamp) && !FORCE) {
+    const stamped = readFileSync(versionStamp, "utf8").trim();
+    if (stamped === PDFIUM_RELEASE) {
+      console.log(`[fetch-pdfium] ${platform}: already present (${PDFIUM_RELEASE}), skipping.`);
+      return;
+    }
+  }
+
+  mkdirSync(libDir, { recursive: true });
+  // Keep the temp tgz under the project (relative path) so Windows `tar`
+  // doesn't misparse a `C:\Users\...` tmpdir as a remote host.
+  const tgzPath = join(libDir, asset);
+  console.log(`[fetch-pdfium] ${platform}: downloading ${url}`);
+  await download(url, tgzPath);
+  console.log(`[fetch-pdfium] ${platform}: extracting to ${libDir}`);
+  // Clean stale extractions first.
+  rmSync(join(libDir, "bin"), { recursive: true, force: true });
+  rmSync(join(libDir, "lib"), { recursive: true, force: true });
+  extractTgz(tgzPath, libDir);
+
+  // The tgz lays out bin/pdfium.dll (win) or lib/libpdfium.dylib/so (mac/linux).
+  // Flatten so the library sits directly under src-tauri/lib/pdfium-<platform>/,
+  // which is where pdf_geometry.rs and the tauri resource config look for it.
+  const nested = {
+    win: join(libDir, "bin", "pdfium.dll"),
+    mac: join(libDir, "lib", "libpdfium.dylib"),
+    linux: join(libDir, "lib", "libpdfium.so"),
+  }[platform];
+  if (existsSync(nested) && nested !== libFile) {
+    rmSync(libFile, { force: true });
+    // Rename across dirs.
+    const content = readFileSync(nested);
+    writeFileSync(libFile, content);
+    rmSync(join(libDir, "bin"), { recursive: true, force: true });
+    rmSync(join(libDir, "lib"), { recursive: true, force: true });
+  }
+
+  if (!existsSync(libFile)) {
+    throw new Error(`expected library not found at ${libFile} after extraction`);
+  }
+  rmSync(tgzPath, { force: true });
+  writeFileSync(versionStamp, PDFIUM_RELEASE);
+  const stats = readFileSync(libFile);
+  console.log(
+    `[fetch-pdfium] ${platform}: installed ${libFile} (${(stats.length / 1024 / 1024).toFixed(1)} MB), version ${PDFIUM_RELEASE}.`
+  );
+}
+
+main().catch((error) => {
+  console.error(`[fetch-pdfium] FAILED: ${error.message}`);
+  process.exit(1);
+});

scripts/generate-docx-corpus.md

@@ -0,0 +1,97 @@
+# PDF-derived Word corpus
+
+`generate-docx-corpus.mjs` converts a directory of PDF IELTS samples into a
+temporary DOCX regression corpus. It uses the application's existing
+`--generate-reading-source` CLI to obtain `DocumentIR`, then rebuilds that
+evidence as native Word paragraphs and tables.
+
+The generated documents preserve:
+
+- `Questions 1-13` range headings;
+- explicit question-number lines;
+- IELTS instructions preceding each question group;
+- consecutive `A-D` and `i-xii` marker runs (two-column Word tables by default);
+- tables already represented in `DocumentIR`;
+- source page boundaries, heading styles, and paragraph order.
+
+Use `--numbering-mode word` to create a native Word-numbering variant. In that
+mode leading question numbers use decimal numbering with the detected start
+value, while structured `A-D` and `i-xii` marker tables use `upperLetter` and
+`lowerRoman` numbering definitions. Consecutive table markers share a Word
+numbering instance; sparse markers such as `A/E/F/G` receive per-row start
+overrides so Word cannot silently renumber them as `A/B/C/D`. The default
+`explicit` mode keeps markers as literal text. The numbering modes use
+different output filenames and can coexist. `--option-layout paragraph` also
+adds a filename suffix, so table and paragraph variants cannot silently reuse
+one another's files.
+
+The output directory contains one `.docx` per source PDF, a latest-run
+`manifest.json`, and a variant manifest such as
+`manifest-word-numbered-table.json`. The manifest maps source PDF paths to DOCX
+paths and records hashes, parser warnings, structure counts,
+omitted-image/manual-review flags, and optional round-trip verification. It
+also stores the normalized AuthoringIR signature for both source and round-trip
+data, including ranges, kinds, instructions, question IDs, prompts,
+interaction types, option labels, `optionTexts`, and option-reuse policy.
+Verification compares all of those semantic fields. A source with zero groups
+only passes when the DOCX conversion also has zero groups.
+
+Existing DOCX files are not trusted only by name. On a repeat run the generator
+rebuilds the deterministic bytes and requires the existing hash to match; it
+then still records source semantics and performs `--verify` when requested. A
+stale or differently configured file fails with a message to rerun using
+`--overwrite`.
+
+Filenames containing markers such as `仅原文无题`, `passage-only`, or
+`no questions` are treated as negative samples with an expected question-group
+count of zero. `--expect-zero <text>` adds another filename substring for a
+corpus-specific negative sample. Generated files for those samples include a
+`passage-only` filename suffix, so the explicit zero-group expectation survives
+the PDF-to-DOCX filename normalization and is honored again during round-trip
+parsing. Unmarked umbrella-only documents still retain the manual question
+import scaffold because they may represent extraction loss rather than a true
+question-free source.
+
+## Small verified sample
+
+```powershell
+node scripts/generate-docx-corpus.mjs `
+  --pdf-dir "E:\tmp\PDF" `
+  --out-dir "$env:TEMP\pdf2test-word-corpus" `
+  --sample 3 `
+  --seed word-smoke `
+  --verify `
+  --numbering-mode explicit `
+  --skip-build `
+  --overwrite
+```
+
+## Full corpus
+
+```powershell
+node scripts/generate-docx-corpus.mjs `
+  --pdf-dir "E:\tmp\PDF" `
+  --out-dir "$env:TEMP\pdf2test-word-corpus" `
+  --verify `
+  --skip-build
+```
+
+Use `--option-layout paragraph` to keep every marker line as a paragraph
+instead of converting consecutive marker runs to tables. Run without
+`--skip-build` when the Rust CLI needs rebuilding. `--strict` makes generation
+or round-trip verification failures return a non-zero exit code.
+
+Run `node scripts/generate-docx-corpus.mjs --self-test` for deterministic
+native-numbering model checks without reading the PDF corpus or building the
+Rust CLI.
+
+`--sample` is seeded random sampling over a stable path order. It is
+reproducible, but it is not stratified by passage number or question type. Use
+the full corpus for regression, or curate a separate smoke list when coverage
+of P1/P2/P3, matching, flow/table/diagram, and passage-only negatives must be
+guaranteed.
+
+This is a cross-format consistency corpus, not OCR ground truth: text defects
+already present in PDF extraction remain visible by design. Real user-authored
+Word samples are still needed to cover floating text boxes, complex merged
+tables, tracked changes, and unusual Word automatic-numbering schemes.