diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 338cb58..aed73da 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -34,6 +34,9 @@ jobs: - name: Validate D1.2 dataset registry run: node scripts/validate-dataset-registry.mjs + - name: Validate D1.3 deterministic data-quality reports + run: node scripts/audit-data-quality.mjs --check + - name: Require a new uNews patchnote for Pull Request changes if: ${{ github.event_name == 'pull_request' }} env: @@ -65,6 +68,7 @@ jobs: node --check scripts/validate-project.mjs node --check scripts/validate-data-provenance.mjs node --check scripts/validate-dataset-registry.mjs + node --check scripts/audit-data-quality.mjs node --check scripts/validate-patchnote-diff.mjs node --check scripts/validate-screenshot-tooling.mjs node --check tools/screenshots/playwright.config.mjs diff --git a/AGENTS.md b/AGENTS.md index cc00569..358779c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,181 +16,171 @@ Before changing files, read: 8. `docs/ARCHITECTURE.md` 9. `docs/FILE_MAP.md` 10. `docs/DATA_PROVENANCE.md` for any data task -11. `data/datasets.json` and `docs/DATASET_REGISTRY.md` for dataset identity, canonical-path or migration work -12. `docs/TRANSLATION_WORKFLOW.md` for any translation or AI-assisted task -13. `docs/NEWS_PUBLISHING.md` -14. `docs/SCREENSHOT_AUTOMATION.md` -15. `docs/CONTENT_AND_RIGHTS.md` -16. the latest commits, tags, branches, open Pull Requests and Issues relevant to the task +11. `data/datasets.json` and `docs/DATASET_REGISTRY.md` for identity, canonical-path or migration work +12. `docs/DATA_QUALITY_AUDIT.md` and `reports/data-quality-audit.md` for quality or content-review work +13. `docs/TRANSLATION_WORKFLOW.md` for translation or AI-assisted work +14. `docs/NEWS_PUBLISHING.md` +15. `docs/SCREENSHOT_AUTOMATION.md` +16. `docs/CONTENT_AND_RIGHTS.md` +17. relevant current commits, branches, PRs, Issues and Actions Do not infer the current application from numbered folders under `_archive/`. -Before implementation, state the verified current status, active branch, open PR, last completed change, exact next action and current prohibitions. Do not change files until these facts agree with GitHub. +Before implementation, state verified status, active branch, open PR, last completed change, exact next action and prohibitions. GitHub facts must agree before files change. ## Sources of truth -- Real GitHub facts — `main`, open Pull Requests, branches, commits, tags, Releases and Actions results — have the highest priority. -- `WORK_STATUS.md` is the live cross-device handoff: active task, branch, actual progress, pause point and exact next action. -- `docs/AI_GITHUB_WORKFLOW.md` defines how devices, chats and agents cooperate without conflicting branches. -- An active GitHub Issue stores the detailed task specification; it does not replace live handoff state. -- `docs/PRODUCT_VISION.md` defines the product mission, final direction and non-destructive data principles. -- `docs/DATA_PROVENANCE.md` defines verified current file evidence, Git history, inferences and unknowns. -- `data/datasets.json` is the machine-readable source for current logical dataset IDs, physical file IDs, roles, hashes and policies. -- `docs/DATASET_REGISTRY.md` defines the approved canonical selection, retained compatibility serialization and future rollback design. -- `docs/TRANSLATION_WORKFLOW.md` defines target translation variants and safe API-assisted workflow. -- `main` is the source for the currently published GitHub Pages site. -- Git tags and GitHub Releases are immutable restoration checkpoints. -- Root `index.html`, `script.js`, `manifest.json`, `version.json`, and `sw.js` are the current runtime. -- `data/divinity_code_ru.json` / `ru-current-v1` is the active database until a documented functional migration changes it. -- `versions//` directories are runnable snapshots and must remain stable. -- `_archive/` is historical material, not current runtime code. -- `ROADMAP.md` distinguishes completed work, the next approved phase and later backlog. -- Old chat messages, AI memory and unpushed local notes are never sources of truth. - -If documents disagree with executable code or GitHub facts, report the conflict before editing and update the documents as part of the approved task. +Priority order: -## Current registered datasets +1. Real GitHub facts: `main`, open PRs, branches, commits, tags, Releases and Actions. +2. `WORK_STATUS.md`: live active-task handoff. +3. Active Issue: detailed task specification. +4. Maintained documentation and executable validators. +5. Chat memory and local unpushed notes are never sources of truth. + +Current sources: -Logical dataset IDs: +- `docs/PRODUCT_VISION.md` — product mission and non-destructive principles. +- `docs/DATA_PROVENANCE.md` — verified file evidence and unknowns. +- `data/datasets.json` — machine-readable logical/physical dataset identities. +- `docs/DATASET_REGISTRY.md` — canonical selection, retention and migration policy. +- `docs/DATA_QUALITY_AUDIT.md` — audit rules, severities and limitations. +- `reports/data-quality-audit.json` / `.md` — current deterministic audit results. +- `docs/TRANSLATION_WORKFLOW.md` — translation and AI-assisted workflow. +- Root runtime files and `data/divinity_code_ru.json` — current published application. +- Git tags/Releases — immutable restoration checkpoints. +- `ROADMAP.md` — completed, next and later work. + +If maintained documents disagree with executable code or GitHub facts, report and correct the conflict in the approved task. + +## Current registered datasets ```text source-divinity-code-en + source-divinity-code-en-bd2 -> data/bd2.json + source-divinity-code-en-db -> data/db.json + ru-current-v1 + ru-current-v1-runtime -> data/divinity_code_ru.json ``` -Physical file IDs: +`data/bd2.json` is the canonical maintained serialization by project-governance decision. This does not prove historical originality. `data/db.json` remains retained equivalent compatibility data. ```text -source-divinity-code-en-bd2 -> data/bd2.json -source-divinity-code-en-db -> data/db.json -ru-current-v1-runtime -> data/divinity_code_ru.json +physical migration: planned-not-executed +remove_or_rename_approved: false ``` -D1.2 selects `data/bd2.json` as the canonical maintained physical serialization through a project-governance decision. This does not prove that it is the historical original or authoritative external edition. +## Current quality baseline + +D1.3 baseline `23.8.10` verifies: -`data/db.json` remains a retained equivalent compatibility serialization. Physical migration status is `planned-not-executed`; deletion, rename or runtime switching is not approved. +- both logical datasets contain 4,086 unique ordered IDs; +- source/current IDs align; +- preserved fields have zero differences; +- structural errors: 0; +- warnings: 0; +- review instances: 5,022 in five aggregated groups. + +Review instances are heuristic candidates, can overlap and are not proven errors. Never auto-fix them in an audit or documentation PR. ## Change discipline -- One task should have one primary goal. -- Use a dedicated branch for structural, data, PWA, automation or multi-file changes. -- Preserve the working site before risky work with a tag/release. -- Do not delete or rewrite `_archive/`, source PDFs, screenshots, data variants, tags or releases without explicit approval. -- Do not edit a released snapshot except for a separately documented path/security repair. -- Keep the project static unless a backend or build system is explicitly approved. -- Prefer relative URLs inside version snapshots so each snapshot remains independently runnable. -- Never place secrets, access tokens, private contact data or credentials in the repository. -- Do not combine a data migration, interface redesign and architecture rewrite in one patch unless an approved plan proves they cannot be separated. +- One task, one primary goal, one active branch and one PR. +- Use a dedicated branch for structural, data, PWA, automation or multi-file work. +- Preserve working code before risky work with an immutable checkpoint. +- Never delete or rewrite `_archive/`, source PDFs, screenshots, data variants, tags or releases as routine cleanup. +- Do not edit released snapshots except for separately documented security/path repairs. +- Keep the project static unless a backend/build system is explicitly approved. +- Never store secrets, access tokens, personal contact data or credentials. +- Separate data migration, content correction, UI work and architecture design. ## AI execution preference -When connected tools permit safe execution, the agent should perform the GitHub work itself rather than send routine commands to the owner. +When connected tools permit safe execution, perform GitHub work directly rather than sending routine commands to the owner. The agent should independently: -- inspect repository facts and documentation; -- create and update the dedicated branch; -- update `WORK_STATUS.md` before implementation; -- edit files, create commits and open Pull Requests; -- run or inspect available validation and GitHub Actions; -- prepare patchnotes and real screenshot evidence; +- inspect GitHub facts and documentation; +- create/update the dedicated branch; +- update and push `WORK_STATUS.md` before substantial work; +- edit, commit, open PRs and inspect CI; +- prepare factual patchnotes and real evidence images; - correct failures and complete the handoff. -Ask the owner only for decisions requiring human judgment, access to an unavailable physical device, secure secret provisioning, translation/content review or an action unavailable to connected tools. Never ask the owner to paste an API key into chat. +Ask the owner only for unavailable physical-device checks, secure secret provisioning, content/translation judgment or unsupported actions. Never ask for an API key in chat. -## Cross-device handoff discipline +## Cross-device handoff -- Before implementation, update `WORK_STATUS.md` with status `IN_PROGRESS`, start time, environment, branch, one goal, planned files and completion criteria. -- Push the handoff before substantial implementation so another chat can see the lock. -- At meaningful checkpoints, record what is actually complete, the last verified commit and the exact next action. -- After every substantial saved stage that changes completed work, the last verified commit, the exact next action, blockers or prohibitions, update and push `WORK_STATUS.md` before continuing. -- Before switching devices or stopping unfinished work, set status to `PAUSED` or `BLOCKED`, commit the handoff and push it to GitHub. -- A local note, chat message or unpushed commit is not a valid handoff. -- When finishing, record planned versus actual work, changed files, checks, PR/merge, risks and the next approved task. -- After merge, return the file to `READY` and record the latest completed PR/commit. -- Do not start unrelated work while `WORK_STATUS.md` describes an unresolved `IN_PROGRESS`, `PAUSED` or `BLOCKED` task unless the owner explicitly closes or supersedes it. -- If `WORK_STATUS.md` conflicts with open PRs or commits, GitHub facts win and the file must be corrected immediately. -- Do not create a competing branch for the same goal from another chat. +- Set `WORK_STATUS.md` to `IN_PROGRESS` before implementation. +- Record start time, environment, branch, one goal, planned files and completion criteria. +- Push the handoff before substantial implementation. +- Update it after each saved stage that changes completed work, last commit, next action, blockers or prohibitions. +- Use `PAUSED` or `BLOCKED` before stopping unfinished work. +- Local notes and chat messages are not valid handoffs. +- After merge, return `main/WORK_STATUS.md` to `READY` and name the next approved task. +- Do not create a competing branch for the same goal. ## Real screenshot requirement -- Every new uNews patchnote must add a new real PNG/JPEG in the same Pull Request. -- Never reuse an existing image from an earlier patchnote. -- User-visible changes should be captured from the exact branch/commit in a real browser, preferably the implemented Playwright Chromium workflow. -- Documentation-only changes may use a real GitHub UI or exact document-render screenshot. -- AI-generated artwork is not proof of a real code, UI, release or documentation change. -- New patchnotes must include `image_source`, `image_target`, `image_commit` and `image_captured_at`. -- Validate that the screenshot visibly proves the claim before merge. -- Never expose secrets, cookies, private data or local home paths in screenshots. -- Follow `docs/SCREENSHOT_AUTOMATION.md` and `tools/screenshots/README.md`. - -For Playwright evidence: - -- use `.github/workflows/capture-screenshots.yml` or the isolated local package; -- inspect the uploaded artifact and `manifest.json`; -- visually inspect the selected PNG; -- copy only an approved PNG into `news/` through an explicit branch change; -- take `image_commit` and `image_captured_at` from matching provenance; -- never make the permanent capture workflow writable. - -## uNews publication requirement - -- User-visible changes, releases, fixes, documentation milestones and meaningful repository changes require a factual Russian patchnote in `news/`. -- Follow `docs/NEWS_PUBLISHING.md` and the canonical rules in `sunpole/uNews`. -- Use `project: uDream` and `series: udream` in YAML. -- Every patchnote requires `type`, `project`, `series`, `title`, next `version`, exact UTC `queued_at`, `repo_url` or `web_url`, `image` and required screenshot provenance fields. -- Name patchnotes `YYYY-MM-DD-udream-short-title.md` using lowercase Latin characters and hyphens. -- Real Telegram publication is performed only by the uNews GitHub Actions workflow. Do not send directly from a local machine. -- Never describe planned work as completed. Build the patchnote from the actual diff and completed checks. -- Before merge, run `node scripts/validate-project.mjs`, all specialized validators and, when practical, the uNews dry-run. -- Merging a new valid file under `news/` into public `main` makes it eligible for automatic publication to `@uNewsLog`. State the exact text and image before merging. - -## Copyright and third-party content - -- The MIT license applies to original uDream software code and original project documentation only. -- Do not claim ownership of source books, excerpts, dictionary interpretations, illustrations or PDFs created by third parties. -- Do not describe third-party content as MIT-licensed unless documented permission proves that it is. -- Keep copyright and provenance statements aligned with `docs/CONTENT_AND_RIGHTS.md` and `THIRD_PARTY_NOTICES.md`. -- Do not add, remove, republish or replace source PDFs or substantial book-derived content without explicit approval and a rights review. - -## Database safety and variant preservation - -- Treat database content changes separately from interface changes. -- Preserve record IDs unless a documented migration requires otherwise. -- Validate JSON syntax, array shape, required fields, record count and duplicate IDs before commit. -- Do not silently replace source wording or biblical references. -- Record source and transformation method for generated or translated data. -- Never overwrite one source database, translation or editorial variant with another. -- Keep every retained data variant identifiable by logical dataset ID, physical file ID, source, language, version/date and transformation history. -- A new translation must receive a new dataset ID and file; the previous translation remains recoverable. -- A future merged search must retain source provenance and must not destroy separate source datasets. -- Raw-distinct files with identical canonical JSON are one logical dataset, not independent translations. -- Do not delete or rename `data/bd2.json`, `data/db.json`, reports or archived data while migration status is `planned-not-executed`. -- The target is one canonical source dataset, one current Russian/localized dataset and up to two genuinely independent alternatives; do not manufacture variants when only one reliable translation exists. -- A future selector must validate the complete target dataset, reload consistently, support safe cache clearing and automatically fall back to the stable dataset on failure. - -Any future physical migration must: - -- use a separate branch and PR; -- create an immutable restoration checkpoint first; -- preserve old raw and canonical hashes in documentation; -- prove runtime/tooling references; -- include exact rollback steps; -- keep the active runtime unchanged unless a separate functional release explicitly changes it. +- Every new uNews patchnote must add a new real PNG/JPEG in the same PR. +- Never reuse an earlier patchnote image. +- UI changes should use the exact branch/commit in real Chromium when possible. +- Documentation/audit milestones may use an exact GitHub document/report page. +- Patchnotes require `image_source`, `image_target`, `image_commit` and `image_captured_at`. +- Validate that the image proves the claim and contains no secrets/private data. +- Permanent capture workflows remain read-only; one-time writable workflows must remove themselves after committing the approved image. + +## uNews publication + +- Meaningful changes require a factual Russian patchnote under `news/`. +- Use `project: uDream`, `series: udream`, next version and exact UTC `queued_at`. +- Describe only completed diff and checks. +- Never publish Telegram manually; uNews handles FIFO publication. +- Before merge, state the exact patchnote image and short Telegram text. +- Run project/specialized validators and patchnote-diff validation. + +## Database safety + +- Treat database content changes separately from UI/documentation changes. +- Preserve IDs unless an approved migration says otherwise. +- Never overwrite one source, translation or editorial variant with another. +- Every variant needs stable logical/physical IDs, source, language, version/date, hashes and transformation history. +- Raw-distinct files with identical canonical JSON are one logical dataset, not separate translations. +- Do not delete/rename `data/bd2.json`, `data/db.json`, reports or archives while migration is `planned-not-executed`. +- A future selector must validate datasets, reload consistently, clear only safe cache scope and fall back automatically. + +A future physical migration must: + +- use a separate branch/PR; +- create an immutable checkpoint first; +- preserve raw/canonical hashes; +- prove all references; +- include exact rollback; +- keep current runtime unchanged unless a separate functional release changes it. + +## Audit safety + +- `scripts/audit-data-quality.mjs` is read-only with respect to data. +- `reports/data-quality-audit.json` and `.md` must be deterministic and current. +- `error` findings fail the structural gate. +- `warning` and `review` findings require human/source evaluation and are not auto-fixes. +- Do not correct audit findings in the same PR that defines or runs the audit. +- Content correction requires exact evidence, a separate data PR, updated identity/version when needed, rollback and human review. ## AI-assisted translation safety -- DeepSeek or another provider may generate only a separate candidate dataset. -- Never call a paid translation API from the public browser application or installed PWA. -- Never commit an API key or place it in browser JavaScript, JSON, logs, artifacts, screenshots or patchnotes. +- DeepSeek or another provider may create only a separate candidate dataset. +- Never call paid translation APIs from the public site/PWA. +- Never commit an API key or expose it in code, JSON, logs, artifacts, screenshots or patchnotes. - Use a local environment variable or GitHub encrypted secret. -- Record source dataset ID/hash, candidate dataset ID, model, prompt-template version, parameters, output hash and review status for every run. -- A translation script must support checkpoints and must never modify the published database in place. +- Record source ID/hash, candidate ID, model, prompt version, parameters, output hash and review status. +- Scripts must be resumable and must not modify the published dataset in place. - AI output requires automatic validation and human review before any data release. ## Required checks -For relevant changes, run and report: +For relevant work: ```bash npm test @@ -198,79 +188,55 @@ jq empty data/divinity_code_ru.json jq 'length' data/divinity_code_ru.json node scripts/validate-data-provenance.mjs node scripts/validate-dataset-registry.mjs +node scripts/audit-data-quality.mjs --check node scripts/validate-project.mjs -python3 -m http.server 8019 ``` -For screenshot-tooling or user-visible UI changes also verify: - -```bash -node scripts/validate-screenshot-tooling.mjs -cd tools/screenshots -npm ci -npx playwright install --with-deps chromium -npm run capture -``` +For screenshot/UI work also use the isolated Playwright tooling and inspect artifacts visually. -Then verify: +Do not claim browser, PWA, offline, mobile or screenshot verification unless actually performed. -- current root page loads; -- active database loads without manual upload; -- search returns a result; -- saved-version launcher opens; -- saved snapshot loads its own database; -- manifest, icons, service worker, PDFs and external assets do not return 404; -- mobile layout remains usable; -- screenshot artifact contains expected scenarios and manifest when applicable; -- no unrelated files changed. +## Versioning -Do not claim a browser, PWA, offline, mobile or screenshot check was completed unless it was actually performed. - -## Versioning and releases - -- Public releases use semantic tags such as `v23.8.0`. -- The historical UI label `v19` is legacy metadata. The maintained application currently uses `v23.8.0`. -- Documentation/provenance/registry baselines may advance without changing the functional application version. -- Update `VERSION.md` and `CHANGELOG.md` for release-worthy or baseline changes. -- Add a matching uNews patchnote for every meaningful change. -- Document rollback steps in `docs/RELEASE_AND_ROLLBACK.md` or the task-specific migration document. -- Never move an existing release tag to a different commit. +- Functional release remains `v23.8.0` until application behavior changes through a release. +- Documentation/provenance/registry/audit baselines may advance without changing runtime version. +- Update `VERSION.md`, `CHANGELOG.md`, `ROADMAP.md`, relevant docs and uNews patchnote together. +- Never move an existing release tag. ## Documentation standard -Keep documentation factual and distinguish: +Clearly distinguish: - verified current behavior; - historical information; - planned work; - governance decisions; -- assumptions that still require testing. - -After a release or major documentation patch, search maintained documents for stale current-version labels, old SHAs and unresolved development-branch wording. +- heuristic audit findings; +- assumptions and unknowns. -End each completed task with files changed, checks run, results, remaining risks and an updated `WORK_STATUS.md` handoff. +End every completed task with changed files, checks, results, risks and updated handoff. ## Next approved planning boundary -D1.1 provenance and D1.2 dataset registry are complete at documentation/data baseline `23.8.9`. +D1.1 provenance, D1.2 registry and D1.3 data-quality audit are complete at baseline `23.8.10`. -The next approved task is D1.3: design and run a non-destructive data-quality audit for all 4,086 records. It may report issues but must not silently rewrite source or localized content. +Next: **D1.4 two-book product architecture**. It is architecture-only unless a separate functional implementation phase is approved. It must define source identity, provenance display, switching/combined/side-by-side options, history/deep-link behavior, validation, cache handling and stable fallback. -D1.4 two-book product architecture and D1.5 AI-assisted translation experiment must not begin before D1.3 is completed and recorded. +D1.5 AI-assisted translation must not begin before D1.4 is completed and recorded. -## uNews / тыНовости — обязательный план публикации +## uNews / тыНовости Этот публичный проект подключён к автоматической очереди [uNews](https://github.com/sunpole/uNews). -После каждого завершённого пользовательского изменения ИИ или программист должен: +После завершённого изменения: -1. Создать русский патчноут в `news/` и новое реальное изображение рядом. -2. Указать в YAML: `type`, `project`, безопасный латинский `series`, `title`, следующую `version`, точное UTC-время `queued_at`, `repo_url` или `web_url`, `image`, а также screenshot provenance. -3. Формат времени: `queued_at: YYYY-MM-DDTHH:MM:SSZ`. -4. Описывать только фактически выполненные изменения; не включать токены, ключи, приватные данные и локальные секреты. -5. Добавлять патчноут и изображение в ту же публичную ветку, что и завершённое изменение. -6. Не публиковать в Telegram вручную: uNews сам проверяет публичные репозитории, выбирает новости в FIFO-порядке и публикует с паузой не менее 61 секунды. -7. При ошибке исправить самый ранний патчноут проекта; после каждого успешного Telegram-поста uNews обязан зафиксировать checkpoint. +1. создать русский патчноут и новое реальное изображение; +2. заполнить обязательный YAML и screenshot provenance; +3. использовать точное UTC-время; +4. не включать секреты и приватные данные; +5. добавить материалы в ту же публичную ветку; +6. не публиковать Telegram вручную; +7. при ошибке исправлять самый ранний неопубликованный патчноут. Репозиторий: `sunpole/udream`. Полная спецификация: https://github.com/sunpole/uNews/blob/main/UNEWS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a2b10b9..3f91869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## Audit and documentation update v23.8.10 — 2026-07-23 + +- Added `scripts/audit-data-quality.mjs` as a deterministic, read-only audit for registered source/current datasets. +- Added `docs/DATA_QUALITY_AUDIT.md` with structural, hygiene and ambiguity rules plus `error`, `warning`, `review` and `info` severities. +- Added deterministic machine-readable and human-readable reports under `reports/`. +- Audited both logical datasets and all 4,086 ordered IDs without changing source or current files. +- Confirmed zero preserved-field differences for `id`, `symbol`, `description`, `source` and `date_added`. +- Confirmed expected changed-field counts: aliases 4,083; notes 4,086; tags 4,086. +- Recorded a passing structural gate with 0 errors and 0 warnings. +- Recorded 5,022 human-review instances in five aggregated groups covering shared aliases, alias-to-primary routing and empty source notes. +- Explicitly documented that review instances are heuristic candidates, can overlap and are not 5,022 proven content errors. +- Required committed report freshness and the structural audit gate in GitHub Actions. +- Synchronized README, roadmap, version, project state, file map, database contract and agent rules. +- Kept all existing data files, application runtime, PWA, package metadata, saved versions and archives unchanged. + ## Documentation and registry update v23.8.9 — 2026-07-23 - Added `data/datasets.json` as the machine-readable D1.2 registry for logical datasets, physical files, hashes, roles and policies. diff --git a/README.md b/README.md index 7972905..08993ca 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ | Продолжить разработку | Сначала открыть [WORK_STATUS.md](WORK_STATUS.md), проверить branch/PR/commits и следовать [GitHub-протоколу](docs/AI_GITHUB_WORKFLOW.md) | | Проверить происхождение баз | [docs/DATA_PROVENANCE.md](docs/DATA_PROVENANCE.md) + `node scripts/validate-data-provenance.mjs` | | Проверить реестр наборов | [docs/DATASET_REGISTRY.md](docs/DATASET_REGISTRY.md) + `node scripts/validate-dataset-registry.mjs` | +| Открыть аудит качества | [reports/data-quality-audit.md](reports/data-quality-audit.md) | +| Проверить актуальность audit reports | `node scripts/audit-data-quality.mjs --check` | | Создать реальные screenshot artifacts | Запустить [Capture uDream screenshots](https://github.com/sunpole/udream/actions/workflows/capture-screenshots.yml) либо использовать `tools/screenshots/` локально | | Скачать стабильный исходный код | [uDream v23.8.0 ZIP](https://github.com/sunpole/udream/archive/refs/tags/v23.8.0.zip) | | Посмотреть точный релиз | [GitHub Release v23.8.0](https://github.com/sunpole/udream/releases/tag/v23.8.0) | @@ -60,7 +62,7 @@ uDream — статическое веб-приложение для поиск - установка на устройство как PWA; - автоматическое обновление PWA без зависания на старом кэше; - каталог сохранённых рабочих версий; -- регрессионные тесты и автоматическая проверка runtime, базы, provenance, dataset registry, handoff и патчноутов; +- регрессионные тесты и автоматическая проверка runtime, базы, provenance, dataset registry, data-quality reports, handoff и патчноутов; - реальные desktop/mobile-скриншоты из точного commit через Playwright Chromium и GitHub Actions artifacts. ## Текущее состояние @@ -69,16 +71,17 @@ uDream — статическое веб-приложение для поиск |---|---| | Стабильная точка восстановления | `v23.8.0` | | Текущая версия приложения | `v23.8.0` | -| Документационный/data baseline | `v23.8.9` — D1.2 dataset registry | +| Документационный/data baseline | `v23.8.10` — D1.3 deterministic data-quality audit | | Оперативная точка продолжения | [WORK_STATUS.md](WORK_STATUS.md) | -| Источник GitHub-процесса | [docs/AI_GITHUB_WORKFLOW.md](docs/AI_GITHUB_WORKFLOW.md) | | Активный logical dataset | `ru-current-v1` | | Активный runtime-файл | `data/divinity_code_ru.json` | | Количество записей | 4 086 | | Canonical source dataset | `source-divinity-code-en` | | Canonical source serialization | `data/bd2.json` | | Retained compatibility serialization | `data/db.json` | -| Следующая утверждённая задача | `D1.3` — non-destructive data-quality audit | +| Structural audit gate | `PASS` — 0 structural errors | +| Human-review findings | 5 022 instances in 5 aggregated groups; не доказанные ошибки | +| Следующая утверждённая задача | `D1.4` — two-book product architecture | | Публикация | GitHub Pages из ветки `main` | | Технологии | HTML, CSS, JavaScript ES Modules, JSON, PWA | @@ -94,14 +97,9 @@ D1.2 закрепил стабильные идентификаторы: source-divinity-code-en ru-current-v1 -source-divinity-code-en-bd2 -data/bd2.json - -source-divinity-code-en-db -data/db.json - -ru-current-v1-runtime -data/divinity_code_ru.json +source-divinity-code-en-bd2 -> data/bd2.json +source-divinity-code-en-db -> data/db.json +ru-current-v1-runtime -> data/divinity_code_ru.json ``` `data/bd2.json` выбран canonical maintained serialization как **project-governance decision**. Это не доказывает, что файл является историческим оригиналом или подтверждённой внешней редакцией. `data/db.json` сохранён как retained equivalent compatibility serialization. @@ -122,7 +120,27 @@ remove_or_rename_approved: false DeepSeek API рассматривается только как инструмент создания отдельного candidate dataset. Ключ никогда не встраивается в сайт или PWA; любой результат получает новый dataset ID и проходит автоматическую и человеческую проверку. -Подробнее: [реестр наборов](docs/DATASET_REGISTRY.md), [проверенное происхождение](docs/DATA_PROVENANCE.md), [переводы и AI-assisted workflow](docs/TRANSLATION_WORKFLOW.md) и [формат данных](docs/DATABASE_FORMAT.md). +## Аудит качества D1.3 + +D1.3 проверяет зарегистрированные source/current datasets в read-only режиме. Отчёты генерируются детерминированно: + +```text +reports/data-quality-audit.json +reports/data-quality-audit.md +``` + +Подтверждено: + +- оба logical datasets содержат 4 086 unique ordered IDs `1–4086`; +- source/current IDs полностью aligned; +- `id`, `symbol`, `description`, `source`, `date_added` имеют 0 различий; +- ожидаемые различия: aliases 4 083, notes 4 086, tags 4 086; +- structural gate прошёл: 0 errors, 0 warnings; +- зарегистрировано 5 022 human-review instances в пяти агрегированных группах. + +5 022 — не число доказанных ошибок. Это пересекающиеся кандидаты на проверку: shared aliases, alias-to-primary routing и пустые source notes. Скрипт ничего не исправляет и не меняет файлы данных. + +Подробнее: [правила аудита](docs/DATA_QUALITY_AUDIT.md), [читаемый отчёт](reports/data-quality-audit.md), [реестр наборов](docs/DATASET_REGISTRY.md), [происхождение](docs/DATA_PROVENANCE.md) и [переводы](docs/TRANSLATION_WORKFLOW.md). ## Структура репозитория @@ -139,11 +157,12 @@ udream/ │ ├── bd2.json # canonical source serialization │ ├── db.json # retained equivalent serialization │ └── report.txt # исторический отчёт +├── reports/ # deterministic machine/human audit reports ├── versions/ # запускаемые контрольные версии ├── news/ # патчноуты и новые реальные изображения для uNews ├── tools/screenshots/ # изолированные Playwright-сценарии -├── docs/ # видение, состояние, процессы, provenance и registry -├── scripts/ # автоматические проверки +├── docs/ # видение, состояние, процессы, provenance, registry и audit +├── scripts/ # автоматические проверки и read-only audit ├── .github/workflows/ # validation, screenshot artifacts и release automation ├── _archive/ # исторические версии и исходные материалы ├── WORK_STATUS.md # текущая задача и GitHub-handoff @@ -153,9 +172,7 @@ udream/ └── VERSION.md # состояние версий ``` -Файлы текущего сайта намеренно находятся в корне: GitHub Pages публикует их напрямую. `data/datasets.json` является metadata registry и не загружается браузером. Исторические эксперименты хранятся отдельно в `_archive/` и не являются рабочим приложением. - -Подробная карта находится в [docs/FILE_MAP.md](docs/FILE_MAP.md). +Файлы текущего сайта намеренно находятся в корне: GitHub Pages публикует их напрямую. `data/datasets.json` и `reports/` не загружаются браузером. Исторические эксперименты хранятся отдельно в `_archive/` и не являются рабочим приложением. ## Локальный запуск и проверки @@ -163,12 +180,13 @@ udream/ python3 -m http.server 8019 ``` -После этого откройте `http://localhost:8019/`. Запуск простым открытием `index.html` через `file://` не считается корректной проверкой: браузеры ограничивают `fetch()` и service worker. +После этого откройте `http://localhost:8019/`. Запуск через `file://` не считается корректной проверкой: браузеры ограничивают `fetch()` и service worker. ```bash npm test node scripts/validate-data-provenance.mjs node scripts/validate-dataset-registry.mjs +node scripts/audit-data-quality.mjs --check node scripts/validate-project.mjs ``` @@ -206,6 +224,8 @@ npm run capture - [Видение и конечная цель продукта](docs/PRODUCT_VISION.md) - [Проверенное происхождение данных](docs/DATA_PROVENANCE.md) - [Реестр наборов и migration policy](docs/DATASET_REGISTRY.md) +- [Аудит качества данных](docs/DATA_QUALITY_AUDIT.md) +- [Читаемый audit report](reports/data-quality-audit.md) - [Переводы и AI-assisted workflow](docs/TRANSLATION_WORKFLOW.md) - [Реальные скриншоты и Playwright-автоматизация](docs/SCREENSHOT_AUTOMATION.md) - [Текущее состояние](docs/PROJECT_STATE.md) diff --git a/ROADMAP.md b/ROADMAP.md index 9268aa3..697f8e5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -41,7 +41,7 @@ See `docs/MODULARIZATION_PLAN.md`. - [x] Reload once safely when the deployed version differs from the running version. - [x] Remove only old `udream-*` caches and preserve unrelated origin caches. - [x] Use network-first with offline fallback for HTML, JavaScript, manifest and JSON. -- [x] Add a visible install banner with system prompt and manual instructions. +- [x] Add a visible install banner with system prompt support and manual instructions. - [x] Hide the install banner in the standalone PWA. - [x] Create the immutable `v23.8.0` tag on the exact functional merge commit. - [x] Add a GitHub Release and documented rollback path. @@ -132,25 +132,34 @@ See `docs/DATA_PROVENANCE.md`. See `data/datasets.json` and `docs/DATASET_REGISTRY.md`. -## Next approved work: D1.3 — data-quality audit design - -D1.3 designs and runs non-destructive quality checks. It must not silently rewrite published content. +## Completed: D1.3 data-quality audit v23.8.10 + +- [x] Define a four-level severity model: error, warning, review and info. +- [x] Add structural checks for arrays, required fields, types, dates, unique/ordered IDs and registered record counts. +- [x] Add data-hygiene and ambiguity checks for whitespace, empty values, duplicate aliases/tags, control characters, HTML-like content and normalized collisions. +- [x] Compare `source-divinity-code-en` with `ru-current-v1` without changing either dataset. +- [x] Confirm 4,086 unique ordered IDs and exact source/current alignment. +- [x] Confirm zero differences in preserved fields `id`, `symbol`, `description`, `source` and `date_added`. +- [x] Record expected changed-field counts: aliases 4,083; notes 4,086; tags 4,086. +- [x] Generate deterministic machine-readable and human-readable reports. +- [x] Add permanent `scripts/audit-data-quality.mjs` with write and `--check` modes. +- [x] Require report freshness and a passing structural gate in GitHub Actions. +- [x] Record 0 structural errors, 0 warnings and 5,022 human-review instances in five aggregated groups. +- [x] State explicitly that heuristic review counts are not proven content errors and are not auto-fixed. +- [x] Preserve all existing data files, runtime, PWA, package metadata, saved versions and archives unchanged. -- [ ] Define automated checks for all 4,086 active records, duplicate IDs, empty fields, invalid types, cross-references and suspicious entries. -- [ ] Classify checks that can be automatic versus checks requiring human/source review. -- [ ] Define severity levels and a stable report format. -- [ ] Compare the current localized dataset with the registered source dataset without changing either file. -- [ ] Produce a machine-readable and human-readable audit report. -- [ ] Add a permanent validator or audit command that does not modify data. -- [ ] Keep content corrections for later separate reviewed data patches. +See `docs/DATA_QUALITY_AUDIT.md` and `reports/data-quality-audit.md`. -## D1.4 — two-book product architecture +## Next approved work: D1.4 — two-book product architecture +- [ ] Identify and register the second source book/dataset as far as retained evidence allows. - [ ] Define the intended relationship between the two source books. - [ ] Compare separate-database switching, combined search and side-by-side comparison. -- [ ] Require visible provenance for every result in any future combined mode. -- [ ] Approve a migration plan before implementing a user-facing selector or merged index. -- [ ] Require full dataset validation, consistent reload, safe cache clearing and automatic stable fallback before any selector release. +- [ ] Define user-visible provenance for every result in every future mode. +- [ ] Define validation, reload, cache-clearing and automatic fallback contracts before any selector implementation. +- [ ] Decide how history, sharing and deep links preserve dataset identity. +- [ ] Approve a migration and rollback plan before implementing a user-facing selector or merged index. +- [ ] Keep D1.4 architecture-only unless a separate functional implementation phase is approved. ## D1.5 — AI-assisted translation experiment diff --git a/VERSION.md b/VERSION.md index 66d1efc..f32bf74 100644 --- a/VERSION.md +++ b/VERSION.md @@ -13,36 +13,46 @@ - Database: the active 4,086-record JSON database is unchanged - Previous checkpoint: `v23.7.0` -## Documentation, provenance and registry baseline +## Documentation, provenance, registry and audit baseline -`v23.8.9` +`v23.8.10` -- Status: D1.2 dataset-registry milestone; it does not create a new application build or move the `v23.8.0` tag -- Purpose: establish stable logical/physical dataset identities, canonical-retained roles and an unexecuted reversible migration plan -- Machine-readable registry: `data/datasets.json`, schema version 1 -- Registry document: `docs/DATASET_REGISTRY.md` -- Registry validator: `scripts/validate-dataset-registry.mjs`, required by `.github/workflows/validate.yml` +- Status: D1.3 non-destructive data-quality audit milestone; it does not create a new application build or move the `v23.8.0` tag +- Purpose: define deterministic audit rules, inspect all registered source/current records and preserve the results without modifying data +- Audit specification: `docs/DATA_QUALITY_AUDIT.md` +- Audit script: `scripts/audit-data-quality.mjs` +- Machine report: `reports/data-quality-audit.json` +- Human report: `reports/data-quality-audit.md` +- CI command: `node scripts/audit-data-quality.mjs --check` +- Logical datasets audited: `source-divinity-code-en`, `ru-current-v1` +- Records audited: 4,086 in each logical dataset +- ID result: unique, ordered and aligned from 1 through 4,086 +- Preserved-field differences: 0 for `id`, `symbol`, `description`, `source` and `date_added` +- Expected changed-field differences: aliases 4,083; notes 4,086; tags 4,086 +- Structural gate: pass +- Structural error instances: 0 +- Warning instances: 0 +- Human-review instances: 5,022 in five aggregated finding groups +- Important limitation: review counts are heuristic candidates, not 5,022 proven content errors; one record may appear in multiple groups +- Data mutation policy: read-only; no finding is corrected by the audit +- Registry baseline: `v23.8.9`, D1.2 completed by PR #27 - Source logical dataset ID: `source-divinity-code-en` - Current localized dataset ID: `ru-current-v1` - Canonical physical serialization: `source-divinity-code-en-bd2` at `data/bd2.json` - Retained compatibility serialization: `source-divinity-code-en-db` at `data/db.json` - Active runtime physical file: `ru-current-v1-runtime` at `data/divinity_code_ru.json` -- Canonical choice type: project-governance decision; it does not prove the historical original or authoritative source edition - Migration status: `planned-not-executed`; no file removal, rename or runtime switch is approved -- Active dataset: 4,086 records, SHA-256 `1def80216e238b0c2a8640aaf1b4e95dd0669d5944a67f4e7c4421fad55a6e64` -- English serialization A: SHA-256 `814c5d33444160e6f1ab20278f9356090ec0e9cc04943cd14ad99d9038be6e28` -- English serialization B: SHA-256 `4e166959d318778be57557349a152c2b466ad9db14e5634f5e5df3c87ca2cdc0` +- Active dataset SHA-256: `1def80216e238b0c2a8640aaf1b4e95dd0669d5944a67f4e7c4421fad55a6e64` - Shared canonical English JSON SHA-256: `5ebe0d973f9cfd1c9db65a9d5abebe0ca16788261219299a710ed9fe78bb25d1` -- Previous provenance baseline: `v23.8.8`, D1.1 completed by PR #25 - Live handoff source: `WORK_STATUS.md` -- Next approved task after D1.2 merge: D1.3 data-quality audit design +- Next approved task after D1.3 merge: D1.4 two-book product architecture - Repository download: https://github.com/sunpole/udream/archive/refs/tags/v23.8.0.zip ## Current development line No functional application version is assigned after `v23.8.0`. -Documentation, provenance and automation updates `23.8.1` through `23.8.9` finalize the immutable release record, product/data baseline, translation safety, cross-device workflow, screenshot provenance, verified data provenance and dataset-registry governance. They do not change application runtime, PWA behavior or existing database records. +Documentation, provenance and automation updates `23.8.1` through `23.8.10` finalize the immutable release record, product/data baseline, translation safety, cross-device workflow, screenshot provenance, verified data provenance, dataset-registry governance and deterministic data-quality reporting. They do not change application runtime, PWA behavior or existing database records. ## Unified version line diff --git a/WORK_STATUS.md b/WORK_STATUS.md index 051c4ea..06a8f21 100644 --- a/WORK_STATUS.md +++ b/WORK_STATUS.md @@ -2,63 +2,91 @@ Этот файл — единая оперативная точка продолжения разработки uDream с телефона, Windows, macOS, другого устройства или любого ИИ-чата. -`ROADMAP.md` хранит общий план, `docs/PROJECT_STATE.md` — проверенное состояние продукта, а этот файл хранит живую текущую задачу и точный следующий шаг. - ## Быстрый сигнал | Поле | Текущее значение | |---|---| -| Состояние | **READY** — незавершённой активной задачи нет | -| Рабочая ветка | `main` | -| Открытый Pull Request | нет | +| Состояние | **COMPLETED** — D1.3 реализован; PR №29 переведён в ready и ожидает зелёный финальный Actions run | +| Рабочая ветка | `audit/d1.3-data-quality-v23.8.10` | +| Открытый Pull Request | `#29` — `https://github.com/sunpole/udream/pull/29` | +| Issue | `#28` — D1.3 data-quality audit | | Стабильный функциональный релиз | `v23.8.0` | -| Документационный/data baseline | `23.8.9` — D1.2 dataset registry завершён | -| Последнее завершённое изменение | PR #27, squash merge `3cb0403016b56a8d5e6fedd9b6367f383ec4a6ba` | -| Закрытая задача | Issue #26 — D1.2 completed | -| Следующая утверждённая задача | D1.3 — non-destructive data-quality audit design | - -## Завершённый этап D1.2 - -- создан machine-readable registry `data/datasets.json`; -- зарегистрированы logical dataset IDs `source-divinity-code-en` и `ru-current-v1`; -- зарегистрированы physical file IDs для обеих английских сериализаций и активной русской базы; -- `data/bd2.json` выбран canonical maintained serialization как project-governance decision; -- выбор canonical path не выдаётся за доказательство исторического оригинала; -- `data/db.json` сохранён как retained equivalent compatibility serialization; -- создан `docs/DATASET_REGISTRY.md` с identity, reference audit, migration и rollback; -- создан permanent validator `scripts/validate-dataset-registry.mjs`; -- GitHub Actions проверяет registry, реальные hashes, canonical JSON, 4 086 ordered IDs, policy и runtime isolation; -- factual uNews patchnote `23.8.9` и новый реальный Chromium document-render сохранены; -- существующие data-файлы, runtime, PWA, package metadata, `versions/` и `_archive/` не изменены. - -## Текущие зарегистрированные данные - -```text -source-divinity-code-en - canonical physical: data/bd2.json - retained equivalent: data/db.json - -ru-current-v1 - active runtime: data/divinity_code_ru.json - -physical migration: planned-not-executed -remove_or_rename_approved: false -``` +| Документационный/data baseline | кандидат `23.8.10` | +| Последний проверенный branch head до ready-review sync | `22fe94598e8de3a4c72ac76ffb9265625249c3eb` | +| Следующая утверждённая задача после merge | D1.4 — two-book product architecture | + +## Выполнено фактически + +- создан permanent read-only `scripts/audit-data-quality.mjs`; +- создан `docs/DATA_QUALITY_AUDIT.md` с severity model, rules, determinism и limitations; +- созданы deterministic reports `reports/data-quality-audit.json` и `reports/data-quality-audit.md`; +- permanent CI запускает `node scripts/audit-data-quality.mjs --check`; +- audit охватывает `source-divinity-code-en` и `ru-current-v1`; +- оба datasets содержат 4 086 unique ordered IDs `1–4086`; +- source/current IDs aligned; +- preserved fields `id`, `symbol`, `description`, `source`, `date_added` имеют 0 differences; +- changed-field counts: aliases 4 083, notes 4 086, tags 4 086; +- structural gate: PASS; +- structural errors: 0; +- warnings: 0; +- human-review instances: 5 022 в пяти aggregated groups; +- findings не исправлялись и не выдавались за доказанные content errors; +- README, VERSION, CHANGELOG, ROADMAP, PRODUCT_VISION, PROJECT_STATE, ARCHITECTURE, FILE_MAP, DATABASE_FORMAT, TRANSLATION_WORKFLOW и AGENTS синхронизированы; +- создан factual patchnote `23.8.10`; +- создан новый real Chromium document-render точной GitHub report page; +- screenshot source commit: `cd6aa539a418861108d58f2206050291642e7fcb`; +- screenshot captured at: `2026-07-23T07:09:49Z`; +- одноразовые report/image workflows удалили себя после сохранения outputs; +- существующие data files, runtime, PWA, package metadata, `versions/` и `_archive/` не изменены. + +## Фактические audit findings + +| Rule | Dataset | Count | Classification | +|---|---|---:|---| +| alias collision across records | `ru-current-v1` | 854 | human/source review | +| alias matches another primary symbol | `ru-current-v1` | 1 145 | routing review | +| alias collision across records | `source-divinity-code-en` | 693 | human/source review | +| alias matches another primary symbol | `source-divinity-code-en` | 1 145 | routing review | +| empty notes | `source-divinity-code-en` | 1 185 | review; may be intentional source structure | + +5 022 — не количество доказанных ошибок. Группы пересекаются; shared aliases могут быть намеренными, а empty source notes могут отражать структуру исходника. + +## Планируемые файлы + +План D1.3 выполнен в пределах: + +- audit script/specification/reports; +- CI validation; +- maintained documentation; +- WORK_STATUS, roadmap, version и changelog; +- factual patchnote и новое real image. + +Незапланированных data-content/runtime изменений нет. + +## Критерии завершения + +- оба registered logical datasets и все 4 086 IDs проверены; +- structural, warning, review и info categories разделены; +- deterministic reports и stale-report check работают; +- source/current alignment и preserved-field equality проверены; +- heuristics не выдаются за доказанные errors; +- protected data/runtime/PWA/saved-version/archive paths отсутствуют в diff; +- provenance, registry, audit и project validators проходят; +- GitHub Actions на final head должны быть зелёными; +- patchnote и real image соответствуют D1.3; +- после squash merge Issue №28 закрывается и `main/WORK_STATUS.md` возвращается в `READY`. ## Следующий точный шаг -Начать D1.3 только в новой отдельной ветке после чтения Issue/ROADMAP и обновления этого файла до `IN_PROGRESS`. - -D1.3 должна спроектировать и выполнить неразрушающий аудит качества всех 4 086 записей: автоматические проверки, severity levels, machine-readable и human-readable reports. Она может находить и описывать проблемы, но не должна молча исправлять или перезаписывать данные. +Дождаться зелёного `Validate uDream` на этом финальном synchronize commit, повторно проверить changed-file list и выполнить squash merge PR №29. Затем закрыть Issue №28 и записать `READY` на `main` для D1.4. -## Главные запреты +## Запреты до merge -- не удалять, не переименовывать и не изменять текущие data-файлы без отдельной обратимой миграции; -- не менять стабильные IDs или тексты в audit-патче; -- не начинать пользовательский selector баз; -- не запускать D1.4 или D1.5 раньше завершения D1.3; -- не выдавать неизвестный translation pipeline за доказанный факт. +- не менять существующие data files или runtime; +- не исправлять findings в D1.3; +- не начинать D1.4/D1.5 до merge; +- не объединять PR при красной или незавершённой проверке. ## Источник истины -Реальные GitHub-факты — `main`, открытые Pull Request, commits, Actions и Issues — имеют приоритет над памятью ИИ, старыми чатами и локальными незапушенными изменениями. +Реальные GitHub-факты — `main`, текущая ветка, commits, Pull Request, Actions и Issues — имеют приоритет над памятью ИИ, старыми чатами и локальными незапушенными изменениями. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 027ca01..d3546b0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -23,144 +23,59 @@ script.js ──import──> src/search.js DOM + localStorage + Web Share APIs ``` -There is no server-side application code. GitHub Pages serves files; the browser performs all search, filtering, rendering, preferences, history management, PWA update checks and installation UI. +There is no server-side application code. GitHub Pages serves files; the browser performs search, rendering, preferences, history, update checks and installation UI. -`data/datasets.json` is not part of this runtime graph. It is repository governance metadata used by documentation and validation only. +The following are repository tooling, not runtime dependencies: -## Current components - -### `index.html` - -- page structure and inline CSS; -- current `v23.8.0` label populated from centralized runtime version metadata; -- menu, search controls, history controls, result area, alphabet rows, tags, install banner host and footer; -- external CDN dependencies; -- links to reference PDFs; -- loads `script.js` as an ES module. - -### `script.js` - -- application orchestration and mutable UI state; -- connects data, search, history, presentation, localization, storage, version and PWA modules; -- autocomplete UI, tag, alphabet, color and digit behavior; -- DOM event binding for prebuilt presentation fragments; -- history and breadcrumb DOM rendering; -- theme and UI preference DOM effects; -- sharing helpers. - -### `src/version.js` - -- defines the runtime application version, visible label and page title; -- updates all `[data-app-version]` labels consistently; -- must agree with `package.json` and `version.json` for release publication. - -### `src/search.js` - -- strict per-field search and autocomplete matching; -- relevance ranking for exact, prefix and substring matches; -- primary-card preference for aliases shared with short redirect records; -- direct-result resolution for exact aliases; -- uses JSDoc and `// @ts-check` without a TypeScript build step; -- is covered by dependency-free Node.js regression tests. - -### `src/data.js` - -- owns ordered database path fallback and manual JSON parsing; -- preserves the runtime contract of a non-empty JSON array; -- returns records plus source metadata without touching the DOM. - -### `src/state.js` - -- creates the initial application state from defaults and the storage module; -- normalizes restored full-history data before the UI starts; -- contains no database transformation logic; -- can be tested without a browser. - -### `src/history.js` - -- owns pure navigation-stack and breadcrumb-window operations; -- appends and normalizes persistent history entries; -- groups full history by day without touching the DOM; -- preserves navigation branch truncation and back/forward boundaries. - -### `src/storage.js` - -- owns string, boolean and JSON reads from browser storage; -- owns browser-compatible serialization and removal; -- returns documented defaults when storage is unavailable or JSON is malformed; -- contains no application UI or database logic. - -### `src/i18n.js` - -- owns the complete RU/EN interface dictionary, language normalization and reviewed instruction HTML; -- falls back to Russian for missing or corrupted stored language values; -- marks the single translation that intentionally contains trusted line-break HTML. - -### `src/pwa.js` - -- owns service-worker registration and PWA installation/update interaction; -- registers after the browser `load` event; -- checks `version.json` with `cache: no-store`; -- requests an update and performs a protected one-time reload when the deployed version differs; -- handles `beforeinstallprompt`, manual installation guidance and standalone-mode suppression; -- safely does nothing when browser capabilities are unavailable; -- reports failures without blocking application startup; -- is covered by dependency-free Node.js regression tests. - -### `src/presentation.js` - -- owns pure HTML builders for records, lists, history, breadcrumbs, tags, autocomplete, statistics and sharing; -- escapes text and attribute values from imported JSON, including both quote types; -- renders notes as safe plain text with paragraphs and line breaks instead of interpreting raw HTML or Markdown; -- contains no DOM queries, event listeners, browser storage or search logic. - -### `data/divinity_code_ru.json` - -The active runtime database and physical file `ru-current-v1-runtime`. It is loaded client-side and kept in browser memory for searching. Its 4,086 records remain the published source until a separately approved functional data migration changes the runtime contract. - -### `data/datasets.json` - -The D1.2 machine-readable registry. It defines: +```text +data/datasets.json +reports/data-quality-audit.json +reports/data-quality-audit.md +scripts/validate-data-provenance.mjs +scripts/validate-dataset-registry.mjs +scripts/audit-data-quality.mjs +``` -- logical dataset IDs; -- physical file IDs and paths; -- exact raw/canonical hashes and record counts; -- canonical, retained and runtime roles; -- source relationships; -- translation-variant policy; -- future migration and rollback policy. +## Current components -The browser, `src/data.js` and Service Worker do not load this file. The registry cannot switch the active database by itself. +### Runtime -### `version.json` +- `index.html` — page structure, inline CSS, controls, install host and footer. +- `script.js` — application orchestration and DOM events. +- `src/version.js` — runtime version, visible labels and title. +- `src/search.js` — strict search, relevance and alias resolution. +- `src/data.js` — active database loading/manual JSON parsing. +- `src/state.js` — initial state and restored preferences. +- `src/history.js` — navigation, breadcrumbs and history grouping. +- `src/storage.js` — typed browser-storage operations. +- `src/i18n.js` — RU/EN interface dictionary. +- `src/presentation.js` — escaped pure HTML builders. +- `src/pwa.js` — Service Worker registration, update and installation flow. +- `version.json` — uncached deployed-version signal. +- `manifest.json` — PWA identity and icons. +- `sw.js` — application shell/active database cache and update strategy. -A small deployment-version file requested without browser HTTP cache. It allows a running page or installed PWA to detect that GitHub Pages has published a newer version. +### Active data -### `manifest.json` +`data/divinity_code_ru.json` is physical file `ru-current-v1-runtime` and the only active runtime dataset. Its 4,086 records remain published until a separate approved functional data release changes the contract. -PWA identity, start URL, scope, theme colors, language, categories and install icons. +### Registry metadata -### `sw.js` +`data/datasets.json` defines logical/physical IDs, roles, hashes, canonical choice, source relationships, migration policy and translation policy. It cannot switch the active runtime database. -- caches the application shell and active database; -- activates new releases immediately through `skipWaiting()` and `clients.claim()`; -- removes only old caches whose names start with `udream-`; -- uses network-first with offline fallback for version-sensitive runtime resources; -- must change its cache name whenever cached runtime assets change; -- does not cache or load `data/datasets.json`. +### Audit reports -### `versions/` +`reports/data-quality-audit.json` and `.md` are deterministic outputs generated from registered canonical source/current datasets. They are not loaded by the browser or cached by the Service Worker. -Contains the saved-version launcher and runnable snapshots. Each snapshot should use relative paths, its own active database, its own manifest and a scoped service worker. +`scripts/audit-data-quality.mjs` supports generation and `--check`. The audit writes reports only and never modifies `data/` inputs. -### `_archive/` +### Saved versions and archive -Historical/reference storage. The current application must not import scripts or databases from archived numbered versions. +- `versions/` contains runnable snapshots with independent relative paths and scoped PWA files. +- `_archive/` contains historical/reference material and is not imported by current runtime. ## Dataset identity boundary -D1.2 registers: - ```text source-divinity-code-en ├─ source-divinity-code-en-bd2 -> data/bd2.json (canonical retained) @@ -170,48 +85,72 @@ ru-current-v1 └─ ru-current-v1-runtime -> data/divinity_code_ru.json (runtime current) ``` -The canonical English choice is a project-governance decision. It does not prove historical originality. Migration remains `planned-not-executed`, and neither English file is a runtime dependency. +The canonical English choice is a project-governance decision and does not prove historical originality. -A future selector or combined search requires a separate reviewed architecture and functional release. The registry alone does not authorize runtime switching. +```text +physical migration: planned-not-executed +remove_or_rename_approved: false +``` + +A future selector or combined search requires separate architecture and a functional release. Registry/audit tooling does not authorize runtime switching. + +## Quality-audit boundary + +D1.3 verifies: + +```text +records per logical dataset: 4086 +ordered/unique/aligned IDs: true +preserved-field differences: 0 +structural errors: 0 +warnings: 0 +review instances: 5022 in five groups +structural gate: pass +``` + +Review instances may overlap and are not proven content errors. The audit cannot decide theological correctness, preferred wording, source intent or whether shared aliases are intentional. Corrections require separate evidence-based data PRs. ## Repository automation GitHub Actions runs: -- dependency-free regression tests; -- `scripts/validate-project.mjs` for runtime assets, active data, handoff, screenshot tooling and patchnotes; -- `scripts/validate-dataset-registry.mjs` for logical/physical IDs, exact files, hashes, canonical identity, roles, policies and runtime isolation; -- Pull Request patchnote/image enforcement; +- dependency-free application regression tests; +- project/runtime/data/handoff/screenshot/patchnote validation; +- D1.1 provenance validation; +- D1.2 registry validation; +- D1.3 report freshness and structural gate; +- PR patchnote/new-image enforcement; - JavaScript syntax checks. -`docs/DATA_PROVENANCE.md` and `scripts/validate-data-provenance.mjs` lock the D1.1 evidence baseline. `data/datasets.json`, `docs/DATASET_REGISTRY.md` and `scripts/validate-dataset-registry.mjs` lock the D1.2 identity and governance baseline. +The `v23.8.0` release workflow separately verifies exact SHA and runtime-version consistency. + +uNews is external to runtime. It discovers valid `news/*.md` from public `main` and publishes through its own Actions workflow. Telegram credentials are not stored in uDream. -The release workflow for `v23.8.0` additionally verifies the exact release SHA and consistency between `package.json`, `src/version.js` and `version.json` before creating or confirming the immutable tag and GitHub Release. +## Screenshot tooling -The uNews publisher is external to the website runtime. Its scheduled workflow lives in `sunpole/uNews`, scans public repositories owned by `sunpole`, discovers `sunpole/udream/news/*.md` and publishes previously unseen valid patchnotes to Telegram. Telegram credentials are never required by the uDream site or repository checks. +Playwright tooling under `tools/screenshots/` is isolated from runtime. The permanent capture workflow is read-only. Documentation/audit patchnotes may use an exact GitHub document/report page captured by a self-removing one-time workflow. ## External dependencies -Loaded from CDNs at runtime: +Runtime CDN dependencies: - Google Fonts (`Inter`); - Font Awesome; - html2canvas. -The core search and note rendering remain repository-hosted, but fonts, icons and image-sharing presentation may degrade offline unless the remaining external resources have already been cached by the browser. +Core search and safe note rendering are repository-hosted. Fonts/icons/image sharing may degrade offline when not already cached by the browser. ## State and privacy -Preferences and browsing history are stored locally in the browser via `localStorage`. There is no project backend receiving this state. Sharing occurs only when the user invokes the share helpers and the browser supports the required API. +Preferences/history use localStorage. There is no project backend receiving them. Sharing occurs only when the user invokes browser share helpers. ## Architectural constraints - Keep GitHub Pages compatibility. - Keep the app usable without a build step unless explicitly changed. -- Do not introduce server requirements for ordinary search. -- Separate content/database migrations from UI changes. -- Preserve old releases and runnable snapshots. -- Preserve each source dataset and translation variant with provenance; do not overwrite one variant with another. -- Keep the dataset registry outside the runtime until a separately approved architecture requires it. -- Do not delete or rename retained physical files while migration status is `planned-not-executed`. -- Follow `docs/PRODUCT_VISION.md` for product boundaries and `docs/DATASET_REGISTRY.md` for current data identity. +- Do not introduce a server for ordinary search. +- Separate data migration, content correction, UI and architecture work. +- Preserve releases, runnable snapshots, sources and translation variants. +- Keep registry and audit tooling outside runtime until a functional architecture explicitly requires otherwise. +- Do not delete/rename retained physical data while migration is `planned-not-executed`. +- D1.4 must define two-book provenance, modes, history/deep links, validation/cache/fallback and rollback before selector implementation. diff --git a/docs/DATABASE_FORMAT.md b/docs/DATABASE_FORMAT.md index 59978e3..c0af0da 100644 --- a/docs/DATABASE_FORMAT.md +++ b/docs/DATABASE_FORMAT.md @@ -8,7 +8,7 @@ data/divinity_code_ru.json At release `v23.8.0` it contains 4,086 records and remains the only active published runtime database. -The D1.2 registry file `data/datasets.json` is governance metadata. It is not a searchable record database and is not loaded by the browser application or Service Worker. +`data/datasets.json` is governance metadata. `reports/` contains deterministic audit outputs. Neither is loaded by the browser application or Service Worker. ## Record schema @@ -36,7 +36,7 @@ The D1.2 registry file `data/datasets.json` is governance metadata. It is not a | `tags` | array of strings | Filtering and category terms | | `notes` | string | Additional plain-text notes rendered with safe paragraphs and line breaks | -The maintained application does not interpret record notes as Markdown or raw HTML. Imported display values are escaped before DOM insertion. +The application does not interpret notes as Markdown or raw HTML. Imported display values are escaped before DOM insertion. ## Dataset registry schema @@ -53,36 +53,14 @@ Top-level sections: | Field | Purpose | |---|---| | `logical_datasets` | Stable logical identities independent of physical file count | -| `physical_files` | Exact JSON serializations, paths, hashes, roles and statuses | +| `physical_files` | Exact serializations, paths, hashes, roles and statuses | | `supporting_files` | Retained reports/evidence that are not record datasets | -| `canonical_selection` | Current project-governance choice and its limitation | +| `canonical_selection` | Current project-governance choice and limitation | | `reference_audit` | Runtime and historical path classification | | `migration_plan` | Future reversible migration design; not execution | -| `translation_variant_policy` | Non-destructive source/current/alternative translation rules | - -Every logical dataset entry records at least: - -- stable dataset ID; -- role/kind; -- source work and language; -- status; -- record count and ordered ID range; -- canonical JSON SHA-256; -- canonical physical file ID; -- all retained physical file IDs; -- source relationship or unknown transformation status when relevant. - -Every physical JSON file entry records at least: - -- stable physical file ID; -- logical dataset ID; -- path and format; -- role and status; -- exact byte count; -- raw SHA-256; -- canonical JSON SHA-256; -- record count; -- retained introduction commit. +| `translation_variant_policy` | Non-destructive translation rules | + +Each logical dataset records ID, role, language, status, record count, ordered ID range, canonical hash, physical files and source relationship. Each physical file records ID, path, role, status, bytes, raw/canonical hashes, record count and introduction commit. ## Current registered identities @@ -90,8 +68,8 @@ Every physical JSON file entry records at least: | ID | Classification | Runtime use | |---|---|---| -| `source-divinity-code-en` | One English source logical dataset | Not used by current runtime | -| `ru-current-v1` | Current mixed-language localized and augmented dataset | Loaded by the application | +| `source-divinity-code-en` | One English source logical dataset | no | +| `ru-current-v1` | Current mixed-language localized and augmented dataset | yes | ### Physical files @@ -102,7 +80,7 @@ Every physical JSON file entry records at least: | `ru-current-v1-runtime` | `data/divinity_code_ru.json` | active runtime serialization | yes | | `data-quality-report-v1` | `data/report.txt` | historical supporting report | no | -Corrected raw SHA-256 values: +Raw SHA-256 values: ```text data/bd2.json 814c5d33444160e6f1ab20278f9356090ec0e9cc04943cd14ad99d9038be6e28 @@ -111,80 +89,103 @@ data/divinity_code_ru.json 1def80216e238b0c2a8640aaf1b4e95dd0669d5944a67f4e7c44 data/report.txt dec064b826ae20b1ded2f9bcbfeed7d1d4d1c94592ef9d0774e495689e59da1d ``` -`data/bd2.json` and `data/db.json` share canonical JSON SHA-256: +The English files share canonical JSON SHA-256: ```text 5ebe0d973f9cfd1c9db65a9d5abebe0ca16788261219299a710ed9fe78bb25d1 ``` -They contain the same ordered records and values but use different physical serialization/formatting. They are one logical dataset, not two translations. - -The active dataset canonical JSON SHA-256 is: +They are one logical dataset, not two translations. The active dataset canonical JSON SHA-256 is: ```text c3682a50dec3303b7a100abefb27befd2dfd5d84e66d219252cf099b1a0fcd64 ``` -It keeps `id`, `symbol`, `description`, `source` and `date_added` unchanged while modifying `aliases`, `notes` and `tags`. The exact generation and translation pipeline remains unknown. - -Full file-level evidence is in `docs/DATA_PROVENANCE.md`. Stable identity, canonical selection and rollback policy are in `docs/DATASET_REGISTRY.md`. +The active dataset keeps `id`, `symbol`, `description`, `source` and `date_added`; `aliases`, `notes` and `tags` differ. The historical transformation pipeline remains unknown. ## Canonical physical decision -D1.2 selects `data/bd2.json` / `source-divinity-code-en-bd2` as the maintained canonical physical serialization. - -This is a project-governance decision based on retained Git order, identical canonical content, smaller physical size and absence of runtime dependency. It does not prove historical originality or a particular external source edition. +D1.2 selects `data/bd2.json` / `source-divinity-code-en-bd2` as the maintained canonical physical serialization through a project-governance decision. -`data/db.json` remains retained and unchanged. Migration status is: +This does not prove historical originality or a particular external edition. `data/db.json` remains retained and unchanged. ```text planned-not-executed remove_or_rename_approved: false ``` -Any later physical migration must be a separate reversible patch with an immutable checkpoint and rollback. +Any later physical migration requires a separate reversible patch, immutable checkpoint and rollback. -## Target dataset model +## D1.3 audit contract -| Logical role | Intended quantity | Notes | -|---|---:|---| -| Canonical source dataset | 1 | Original-language or closest retained source representation | -| Current Russian translation | 1 | Published runtime version | -| Alternative Russian translation A | 0–1 | Independent candidate for comparison | -| Alternative Russian translation B | 0–1 | Second independent candidate only when quality justifies it | +Read-only audit inputs are the registered canonical source and current datasets. Outputs: -Two semantically identical serializations are one logical dataset, not two translations. The project does not require two alternative translations at any cost. When only one reliable Russian translation exists, the correct product state is one source dataset plus one Russian translation. +```text +reports/data-quality-audit.json +reports/data-quality-audit.md +``` -## Dataset and translation preservation +The permanent command is: -The current single-file runtime is not permission to overwrite other sources or translation variants. +```bash +node scripts/audit-data-quality.mjs +node scripts/audit-data-quality.mjs --check +``` -A new translation or corrected edition must be introduced as a new recoverable logical dataset and physical file. The prior version remains available until an explicit retention decision is documented. +Audit report schema version: `1`. The report records: -A future merged search may combine results for users, but it must preserve the source identity of every record and must not destroy separate source datasets. +- audit and registry baselines; +- exact dataset paths, physical IDs, bytes and hashes; +- record count, ordered/unique ID status and range; +- field population, source/date distributions and selected text statistics; +- source/current alignment and difference counts; +- aggregated findings with severity, rule ID, count and sample IDs; +- structural gate and limitations. -## AI-assisted candidate datasets +Severity meanings: + +| Severity | Meaning | +|---|---| +| `error` | Structural or registered-identity violation; fails CI | +| `warning` | Likely hygiene issue requiring review | +| `review` | Ambiguity/content candidate not safe to auto-fix | +| `info` | Statistics/observation | -An API-generated translation is stored as a separate candidate dataset. It must never overwrite `data/divinity_code_ru.json` in place. +Current verified result: -For every AI-assisted run, record: +```text +records per logical dataset: 4086 +ordered/unique/aligned IDs: true +preserved-field differences: 0 +changed fields: aliases 4083, notes 4086, tags 4086 +structural errors: 0 +warnings: 0 +review instances: 5022 in 5 groups +structural gate: pass +``` -- exact input dataset and hash; -- fields submitted for translation; -- provider and model identifier; -- prompt-template version; -- date and relevant parameters; -- checkpoint/resume state; -- raw-output location when retained; -- normalized candidate hash; -- automatic validation report; -- human-review status. +Review instances can overlap and do not equal proven errors. Shared aliases and empty source notes may be intentional. The audit never modifies data. -The API key itself is never stored in Git, JSON data, browser code, logs, reports or patchnotes. Full rules are in `docs/TRANSLATION_WORKFLOW.md`. +Full rules are in `docs/DATA_QUALITY_AUDIT.md`; factual results are in `reports/data-quality-audit.md`. -## Validation checklist +## Target dataset model + +| Logical role | Intended quantity | Notes | +|---|---:|---| +| Canonical source dataset | 1 | Original-language or closest retained source representation | +| Current Russian/localized dataset | 1 | Published runtime version | +| Alternative Russian translation A | 0–1 | Independent candidate | +| Alternative Russian translation B | 0–1 | Second candidate only when quality justifies it | -For the current protected baseline: +Equivalent serializations are one logical dataset. A new translation or corrected edition must use a new recoverable logical dataset and physical file; the prior version remains available. + +## AI-assisted candidate datasets + +An API-generated translation must be a separate candidate and must never overwrite `data/divinity_code_ru.json`. + +For every run, record source ID/hash, fields, provider/model, prompt version, parameters, checkpoint state, candidate hash, validation and human-review status. API keys never enter Git, JSON, browser code, logs, reports or patchnotes. + +## Validation checklist ```bash jq empty data/divinity_code_ru.json @@ -192,19 +193,19 @@ jq 'length' data/divinity_code_ru.json jq 'map(.id) | length == (unique | length)' data/divinity_code_ru.json node scripts/validate-data-provenance.mjs node scripts/validate-dataset-registry.mjs +node scripts/audit-data-quality.mjs --check node scripts/validate-project.mjs ``` -The registry validator checks real file bytes and content rather than trusting registry text. It verifies exact paths, sizes, raw/canonical hashes, record counts, ordered IDs, semantic identity, roles, policies and runtime isolation. - -Record-count changes, ID changes, source changes, translation changes or registry identity changes must be explained in the commit, changelog and matching migration/data document. +Record-count, ID, source, translation, registry or audit-contract changes require a separate reviewed patch with changelog and rollback documentation. ## Content constraints -- Preserve original source meaning and references. +- Preserve source meaning and references. - Distinguish source text, translation, editorial notes, aliases and generated tags. - Do not invent biblical references or silently strengthen interpretations. -- Record the tool/script and source revision used for bulk transformations. +- Record tools and source revisions for bulk transformations. - Do not silently merge records from different books or editions. -- Do not delete retained source or translation variants as routine cleanup. +- Do not delete retained source/translation variants as routine cleanup. - Do not promote an AI-generated candidate without human review and a separate data release. +- Do not fix D1.3 findings in the audit PR itself. diff --git a/docs/DATA_QUALITY_AUDIT.md b/docs/DATA_QUALITY_AUDIT.md new file mode 100644 index 0000000..33e0498 --- /dev/null +++ b/docs/DATA_QUALITY_AUDIT.md @@ -0,0 +1,189 @@ +# Data-quality audit + +## Scope + +D1.3 audits the registered uDream datasets without modifying them. + +Read-only inputs: + +```text +data/datasets.json +data/bd2.json +data/db.json +data/divinity_code_ru.json +data/report.txt +``` + +The audit uses the canonical physical serialization of each registered logical dataset: + +```text +source-divinity-code-en -> data/bd2.json +ru-current-v1 -> data/divinity_code_ru.json +``` + +`data/db.json` remains protected by provenance and registry validation as an equivalent retained serialization. The quality audit does not count it as a third logical dataset. + +## Outputs + +```text +reports/data-quality-audit.json +reports/data-quality-audit.md +``` + +Both files are generated deterministically by: + +```bash +node scripts/audit-data-quality.mjs +``` + +CI verifies that committed reports are current: + +```bash +node scripts/audit-data-quality.mjs --check +``` + +The script never writes to `data/`, runtime, PWA, saved versions or archives. + +## Severity model + +| Severity | Meaning | Automatic action | +|---|---|---| +| `error` | Structural or registered-identity contract violation | Fails the audit gate and CI | +| `warning` | Likely formatting or data-hygiene problem | Report only; requires review before correction | +| `review` | Ambiguity or content question that cannot be resolved safely by a heuristic | Human/source review | +| `info` | Statistics and observations | Informational | + +A warning or review finding is not proof that the content is wrong. The audit reports candidates; it does not rewrite them. + +## Structural checks + +The audit verifies for each logical dataset: + +- JSON root is an array; +- record count is 4,086; +- IDs are integers, unique and ordered from 1 through 4,086; +- required fields exist with expected types; +- `symbol`, `description`, `source`, `date_added` and `notes` are strings; +- `aliases` and `tags` are arrays of strings; +- date values use a valid `YYYY-MM-DD` calendar date; +- exact raw and canonical hashes are recorded in the report; +- source/current IDs align; +- registered preserved fields remain equal between source and current datasets. + +Any structural or preserved-field mismatch is an `error`. + +## Data-hygiene checks + +The audit detects and aggregates: + +- empty or whitespace-only required text; +- leading/trailing whitespace; +- empty array items; +- duplicate normalized aliases or tags inside one record; +- Unicode replacement characters; +- unexpected control characters; +- HTML-like markup in imported data; +- unusually long text values; +- invalid dates. + +These checks use `warning` or `review` because automatic correction may change meaning. + +## Identity and ambiguity checks + +The audit reports: + +- duplicate normalized primary symbols across records; +- normalized aliases shared by multiple records; +- aliases that match another record's primary symbol; +- aliases equal to their own primary symbol; +- records with identical content except for ID. + +Such cases may be intentional redirects, synonyms or source-book structure. They require human/source review and are never auto-merged. + +## Source/current comparison + +The registered preserved fields are: + +```text +id +symbol +description +source +date_added +``` + +Differences in those fields are structural `error` findings. + +The expected transformable fields are: + +```text +aliases +notes +tags +``` + +Their difference counts are reported as metrics, not automatically classified as errors. + +## Statistics + +For each dataset the report includes: + +- path, physical ID, byte size and hashes; +- record count and ID range; +- number of records without aliases or tags; +- non-empty field counts; +- source and date distributions; +- counts of cross-reference-like and biblical-reference-like text; +- number of distinct normalized symbols and aliases. + +These statistics help plan later human review without changing the data. + +## Determinism + +The report intentionally omits a generation timestamp. It is derived only from: + +- registered inputs; +- audit version and rules; +- deterministic sorting and sampling. + +Repeated runs on the same commit must produce byte-identical JSON and Markdown reports. `--check` fails when committed reports are missing or stale. + +## Limitations + +The audit cannot automatically determine: + +- theological correctness; +- preferred Russian wording; +- whether a shared alias is intentional; +- whether two similar interpretations should be merged; +- whether a biblical reference is semantically correct; +- the exact historical translation pipeline; +- the completeness of previous human review. + +Cross-reference-like and biblical-reference-like strings are counted but not validated against the source PDFs in D1.3. + +## Correction workflow + +D1.3 does not correct findings. + +Any later fix must: + +1. identify the exact finding and source evidence; +2. use a separate branch and data Pull Request; +3. preserve old file hashes and rollback; +4. update the relevant dataset version/identity when content changes materially; +5. run provenance, registry, audit and project validation; +6. receive human content review; +7. publish a factual data patchnote. + +## Required commands + +```bash +npm test +node scripts/validate-data-provenance.mjs +node scripts/validate-dataset-registry.mjs +node scripts/audit-data-quality.mjs --check +node scripts/validate-project.mjs +``` + +The source files remain untouched throughout the audit. \ No newline at end of file diff --git a/docs/FILE_MAP.md b/docs/FILE_MAP.md index 0feeb92..34a3ac9 100644 --- a/docs/FILE_MAP.md +++ b/docs/FILE_MAP.md @@ -25,19 +25,23 @@ | `preview.jpg` | Social sharing preview | Low | | `.nojekyll` | GitHub Pages behavior | Medium | -## Dataset registry and data evidence +## Dataset registry, provenance and audit | Path | Status | |---|---| -| `data/datasets.json` | Machine-readable D1.2 registry; not loaded by the browser runtime | -| `data/divinity_code_ru.json` | Physical ID `ru-current-v1-runtime`; active mixed-language localized and augmented runtime dataset; 4,086 records | -| `data/bd2.json` | Physical ID `source-divinity-code-en-bd2`; canonical retained serialization of logical dataset `source-divinity-code-en` | -| `data/db.json` | Physical ID `source-divinity-code-en-db`; retained equivalent compatibility serialization of the same logical dataset | -| `data/report.txt` | Supporting ID `data-quality-report-v1`; historical generation/quality summary | -| `docs/DATA_PROVENANCE.md` | D1.1 evidence: hashes, Git history, comparisons, corrections, inferences and unknowns | -| `docs/DATASET_REGISTRY.md` | D1.2 identity, canonical selection, reference-audit, migration and rollback rules | -| `scripts/validate-data-provenance.mjs` | Permanent byte/hash/schema/canonical-identity and field-difference validator | -| `scripts/validate-dataset-registry.mjs` | Permanent registry/file/role/policy/runtime-isolation validator | +| `data/datasets.json` | Machine-readable D1.2 registry; not loaded by browser runtime | +| `data/divinity_code_ru.json` | `ru-current-v1-runtime`; active dataset; 4,086 records | +| `data/bd2.json` | `source-divinity-code-en-bd2`; canonical retained source serialization | +| `data/db.json` | `source-divinity-code-en-db`; retained equivalent compatibility serialization | +| `data/report.txt` | `data-quality-report-v1`; historical generation/quality summary | +| `docs/DATA_PROVENANCE.md` | D1.1 file evidence, history, hashes and unknowns | +| `docs/DATASET_REGISTRY.md` | D1.2 identities, canonical selection, migration and rollback | +| `docs/DATA_QUALITY_AUDIT.md` | D1.3 rules, severity model, determinism and limitations | +| `reports/data-quality-audit.json` | Deterministic machine-readable D1.3 report | +| `reports/data-quality-audit.md` | Deterministic human-readable D1.3 report | +| `scripts/validate-data-provenance.mjs` | File-level provenance validator | +| `scripts/validate-dataset-registry.mjs` | Registry/file/role/policy/runtime-isolation validator | +| `scripts/audit-data-quality.mjs` | Read-only report generator and `--check` stale-report/structural gate | Logical dataset IDs: @@ -54,15 +58,17 @@ source-divinity-code-en-db ru-current-v1-runtime ``` -`data/bd2.json` and `data/db.json` are raw-distinct but parsed/canonical-JSON equal. D1.2 selects `data/bd2.json` as the canonical maintained physical serialization through a project-governance decision. This does not prove historical originality. `data/db.json` remains retained and unchanged. +D1.3 confirms both logical datasets have 4,086 unique ordered IDs and zero preserved-field differences. Reports contain 0 structural errors, 0 warnings and 5,022 overlapping human-review instances in five aggregated groups. These are candidates for review, not proven errors. -No retained database or translation variant may be destructively replaced. The target logical topology is one canonical source dataset, one current localized dataset and up to two independent alternative Russian translations. Equivalent serializations are one logical source dataset, not separate variants. +`data/bd2.json` and `data/db.json` are raw-distinct but parsed/canonical-JSON equal. `data/bd2.json` is the canonical maintained serialization by project-governance decision; historical originality is not claimed. `data/db.json` remains retained and unchanged. + +No retained database or translation variant may be destructively replaced. Physical migration remains `planned-not-executed`. ## Planned translation tooling No translation API client or batch script is part of the current runtime. -Future D1 tooling may add: +Future tooling may add: | Planned area | Rule | |---|---| @@ -71,61 +77,61 @@ Future D1 tooling may add: | `.env.example` | May contain only an empty variable name, never a real key | | translation manifests and reports | Must record source hash, model, prompt version, output hash and review status | -Any such tooling belongs under a separate implementation PR and must not be imported by the public browser application. +Such tooling belongs in a separate PR and must not be imported by the public browser application. ## Versioning, handoff and restoration | Path or reference | Role | |---|---| -| `WORK_STATUS.md` | Live cross-device and cross-chat task lock: active branch, actual progress, pause point and exact next action | -| `docs/AI_GITHUB_WORKFLOW.md` | Binding priority order and execution protocol for devices, chats and AI agents | -| `VERSION.md` | Current release and documentation/provenance/registry baseline | +| `WORK_STATUS.md` | Live cross-device task lock and exact next action | +| `docs/AI_GITHUB_WORKFLOW.md` | Binding device/chat/agent protocol | +| `VERSION.md` | Current release and documentation/data baseline | | `version.json` | Deployed runtime version check | | `versions/index.html` | User-facing saved-version launcher | | `versions/v3.0.0/` | Independently runnable pre-cleanup fallback | | Git tag `v3.0.0` | Exact immutable pre-cleanup source | | Git tag `v3.6.0` | Exact immutable completed M5 source | | Git tag `v23.7.0` | Previous unified-version restoration checkpoint | -| Git tag `v23.8.0` | Current immutable functional restoration checkpoint at `24dece593bea679485057d7551a2583f7f1f5acf` | +| Git tag `v23.8.0` | Current functional restoration checkpoint at `24dece593bea679485057d7551a2583f7f1f5acf` | -`WORK_STATUS.md` is intentionally mutable and records current work. Git tags and Releases are immutable restoration sources. These roles must not be confused. +`WORK_STATUS.md` is mutable current state. Tags and Releases are immutable restoration sources. -The direct stable source download is `https://github.com/sunpole/udream/archive/refs/tags/v23.8.0.zip`. +Direct stable download: `https://github.com/sunpole/udream/archive/refs/tags/v23.8.0.zip`. ## Playwright screenshot tooling -These files are development and CI tooling only. They are not imported by the public site, PWA or Service Worker. +These files are development/CI tooling only and are not imported by the site or PWA. | Path | Role | |---|---| -| `tools/screenshots/package.json` | Private isolated package, capture scripts and pinned Playwright dependency | -| `tools/screenshots/package-lock.json` | Exact lock for screenshot tooling | -| `tools/screenshots/playwright.config.mjs` | One-worker Chromium configuration and local HTTP server | +| `tools/screenshots/package.json` | Isolated package and pinned Playwright dependency | +| `tools/screenshots/package-lock.json` | Exact screenshot-tooling lock | +| `tools/screenshots/playwright.config.mjs` | Chromium configuration and local HTTP server | | `tools/screenshots/prepare-artifacts.mjs` | Guarded artifact cleanup | -| `tools/screenshots/capture.spec.mjs` | Allowlisted scenario runner, assertions, PNG capture and manifest generation | -| `tools/screenshots/scenarios/*.json` | Factual desktop/mobile startup, search and alias scenarios | -| `tools/screenshots/README.md` | Local and GitHub Actions usage guide | -| `tools/screenshots/v23.8.7-selected-image.json` | Exact provenance of the approved screenshot | -| `.github/workflows/capture-screenshots.yml` | Permanent read-only Chromium capture and artifact workflow | -| `scripts/validate-screenshot-tooling.mjs` | Package, workflow, scenario and runtime-isolation validation | -| `.gitignore` | Excludes browser dependencies, reports and generated artifacts | +| `tools/screenshots/capture.spec.mjs` | Allowlisted scenario runner, assertions and manifest generation | +| `tools/screenshots/scenarios/*.json` | Factual desktop/mobile scenarios | +| `tools/screenshots/README.md` | Local and Actions usage guide | +| `.github/workflows/capture-screenshots.yml` | Permanent read-only capture workflow | +| `scripts/validate-screenshot-tooling.mjs` | Package/workflow/scenario/runtime-isolation validation | +| `.gitignore` | Excludes dependencies, reports and generated artifacts | -Generated artifacts are stored under `artifacts/screenshots/` and are never committed automatically. +Generated browser artifacts are stored under `artifacts/screenshots/` and are never committed automatically. ## News, screenshot evidence and automation | Path | Role | |---|---| | `news/*.md` | Factual patchnotes discovered by uNews | -| `news/*.{jpg,png}` | New real Telegram visuals stored beside their patchnotes | -| `docs/SCREENSHOT_AUTOMATION.md` | Real-screenshot definition, Playwright workflow and review contract | -| `scripts/validate-project.mjs` | Repository, handoff, screenshot tooling, provenance, active database, patchnote and image validation | +| `news/*.{jpg,png}` | New real Telegram visuals beside patchnotes | +| `docs/SCREENSHOT_AUTOMATION.md` | Screenshot evidence contract | +| `scripts/validate-project.mjs` | Repository, handoff, active data, screenshot, provenance and patchnote validation | | `scripts/validate-dataset-registry.mjs` | D1.2 registry validation required by CI | -| `scripts/validate-patchnote-diff.mjs` | Requires a new patchnote and newly added screenshot evidence in each Pull Request | -| `.github/workflows/validate.yml` | Automatic tests, project validation, registry validation, PR patchnote enforcement and syntax checks | -| `.github/workflows/publish-v23.8.0.yml` | Immutable one-release workflow for tag and GitHub Release publication | -| `.github/pull_request_template.md` | Review checklist including handoff and screenshot provenance | -| `.github/CODEOWNERS` | Default repository owner for review routing | +| `scripts/audit-data-quality.mjs` | D1.3 report freshness and structural audit gate | +| `scripts/validate-patchnote-diff.mjs` | Requires new patchnote and image in each PR | +| `.github/workflows/validate.yml` | Tests, project/registry/audit validation, patchnote enforcement and syntax checks | +| `.github/workflows/publish-v23.8.0.yml` | Immutable release workflow | +| `.github/pull_request_template.md` | Review checklist | +| `.github/CODEOWNERS` | Review routing | ## Archive @@ -133,35 +139,37 @@ Generated artifacts are stored under `artifacts/screenshots/` and are never comm |---|---| | `_archive/legacy-versions/` | UI iterations `002–019` | | `_archive/admin-versions/` | Admin prototypes `admin1–admin7` | -| `_archive/old-data/` | Older database files and variants, including historical `db_v2` paths | +| `_archive/old-data/` | Older database variants and historical paths | | `_archive/source-files/` | PDFs and historical screenshots | -Do not delete archived files as routine cleanup. Git history is valuable, but the archive also documents intermediate product decisions and may contain files that were never released separately. +Do not delete archived files as routine cleanup. ## Project governance and documentation | Path | Role | |---|---| -| `WORK_STATUS.md` | Mandatory start/pause/completion record for continuing work across devices, chats and agents | -| `docs/AI_GITHUB_WORKFLOW.md` | Unified GitHub-centered operating protocol and conflict recovery | -| `AGENTS.md` | Binding instructions for future coding agents | -| `README.md` | Entry point, current overview, installation, screenshots and stable download links | -| `docs/PRODUCT_VISION.md` | Current mission, final product direction and data-preservation rules | -| `docs/TRANSLATION_WORKFLOW.md` | Target translation variants and safe AI-assisted workflow | -| `docs/DATA_PROVENANCE.md` | Verified D1.1 provenance, hashes, Git history and corrected dataset classification | -| `docs/DATASET_REGISTRY.md` | Verified D1.2 logical/physical identities and migration policy | -| `docs/SCREENSHOT_AUTOMATION.md` | Real screenshot and implemented Playwright workflow contract | +| `WORK_STATUS.md` | Mandatory start/pause/completion handoff | +| `docs/AI_GITHUB_WORKFLOW.md` | Unified GitHub-centered protocol | +| `AGENTS.md` | Binding agent instructions | +| `README.md` | Entry point, installation, state and downloads | +| `docs/PRODUCT_VISION.md` | Mission and data-preservation rules | +| `docs/TRANSLATION_WORKFLOW.md` | Translation variants and safe AI-assisted workflow | +| `docs/DATA_PROVENANCE.md` | D1.1 provenance evidence | +| `docs/DATASET_REGISTRY.md` | D1.2 logical/physical identities and migration policy | +| `docs/DATA_QUALITY_AUDIT.md` | D1.3 quality-audit contract | +| `reports/data-quality-audit.md` | Current readable audit results | +| `docs/SCREENSHOT_AUTOMATION.md` | Playwright evidence contract | | `VERSION.md` | Release and baseline state | -| `CHANGELOG.md` | User/project-visible change history | -| `ROADMAP.md` | Completed work, next approved phase and later backlog | -| `docs/PROJECT_STATE.md` | Detailed verified state and risks | +| `CHANGELOG.md` | Change history | +| `ROADMAP.md` | Completed work, next phase and backlog | +| `docs/PROJECT_STATE.md` | Verified state and risks | | `docs/ARCHITECTURE.md` | Runtime design | -| `docs/MODULARIZATION_PLAN.md` | Completed staged ES-module migration plan | -| `docs/DATABASE_FORMAT.md` | Current data contract, registry classification and checks | -| `docs/RELEASE_AND_ROLLBACK.md` | Safe release and recovery procedure | -| `docs/HISTORICAL_CONTEXT.md` | Earlier project description and relationship to current docs | -| `docs/NEWS_PUBLISHING.md` | uNews, Telegram and screenshot-provenance workflow | -| `docs/CONTENT_AND_RIGHTS.md` | Boundary between original code and third-party content | -| `THIRD_PARTY_NOTICES.md` | Public notice for source works and other third-party material | -| `CONTRIBUTING.md` | Contribution and Pull Request rules | -| `CITATION.cff` | GitHub citation metadata for the software project | +| `docs/MODULARIZATION_PLAN.md` | Completed ES-module migration | +| `docs/DATABASE_FORMAT.md` | Data contract and registered identities | +| `docs/RELEASE_AND_ROLLBACK.md` | Release/recovery procedure | +| `docs/HISTORICAL_CONTEXT.md` | Historical relationship to current docs | +| `docs/NEWS_PUBLISHING.md` | uNews and image provenance workflow | +| `docs/CONTENT_AND_RIGHTS.md` | Original/third-party boundary | +| `THIRD_PARTY_NOTICES.md` | Third-party material notice | +| `CONTRIBUTING.md` | Contribution rules | +| `CITATION.cff` | Citation metadata | diff --git a/docs/PRODUCT_VISION.md b/docs/PRODUCT_VISION.md index e5cd7e0..ead4293 100644 --- a/docs/PRODUCT_VISION.md +++ b/docs/PRODUCT_VISION.md @@ -4,16 +4,16 @@ uDream превращает трудоёмкий ручной поиск по христианской литературе о снах и видениях в быстрый, понятный и проверяемый мобильный справочник. -Проект должен помогать пользователю: +Проект помогает: - найти символ, синоним или связанную тему; -- увидеть толкование и его источник; -- различать материалы разных книг и редакций; +- увидеть толкование и источник; +- различать книги, редакции и переводы; - возвращаться к просмотренным карточкам; -- пользоваться справочником как сайтом и установленной PWA; -- сохранять доступ к проверенным прежним версиям проекта. +- пользоваться справочником как сайтом и PWA; +- сохранять доступ к проверенным прежним версиям. -uDream является поисковым интерфейсом к источникам, а не заменой исходных книг и не официальным изданием их содержания. +uDream является поисковым интерфейсом к источникам, а не заменой книг и не официальным изданием их содержания. ## Для кого создаётся продукт @@ -24,177 +24,188 @@ uDream является поисковым интерфейсом к источ - изучение связанных символов и библейских ссылок; - сравнение толкований из разных источников; - повторное открытие карточек из локальной истории; -- работа без постоянного соединения после установки PWA; -- подготовка и проверка новых вариантов базы без разрушения опубликованной версии. +- работа после установки PWA; +- безопасная подготовка и проверка новых вариантов данных. ## Конечный образ продукта Завершённый uDream должен включать: -1. быстрый и предсказуемый поиск по символам, алиасам, описаниям и тегам; +1. быстрый предсказуемый поиск; 2. понятное указание источника каждого толкования; -3. поддержку двух исходных книг без смешивания происхождения текста; -4. безопасный выбор отдельной базы, варианта перевода или согласованного объединённого поиска; +3. поддержку двух книг без потери происхождения; +4. безопасный выбор базы/перевода или согласованный объединённый поиск; 5. мобильный, доступный и устанавливаемый интерфейс; -6. документированный процесс подготовки, перевода, проверки и публикации данных; -7. административный процесс, который не подвергает риску публичную базу; +6. воспроизводимый процесс подготовки, перевода, проверки и публикации данных; +7. административный процесс вне риска для публичной базы; 8. неизменяемые релизы и понятный откат; -9. автоматические тесты данных, поиска, PWA и запуска приложения; -10. прозрачные правила прав, происхождения и использования сторонних материалов. +9. автоматические тесты данных, поиска, PWA и запуска; +10. прозрачные правила прав и происхождения материалов. ## Текущее состояние -Стабильная версия `v23.8.0` уже предоставляет: +Стабильная версия `v23.8.0` предоставляет: - статический сайт на GitHub Pages; -- PWA с обновлением без зависания на старом кэше; +- PWA с безопасным обновлением; - активную базу из 4 086 записей; -- поиск, релевантность, строгие режимы фильтрации и русские алиасы; +- поиск, релевантность, строгие фильтры и русские алиасы; - историю, темы, RU/EN-интерфейс и sharing; -- нативные ES Modules без сборщика и сервера; -- регрессионные тесты, валидатор, GitHub Actions и uNews; -- неизменяемые Git-релизы и запускаемый fallback `versions/v3.0.0/`. +- ES Modules без сборщика и сервера; +- тесты, валидаторы, GitHub Actions и uNews; +- неизменяемые релизы и runnable fallback. -Документационный/data baseline `23.8.9` дополнительно фиксирует: +Data baseline `23.8.10` дополнительно фиксирует: -- проверенное происхождение текущих файлов; -- стабильные logical dataset IDs; -- стабильные physical file IDs; -- одну canonical и одну retained equivalent английскую сериализацию; -- отдельный текущий localized dataset; -- машинно-читаемый registry и permanent validator; -- обратимый будущий migration plan без выполнения миграции. +- доказанное происхождение текущих файлов; +- stable logical/physical dataset IDs; +- canonical и retained equivalent английские сериализации; +- current localized dataset; +- machine-readable registry и permanent validators; +- deterministic read-only quality reports; +- passing structural audit gate без изменения данных. ## Обязательная стратегия данных ### Никаких разрушительных замен -Ни одна исходная база, редакция или версия перевода не должна молча перезаписывать другую. +Ни одна база, редакция или версия перевода не должна молча перезаписывать другую. -Для каждой базы или редакции должны сохраняться: +Для каждого набора сохраняются: -- стабильный logical dataset ID; -- один или несколько stable physical file IDs; -- собственный файл или каталог; -- название источника; -- язык; -- версия или дата; -- способ получения; -- инструмент и правила преобразования; -- связь с исходным материалом; +- stable logical dataset ID; +- stable physical file IDs; +- source/book и language; +- version/date; +- получение и transformation method; - exact hashes; -- результаты проверки. +- validation/review status; +- связь с предыдущими версиями. -### Текущие утверждённые идентификаторы +### Текущие идентификаторы ```text source-divinity-code-en ru-current-v1 -source-divinity-code-en-bd2 -source-divinity-code-en-db -ru-current-v1-runtime +source-divinity-code-en-bd2 -> data/bd2.json +source-divinity-code-en-db -> data/db.json +ru-current-v1-runtime -> data/divinity_code_ru.json ``` -`data/bd2.json` и `data/db.json` физически различаются, но содержат одинаковый parsed/canonical JSON. Они являются двумя сериализациями одного logical dataset `source-divinity-code-en`, а не двумя переводами. +`data/bd2.json` и `data/db.json` — raw-distinct сериализации одного canonical JSON. `data/bd2.json` выбран canonical maintained serialization как project-governance decision, а не доказанный исторический оригинал. `data/db.json` остаётся retained equivalent compatibility serialization. -D1.2 выбирает `data/bd2.json` как canonical maintained physical serialization. Это project-governance decision, а не доказательство исторического оригинала или подтверждённой исходной редакции. `data/db.json` остаётся retained equivalent compatibility serialization. +```text +physical migration: planned-not-executed +remove_or_rename_approved: false +``` -### Сохранение вариантов перевода +### Сохранение переводов -Если создаётся новый вариант перевода, прежний вариант сохраняется. Новый перевод получает отдельный dataset ID, physical file ID, версию и описание отличий. +Новый перевод получает новый dataset ID и physical file. Прежний вариант остаётся recoverable. Запрещено: -- массово менять опубликованный перевод без отдельного data-релиза; -- смешивать исходный текст, перевод и редакторские дополнения без маркировки; -- удалять старый вариант только потому, что появился более новый; -- менять стабильные ID без документированной миграции. +- массово менять опубликованный перевод без data release; +- смешивать источник, перевод и редакторские дополнения без маркировки; +- удалять старый вариант только из-за появления нового; +- менять stable IDs без migration. + +Целевая модель: -### Целевая модель переводов +1. один canonical source dataset; +2. один current localized dataset; +3. до двух действительно независимых альтернативных русских переводов; +4. меньше вариантов, если качественно существует только один перевод. -После завершения D1 проект стремится иметь: +### AI как вспомогательный инструмент -1. один canonical logical source dataset `source-divinity-code-en`; -2. один текущий опубликованный русский/localized dataset `ru-current-v1`; -3. до двух самостоятельных альтернативных русских переводов для сравнения качества. +DeepSeek или другой API может создать только отдельный candidate dataset. Результат не считается автоматически правильным или опубликованным. -Если два качественно независимых альтернативных перевода получить не удастся, проект не создаёт искусственные варианты ради количества. В таком случае сохраняются исходная база и один надёжный русский перевод. +API key не попадает в браузер, JSON, Git, патчноуты, screenshots или logs. Каждый запуск фиксирует source hash, candidate ID, model, prompt version, parameters, output hash, validation и human review. -Подробный порядок зафиксирован в [документе о переводах и AI-assisted workflow](TRANSLATION_WORKFLOW.md), а текущая identity-модель — в [реестре наборов](DATASET_REGISTRY.md). +### Две исходные книги -### DeepSeek как вспомогательный инструмент +До утверждения D1.4 материалы двух книг должны оставаться различимыми. -DeepSeek API может использоваться только для создания отдельного кандидата перевода или расширения непереведённых полей. Результат модели не считается автоматически правильным, опубликованным или богословски проверенным. +Возможные будущие режимы: -Ключ API не должен попадать в браузерный код, JSON-базы, Git, патчноуты или логи. Каждый запуск создаёт новую кандидатную версию и не перезаписывает активную базу. +- поиск по одной книге; +- поиск по выбранной редакции/переводу; +- объединённый поиск с visible provenance; +- side-by-side comparison. -### Две исходные книги +Объединённый индекс не должен уничтожать отдельные source datasets. + +## Проверяемость данных -До утверждения продуктовой модели данные двух книг должны оставаться различимыми по происхождению. +D1.3 добавил deterministic read-only audit: -Будущая реализация может поддерживать: +```text +scripts/audit-data-quality.mjs +reports/data-quality-audit.json +reports/data-quality-audit.md +``` + +Подтверждено: -- поиск только по одной книге; -- поиск по выбранной редакции или переводу; -- объединённый поиск с обязательным указанием источника; -- сравнение нескольких карточек по одному символу. +- 4 086 unique ordered IDs в source/current datasets; +- полное ID alignment; +- 0 preserved-field differences; +- 0 structural errors и 0 warnings; +- 5 022 overlapping human-review instances в пяти aggregated groups. -Объединённая база не должна уничтожать отдельные исходные наборы. +Review findings — не доказанные content errors. Audit не исправляет данные. Любая коррекция требует отдельного evidence-based data PR и human review. ## Технические принципы - GitHub `main` — источник опубликованного сайта. -- Git-теги и GitHub Releases — неизменяемые точки восстановления. -- Проект остаётся статическим и совместимым с GitHub Pages, пока реальная задача не потребует backend. -- JavaScript ES Modules, JSDoc и `@ts-check` остаются основой; TypeScript вводится только при доказанной пользе. -- Изменения интерфейса, данных, PWA и архитектуры выполняются отдельными контролируемыми этапами. -- Активная база не меняется в PR, цель которого относится только к интерфейсу или документации. -- Исторические каталоги, исходные PDF и сохранённые версии не удаляются как обычная очистка. +- Tags/Releases — неизменяемые restoration points. +- Проект остаётся статическим, пока backend не доказан задачей. +- JavaScript ES Modules, JSDoc и `@ts-check` остаются основой. +- UI, data, PWA, architecture и content corrections выполняются раздельно. +- Исторические каталоги, PDFs и saved versions не удаляются как cleanup. - Секреты и персональные данные не хранятся в репозитории. -- Любое утверждение о проверке должно соответствовать реально выполненной проверке. +- Утверждения о проверке должны соответствовать реально выполненным checks. -## Критерии готовности следующего этапа +## Готовность к следующему этапу -Перед началом следующей серии должны быть выполнены следующие условия: +После D1.3 baseline `23.8.10`: -- документация не противоречит текущему runtime; -- текущий релиз и точный SHA записаны одинаково во всех документах; -- открытые задачи разделены на завершённое, следующий этап и более поздний backlog; -- определена граница следующего этапа; -- сохранена точка безопасного отката; -- изменения следующего этапа не смешивают документацию, данные и новый интерфейс без необходимости. - -После D1.2 registry baseline `23.8.9` эти условия выполнены для перехода к D1.3. +- documentation согласована с runtime; +- current release и restoration SHA зафиксированы; +- provenance, registry и quality reports воспроизводимы; +- open work разделено на next phase и backlog; +- protected data не изменены; +- next phase имеет отдельную architecture-only границу. ## Следующий этап -Следующая утверждённая задача — **D1.3: non-destructive data-quality audit design**. +Следующая утверждённая задача — **D1.4: two-book product architecture**. -D1.3 должна: +D1.4 должна: -1. определить автоматические проверки всех 4 086 записей; -2. разделить машинно проверяемые ошибки и вопросы, требующие сверки человеком; -3. определить severity levels и стабильный report format; -4. сравнить зарегистрированные source/current datasets без изменения файлов; -5. создать machine-readable и human-readable audit reports; -6. не исправлять тексты автоматически в том же патче. +1. идентифицировать и зарегистрировать вторую книгу настолько, насколько позволяет evidence; +2. сравнить switching, combined search и side-by-side comparison; +3. определить visible provenance для каждого результата; +4. определить dataset-aware history, sharing и deep links; +5. определить validation, reload, cache-clearing и automatic fallback contracts; +6. подготовить migration/rollback plan; +7. не реализовывать selector без отдельной functional phase. -После D1.3 отдельно проектируются two-book architecture и AI-assisted translation experiment. +После D1.4 отдельно выполняется D1.5 AI-assisted translation experiment. ## Что не считается завершённым -Следующие направления остаются будущей работой и не должны ошибочно описываться как готовые: - -- восстановленный и воспроизводимый pipeline перевода активной базы; -- альтернативные русские переводы; -- полная содержательная проверка всех 4 086 записей; -- фактическая migration/removal retained English serialization; -- подтверждённые URL и формулировки разрешений для каждого PDF; -- пользовательский выбор между двумя книгами и вариантами перевода; -- восстановленный административный редактор; -- полный accessibility-аудит; -- автоматические браузерные smoke-тесты; -- итоговый аудит приватности и sharing. - -Этот документ является источником истины для продуктовой цели. Текущее техническое состояние фиксирует `docs/PROJECT_STATE.md`, dataset identity — `data/datasets.json` и `docs/DATASET_REGISTRY.md`, архитектуру — `docs/ARCHITECTURE.md`, а очередность работ — `ROADMAP.md`. +- воспроизводимый historical translation pipeline; +- reviewed alternative Russian translations; +- полная human/source review всех D1.3 findings; +- physical migration/removal retained serialization; +- подтверждённые source URLs/permissions для PDFs; +- user-facing two-book/translation selector; +- восстановленный admin editor; +- полный accessibility audit; +- расширенные browser smoke tests; +- итоговый privacy/sharing audit. + +Этот документ задаёт продуктовую цель. Current technical state хранится в `docs/PROJECT_STATE.md`, dataset identity — в `data/datasets.json` и `docs/DATASET_REGISTRY.md`, audit — в `docs/DATA_QUALITY_AUDIT.md` и `reports/`, architecture — в `docs/ARCHITECTURE.md`, очередь работ — в `ROADMAP.md`. diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index 27d10eb..bc36abd 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -2,13 +2,13 @@ The current restoration checkpoint is release `v23.8.0` at commit `24dece593bea679485057d7551a2583f7f1f5acf`, published on 2026-07-20 after verification of the PWA update, cache migration, uncached version checks and installation flow. The previous checkpoint remains `v23.7.0`, while the independently runnable pre-cleanup fallback remains `v3.0.0`. -Documentation, provenance and registry baseline `23.8.9` completes D1.2. It adds stable dataset identities, a machine-readable registry, a permanent validator and a reversible future migration design. It does not change application runtime, PWA behavior, package metadata or any existing database record. +Documentation, provenance, registry and audit baseline `23.8.10` completes D1.3. It adds deterministic read-only quality reports and a permanent audit command without changing application runtime, PWA behavior, package metadata or any existing database record. ## What UDREAM is UDREAM is a public, static GitHub Pages application for searching Christian dream-symbol reference material. It has no backend, account system, database server or browser build step. -The current product mission and final direction are defined in `docs/PRODUCT_VISION.md`. Dataset identities and canonical/retained roles are defined in `data/datasets.json` and `docs/DATASET_REGISTRY.md`. Translation variants and API-assisted data preparation are governed by `docs/TRANSLATION_WORKFLOW.md`. +The product mission is defined in `docs/PRODUCT_VISION.md`. Dataset identities and canonical/retained roles are defined in `data/datasets.json` and `docs/DATASET_REGISTRY.md`. Translation variants are governed by `docs/TRANSLATION_WORKFLOW.md`. Audit rules and factual reports are in `docs/DATA_QUALITY_AUDIT.md` and `reports/`. ## GitHub-centered work status @@ -20,12 +20,10 @@ The completed process baseline includes: - cross-device handoff through `WORK_STATUS.md`; - one task, one branch and one Pull Request; -- required factual uNews patchnotes and new real images; +- factual uNews patchnotes and new real images; - read-only Playwright Chromium screenshot artifacts; -- permanent provenance and dataset-registry validators; -- automatic GitHub Actions checks. - -`ROADMAP.md` remains the long-term plan and this file remains the detailed verified state. Neither replaces the live handoff. +- permanent provenance, registry and data-quality checks; +- automatic GitHub Actions validation. ## Published runtime @@ -41,9 +39,9 @@ sw.js data/divinity_code_ru.json ``` -The maintained page is labelled `v23.8.0` in its title, header, menu and footer. The former `v19` label remains only in historical material. Release `v23.8.0` is the current immutable Git restoration checkpoint. +The maintained page is labelled `v23.8.0`. Release `v23.8.0` is the current immutable Git restoration checkpoint. -The new registry `data/datasets.json` is development/governance metadata. It is not loaded by the browser application or Service Worker. +`data/datasets.json` and `reports/` are development/governance artifacts. They are not loaded by the browser application or Service Worker. ## Verified capabilities in code @@ -55,46 +53,22 @@ The new registry `data/datasets.json` is development/governance metadata. It is - tag cloud and tag filtering; - back/forward history and breadcrumbs; - persistent history in `localStorage`; -- safe recovery from malformed or partially written saved history; +- safe recovery from malformed saved history; - light/dark themes; - Russian/English interface text; - text-selection and scrollbar preferences; - text and image sharing helpers; - automatic JSON loading with manual-file fallback; - PWA manifest and isolated service-worker registration; -- immediate Service Worker activation through `skipWaiting()` and `clients.claim()`; -- uncached checks through `version.json` and protected one-time reload when an update is detected; -- migration from older `udream-*` caches without affecting other projects on the same origin; -- install banner with system prompt support, manual instructions and standalone-app suppression. - -Release `v3.6.0` completed M5. Release `v23.7.0` added centralized version metadata, strict filters, relevance ranking, Enter submission and alias resolution. Release `v23.8.0` added the verified PWA update and installation flow and remains the current restoration checkpoint. - -Imported display values are escaped before DOM insertion. Notes are rendered as plain text with safe paragraphs and line breaks; raw HTML and Markdown from a manually selected JSON file are not interpreted. - -The current root site and installed PWA were verified on Android during development of `v23.8.0`. D1.1 and D1.2 do not alter that runtime. - -## Screenshot automation - -The screenshot system is isolated under `tools/screenshots/` and is not imported by the public site or PWA. - -Verified implementation includes: +- immediate Service Worker activation; +- uncached deployed-version checks and protected one-time reload; +- migration from older `udream-*` caches without affecting other projects; +- installation banner and standalone-mode suppression. -- a dedicated package and lockfile; -- pinned Playwright/Chromium tooling; -- one-worker deterministic scenarios; -- local HTTP serving of the exact checkout; -- JSON scenarios with allowlisted actions and assertions; -- storage cleanup, blocked Service Worker registration and disabled animations; -- PNG validation and per-scenario provenance; -- read-only `.github/workflows/capture-screenshots.yml`; -- structural validation through `scripts/validate-screenshot-tooling.mjs`. - -Documentation-only milestones may use an exact GitHub UI or document-render screenshot when the image truthfully proves the documented change. +The current root site and installed PWA were verified on Android during development of `v23.8.0`. D1.1–D1.3 do not alter that runtime. ## Data and dataset registry -D1.1 established the corrected file-level provenance. D1.2 now assigns stable identities without changing any existing file. - ### Logical datasets | Logical dataset ID | Role | Status | @@ -112,44 +86,60 @@ D1.1 established the corrected file-level provenance. D1.2 now assigns stable id `data/bd2.json` and `data/db.json` have different raw bytes and SHA-256 values but identical parsed and canonical JSON. They represent one logical English dataset, not two translations. -D1.2 selects `data/bd2.json` as the maintained canonical physical serialization through a project-governance decision. This does not prove that it is the historical original or authoritative source edition. `data/db.json` remains in place as a retained equivalent compatibility serialization. +D1.2 selects `data/bd2.json` as the maintained canonical physical serialization through a project-governance decision. This does not prove historical originality. `data/db.json` remains a retained equivalent compatibility serialization. + +The physical migration status remains `planned-not-executed`. No deletion, rename, runtime switch or selector is approved. + +## D1.3 quality-audit results -The active dataset contains 4,086 ordered IDs and preserves `symbol`, `description`, `source` and `date_added`; it differs from the English logical dataset in `aliases`, `notes` and `tags`. The exact generation and translation pipeline remains unproven. +D1.3 audits the canonical source dataset and current runtime dataset with a deterministic, read-only script. Permanent sources: -- `docs/DATA_PROVENANCE.md` — evidence, hashes, history, corrections and unknowns; -- `data/datasets.json` — machine-readable identity and policy registry; -- `docs/DATASET_REGISTRY.md` — canonical decision, retention and rollback; -- `scripts/validate-data-provenance.mjs` — file-level provenance validation; -- `scripts/validate-dataset-registry.mjs` — registry/file/policy validation. +- `docs/DATA_QUALITY_AUDIT.md` — rules, severity model and limitations; +- `scripts/audit-data-quality.mjs` — report generator and `--check` validator; +- `reports/data-quality-audit.json` — machine-readable facts; +- `reports/data-quality-audit.md` — human-readable report. -The physical migration status is `planned-not-executed`. No deletion, rename, runtime switch or selector is approved by D1.2. +Verified structural facts: -The intended target remains one canonical source dataset, one current localized dataset and up to two independent alternative Russian translations. Equivalent serializations do not count as separate variants. When only one reliable Russian translation exists, the smaller set is preferred over artificial duplication. +- both logical datasets contain 4,086 records; +- IDs are unique, ordered and aligned from 1 through 4,086; +- preserved-field differences are 0 for `id`, `symbol`, `description`, `source` and `date_added`; +- expected changed-field counts are aliases 4,083, notes 4,086 and tags 4,086; +- structural gate: pass; +- structural errors: 0; +- warnings: 0. -DeepSeek remains only a possible candidate-generation tool. No API client or key is part of the browser runtime. Any future experiment must create a separate recoverable candidate dataset and require validation plus human review. +The audit reports 5,022 human-review instances in five aggregated groups: + +- source alias collisions: 693; +- current alias collisions: 854; +- source alias-to-primary matches: 1,145; +- current alias-to-primary matches: 1,145; +- empty source notes: 1,185. + +These are not 5,022 proven errors. Findings can overlap, shared aliases may be intentional, and empty source notes may reflect source structure. The audit never changes data. ## Documentation status -The maintained documentation set now covers: +The maintained documentation covers: - GitHub-centered multi-chat and multi-device execution; - product mission and final direction; -- verified current runtime and risks; +- verified runtime and risks; - architecture and file ownership; -- active data contract and file-level provenance; +- file-level provenance; - stable logical and physical dataset identity; - canonical/retained serialization roles; +- deterministic audit rules and reports; - target translation topology and safe AI-assisted workflow; -- implemented real Chromium screenshot evidence; +- real Chromium screenshot evidence; - completed M1–M5 modularization; - versioning, releases, download and rollback; - uNews publication; - rights and third-party materials; -- next approved series and later backlog. - -The sources of truth are listed in `AGENTS.md` and mapped in `docs/FILE_MAP.md`. +- next approved work and later backlog. ## Historical material @@ -160,32 +150,32 @@ The repository retains: - old databases in `_archive/old-data/`; - PDFs and screenshots in `_archive/source-files/`. -Historical files are deliberately preserved and are not the source of truth for the current site. +Historical files are preserved and are not the source of truth for the current site. ## Known risks and unfinished work -- The exact generation and translation pipeline is still not proven. +- The exact generation and translation pipeline remains unproven. - The original external source URL and exact source edition remain unknown. -- D1.2 designs but does not execute the equivalent-serialization migration. +- The equivalent-serialization migration is designed but not executed. +- Audit review findings have not received full human/source review. - Alternative Russian translations do not yet exist as reviewed datasets. -- The full quality audit of all 4,086 active records is not yet complete. - The relationship between the two source books is not yet implemented as a user-selectable architecture. -- The administration workflow is archived rather than part of the maintained runtime. -- External CDN dependencies remain for fonts, icons and image capture. +- The administration workflow is archived rather than maintained runtime. +- External CDN dependencies remain. - Accessibility has not been formally audited. - Bundled source-PDF URLs and exact distribution statements still require archival evidence. -- Privacy and sharing behavior require a final documented review. +- Privacy and sharing behavior require final review. -These are tracked as future work in `ROADMAP.md`; they are not hidden defects in the `v23.8.0` release. +These are tracked in `ROADMAP.md`; they are not hidden defects in `v23.8.0`. ## Next approved work -D1.3 — non-destructive data-quality audit design and reporting. +D1.4 — two-book product architecture. -D1.3 may inspect and report problems but must not silently rewrite the current source or localized datasets. Content corrections require later separate reviewed data patches. +D1.4 is architecture-only unless a separate functional implementation phase is approved. It must define source identity, user-visible provenance, switching/combined/side-by-side options, history and deep-link behavior, validation, cache handling and stable fallback before any selector is implemented. -No user-facing database selector, physical data migration or AI-generated replacement is approved by D1.2. +No physical data migration, user-facing selector or AI-generated replacement is approved by D1.3. ## Safe restoration point -Release `v23.8.0` and its tag preserve the exact verified functional source at `24dece593bea679485057d7551a2583f7f1f5acf`. Release `v23.7.0` remains the previous unified-version checkpoint. The independently runnable fallback remains `versions/v3.0.0/`; no separate runnable `v23.8.0` snapshot was created. +Release `v23.8.0` and its tag preserve the exact verified functional source at `24dece593bea679485057d7551a2583f7f1f5acf`. Release `v23.7.0` remains the previous unified-version checkpoint. The independently runnable fallback remains `versions/v3.0.0/`. diff --git a/docs/TRANSLATION_WORKFLOW.md b/docs/TRANSLATION_WORKFLOW.md index b971e90..c191745 100644 --- a/docs/TRANSLATION_WORKFLOW.md +++ b/docs/TRANSLATION_WORKFLOW.md @@ -4,69 +4,69 @@ Этот документ фиксирует целевую модель переводов uDream и безопасный порядок экспериментов с API-переводом. Он не означает, что новые переводы уже созданы или что активная база должна быть немедленно изменена. -Стабильные текущие dataset IDs определены D1.2 в `data/datasets.json` и `docs/DATASET_REGISTRY.md`. +Стабильные IDs определены в `data/datasets.json` и `docs/DATASET_REGISTRY.md`. Current quality baseline хранится в `docs/DATA_QUALITY_AUDIT.md` и `reports/`. ## Текущие зарегистрированные наборы | Dataset ID | Роль | Статус | |---|---|---| -| `source-divinity-code-en` | один логический английский исходный набор | retained source | -| `ru-current-v1` | текущий локализованный и дополненный набор | runtime current | +| `source-divinity-code-en` | один logical English source dataset | retained source | +| `ru-current-v1` | текущий localized/augmented dataset | runtime current | -Текущий русский набор связан с `source-divinity-code-en`, но точный исторический translation pipeline, prompts, последовательность провайдеров и запись полной человеческой проверки остаются неизвестными. +Точный historical translation pipeline, prompts, provider sequence и полный human-review record остаются неизвестными. -## Целевой набор данных +## Целевая модель -После завершения D1 проект стремится иметь следующие логически независимые наборы: +1. `source-divinity-code-en` — canonical logical source dataset. +2. `ru-current-v1` — current published localized dataset. +3. Alternative Russian translation A — новый independent dataset ID. +4. Alternative Russian translation B — только если качество и проверяемость это оправдывают. -1. **Исходная база** — `source-divinity-code-en`, канонический логический набор на языке исходного материала. -2. **Текущий русский вариант** — `ru-current-v1`, опубликованный runtime-набор. -3. **Альтернативный русский перевод A** — отдельный вариант с новым dataset ID. -4. **Альтернативный русский перевод B** — второй независимый вариант, только если его удастся получить и проверить с достаточным качеством. - -Это целевая модель, а не требование искусственно создавать лишние копии. Если надёжно получается только один русский перевод, проект сохраняет одну русскую версию и исходную базу. Количество вариантов не должно быть важнее качества, происхождения и проверяемости. +Если надёжно получается только один русский перевод, проект сохраняет один. Количество вариантов не важнее качества и provenance. ## Эквивалентные английские сериализации -D1.1 доказал, что `data/bd2.json` и `data/db.json`: +D1.1 доказал: -- имеют разные размеры и raw SHA-256; -- после JSON-разбора и canonical normalization полностью совпадают; -- содержат одинаковые 4 086 записей, порядок, поля и значения; -- являются двумя физическими сериализациями одного logical dataset `source-divinity-code-en`. +- `data/bd2.json` и `data/db.json` имеют разные raw bytes/hashes; +- parsed/canonical JSON полностью одинаков; +- оба файла содержат одинаковые 4 086 ordered records; +- это две physical serializations одного `source-divinity-code-en`. -D1.2 утвердил физические IDs: +D1.2 утвердил: ```text -source-divinity-code-en-bd2 -> data/bd2.json -source-divinity-code-en-db -> data/db.json +source-divinity-code-en-bd2 -> data/bd2.json +source-divinity-code-en-db -> data/db.json ``` -`source-divinity-code-en-bd2` выбран canonical physical serialization через project-governance decision. Это не доказывает, что `data/bd2.json` является историческим оригиналом или подтверждённой исходной редакцией. - -`source-divinity-code-en-db` остаётся retained equivalent compatibility serialization. Оба файла сохраняются без изменений. Фактическое удаление, переименование или перенос не одобрены и могут выполняться только отдельным будущим migration-патчем с rollback. +`data/bd2.json` — canonical maintained serialization по project-governance decision. Это не доказательство historical original. `data/db.json` остаётся retained equivalent compatibility serialization. -## Идентичность каждого варианта - -Каждый самостоятельный набор или перевод должен иметь собственный стабильный logical dataset ID и метаданные: +```text +migration: planned-not-executed +remove_or_rename_approved: false +``` -- источник и книга; -- исходный язык; -- язык перевода; -- название варианта; -- версия или дата; -- исходный commit или SHA-256; -- способ получения; -- переводчик, инструмент или модель; -- шаблон инструкции для преобразования; -- правила редакторской обработки; -- результаты автоматической проверки; -- статус человеческой проверки; +## Идентичность варианта + +Каждый самостоятельный dataset/translation должен иметь: + +- stable logical dataset ID; +- stable physical file ID; +- source/book и language; +- version/date; +- source commit/hash; +- получение и transformation method; +- translator/tool/model; +- prompt-template version; +- editorial rules; +- validation result; +- human-review status; - связь с предыдущей версией. -Каждый физический файл получает отдельный physical file ID, path, exact raw hash и роль. Несколько физически разных файлов могут относиться к одному logical dataset только когда их semantic/canonical identity доказана. +Несколько physical files относятся к одному logical dataset только при доказанной canonical identity. -Примеры будущих идентификаторов: +Примеры будущих IDs: ```text ru-deepseek-candidate-v1 @@ -76,106 +76,103 @@ ru-alternative-reviewed-v2 Новый перевод не может использовать ID `ru-current-v1` и не может перезаписывать его файл. -## Использование DeepSeek API +## D1.3 quality baseline + +D1.3 проверил `source-divinity-code-en` и `ru-current-v1` read-only аудитом. + +Подтверждено: -DeepSeek рассматривается как один из возможных инструментов для создания дополнительного кандидата перевода или расширения непереведённых полей. Результат API не считается автоматически правильным, опубликованным или богословски проверенным. +- по 4 086 unique ordered IDs; +- source/current IDs aligned; +- preserved-field differences: 0; +- structural errors: 0; +- warnings: 0; +- review instances: 5 022 in 5 aggregated groups. -### Где допускается запуск +Review findings не являются доказанными translation errors. Они могут пересекаться и требуют source/human review. Audit не меняет данные и не выбирает «лучший» перевод. -Перевод через API выполняется только: +## Использование DeepSeek API + +DeepSeek может создавать только отдельный candidate dataset или расширять непереведённые поля в отдельном кандидате. Результат не считается автоматически правильным, опубликованным или богословски проверенным. -- локальным служебным скриптом разработчика; или -- отдельным GitHub Actions workflow с ключом в GitHub Secrets. +### Допустимое место запуска -Статический сайт и установленная PWA не должны обращаться к переводческому API напрямую. Встраивание ключа в браузерный JavaScript неизбежно раскроет его пользователям. +- локальный служебный script; или +- отдельный GitHub Actions workflow с encrypted secret. + +Public site и PWA никогда не обращаются к paid translation API напрямую. ### Хранение ключа -Ключ API: +Ключ: - не записывается в Git; -- не помещается в `index.html`, `script.js`, `src/`, JSON-базы, патчноуты или документацию; -- локально передаётся через переменную окружения или игнорируемый `.env`; -- в GitHub Actions хранится только как encrypted secret; -- не выводится в логи, отчёты об ошибках и артефакты; -- при подозрении на утечку немедленно отзывается и заменяется. - -Рекомендуемое имя локальной переменной: +- не попадает в browser code, JSON, patchnotes, docs, screenshots, logs или artifacts; +- локально передаётся через environment variable/ignored `.env`; +- в Actions хранится только как encrypted secret; +- при подозрении на утечку отзывается. ```text DEEPSEEK_API_KEY ``` -В репозиторий может быть добавлен только безопасный `.env.example` без значения. +`.env.example` может содержать только пустое имя variable. -## Безопасный конвейер AI-перевода +## Безопасный AI pipeline -Экспериментальный перевод должен идти отдельными контролируемыми шагами: +1. выбрать registered source dataset ID и exact hash; +2. назначить новый candidate dataset ID; +3. определить разрешённые fields; +4. запретить изменение IDs, sources и Bible references; +5. обрабатывать resumable batches; +6. сохранять raw response отдельно от normalized candidate; +7. записывать model, date, parameters и prompt version; +8. проверять count, IDs, types и references; +9. формировать diff с `ru-current-v1`; +10. выполнять human review; +11. регистрировать candidate file/hashes после validation; +12. публиковать только отдельным data release. -1. выбрать exact registered source dataset ID и source hash; -2. назначить новый candidate dataset ID до начала генерации; -3. определить поля, которые разрешено переводить; -4. запретить модели изменять ID, источники и библейские ссылки; -5. обрабатывать данные небольшими возобновляемыми пакетами; -6. сохранять сырой ответ отдельно от нормализованного кандидата; -7. записывать модель, дату, параметры и версию prompt-шаблона; -8. повторно проверять число записей, ID, типы полей и ссылки; -9. формировать отчёт различий с `ru-current-v1`; -10. проводить человеческую выборочную и содержательную проверку; -11. регистрировать candidate file и hashes только после автоматической проверки; -12. публиковать только через отдельный data-релиз после утверждения. +Сбой не должен портить опубликованный файл. Script обязан иметь checkpoint/resume. -Сбой посередине не должен портить уже опубликованный файл. Скрипт обязан поддерживать checkpoint и продолжение с последнего подтверждённого пакета. +## Что нельзя автоматизировать без проверки -## Что нельзя поручать модели без проверки +- новые Bible references; +- усиление/изменение смысла; +- объединение книг в одну запись; +- удаление неоднозначности; +- замена source; +- изменение stable ID; +- выбор окончательного перевода. -AI-перевод нельзя автоматически использовать для: +## Future UI modes -- добавления новых библейских ссылок; -- усиления или изменения смысла толкования; -- объединения разных книг в одну запись; -- удаления неоднозначных мест; -- замены источника; -- изменения стабильного ID; -- принятия решения, какой перевод является окончательным. +Будущая архитектура может поддерживать: -Такие изменения требуют сверки с исходником и отдельного редакторского решения. +- выбор source dataset; +- выбор current/alternative translation; +- side-by-side comparison; +- combined search with visible source/variant. -## Сравнение вариантов - -Будущий интерфейс может позволить: - -- выбрать исходную базу; -- выбрать текущий или альтернативный перевод; -- открыть два варианта рядом; -- выполнить объединённый поиск с явной подписью источника и варианта. - -Переключатель не реализуется до завершения D1.3–D1.4, полного validation контракта и отдельного функционального релиза. Активная база `data/divinity_code_ru.json` / `ru-current-v1` остаётся единственным опубликованным runtime-набором. +Selector не реализуется до завершения D1.4, полного validation contract и отдельного functional release. `ru-current-v1` остаётся единственным runtime dataset. ## Критерии допуска нового перевода -Новый вариант можно считать кандидатом на публикацию только когда: - -- сохранён зарегистрированный исходный набор и его hash; -- создан новый logical dataset ID; -- создана отдельная физическая версия, не перезаписывающая текущую; -- число и идентичность записей объяснены; -- автоматические проверки прошли; -- сформирован отчёт различий; -- спорные формулировки и ссылки проверены человеком; -- права и происхождение данных не стали менее ясными; -- существует точный откат; -- изменение оформлено отдельным Pull Request, data-релизом и uNews-патчноутом. +- зарегистрирован exact source ID/hash; +- создан новый logical dataset ID и separate physical file; +- current dataset не перезаписан; +- count/identity объяснены; +- automatic checks прошли; +- diff report создан; +- disputed wording/references проверены человеком; +- provenance/rights не ухудшились; +- существует rollback; +- оформлены separate PR, data release и uNews patchnote. -## Текущее решение D1.2 +## Текущий этап -D1.2 дал однозначный ответ на текущие identity-вопросы: +D1.1 provenance, D1.2 registry и D1.3 quality audit завершены. -1. логический исходный набор: `source-divinity-code-en`; -2. текущий опубликованный локализованный набор: `ru-current-v1`; -3. canonical physical serialization: `source-divinity-code-en-bd2` / `data/bd2.json`; -4. retained equivalent serialization: `source-divinity-code-en-db` / `data/db.json`; -5. physical migration status: `planned-not-executed`; -6. существующие данные не удаляются и не переименовываются. +Следующий этап — **D1.4 two-book product architecture**. Он должен определить relationship двух книг, visible provenance, switching/combined/side-by-side UX, dataset-aware history/deep links, validation/cache/fallback и migration plan. Реализация selector требует отдельной functional phase. -Будущие альтернативные переводы и пользовательский selector ещё не реализованы. Следующий этап — D1.3 data-quality audit design. +D1.5 AI-assisted translation experiment начинается только после завершения D1.4. diff --git a/news/2026-07-23-udream-v23-8-10-data-quality-audit.md b/news/2026-07-23-udream-v23-8-10-data-quality-audit.md new file mode 100644 index 0000000..1a550ea --- /dev/null +++ b/news/2026-07-23-udream-v23-8-10-data-quality-audit.md @@ -0,0 +1,38 @@ +--- +type: audit +project: uDream +series: udream +title: Проверены все 4 086 записей без изменения базы +version: 23.8.10 +queued_at: 2026-07-23T07:10:00Z +repo_url: https://github.com/sunpole/udream +web_url: https://sunpole.github.io/udream/ +image: 2026-07-23-udream-v23-8-10-data-quality-audit.png +image_source: document-render +image_target: reports/data-quality-audit.md +image_commit: cd6aa539a418861108d58f2206050291642e7fcb +image_captured_at: 2026-07-23T07:09:49Z +--- + +# uDream 23.8.10: неразрушающий аудит качества данных + +Завершён D1.3 — зарегистрированные source/current datasets проверены детерминированным read-only аудитом без исправления или перезаписи данных. + +- Проверены logical datasets `source-divinity-code-en` и `ru-current-v1`. +- В каждом наборе подтверждены 4 086 unique ordered IDs `1–4086`. +- Source/current IDs полностью aligned. +- Поля `id`, `symbol`, `description`, `source` и `date_added` имеют 0 различий. +- Подтверждены ожидаемые differences: aliases 4 083, notes 4 086, tags 4 086. +- Structural gate прошёл: 0 errors и 0 warnings. +- Зафиксированы 5 022 human-review instances в пяти агрегированных группах. +- Эти 5 022 instances не являются 5 022 доказанными ошибками: группы пересекаются, shared aliases могут быть намеренными, а пустые source notes могут отражать структуру исходника. +- Добавлены deterministic reports `reports/data-quality-audit.json` и `reports/data-quality-audit.md`. +- Добавлен permanent `scripts/audit-data-quality.mjs` с режимом `--check`. +- GitHub Actions проверяет актуальность reports и structural gate. +- Ни один существующий data-файл, runtime, PWA, package metadata, saved version или archive не изменён. + +Изображение к публикации — новый реальный Chromium document-render точного GitHub report page. + +Короткий текст для Telegram: + +uDream 23.8.10 завершает D1.3: все 4 086 записей source/current datasets проверены детерминированным read-only аудитом. Structural gate прошёл с 0 errors и 0 warnings. Отчёт содержит 5 022 пересекающихся human-review instances — это кандидаты на проверку, а не доказанные ошибки. Базы, runtime и PWA не изменялись. diff --git a/news/2026-07-23-udream-v23-8-10-data-quality-audit.png b/news/2026-07-23-udream-v23-8-10-data-quality-audit.png new file mode 100644 index 0000000..c225b2a Binary files /dev/null and b/news/2026-07-23-udream-v23-8-10-data-quality-audit.png differ diff --git a/reports/data-quality-audit.json b/reports/data-quality-audit.json new file mode 100644 index 0000000..e9f2768 --- /dev/null +++ b/reports/data-quality-audit.json @@ -0,0 +1,330 @@ +{ + "schema_version": 1, + "audit_version": "23.8.10", + "registry_baseline": "23.8.9", + "registry_path": "data/datasets.json", + "registry_raw_sha256": "6da823d20eb5f6018da6f5fcd3a2c98f1ded3dd0e0f354bd06344a0c4742f338", + "deterministic": true, + "mutation_policy": "read-only", + "datasets": [ + { + "dataset_id": "source-divinity-code-en", + "physical_file_id": "source-divinity-code-en-bd2", + "path": "data/bd2.json", + "bytes": 2141655, + "raw_sha256": "814c5d33444160e6f1ab20278f9356090ec0e9cc04943cd14ad99d9038be6e28", + "canonical_json_sha256": "5ebe0d973f9cfd1c9db65a9d5abebe0ca16788261219299a710ed9fe78bb25d1", + "record_count": 4086, + "ordered_ids": true, + "unique_ids": true, + "id_min": 1, + "id_max": 4086, + "records_without_aliases": 2482, + "records_without_tags": 153, + "records_with_cross_reference_like_text": 1637, + "records_with_bible_reference_like_text": 2448, + "field_non_empty_counts": { + "id": 4086, + "symbol": 4086, + "aliases": 1604, + "description": 4086, + "source": 4086, + "date_added": 4086, + "tags": 3933, + "notes": 2901 + }, + "source_distribution": { + "DivinityCode": 4086 + }, + "date_distribution": { + "2026-05-24": 4086 + }, + "normalized_primary_symbol_count": 4086, + "normalized_alias_count": 1517 + }, + { + "dataset_id": "ru-current-v1", + "physical_file_id": "ru-current-v1-runtime", + "path": "data/divinity_code_ru.json", + "bytes": 4688773, + "raw_sha256": "1def80216e238b0c2a8640aaf1b4e95dd0669d5944a67f4e7c4421fad55a6e64", + "canonical_json_sha256": "c3682a50dec3303b7a100abefb27befd2dfd5d84e66d219252cf099b1a0fcd64", + "record_count": 4086, + "ordered_ids": true, + "unique_ids": true, + "id_min": 1, + "id_max": 4086, + "records_without_aliases": 2, + "records_without_tags": 0, + "records_with_cross_reference_like_text": 1637, + "records_with_bible_reference_like_text": 2453, + "field_non_empty_counts": { + "id": 4086, + "symbol": 4086, + "aliases": 4084, + "description": 4086, + "source": 4086, + "date_added": 4086, + "tags": 4086, + "notes": 4086 + }, + "source_distribution": { + "DivinityCode": 4086 + }, + "date_distribution": { + "2026-05-24": 4086 + }, + "normalized_primary_symbol_count": 4086, + "normalized_alias_count": 5429 + } + ], + "comparison": { + "source_dataset_id": "source-divinity-code-en", + "current_dataset_id": "ru-current-v1", + "source_record_count": 4086, + "current_record_count": 4086, + "ids_aligned": true, + "preserved_field_difference_counts": { + "id": 0, + "symbol": 0, + "description": 0, + "source": 0, + "date_added": 0 + }, + "changed_field_difference_counts": { + "aliases": 4083, + "notes": 4086, + "tags": 4086 + } + }, + "findings": [ + { + "severity": "review", + "rule_id": "alias_collision_across_records", + "dataset_id": "ru-current-v1", + "count": 854, + "sample_ids": [ + 3, + 4, + 5, + 8, + 10, + 13, + 14, + 19, + 23, + 24, + 25, + 26, + 27, + 28, + 35, + 36, + 37, + 38, + 39, + 40 + ], + "message": "A normalized alias points to more than one record and may be intentionally shared or ambiguous.", + "details": { + "sample_keys": [ + "black man", + "first nations peoples", + "foreigner", + "native/s", + "birth", + "under", + "underwater", + "seed", + "snake", + "boat" + ] + } + }, + { + "severity": "review", + "rule_id": "alias_matches_other_primary_symbol", + "dataset_id": "ru-current-v1", + "count": 1145, + "sample_ids": [ + 2, + 3, + 5, + 8, + 10, + 12, + 13, + 14, + 19, + 23, + 24, + 25, + 26, + 27, + 28, + 32, + 35, + 36, + 37, + 38 + ], + "message": "An alias matches another record's normalized primary symbol and requires routing review.", + "details": { + "sample_keys": [ + "bigfoot", + "black man", + "first nations peoples", + "foreigner", + "native/s", + "under", + "underwater", + "seed", + "snake", + "boat" + ] + } + }, + { + "severity": "review", + "rule_id": "alias_collision_across_records", + "dataset_id": "source-divinity-code-en", + "count": 693, + "sample_ids": [ + 3, + 4, + 5, + 8, + 10, + 13, + 14, + 23, + 24, + 25, + 26, + 27, + 28, + 35, + 36, + 37, + 38, + 39, + 40, + 52 + ], + "message": "A normalized alias points to more than one record and may be intentionally shared or ambiguous.", + "details": { + "sample_keys": [ + "black man", + "first nations peoples", + "foreigner", + "native/s", + "birth", + "under", + "underwater", + "seed", + "snake", + "boat" + ] + } + }, + { + "severity": "review", + "rule_id": "alias_matches_other_primary_symbol", + "dataset_id": "source-divinity-code-en", + "count": 1145, + "sample_ids": [ + 2, + 3, + 5, + 8, + 10, + 12, + 13, + 14, + 19, + 23, + 24, + 25, + 26, + 27, + 28, + 32, + 35, + 36, + 37, + 38 + ], + "message": "An alias matches another record's normalized primary symbol and requires routing review.", + "details": { + "sample_keys": [ + "bigfoot", + "black man", + "first nations peoples", + "foreigner", + "native/s", + "under", + "underwater", + "seed", + "snake", + "boat" + ] + } + }, + { + "severity": "review", + "rule_id": "notes_empty", + "dataset_id": "source-divinity-code-en", + "count": 1185, + "sample_ids": [ + 47, + 59, + 72, + 116, + 320, + 408, + 447, + 448, + 449, + 477, + 553, + 576, + 585, + 590, + 857, + 858, + 886, + 909, + 924, + 945 + ], + "message": "notes is empty or whitespace-only." + } + ], + "summary": { + "finding_instances_by_severity": { + "error": 0, + "warning": 0, + "review": 5022, + "info": 0 + }, + "finding_groups_by_severity": { + "error": 0, + "warning": 0, + "review": 5, + "info": 0 + }, + "finding_instances_by_rule": { + "alias_collision_across_records": 1547, + "alias_matches_other_primary_symbol": 2290, + "notes_empty": 1185 + }, + "structural_gate": "pass" + }, + "limitations": [ + "Heuristic warnings and review findings are not proven semantic errors.", + "The exact historical translation pipeline and complete human-review record remain unknown.", + "Cross-reference-like and biblical-reference-like text is counted but not automatically validated against source books.", + "The audit does not decide theological correctness, preferred wording or final translation quality.", + "No finding is corrected by this script; fixes require separate reviewed data patches." + ] +} diff --git a/reports/data-quality-audit.md b/reports/data-quality-audit.md new file mode 100644 index 0000000..eb91215 --- /dev/null +++ b/reports/data-quality-audit.md @@ -0,0 +1,111 @@ +# uDream data-quality audit + +Audit baseline: `23.8.10` +Registry baseline: `23.8.9` +Structural gate: **PASS** + +This report is generated deterministically by `scripts/audit-data-quality.mjs`. It does not modify data and does not treat heuristic findings as proven content errors. + +## Scope + +- Logical datasets audited: 2 +- Source/current record count: 4086 / 4086 +- Ordered ID range expected: 1–4086 +- Existing source/current files are read-only inputs. + +## Severity model + +| Severity | Meaning | CI behavior | +|---|---|---| +| `error` | Structural or identity contract violation | Fails audit gate | +| `warning` | Likely formatting/data hygiene problem | Reported; requires review | +| `review` | Ambiguity or content question not safely auto-fixable | Human/source review | +| `info` | Observation or statistics | Informational | + +## Summary + +| Severity | Finding instances | Finding groups | +|---|---:|---:| +| error | 0 | 0 | +| warning | 0 | 0 | +| review | 5022 | 5 | +| info | 0 | 0 | + +## Dataset metrics + +| Dataset | Path | Records | Ordered IDs | Unique IDs | No aliases | No tags | +|---|---|---:|---|---|---:|---:| +| `source-divinity-code-en` | `data/bd2.json` | 4086 | yes | yes | 2482 | 153 | +| `ru-current-v1` | `data/divinity_code_ru.json` | 4086 | yes | yes | 2 | 0 | + +## Source/current comparison + +- IDs aligned: **yes** +- Preserved-field differences: `{"id":0,"symbol":0,"description":0,"source":0,"date_added":0}` +- Allowed changed-field differences: `{"aliases":4083,"notes":4086,"tags":4086}` + +## Findings + +| Severity | Rule | Dataset | Count | Sample IDs | +|---|---|---|---:|---| +| review | `alias_collision_across_records` | `ru-current-v1` | 854 | 3, 4, 5, 8, 10, 13, 14, 19, 23, 24, 25, 26, 27, 28, 35, 36, 37, 38, 39, 40 | +| review | `alias_matches_other_primary_symbol` | `ru-current-v1` | 1145 | 2, 3, 5, 8, 10, 12, 13, 14, 19, 23, 24, 25, 26, 27, 28, 32, 35, 36, 37, 38 | +| review | `alias_collision_across_records` | `source-divinity-code-en` | 693 | 3, 4, 5, 8, 10, 13, 14, 23, 24, 25, 26, 27, 28, 35, 36, 37, 38, 39, 40, 52 | +| review | `alias_matches_other_primary_symbol` | `source-divinity-code-en` | 1145 | 2, 3, 5, 8, 10, 12, 13, 14, 19, 23, 24, 25, 26, 27, 28, 32, 35, 36, 37, 38 | +| review | `notes_empty` | `source-divinity-code-en` | 1185 | 47, 59, 72, 116, 320, 408, 447, 448, 449, 477, 553, 576, 585, 590, 857, 858, 886, 909, 924, 945 | + +### review: alias_collision_across_records + +A normalized alias points to more than one record and may be intentionally shared or ambiguous. + +Dataset/comparison: `ru-current-v1`. Count: **854**. Sample IDs: 3, 4, 5, 8, 10, 13, 14, 19, 23, 24, 25, 26, 27, 28, 35, 36, 37, 38, 39, 40. + +Details: `{"sample_keys":["black man","first nations peoples","foreigner","native/s","birth","under","underwater","seed","snake","boat"]}` + +### review: alias_matches_other_primary_symbol + +An alias matches another record's normalized primary symbol and requires routing review. + +Dataset/comparison: `ru-current-v1`. Count: **1145**. Sample IDs: 2, 3, 5, 8, 10, 12, 13, 14, 19, 23, 24, 25, 26, 27, 28, 32, 35, 36, 37, 38. + +Details: `{"sample_keys":["bigfoot","black man","first nations peoples","foreigner","native/s","under","underwater","seed","snake","boat"]}` + +### review: alias_collision_across_records + +A normalized alias points to more than one record and may be intentionally shared or ambiguous. + +Dataset/comparison: `source-divinity-code-en`. Count: **693**. Sample IDs: 3, 4, 5, 8, 10, 13, 14, 23, 24, 25, 26, 27, 28, 35, 36, 37, 38, 39, 40, 52. + +Details: `{"sample_keys":["black man","first nations peoples","foreigner","native/s","birth","under","underwater","seed","snake","boat"]}` + +### review: alias_matches_other_primary_symbol + +An alias matches another record's normalized primary symbol and requires routing review. + +Dataset/comparison: `source-divinity-code-en`. Count: **1145**. Sample IDs: 2, 3, 5, 8, 10, 12, 13, 14, 19, 23, 24, 25, 26, 27, 28, 32, 35, 36, 37, 38. + +Details: `{"sample_keys":["bigfoot","black man","first nations peoples","foreigner","native/s","under","underwater","seed","snake","boat"]}` + +### review: notes_empty + +notes is empty or whitespace-only. + +Dataset/comparison: `source-divinity-code-en`. Count: **1185**. Sample IDs: 47, 59, 72, 116, 320, 408, 447, 448, 449, 477, 553, 576, 585, 590, 857, 858, 886, 909, 924, 945. + +## Important limitations + +- Heuristic warnings and review findings are not proven semantic errors. +- The exact historical translation pipeline and complete human-review record remain unknown. +- Cross-reference-like and biblical-reference-like text is counted but not automatically validated against source books. +- The audit does not decide theological correctness, preferred wording or final translation quality. +- No finding is corrected by this script; fixes require separate reviewed data patches. + +## Reproduce + +```bash +node scripts/audit-data-quality.mjs +node scripts/audit-data-quality.mjs --check +``` + +Content corrections must be handled in later separate reviewed data patches. This audit never rewrites the source or current dataset. + diff --git a/scripts/audit-data-quality.mjs b/scripts/audit-data-quality.mjs new file mode 100644 index 0000000..2d1044f --- /dev/null +++ b/scripts/audit-data-quality.mjs @@ -0,0 +1,633 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; + +const ROOT = process.cwd(); +const REGISTRY_PATH = "data/datasets.json"; +const JSON_REPORT_PATH = "reports/data-quality-audit.json"; +const MARKDOWN_REPORT_PATH = "reports/data-quality-audit.md"; +const AUDIT_VERSION = "23.8.10"; +const REGISTRY_BASELINE = "23.8.9"; +const EXPECTED_RECORDS = 4_086; +const SAMPLE_LIMIT = 20; +const REQUIRED_FIELDS = { + id: "number", + symbol: "string", + aliases: "array", + description: "string", + source: "string", + date_added: "string", + tags: "array", + notes: "string", +}; +const TEXT_FIELDS = ["symbol", "description", "source", "date_added", "notes"]; +const PRESERVED_FIELDS = ["id", "symbol", "description", "source", "date_added"]; +const CHANGED_FIELDS = ["aliases", "notes", "tags"]; +const SEVERITY_ORDER = new Map([ + ["error", 0], + ["warning", 1], + ["review", 2], + ["info", 3], +]); +const CONTROL_CHARACTER_RE = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/; +const HTML_LIKE_RE = /<\/?[a-z][^>]*>/i; +const CROSS_REFERENCE_RE = /\b(?:see|refer(?:ence)?|см\.?|смотри|смотрите)\b/i; +const BIBLE_REFERENCE_RE = /\b(?:[1-3]\s*)?[A-Za-zА-Яа-яЁё]{2,}\.?\s+\d{1,3}:\d{1,3}(?:[-–]\d{1,3})?\b/u; +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; + +function fail(message) { + throw new Error(message); +} + +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function stableStringify(value) { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; + const keys = Object.keys(value).sort(); + return `{${keys.map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`; +} + +function normalizeText(value) { + return String(value ?? "") + .normalize("NFKC") + .trim() + .replace(/\s+/g, " ") + .toLocaleLowerCase("ru-RU"); +} + +function valueType(value) { + if (Array.isArray(value)) return "array"; + if (value === null) return "null"; + return typeof value; +} + +function isValidDate(value) { + if (!DATE_RE.test(value)) return false; + const [year, month, day] = value.split("-").map(Number); + const date = new Date(Date.UTC(year, month - 1, day)); + return date.getUTCFullYear() === year + && date.getUTCMonth() === month - 1 + && date.getUTCDate() === day; +} + +function increment(map, key, amount = 1) { + map.set(key, (map.get(key) || 0) + amount); +} + +function sortedObjectFromMap(map) { + return Object.fromEntries([...map.entries()].sort(([a], [b]) => String(a).localeCompare(String(b), "en"))); +} + +function uniqueSortedNumbers(values) { + return [...new Set(values)].sort((a, b) => a - b); +} + +function sample(values) { + return uniqueSortedNumbers(values).slice(0, SAMPLE_LIMIT); +} + +function createCollector(datasetId) { + const buckets = new Map(); + + function record(severity, ruleId, recordId, message) { + if (!SEVERITY_ORDER.has(severity)) fail(`Unknown severity: ${severity}`); + const key = `${severity}\u0000${ruleId}`; + if (!buckets.has(key)) { + buckets.set(key, { + severity, + rule_id: ruleId, + dataset_id: datasetId, + count: 0, + sample_ids: [], + message, + }); + } + const finding = buckets.get(key); + finding.count += 1; + if (Number.isInteger(recordId) && finding.sample_ids.length < SAMPLE_LIMIT) { + finding.sample_ids.push(recordId); + } + } + + function recordGroup(severity, ruleId, count, recordIds, message, details = undefined) { + if (!count) return; + const key = `${severity}\u0000${ruleId}`; + const finding = { + severity, + rule_id: ruleId, + dataset_id: datasetId, + count, + sample_ids: sample(recordIds), + message, + }; + if (details !== undefined) finding.details = details; + buckets.set(key, finding); + } + + function findings() { + return [...buckets.values()].sort((a, b) => { + return SEVERITY_ORDER.get(a.severity) - SEVERITY_ORDER.get(b.severity) + || a.rule_id.localeCompare(b.rule_id, "en") + || a.dataset_id.localeCompare(b.dataset_id, "en"); + }); + } + + return { record, recordGroup, findings }; +} + +async function readJson(relativePath) { + const raw = await readFile(path.join(ROOT, relativePath)); + let parsed; + try { + parsed = JSON.parse(raw.toString("utf8")); + } catch (error) { + fail(`${relativePath}: invalid JSON (${error.message})`); + } + return { raw, parsed }; +} + +function inspectString(record, field, collector, metrics) { + const value = record[field]; + if (typeof value !== "string") return; + if (value !== value.trim()) collector.record("warning", `${field}_outer_whitespace`, record.id, `${field} has leading or trailing whitespace.`); + if (!value.trim()) { + const severity = field === "symbol" ? "error" : field === "description" || field === "source" ? "warning" : "review"; + collector.record(severity, `${field}_empty`, record.id, `${field} is empty or whitespace-only.`); + } + if (value.includes("\ufffd")) collector.record("warning", `${field}_replacement_character`, record.id, `${field} contains Unicode replacement characters.`); + if (CONTROL_CHARACTER_RE.test(value)) collector.record("warning", `${field}_control_character`, record.id, `${field} contains unexpected control characters.`); + if (HTML_LIKE_RE.test(value)) collector.record("review", `${field}_html_like`, record.id, `${field} contains HTML-like markup and requires source review.`); + if (value.length > 10_000) collector.record("review", `${field}_very_long`, record.id, `${field} is unusually long (>10,000 characters).`); + if (CROSS_REFERENCE_RE.test(value)) metrics.cross_reference_like_records.add(record.id); + if (BIBLE_REFERENCE_RE.test(value)) metrics.bible_reference_like_records.add(record.id); +} + +function inspectStringArray(record, field, collector, metrics) { + const value = record[field]; + if (!Array.isArray(value)) return; + const normalized = new Map(); + for (const [index, item] of value.entries()) { + if (typeof item !== "string") { + collector.record("error", `${field}_non_string_item`, record.id, `${field} contains a non-string item.`); + continue; + } + if (!item.trim()) collector.record("warning", `${field}_empty_item`, record.id, `${field} contains an empty item.`); + if (item !== item.trim()) collector.record("warning", `${field}_item_outer_whitespace`, record.id, `${field} contains an item with outer whitespace.`); + if (item.includes("\ufffd")) collector.record("warning", `${field}_replacement_character`, record.id, `${field} contains Unicode replacement characters.`); + if (CONTROL_CHARACTER_RE.test(item)) collector.record("warning", `${field}_control_character`, record.id, `${field} contains unexpected control characters.`); + if (HTML_LIKE_RE.test(item)) collector.record("review", `${field}_html_like`, record.id, `${field} contains HTML-like markup.`); + const key = normalizeText(item); + if (key) { + if (normalized.has(key)) collector.record("warning", `${field}_duplicate_inside_record`, record.id, `${field} contains duplicate normalized values.`); + else normalized.set(key, index); + } + } + if (field === "aliases" && value.length === 0) metrics.records_without_aliases += 1; + if (field === "tags" && value.length === 0) metrics.records_without_tags += 1; +} + +function inspectDataset(datasetId, physicalId, relativePath, raw, records) { + const collector = createCollector(datasetId); + const metrics = { + record_count: Array.isArray(records) ? records.length : 0, + ordered_ids: true, + unique_ids: true, + id_min: null, + id_max: null, + records_without_aliases: 0, + records_without_tags: 0, + cross_reference_like_records: new Set(), + bible_reference_like_records: new Set(), + source_distribution: new Map(), + date_distribution: new Map(), + field_non_empty_counts: Object.fromEntries(Object.keys(REQUIRED_FIELDS).map((field) => [field, 0])), + }; + + if (!Array.isArray(records)) { + collector.recordGroup("error", "dataset_not_array", 1, [], "Dataset root is not an array."); + return { collector, metrics, symbolIndex: new Map(), aliasIndex: new Map(), signatureIndex: new Map() }; + } + + if (records.length !== EXPECTED_RECORDS) { + collector.recordGroup("error", "record_count_mismatch", 1, [], `Expected ${EXPECTED_RECORDS} records, found ${records.length}.`, { + expected: EXPECTED_RECORDS, + actual: records.length, + }); + } + + const seenIds = new Set(); + const symbolIndex = new Map(); + const aliasIndex = new Map(); + const signatureIndex = new Map(); + const ids = []; + + for (const [index, record] of records.entries()) { + if (!record || typeof record !== "object" || Array.isArray(record)) { + collector.record("error", "record_not_object", index + 1, "Record is not an object."); + metrics.ordered_ids = false; + continue; + } + + const id = record.id; + if (!Number.isInteger(id)) { + collector.record("error", "id_not_integer", index + 1, "Record ID is not an integer."); + metrics.ordered_ids = false; + } else { + ids.push(id); + if (seenIds.has(id)) { + collector.record("error", "id_duplicate", id, "Record ID is duplicated."); + metrics.unique_ids = false; + } + seenIds.add(id); + if (id !== index + 1) { + collector.record("error", "id_not_ordered", id, "Record ID does not match its one-based array position."); + metrics.ordered_ids = false; + } + } + + for (const [field, expectedType] of Object.entries(REQUIRED_FIELDS)) { + const actualType = valueType(record[field]); + if (actualType !== expectedType) { + collector.record("error", `${field}_wrong_type`, Number.isInteger(id) ? id : index + 1, `${field} must be ${expectedType}, found ${actualType}.`); + } else if ( + (typeof record[field] === "string" && record[field].trim()) + || (Array.isArray(record[field]) && record[field].length) + || field === "id" + ) { + metrics.field_non_empty_counts[field] += 1; + } + } + + for (const field of TEXT_FIELDS) inspectString(record, field, collector, metrics); + inspectStringArray(record, "aliases", collector, metrics); + inspectStringArray(record, "tags", collector, metrics); + + if (typeof record.date_added === "string") { + if (!isValidDate(record.date_added)) collector.record("warning", "date_added_invalid", id, "date_added is not a valid YYYY-MM-DD calendar date."); + else increment(metrics.date_distribution, record.date_added); + } + if (typeof record.source === "string") increment(metrics.source_distribution, record.source.trim() || ""); + + if (typeof record.symbol === "string") { + const symbolKey = normalizeText(record.symbol); + if (symbolKey) { + if (!symbolIndex.has(symbolKey)) symbolIndex.set(symbolKey, []); + symbolIndex.get(symbolKey).push(id); + } + } + + if (Array.isArray(record.aliases)) { + const symbolKey = normalizeText(record.symbol); + for (const alias of record.aliases) { + if (typeof alias !== "string") continue; + const aliasKey = normalizeText(alias); + if (!aliasKey) continue; + if (!aliasIndex.has(aliasKey)) aliasIndex.set(aliasKey, []); + aliasIndex.get(aliasKey).push(id); + if (aliasKey === symbolKey) collector.record("review", "alias_equals_primary_symbol", id, "An alias normalizes to the primary symbol."); + } + } + + const signature = sha256(Buffer.from(stableStringify({ + symbol: record.symbol, + aliases: record.aliases, + description: record.description, + source: record.source, + date_added: record.date_added, + tags: record.tags, + notes: record.notes, + }), "utf8")); + if (!signatureIndex.has(signature)) signatureIndex.set(signature, []); + signatureIndex.get(signature).push(id); + } + + if (ids.length) { + metrics.id_min = Math.min(...ids); + metrics.id_max = Math.max(...ids); + } + + const duplicateSymbols = [...symbolIndex.entries()].filter(([, recordIds]) => new Set(recordIds).size > 1); + collector.recordGroup( + "review", + "duplicate_normalized_symbol_across_records", + duplicateSymbols.length, + duplicateSymbols.flatMap(([, recordIds]) => recordIds), + "Multiple records share the same normalized primary symbol and require content/source review.", + { sample_keys: duplicateSymbols.slice(0, 10).map(([key]) => key) }, + ); + + const aliasCollisions = [...aliasIndex.entries()].filter(([, recordIds]) => new Set(recordIds).size > 1); + collector.recordGroup( + "review", + "alias_collision_across_records", + aliasCollisions.length, + aliasCollisions.flatMap(([, recordIds]) => recordIds), + "A normalized alias points to more than one record and may be intentionally shared or ambiguous.", + { sample_keys: aliasCollisions.slice(0, 10).map(([key]) => key) }, + ); + + const symbolAliasCollisions = []; + for (const [key, aliasIds] of aliasIndex.entries()) { + const symbolIds = symbolIndex.get(key) || []; + const combined = new Set([...aliasIds, ...symbolIds]); + if (symbolIds.length && combined.size > 1) symbolAliasCollisions.push([key, [...combined]]); + } + collector.recordGroup( + "review", + "alias_matches_other_primary_symbol", + symbolAliasCollisions.length, + symbolAliasCollisions.flatMap(([, recordIds]) => recordIds), + "An alias matches another record's normalized primary symbol and requires routing review.", + { sample_keys: symbolAliasCollisions.slice(0, 10).map(([key]) => key) }, + ); + + const duplicateContentGroups = [...signatureIndex.values()].filter((recordIds) => new Set(recordIds).size > 1); + collector.recordGroup( + "review", + "duplicate_record_content_except_id", + duplicateContentGroups.length, + duplicateContentGroups.flat(), + "Multiple records have identical content except for ID and may be intentional redirects or duplicates.", + ); + + const normalizedMetrics = { + dataset_id: datasetId, + physical_file_id: physicalId, + path: relativePath, + bytes: raw.length, + raw_sha256: sha256(raw), + canonical_json_sha256: sha256(Buffer.from(stableStringify(records), "utf8")), + record_count: metrics.record_count, + ordered_ids: metrics.ordered_ids, + unique_ids: metrics.unique_ids, + id_min: metrics.id_min, + id_max: metrics.id_max, + records_without_aliases: metrics.records_without_aliases, + records_without_tags: metrics.records_without_tags, + records_with_cross_reference_like_text: metrics.cross_reference_like_records.size, + records_with_bible_reference_like_text: metrics.bible_reference_like_records.size, + field_non_empty_counts: metrics.field_non_empty_counts, + source_distribution: sortedObjectFromMap(metrics.source_distribution), + date_distribution: sortedObjectFromMap(metrics.date_distribution), + normalized_primary_symbol_count: symbolIndex.size, + normalized_alias_count: aliasIndex.size, + }; + + return { collector, metrics: normalizedMetrics, symbolIndex, aliasIndex, signatureIndex }; +} + +function compareDatasets(sourceDatasetId, sourceRecords, currentDatasetId, currentRecords) { + const collector = createCollector(`${sourceDatasetId}->${currentDatasetId}`); + const metrics = { + source_dataset_id: sourceDatasetId, + current_dataset_id: currentDatasetId, + source_record_count: Array.isArray(sourceRecords) ? sourceRecords.length : 0, + current_record_count: Array.isArray(currentRecords) ? currentRecords.length : 0, + ids_aligned: true, + preserved_field_difference_counts: Object.fromEntries(PRESERVED_FIELDS.map((field) => [field, 0])), + changed_field_difference_counts: Object.fromEntries(CHANGED_FIELDS.map((field) => [field, 0])), + }; + + if (!Array.isArray(sourceRecords) || !Array.isArray(currentRecords)) { + collector.recordGroup("error", "comparison_dataset_not_array", 1, [], "Source/current comparison requires JSON arrays."); + return { metrics, findings: collector.findings() }; + } + + if (sourceRecords.length !== currentRecords.length) { + collector.recordGroup("error", "comparison_record_count_mismatch", 1, [], "Source and current datasets have different record counts.", { + source: sourceRecords.length, + current: currentRecords.length, + }); + metrics.ids_aligned = false; + } + + const maxLength = Math.max(sourceRecords.length, currentRecords.length); + for (let index = 0; index < maxLength; index += 1) { + const source = sourceRecords[index]; + const current = currentRecords[index]; + const sampleId = Number.isInteger(current?.id) ? current.id : Number.isInteger(source?.id) ? source.id : index + 1; + if (!source || !current || source.id !== current.id) { + collector.record("error", "comparison_id_alignment", sampleId, "Source/current records are missing or have different IDs at the same position."); + metrics.ids_aligned = false; + continue; + } + for (const field of PRESERVED_FIELDS) { + if (stableStringify(source[field]) !== stableStringify(current[field])) { + metrics.preserved_field_difference_counts[field] += 1; + collector.record("error", `preserved_field_changed_${field}`, current.id, `Preserved field ${field} differs between source and current datasets.`); + } + } + for (const field of CHANGED_FIELDS) { + if (stableStringify(source[field]) !== stableStringify(current[field])) { + metrics.changed_field_difference_counts[field] += 1; + } + } + } + + return { metrics, findings: collector.findings() }; +} + +function summarizeFindings(findings) { + const bySeverity = new Map([...SEVERITY_ORDER.keys()].map((severity) => [severity, 0])); + const groupsBySeverity = new Map([...SEVERITY_ORDER.keys()].map((severity) => [severity, 0])); + const byRule = new Map(); + for (const finding of findings) { + increment(bySeverity, finding.severity, finding.count); + increment(groupsBySeverity, finding.severity, 1); + increment(byRule, finding.rule_id, finding.count); + } + return { + finding_instances_by_severity: Object.fromEntries(bySeverity), + finding_groups_by_severity: Object.fromEntries(groupsBySeverity), + finding_instances_by_rule: sortedObjectFromMap(byRule), + structural_gate: (bySeverity.get("error") || 0) === 0 ? "pass" : "fail", + }; +} + +function buildMarkdown(report) { + const lines = []; + lines.push("# uDream data-quality audit", ""); + lines.push(`Audit baseline: \`${report.audit_version}\``); + lines.push(`Registry baseline: \`${report.registry_baseline}\``); + lines.push(`Structural gate: **${report.summary.structural_gate.toUpperCase()}**`, ""); + lines.push("This report is generated deterministically by `scripts/audit-data-quality.mjs`. It does not modify data and does not treat heuristic findings as proven content errors.", ""); + + lines.push("## Scope", ""); + lines.push(`- Logical datasets audited: ${report.datasets.length}`); + lines.push(`- Source/current record count: ${report.comparison.source_record_count} / ${report.comparison.current_record_count}`); + lines.push(`- Ordered ID range expected: 1–${EXPECTED_RECORDS}`); + lines.push("- Existing source/current files are read-only inputs.", ""); + + lines.push("## Severity model", ""); + lines.push("| Severity | Meaning | CI behavior |", "|---|---|---|"); + lines.push("| `error` | Structural or identity contract violation | Fails audit gate |"); + lines.push("| `warning` | Likely formatting/data hygiene problem | Reported; requires review |"); + lines.push("| `review` | Ambiguity or content question not safely auto-fixable | Human/source review |"); + lines.push("| `info` | Observation or statistics | Informational |", ""); + + lines.push("## Summary", ""); + lines.push("| Severity | Finding instances | Finding groups |", "|---|---:|---:|"); + for (const severity of ["error", "warning", "review", "info"]) { + lines.push(`| ${severity} | ${report.summary.finding_instances_by_severity[severity]} | ${report.summary.finding_groups_by_severity[severity]} |`); + } + lines.push(""); + + lines.push("## Dataset metrics", ""); + lines.push("| Dataset | Path | Records | Ordered IDs | Unique IDs | No aliases | No tags |", "|---|---|---:|---|---|---:|---:|"); + for (const dataset of report.datasets) { + lines.push(`| \`${dataset.dataset_id}\` | \`${dataset.path}\` | ${dataset.record_count} | ${dataset.ordered_ids ? "yes" : "no"} | ${dataset.unique_ids ? "yes" : "no"} | ${dataset.records_without_aliases} | ${dataset.records_without_tags} |`); + } + lines.push(""); + + lines.push("## Source/current comparison", ""); + lines.push(`- IDs aligned: **${report.comparison.ids_aligned ? "yes" : "no"}**`); + lines.push(`- Preserved-field differences: \`${JSON.stringify(report.comparison.preserved_field_difference_counts)}\``); + lines.push(`- Allowed changed-field differences: \`${JSON.stringify(report.comparison.changed_field_difference_counts)}\``, ""); + + lines.push("## Findings", ""); + if (!report.findings.length) { + lines.push("No findings.", ""); + } else { + lines.push("| Severity | Rule | Dataset | Count | Sample IDs |", "|---|---|---|---:|---|"); + for (const finding of report.findings) { + const samples = finding.sample_ids.length ? finding.sample_ids.join(", ") : "—"; + lines.push(`| ${finding.severity} | \`${finding.rule_id}\` | \`${finding.dataset_id}\` | ${finding.count} | ${samples} |`); + } + lines.push(""); + for (const finding of report.findings) { + lines.push(`### ${finding.severity}: ${finding.rule_id}`, ""); + lines.push(finding.message, ""); + lines.push(`Dataset/comparison: \`${finding.dataset_id}\`. Count: **${finding.count}**. Sample IDs: ${finding.sample_ids.length ? finding.sample_ids.join(", ") : "none"}.`, ""); + if (finding.details) lines.push(`Details: \`${JSON.stringify(finding.details)}\``, ""); + } + } + + lines.push("## Important limitations", ""); + for (const limitation of report.limitations) lines.push(`- ${limitation}`); + lines.push(""); + + lines.push("## Reproduce", "", "```bash", "node scripts/audit-data-quality.mjs", "node scripts/audit-data-quality.mjs --check", "```", ""); + lines.push("Content corrections must be handled in later separate reviewed data patches. This audit never rewrites the source or current dataset.", ""); + return `${lines.join("\n")}\n`; +} + +async function generateReport() { + const { raw: registryRaw, parsed: registry } = await readJson(REGISTRY_PATH); + if (!registry || !Array.isArray(registry.logical_datasets) || !Array.isArray(registry.physical_files)) { + fail("Dataset registry shape is invalid"); + } + + const physicalById = new Map(registry.physical_files.map((entry) => [entry.id, entry])); + const logicalById = new Map(registry.logical_datasets.map((entry) => [entry.id, entry])); + const datasetIds = ["source-divinity-code-en", "ru-current-v1"]; + const inspected = []; + const recordsByDataset = new Map(); + + for (const datasetId of datasetIds) { + const logical = logicalById.get(datasetId); + if (!logical) fail(`Registry is missing logical dataset ${datasetId}`); + const physical = physicalById.get(logical.canonical_physical_file_id); + if (!physical) fail(`Registry is missing canonical physical file for ${datasetId}`); + const { raw, parsed } = await readJson(physical.path); + const result = inspectDataset(datasetId, physical.id, physical.path, raw, parsed); + inspected.push({ ...result.metrics, findings: result.collector.findings() }); + recordsByDataset.set(datasetId, parsed); + } + + const comparisonResult = compareDatasets( + "source-divinity-code-en", + recordsByDataset.get("source-divinity-code-en"), + "ru-current-v1", + recordsByDataset.get("ru-current-v1"), + ); + + const findings = [ + ...inspected.flatMap((dataset) => dataset.findings), + ...comparisonResult.findings, + ].sort((a, b) => { + return SEVERITY_ORDER.get(a.severity) - SEVERITY_ORDER.get(b.severity) + || a.dataset_id.localeCompare(b.dataset_id, "en") + || a.rule_id.localeCompare(b.rule_id, "en"); + }); + + const datasets = inspected.map(({ findings: _findings, ...metrics }) => metrics); + const report = { + schema_version: 1, + audit_version: AUDIT_VERSION, + registry_baseline: REGISTRY_BASELINE, + registry_path: REGISTRY_PATH, + registry_raw_sha256: sha256(registryRaw), + deterministic: true, + mutation_policy: "read-only", + datasets, + comparison: comparisonResult.metrics, + findings, + summary: summarizeFindings(findings), + limitations: [ + "Heuristic warnings and review findings are not proven semantic errors.", + "The exact historical translation pipeline and complete human-review record remain unknown.", + "Cross-reference-like and biblical-reference-like text is counted but not automatically validated against source books.", + "The audit does not decide theological correctness, preferred wording or final translation quality.", + "No finding is corrected by this script; fixes require separate reviewed data patches.", + ], + }; + + const json = `${JSON.stringify(report, null, 2)}\n`; + const markdown = buildMarkdown(report); + return { report, json, markdown }; +} + +async function checkReport(expected, relativePath) { + let actual; + try { + actual = await readFile(path.join(ROOT, relativePath), "utf8"); + } catch (error) { + fail(`${relativePath} is missing (${error.message})`); + } + if (actual !== expected) fail(`${relativePath} is stale; run node scripts/audit-data-quality.mjs`); +} + +async function main() { + const checkOnly = process.argv.includes("--check"); + const { report, json, markdown } = await generateReport(); + + if (report.summary.structural_gate !== "pass") { + fail(`Structural audit gate failed with ${report.summary.finding_instances_by_severity.error} error finding instance(s)`); + } + + if (checkOnly) { + await checkReport(json, JSON_REPORT_PATH); + await checkReport(markdown, MARKDOWN_REPORT_PATH); + console.log( + `Data-quality audit reports are current: ${report.datasets.length} datasets, ` + + `${report.datasets[0].record_count} source records, ` + + `${report.summary.finding_instances_by_severity.warning} warning, ` + + `${report.summary.finding_instances_by_severity.review} review instance(s).`, + ); + return; + } + + await mkdir(path.join(ROOT, "reports"), { recursive: true }); + await writeFile(path.join(ROOT, JSON_REPORT_PATH), json, "utf8"); + await writeFile(path.join(ROOT, MARKDOWN_REPORT_PATH), markdown, "utf8"); + console.log( + `Generated ${JSON_REPORT_PATH} and ${MARKDOWN_REPORT_PATH}: ` + + `${report.summary.finding_instances_by_severity.error} error, ` + + `${report.summary.finding_instances_by_severity.warning} warning, ` + + `${report.summary.finding_instances_by_severity.review} review instance(s).`, + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + main().catch((error) => { + console.error(`Data-quality audit failed: ${error.message}`); + process.exit(1); + }); +}